From 5510374421260dd844cfaceff74b4288270b5ad2 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 8 Jan 2025 08:07:56 +0100 Subject: [PATCH 01/52] use books dataset to ensure scoring consistency (#119676) --- .../src/main/resources/scoring.csv-spec | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index 9d3526982f9ef..f1fe4ce5484dc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -92,26 +92,26 @@ testMultiValuedFieldWithConjunctionWithScore required_capability: match_function required_capability: metadata_score -from employees metadata _score -| where match(job_positions, "Data Scientist") and match(job_positions, "Support Engineer") -| keep emp_no, first_name, last_name, job_positions, _score; +from books metadata _score +| where match(author, "Keith Faulkner") and match(author, "Rory Tyger") +| keep book_no, title, author, _score; -emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword | _score:double -10043 | Yishay | Tzvieli | [Data Scientist, Python Developer, Support Engineer] | 5.233309745788574 +book_no:keyword | title:text | author:text | _score:double +6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 8.5822172164917 ; testMatchAndQueryStringFunctionsWithScore required_capability: match_function required_capability: metadata_score -from employees metadata _score -| where match(job_positions, "Data Scientist") and qstr("job_positions: (Support Engineer) and gender: F") -| keep emp_no, first_name, last_name, job_positions, _score; +from books metadata _score +| where match(author, "Keith Faulkner") and qstr("author:Rory or author:Beverlie") +| keep book_no, title, author, _score; ignoreOrder:true -emp_no:integer | first_name:keyword | last_name:keyword | job_positions:keyword | _score:double -10041 | Uri | Lenart | [Data Scientist, Head Human Resources, Internship, Senior Team Lead] | 3.509873867034912 -10043 | Yishay | Tzvieli | [Data Scientist, Python Developer, Support Engineer] | 5.233309745788574 +book_no:keyword | title:text | author:text | _score:double +3535 | Rainbow's End: A Magical Story and Moneybox | [Beverlie Manson, Keith Faulkner] | 6.5579609870910645 +6151 | Pop! Went Another Balloon: A Magical Counting Storybook (Magical Counting Storybooks) | [Keith Faulkner, Rory Tyger] | 5.975414276123047 ; multipleWhereWithMatchScoringNoSort From 17353fa2c1057adfc248693dd7b39dfbc417ffe2 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 8 Jan 2025 02:44:25 -0500 Subject: [PATCH 02/52] Mute HighlightBuilderTests.testInvalidMaxAnalyzedOffset (#119725) --- .../search/fetch/subphase/highlight/HighlightBuilderTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 62098faddb618..0f73c367ff2ef 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -574,6 +574,7 @@ public void testPreTagsWithoutPostTags() throws IOException { assertEquals("pre_tags are set but post_tags are not set", e.getCause().getCause().getMessage()); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/119723") public void testInvalidMaxAnalyzedOffset() throws IOException { XContentParseException e = expectParseThrows( XContentParseException.class, From 9fdfc79fad8690b2ff24939c37c21701db12e021 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 8 Jan 2025 09:07:52 +0100 Subject: [PATCH 03/52] Simplify test class hierarchy in old-repository-versions-compatibility (#119670) The test method does not need to be repeated in every test class, and the two base test classes can be abstract. Meanwhile I also corrected javadocs on DocValueOnlyFieldsIT that were outdated. --- .../oldrepos/AbstractUpgradeCompatibilityTestCase.java | 8 +++++++- .../oldrepos/archiveindex/ArchiveIndexTestCase.java | 6 +++--- .../oldrepos/archiveindex/RestoreFromVersion5IT.java | 6 +----- .../oldrepos/archiveindex/RestoreFromVersion6IT.java | 6 +----- .../oldrepos/searchablesnapshot/MountFromVersion5IT.java | 6 +----- .../oldrepos/searchablesnapshot/MountFromVersion6IT.java | 6 +----- .../searchablesnapshot/SearchableSnapshotTestCase.java | 6 +++--- .../org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java | 2 +- 8 files changed, 18 insertions(+), 28 deletions(-) diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java index 4ff2b80aa29cc..83473e7b990d4 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java @@ -73,9 +73,11 @@ public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCas private static boolean upgradeFailed = false; private final Version clusterVersion; + private final String indexCreatedVersion; - public AbstractUpgradeCompatibilityTestCase(@Name("cluster") Version clusterVersion) { + public AbstractUpgradeCompatibilityTestCase(@Name("cluster") Version clusterVersion, String indexCreatedVersion) { this.clusterVersion = clusterVersion; + this.indexCreatedVersion = indexCreatedVersion; } @ParametersFactory @@ -208,4 +210,8 @@ private static void unzip(Path zipFilePath, Path outputDir) throws IOException { } } } + + public final void testArchiveIndex() throws Exception { + verifyCompatibility(indexCreatedVersion); + } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java index 17bdb76e0eae5..fb6f0c4140b62 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java @@ -25,14 +25,14 @@ * when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index * is successfully restored. */ -public class ArchiveIndexTestCase extends AbstractUpgradeCompatibilityTestCase { +abstract class ArchiveIndexTestCase extends AbstractUpgradeCompatibilityTestCase { static { clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); } - public ArchiveIndexTestCase(Version version) { - super(version); + protected ArchiveIndexTestCase(Version version, String indexCreatedVersion) { + super(version, indexCreatedVersion); } /** diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java index 9f62d65592a37..8606d1a467a29 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java @@ -12,10 +12,6 @@ public class RestoreFromVersion5IT extends ArchiveIndexTestCase { public RestoreFromVersion5IT(Version version) { - super(version); - } - - public void testArchiveIndex() throws Exception { - verifyCompatibility("5"); + super(version, "5"); } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java index b3cca45c205f6..ae75630f9e636 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java @@ -12,10 +12,6 @@ public class RestoreFromVersion6IT extends ArchiveIndexTestCase { public RestoreFromVersion6IT(Version version) { - super(version); - } - - public void testArchiveIndex() throws Exception { - verifyCompatibility("6"); + super(version, "6"); } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java index 3e371b5128b6a..49ac218bfd646 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java @@ -12,10 +12,6 @@ public class MountFromVersion5IT extends SearchableSnapshotTestCase { public MountFromVersion5IT(Version version) { - super(version); - } - - public void testSearchableSnapshot() throws Exception { - verifyCompatibility("5"); + super(version, "5"); } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java index 29b81fe595e5f..10b73b42df6b3 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java @@ -12,10 +12,6 @@ public class MountFromVersion6IT extends SearchableSnapshotTestCase { public MountFromVersion6IT(Version version) { - super(version); - } - - public void testSearchableSnapshot() throws Exception { - verifyCompatibility("6"); + super(version, "6"); } } diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java index 08a5db2111904..ddc5c4a70f891 100644 --- a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java @@ -21,14 +21,14 @@ * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version. * Test methods are executed after each upgrade. */ -public class SearchableSnapshotTestCase extends AbstractUpgradeCompatibilityTestCase { +abstract class SearchableSnapshotTestCase extends AbstractUpgradeCompatibilityTestCase { static { clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); } - public SearchableSnapshotTestCase(Version version) { - super(version); + protected SearchableSnapshotTestCase(Version version, String indexCreatedVersion) { + super(version, indexCreatedVersion); } /** diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java index 968262448c87e..8cf58f0c2c082 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java @@ -30,7 +30,7 @@ import java.io.IOException; /** - * Tests doc-value-based searches against indices imported from clusters older than N-1. + * Tests doc-value-based searches against archive indices, imported from clusters older than N-2. * We reuse the YAML tests in search/390_doc_values_search.yml but have to do the setup * manually here as the setup is done on the old cluster for which we have to use the * low-level REST client instead of the YAML set up that only knows how to talk to From c88eef308c9a32f80fb5f86025bd962d2f7a8c1a Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 8 Jan 2025 09:59:26 +0100 Subject: [PATCH 04/52] Restrict Connector APIs to manage/monitor_connector privileges (#119389) * [Connector API] Use monitor/manage_connector privilege * Update docs/changelog/119389.yaml * Remove index-level permissions from sync job actions * Don't keep client in class instance variable --------- Co-authored-by: Elastic Machine --- docs/changelog/119389.yaml | 5 + x-pack/plugin/ent-search/qa/rest/roles.yml | 8 +- .../connector/ConnectorIndexService.java | 242 ++++++++++-------- .../action/ConnectorActionRequest.java | 18 +- .../action/DeleteConnectorAction.java | 11 +- .../connector/action/GetConnectorAction.java | 2 +- .../connector/action/ListConnectorAction.java | 2 +- .../connector/action/PostConnectorAction.java | 2 +- .../connector/action/PutConnectorAction.java | 5 +- .../UpdateConnectorActiveFilteringAction.java | 2 +- .../action/UpdateConnectorApiKeyIdAction.java | 2 +- .../UpdateConnectorConfigurationAction.java | 2 +- .../action/UpdateConnectorErrorAction.java | 2 +- .../action/UpdateConnectorFeaturesAction.java | 2 +- .../UpdateConnectorFilteringAction.java | 2 +- ...ateConnectorFilteringValidationAction.java | 2 +- .../UpdateConnectorIndexNameAction.java | 2 +- .../action/UpdateConnectorLastSeenAction.java | 2 +- .../UpdateConnectorLastSyncStatsAction.java | 2 +- .../action/UpdateConnectorNameAction.java | 2 +- .../action/UpdateConnectorNativeAction.java | 2 +- .../action/UpdateConnectorPipelineAction.java | 2 +- .../UpdateConnectorSchedulingAction.java | 2 +- .../UpdateConnectorServiceTypeAction.java | 2 +- .../action/UpdateConnectorStatusAction.java | 2 +- .../syncjob/ConnectorSyncJobIndexService.java | 29 ++- .../action/CancelConnectorSyncJobAction.java | 2 +- .../action/CheckInConnectorSyncJobAction.java | 2 +- .../action/ClaimConnectorSyncJobAction.java | 2 +- .../action/ConnectorSyncJobActionRequest.java | 17 +- .../action/DeleteConnectorSyncJobAction.java | 2 +- .../action/GetConnectorSyncJobAction.java | 2 +- .../action/ListConnectorSyncJobsAction.java | 2 +- .../action/PostConnectorSyncJobAction.java | 11 +- .../UpdateConnectorSyncJobErrorAction.java | 2 +- ...eConnectorSyncJobIngestionStatsAction.java | 2 +- .../xpack/security/operator/Constants.java | 60 ++--- 37 files changed, 230 insertions(+), 230 deletions(-) create mode 100644 docs/changelog/119389.yaml diff --git a/docs/changelog/119389.yaml b/docs/changelog/119389.yaml new file mode 100644 index 0000000000000..267eaa345b9fd --- /dev/null +++ b/docs/changelog/119389.yaml @@ -0,0 +1,5 @@ +pr: 119389 +summary: Restrict Connector APIs to manage/monitor_connector privileges +area: Extract&Transform +type: feature +issues: [] diff --git a/x-pack/plugin/ent-search/qa/rest/roles.yml b/x-pack/plugin/ent-search/qa/rest/roles.yml index d32f05b7b749e..661ba482c6367 100644 --- a/x-pack/plugin/ent-search/qa/rest/roles.yml +++ b/x-pack/plugin/ent-search/qa/rest/roles.yml @@ -4,13 +4,12 @@ admin: - manage_behavioral_analytics - manage - monitor + - manage_connector indices: - names: [ # indices and search applications "test-*", "another-test-search-application", - ".elastic-connectors-v1", - ".elastic-connectors-sync-jobs-v1" ] privileges: [ "manage", "write", "read" ] @@ -20,6 +19,7 @@ user: - manage_api_key - read_connector_secrets - write_connector_secrets + - monitor_connector indices: - names: [ "test-index1", @@ -27,9 +27,7 @@ user: "test-search-application-1", "test-search-application-with-aggs", "test-search-application-with-list", - "test-search-application-with-list-invalid", - ".elastic-connectors-v1", - ".elastic-connectors-sync-jobs-v1" + "test-search-application-with-list-invalid" ] privileges: [ "read" ] diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index d5d2159d8f373..14d6c0e103689 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -77,13 +78,15 @@ import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link Connector} configurations. */ public class ConnectorIndexService { - private final Client client; + // The client to interact with the system index (internal user). + private final Client clientWithOrigin; public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; @@ -91,7 +94,7 @@ public class ConnectorIndexService { * @param client A client for executing actions on the connector index */ public ConnectorIndexService(Client client) { - this.client = client; + this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } /** @@ -137,7 +140,7 @@ public void createConnector( indexRequest = indexRequest.id(connectorId); } - client.index( + clientWithOrigin.index( indexRequest, listener.delegateFailureAndWrap( (ll, indexResponse) -> ll.onResponse( @@ -201,7 +204,7 @@ public void getConnector(String connectorId, ActionListener(connectorId, listener, (l, getResponse) -> { + clientWithOrigin.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -235,17 +238,23 @@ public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, Ac .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); try { - client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> { - if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - if (shouldDeleteSyncJobs) { - new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, l.map(r -> deleteResponse)); - } else { - l.onResponse(deleteResponse); - } - })); + clientWithOrigin.delete( + deleteRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> { + if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + if (shouldDeleteSyncJobs) { + new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId( + connectorId, + l.map(r -> deleteResponse) + ); + } else { + l.onResponse(deleteResponse); + } + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -279,7 +288,7 @@ public void listConnectors( .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); - client.search(req, new ActionListener<>() { + clientWithOrigin.search(req, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { @@ -454,7 +463,7 @@ else if (configurationValues != null) { return; } - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -491,13 +500,16 @@ public void updateConnectorError(String connectorId, String error, ActionListene } }) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -519,13 +531,16 @@ public void updateConnectorNameOrDescription(UpdateConnectorNameAction.Request r .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -546,13 +561,16 @@ public void updateConnectorFiltering(String connectorId, List(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -573,13 +591,16 @@ public void updateConnectorFeatures(String connectorId, ConnectorFeatures featur .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.FEATURES_FIELD.getPreferredName(), features)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -635,13 +656,16 @@ public void updateConnectorFilteringDraft( .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(connectorFilteringWithUpdatedDraft))) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - ll.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + ll.onResponse(updateResponse); + }) + ); })); } catch (Exception e) { @@ -683,7 +707,7 @@ public void updateConnectorDraftFilteringValidation( .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(activatedConnectorFiltering))) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -740,7 +764,7 @@ public void activateConnectorDraftFiltering(String connectorId, ActionListener(connectorId, l, (ll, updateResponse) -> { + clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -768,13 +792,16 @@ public void checkInConnector(String connectorId, ActionListener .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.LAST_SEEN_FIELD.getPreferredName(), Instant.now())) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -795,13 +822,16 @@ public void updateConnectorLastSyncStats(UpdateConnectorLastSyncStatsAction.Requ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -865,13 +895,16 @@ public void updateConnectorNative(UpdateConnectorNativeAction.Request request, A ) ) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - ll.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + ll.onResponse(updateResponse); + }) + ); })); } catch (Exception e) { listener.onFailure(e); @@ -894,13 +927,16 @@ public void updateConnectorPipeline(UpdateConnectorPipelineAction.Request reques .source(Map.of(Connector.PIPELINE_FIELD.getPreferredName(), request.getPipeline())) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -959,7 +995,7 @@ public void updateConnectorIndexName(UpdateConnectorIndexNameAction.Request requ } }) ); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (lll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -992,13 +1028,16 @@ public void updateConnectorScheduling(UpdateConnectorSchedulingAction.Request re .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.SCHEDULING_FIELD.getPreferredName(), request.getScheduling())) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -1034,7 +1073,7 @@ public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request ) ); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -1077,7 +1116,7 @@ public void updateConnectorStatus(UpdateConnectorStatusAction.Request request, A .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.STATUS_FIELD.getPreferredName(), request.getStatus())) ); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -1105,13 +1144,16 @@ public void updateConnectorApiKeyIdOrApiKeySecretId( .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - })); + clientWithOrigin.update( + updateRequest, + new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + }) + ); } catch (Exception e) { listener.onFailure(e); } @@ -1181,7 +1223,7 @@ private void isDataIndexNameAlreadyInUse(String indexName, String connectorId, A final SearchSourceBuilder searchSource = new SearchSourceBuilder().query(boolFilterQueryBuilder); final SearchRequest searchRequest = new SearchRequest(CONNECTOR_INDEX_NAME).source(searchSource); - client.search(searchRequest, new ActionListener<>() { + clientWithOrigin.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { boolean indexNameIsInUse = searchResponse.getHits().getTotalHits().value() > 0L; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java index 66f347bc4dbb4..723ff12b3b1e1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java @@ -9,12 +9,9 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.indices.InvalidIndexNameException; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; @@ -22,10 +19,9 @@ import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX; /** - * Abstract base class for action requests targeting the connectors index. Implements {@link org.elasticsearch.action.IndicesRequest} - * to ensure index-level privilege support. This class defines the connectors index as the target for all derived action requests. + * Abstract base class for action requests targeting the connectors index. */ -public abstract class ConnectorActionRequest extends ActionRequest implements IndicesRequest { +public abstract class ConnectorActionRequest extends ActionRequest { public ConnectorActionRequest() { super(); @@ -78,14 +74,4 @@ public ActionRequestValidationException validateManagedConnectorIndexPrefix( } return validationException; } - - @Override - public String[] indices() { - return new String[] { ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; - } - - @Override - public IndicesOptions indicesOptions() { - return IndicesOptions.lenientExpandHidden(); - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java index 930068a2a46ef..5d98f9703ecea 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; import java.util.Objects; @@ -28,7 +27,7 @@ public class DeleteConnectorAction { - public static final String NAME = "indices:data/write/xpack/connector/delete"; + public static final String NAME = "cluster:admin/xpack/connector/delete"; public static final ActionType INSTANCE = new ActionType<>(NAME); private DeleteConnectorAction() {/* no instances */} @@ -71,14 +70,6 @@ public boolean shouldDeleteSyncJobs() { return deleteSyncJobs; } - @Override - public String[] indices() { - // When deleting a connector, corresponding sync jobs can also be deleted - return new String[] { - ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN, - ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java index 2edd47b1fce3a..a976e97adc0cf 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java @@ -28,7 +28,7 @@ public class GetConnectorAction { - public static final String NAME = "indices:data/read/xpack/connector/get"; + public static final String NAME = "cluster:admin/xpack/connector/get"; public static final ActionType INSTANCE = new ActionType<>(NAME); private GetConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index e543d805b7099..c5de4f0545353 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -34,7 +34,7 @@ public class ListConnectorAction { - public static final String NAME = "indices:data/read/xpack/connector/list"; + public static final String NAME = "cluster:admin/xpack/connector/list"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ListConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java index b1c38637298c4..b5087634a77f2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java @@ -25,7 +25,7 @@ public class PostConnectorAction { - public static final String NAME = "indices:data/write/xpack/connector/post"; + public static final String NAME = "cluster:admin/xpack/connector/post"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PostConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java index f3e8ed6b6e76d..c5922ebfafe1b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -27,12 +26,12 @@ public class PutConnectorAction { - public static final String NAME = "indices:data/write/xpack/connector/put"; + public static final String NAME = "cluster:admin/xpack/connector/put"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PutConnectorAction() {/* no instances */} - public static class Request extends ConnectorActionRequest implements IndicesRequest, ToXContentObject { + public static class Request extends ConnectorActionRequest implements ToXContentObject { @Nullable private final String connectorId; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java index 7b4ce08ef8320..a7bd9c2382331 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java @@ -22,7 +22,7 @@ public class UpdateConnectorActiveFilteringAction { - public static final String NAME = "indices:data/write/xpack/connector/update_filtering/activate"; + public static final String NAME = "cluster:admin/xpack/connector/update_filtering/activate"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorActiveFilteringAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java index 7f726f21ce225..e76c5191a58a6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorApiKeyIdAction { - public static final String NAME = "indices:data/write/xpack/connector/update_api_key_id"; + public static final String NAME = "cluster:admin/xpack/connector/update_api_key_id"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorApiKeyIdAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java index 5d36c5f886ea0..6948667fa7351 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java @@ -32,7 +32,7 @@ public class UpdateConnectorConfigurationAction { - public static final String NAME = "indices:data/write/xpack/connector/update_configuration"; + public static final String NAME = "cluster:admin/xpack/connector/update_configuration"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorConfigurationAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java index 3e506fc835f65..a8c2b334cfee3 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorErrorAction { - public static final String NAME = "indices:data/write/xpack/connector/update_error"; + public static final String NAME = "cluster:admin/xpack/connector/update_error"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorErrorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java index 56656855583aa..4bd51794c1f9e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorFeaturesAction { - public static final String NAME = "indices:data/write/xpack/connector/update_features"; + public static final String NAME = "cluster:admin/xpack/connector/update_features"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFeaturesAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java index 660956b2e9d7f..e527e79161f2f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java @@ -32,7 +32,7 @@ public class UpdateConnectorFilteringAction { - public static final String NAME = "indices:data/write/xpack/connector/update_filtering"; + public static final String NAME = "cluster:admin/xpack/connector/update_filtering"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFilteringAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java index 92291506d0719..a22b08152f509 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorFilteringValidationAction { - public static final String NAME = "indices:data/write/xpack/connector/update_filtering/draft_validation"; + public static final String NAME = "cluster:admin/xpack/connector/update_filtering/draft_validation"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFilteringValidationAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java index e7840e1f84fad..ddc98d4756dc4 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorIndexNameAction { - public static final String NAME = "indices:data/write/xpack/connector/update_index_name"; + public static final String NAME = "cluster:admin/xpack/connector/update_index_name"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorIndexNameAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java index f72938ec8dba2..deae10d901c13 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java @@ -23,7 +23,7 @@ public class UpdateConnectorLastSeenAction { - public static final String NAME = "indices:data/write/xpack/connector/update_last_seen"; + public static final String NAME = "cluster:admin/xpack/connector/update_last_seen"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorLastSeenAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java index ae3be3801786c..029e261e51fab 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java @@ -32,7 +32,7 @@ public class UpdateConnectorLastSyncStatsAction { - public static final String NAME = "indices:data/write/xpack/connector/update_last_sync_stats"; + public static final String NAME = "cluster:admin/xpack/connector/update_last_sync_stats"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorLastSyncStatsAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java index bbc1f992b48e2..5a63bb106747f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorNameAction { - public static final String NAME = "indices:data/write/xpack/connector/update_name"; + public static final String NAME = "cluster:admin/xpack/connector/update_name"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorNameAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java index 7b3f2e4577f4e..d20b535d5cc57 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java @@ -26,7 +26,7 @@ public class UpdateConnectorNativeAction { - public static final String NAME = "indices:data/write/xpack/connector/update_native"; + public static final String NAME = "cluster:admin/xpack/connector/update_native"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorNativeAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java index e58d614f4ef21..2e738033c3384 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java @@ -27,7 +27,7 @@ public class UpdateConnectorPipelineAction { - public static final String NAME = "indices:data/write/xpack/connector/update_pipeline"; + public static final String NAME = "cluster:admin/xpack/connector/update_pipeline"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorPipelineAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java index 578639f065a0b..65cfd645ea60b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java @@ -32,7 +32,7 @@ public class UpdateConnectorSchedulingAction { - public static final String NAME = "indices:data/write/xpack/connector/update_scheduling"; + public static final String NAME = "cluster:admin/xpack/connector/update_scheduling"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSchedulingAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java index de07a6db21bab..1d4df12e10eba 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java @@ -26,7 +26,7 @@ public class UpdateConnectorServiceTypeAction { - public static final String NAME = "indices:data/write/xpack/connector/update_service_type"; + public static final String NAME = "cluster:admin/xpack/connector/update_service_type"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorServiceTypeAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java index aebaa0afb9052..79f097db2dec9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java @@ -28,7 +28,7 @@ public class UpdateConnectorStatusAction { - public static final String NAME = "indices:data/write/xpack/connector/update_status"; + public static final String NAME = "cluster:admin/xpack/connector/update_status"; public static final ActionType INSTANCE = new ActionType<>(NAME); public UpdateConnectorStatusAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index ce6f7f0dbf2b2..f46d915a7123f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.DocumentMissingException; @@ -68,6 +69,7 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorIndexService.CONNECTOR_INDEX_NAME; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link ConnectorSyncJob} configurations. @@ -76,7 +78,8 @@ public class ConnectorSyncJobIndexService { private static final Long ZERO = 0L; - private final Client client; + // The client to interact with the system index (internal user). + private final Client clientWithOrigin; public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; @@ -84,7 +87,7 @@ public class ConnectorSyncJobIndexService { * @param client A client for executing actions on the connectors sync jobs index. */ public ConnectorSyncJobIndexService(Client client) { - this.client = client; + this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); } /** @@ -149,7 +152,7 @@ public void createConnectorSyncJob( indexRequest.source(syncJob.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - client.index( + clientWithOrigin.index( indexRequest, l.delegateFailureAndWrap( (ll, indexResponse) -> ll.onResponse(new PostConnectorSyncJobAction.Response(indexResponse.getId())) @@ -175,7 +178,7 @@ public void deleteConnectorSyncJob(String connectorSyncJobId, ActionListener(connectorSyncJobId, listener, (l, deleteResponse) -> { if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -205,7 +208,7 @@ public void checkInConnectorSyncJob(String connectorSyncJobId, ActionListener(connectorSyncJobId, listener, (l, updateResponse) -> { if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -230,7 +233,7 @@ public void getConnectorSyncJob(String connectorSyncJobId, ActionListener(connectorSyncJobId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { @@ -306,7 +309,7 @@ public void cancelConnectorSyncJob(String connectorSyncJobId, ActionListener( connectorSyncJobId, @@ -355,7 +358,7 @@ public void listConnectorSyncJobs( final SearchRequest searchRequest = new SearchRequest(CONNECTOR_SYNC_JOB_INDEX_NAME).source(searchSource); - client.search(searchRequest, new ActionListener<>() { + clientWithOrigin.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { @@ -474,7 +477,7 @@ public void updateConnectorSyncJobIngestionStats( ).doc(fieldsToUpdate); try { - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(syncJobId, listener, (l, updateResponse) -> { if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -501,7 +504,7 @@ private void getSyncJobConnectorInfo(String connectorId, ConnectorSyncJobType jo final GetRequest request = new GetRequest(CONNECTOR_INDEX_NAME, connectorId); - client.get(request, new ActionListener<>() { + clientWithOrigin.get(request, new ActionListener<>() { @Override public void onResponse(GetResponse response) { final boolean connectorDoesNotExist = response.isExists() == false; @@ -594,7 +597,7 @@ public void updateConnectorSyncJobError(String connectorSyncJobId, String error, ) ); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>( connectorSyncJobId, @@ -629,7 +632,7 @@ public void deleteAllSyncJobsByConnectorId(String connectorId, ActionListener { + clientWithOrigin.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, listener.delegateFailureAndWrap((l, r) -> { final List bulkDeleteFailures = r.getBulkFailures(); if (bulkDeleteFailures.isEmpty() == false) { l.onFailure( @@ -681,7 +684,7 @@ public void claimConnectorSyncJob( WriteRequest.RefreshPolicy.IMMEDIATE ).doc(document); - client.update( + clientWithOrigin.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>( connectorSyncJobId, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java index 160eda3aeef7c..658d48e9752d0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java @@ -28,7 +28,7 @@ public class CancelConnectorSyncJobAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/cancel"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/cancel"; public static final ActionType INSTANCE = new ActionType(NAME); private CancelConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java index aa6dea50de464..ae44813354eb9 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java @@ -28,7 +28,7 @@ public class CheckInConnectorSyncJobAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/check_in"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/check_in"; public static final ActionType INSTANCE = new ActionType<>(NAME); private CheckInConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java index b108116a5e68c..84e3183c2830c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java @@ -31,7 +31,7 @@ public class ClaimConnectorSyncJobAction { public static final ParseField CONNECTOR_SYNC_JOB_ID_FIELD = new ParseField("connector_sync_job_id"); - public static final String NAME = "indices:data/write/xpack/connector/sync_job/claim"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/claim"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ClaimConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java index bb83fd78151df..11a3334aed4e6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java @@ -8,19 +8,14 @@ package org.elasticsearch.xpack.application.connector.syncjob.action; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; /** * Abstract base class for action requests targeting the connector sync job index. - * Implements {@link org.elasticsearch.action.IndicesRequest} to ensure index-level privilege support. - * This class defines the connectors sync job index as the target for all derived action requests. */ -public abstract class ConnectorSyncJobActionRequest extends ActionRequest implements IndicesRequest { +public abstract class ConnectorSyncJobActionRequest extends ActionRequest { public ConnectorSyncJobActionRequest() { super(); @@ -29,14 +24,4 @@ public ConnectorSyncJobActionRequest() { public ConnectorSyncJobActionRequest(StreamInput in) throws IOException { super(in); } - - @Override - public String[] indices() { - return new String[] { ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN }; - } - - @Override - public IndicesOptions indicesOptions() { - return IndicesOptions.lenientExpandHidden(); - } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java index c9c84ead2e4b0..935aa1fd6b7d1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java @@ -28,7 +28,7 @@ public class DeleteConnectorSyncJobAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/delete"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/delete"; public static final ActionType INSTANCE = new ActionType<>(NAME); private DeleteConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java index 7ff82b9881fb4..485e516c80c39 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java @@ -29,7 +29,7 @@ public class GetConnectorSyncJobAction { - public static final String NAME = "indices:data/read/xpack/connector/sync_job/get"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/get"; public static final ActionType INSTANCE = new ActionType<>(NAME); private GetConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java index 5b34e05ae37f0..04b765f3b5687 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java @@ -33,7 +33,7 @@ public class ListConnectorSyncJobsAction { - public static final String NAME = "indices:data/read/xpack/connector/sync_job/list"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/list"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ListConnectorSyncJobsAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java index 8c1d24e466daa..0d17a6dba6c35 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.Connector; -import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTriggerMethod; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobType; @@ -32,7 +31,7 @@ public class PostConnectorSyncJobAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/post"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/post"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PostConnectorSyncJobAction() {/* no instances */} @@ -140,14 +139,6 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(id, jobType, triggerMethod); } - - @Override - public String[] indices() { - // Creating a new sync job requires reading from connector index - return new String[] { - ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN, - ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; - } } public static class Response extends ActionResponse implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java index 2235ba7cfe720..0d5f57c202d9f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java @@ -28,7 +28,7 @@ public class UpdateConnectorSyncJobErrorAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/update_error"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_error"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSyncJobErrorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java index 0fd9b6dec8184..c890ca0d69bc6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java @@ -35,7 +35,7 @@ public class UpdateConnectorSyncJobIngestionStatsAction { - public static final String NAME = "indices:data/write/xpack/connector/sync_job/update_stats"; + public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_stats"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSyncJobIngestionStatsAction() {/* no instances */} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 62687e42b0912..82bad85fa34dc 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -134,40 +134,40 @@ public class Constants { "cluster:admin/xpack/ccr/auto_follow_pattern/put", "cluster:admin/xpack/ccr/pause_follow", "cluster:admin/xpack/ccr/resume_follow", - "indices:data/write/xpack/connector/delete", - "indices:data/read/xpack/connector/get", - "indices:data/read/xpack/connector/list", - "indices:data/write/xpack/connector/post", - "indices:data/write/xpack/connector/put", - "indices:data/write/xpack/connector/update_api_key_id", - "indices:data/write/xpack/connector/update_configuration", - "indices:data/write/xpack/connector/update_error", - "indices:data/write/xpack/connector/update_features", - "indices:data/write/xpack/connector/update_filtering", - "indices:data/write/xpack/connector/update_filtering/activate", - "indices:data/write/xpack/connector/update_filtering/draft_validation", - "indices:data/write/xpack/connector/update_index_name", - "indices:data/write/xpack/connector/update_last_seen", - "indices:data/write/xpack/connector/update_last_sync_stats", - "indices:data/write/xpack/connector/update_name", - "indices:data/write/xpack/connector/update_native", - "indices:data/write/xpack/connector/update_pipeline", - "indices:data/write/xpack/connector/update_scheduling", - "indices:data/write/xpack/connector/update_service_type", - "indices:data/write/xpack/connector/update_status", + "cluster:admin/xpack/connector/delete", + "cluster:admin/xpack/connector/get", + "cluster:admin/xpack/connector/list", + "cluster:admin/xpack/connector/post", + "cluster:admin/xpack/connector/put", + "cluster:admin/xpack/connector/update_api_key_id", + "cluster:admin/xpack/connector/update_configuration", + "cluster:admin/xpack/connector/update_error", + "cluster:admin/xpack/connector/update_features", + "cluster:admin/xpack/connector/update_filtering", + "cluster:admin/xpack/connector/update_filtering/activate", + "cluster:admin/xpack/connector/update_filtering/draft_validation", + "cluster:admin/xpack/connector/update_index_name", + "cluster:admin/xpack/connector/update_last_seen", + "cluster:admin/xpack/connector/update_last_sync_stats", + "cluster:admin/xpack/connector/update_name", + "cluster:admin/xpack/connector/update_native", + "cluster:admin/xpack/connector/update_pipeline", + "cluster:admin/xpack/connector/update_scheduling", + "cluster:admin/xpack/connector/update_service_type", + "cluster:admin/xpack/connector/update_status", "cluster:admin/xpack/connector/secret/delete", "cluster:admin/xpack/connector/secret/get", "cluster:admin/xpack/connector/secret/post", "cluster:admin/xpack/connector/secret/put", - "indices:data/write/xpack/connector/sync_job/cancel", - "indices:data/write/xpack/connector/sync_job/check_in", - "indices:data/write/xpack/connector/sync_job/claim", - "indices:data/write/xpack/connector/sync_job/delete", - "indices:data/read/xpack/connector/sync_job/get", - "indices:data/read/xpack/connector/sync_job/list", - "indices:data/write/xpack/connector/sync_job/post", - "indices:data/write/xpack/connector/sync_job/update_error", - "indices:data/write/xpack/connector/sync_job/update_stats", + "cluster:admin/xpack/connector/sync_job/cancel", + "cluster:admin/xpack/connector/sync_job/check_in", + "cluster:admin/xpack/connector/sync_job/claim", + "cluster:admin/xpack/connector/sync_job/delete", + "cluster:admin/xpack/connector/sync_job/get", + "cluster:admin/xpack/connector/sync_job/list", + "cluster:admin/xpack/connector/sync_job/post", + "cluster:admin/xpack/connector/sync_job/update_error", + "cluster:admin/xpack/connector/sync_job/update_stats", "cluster:admin/xpack/deprecation/info", "cluster:admin/xpack/deprecation/nodes/info", "cluster:admin/xpack/enrich/delete", From eb5ce98bfd1b8198d68b74faaea6b80ad7d73fa8 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Wed, 8 Jan 2025 10:12:10 +0100 Subject: [PATCH 05/52] Bump deprecated tracing.apm settings to critical to be shown in the Upgrade assistent (#119677) --- .../telemetry/apm/internal/APMAgentSettings.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java index 99b2a4510bf93..f66683a787bc0 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java @@ -27,7 +27,7 @@ import java.util.Set; import java.util.function.Function; -import static org.elasticsearch.common.settings.Setting.Property.DeprecatedWarning; +import static org.elasticsearch.common.settings.Setting.Property.Deprecated; import static org.elasticsearch.common.settings.Setting.Property.NodeScope; import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic; @@ -250,7 +250,7 @@ private static Setting concreteAgentSetting(String namespace, String qua TELEMETRY_SETTING_PREFIX + "agent.", LEGACY_TRACING_APM_SETTING_PREFIX + "agent.", (namespace, qualifiedKey) -> qualifiedKey.startsWith(LEGACY_TRACING_APM_SETTING_PREFIX) - ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, DeprecatedWarning) + ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, Deprecated) : concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic) ); @@ -262,7 +262,7 @@ private static Setting concreteAgentSetting(String namespace, String qua LEGACY_TRACING_APM_SETTING_PREFIX + "names.include", OperatorDynamic, NodeScope, - DeprecatedWarning + Deprecated ); public static final Setting> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.listSetting( @@ -281,7 +281,7 @@ private static Setting concreteAgentSetting(String namespace, String qua LEGACY_TRACING_APM_SETTING_PREFIX + "names.exclude", OperatorDynamic, NodeScope, - DeprecatedWarning + Deprecated ); public static final Setting> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.listSetting( @@ -314,7 +314,7 @@ private static Setting concreteAgentSetting(String namespace, String qua ), OperatorDynamic, NodeScope, - DeprecatedWarning + Deprecated ); public static final Setting> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting( @@ -334,7 +334,7 @@ private static Setting concreteAgentSetting(String namespace, String qua false, OperatorDynamic, NodeScope, - DeprecatedWarning + Deprecated ); public static final Setting TELEMETRY_TRACING_ENABLED_SETTING = Setting.boolSetting( @@ -358,7 +358,7 @@ private static Setting concreteAgentSetting(String namespace, String qua public static final Setting TRACING_APM_SECRET_TOKEN_SETTING = SecureSetting.secureString( LEGACY_TRACING_APM_SETTING_PREFIX + "secret_token", null, - DeprecatedWarning + Deprecated ); public static final Setting TELEMETRY_SECRET_TOKEN_SETTING = SecureSetting.secureString( @@ -373,7 +373,7 @@ private static Setting concreteAgentSetting(String namespace, String qua public static final Setting TRACING_APM_API_KEY_SETTING = SecureSetting.secureString( LEGACY_TRACING_APM_SETTING_PREFIX + "api_key", null, - DeprecatedWarning + Deprecated ); public static final Setting TELEMETRY_API_KEY_SETTING = SecureSetting.secureString( From 66f7c7bf9edef539e8be56494b054b9ee6d66f3c Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 8 Jan 2025 13:19:29 +0400 Subject: [PATCH 06/52] Remove unsupported timeout from rest-api-spec license API (#118919) --- docs/changelog/118919.yaml | 5 +++++ .../rest-api-spec/api/license.post_start_trial.json | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/118919.yaml diff --git a/docs/changelog/118919.yaml b/docs/changelog/118919.yaml new file mode 100644 index 0000000000000..832fd86fe08ba --- /dev/null +++ b/docs/changelog/118919.yaml @@ -0,0 +1,5 @@ +pr: 118919 +summary: Remove unsupported timeout from rest-api-spec license API +area: License +type: bug +issues: [] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json b/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json index 986040d69cb4f..9fb85807d611f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/license.post_start_trial.json @@ -31,10 +31,6 @@ "master_timeout": { "type": "time", "description": "Timeout for processing on master node" - }, - "timeout": { - "type": "time", - "description": "Timeout for acknowledgement of update from all nodes in cluster" } } } From 3464adb3aed66bc8a1c91c1a754479d6f9da18d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Wed, 8 Jan 2025 10:24:03 +0100 Subject: [PATCH 07/52] Adjust Bootstrap and JVM options to ensure the SM is never used when entitlements are enabled (#119689) --- .../elasticsearch/server/cli/SystemJvmOptions.java | 6 +++--- .../org/elasticsearch/bootstrap/Bootstrap.java | 8 +++++++- .../elasticsearch/bootstrap/BootstrapChecks.java | 1 - .../org/elasticsearch/bootstrap/Elasticsearch.java | 14 ++++++++++---- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index ce951715939f0..928a8ba43cae1 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -71,7 +71,7 @@ static List systemJvmOptions(Settings nodeSettings, final Map s).toList(); } @@ -140,8 +140,8 @@ private static Stream maybeWorkaroundG1Bug() { } @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - private static Stream maybeAllowSecurityManager() { - if (RuntimeVersionFeature.isSecurityManagerAvailable()) { + private static Stream maybeAllowSecurityManager(boolean useEntitlements) { + if (useEntitlements == false && RuntimeVersionFeature.isSecurityManagerAvailable()) { // Will become conditional on useEntitlements once entitlements can run without SM return Stream.of("-Djava.security.manager=allow"); } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index 56d185645e149..4c7fb96c5b1d5 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -33,6 +33,7 @@ class Bootstrap { // arguments from the CLI process private final ServerArgs args; + private final boolean useEntitlements; // controller for spawning component subprocesses private final Spawner spawner = new Spawner(); @@ -46,10 +47,11 @@ class Bootstrap { // loads information about plugins required for entitlements in phase 2, used by plugins service in phase 3 private final SetOnce pluginsLoader = new SetOnce<>(); - Bootstrap(PrintStream out, PrintStream err, ServerArgs args) { + Bootstrap(PrintStream out, PrintStream err, ServerArgs args, boolean useEntitlements) { this.out = out; this.err = err; this.args = args; + this.useEntitlements = useEntitlements; } ServerArgs args() { @@ -60,6 +62,10 @@ Spawner spawner() { return spawner; } + public boolean useEntitlements() { + return useEntitlements; + } + void setSecureSettings(SecureSettings secureSettings) { this.secureSettings.set(secureSettings); } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java index 7b2f0c2c894be..b5b616fff0182 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java @@ -212,7 +212,6 @@ static List checks() { checks.add(new OnErrorCheck()); checks.add(new OnOutOfMemoryErrorCheck()); checks.add(new EarlyAccessCheck()); - checks.add(new AllPermissionCheck()); checks.add(new DiscoveryConfiguredCheck()); checks.add(new ByteOrderCheck()); return Collections.unmodifiableList(checks); diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 6822c201ab030..f26bf96cc2211 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -54,6 +54,7 @@ import java.nio.file.Path; import java.security.Permission; import java.security.Security; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.concurrent.CountDownLatch; @@ -108,6 +109,7 @@ private static Bootstrap initPhase1() { final PrintStream out = getStdout(); final PrintStream err = getStderr(); final ServerArgs args; + final boolean useEntitlements = Boolean.parseBoolean(System.getProperty("es.entitlements.enabled")); try { initSecurityProperties(); @@ -116,7 +118,7 @@ private static Bootstrap initPhase1() { * the presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy). * This forces such policies to take effect immediately. */ - if (RuntimeVersionFeature.isSecurityManagerAvailable()) { + if (useEntitlements == false && RuntimeVersionFeature.isSecurityManagerAvailable()) { org.elasticsearch.bootstrap.Security.setSecurityManager(new SecurityManager() { @Override public void checkPermission(Permission perm) { @@ -149,7 +151,7 @@ public void checkPermission(Permission perm) { return null; // unreachable, to satisfy compiler } - return new Bootstrap(out, err, args); + return new Bootstrap(out, err, args, useEntitlements); } /** @@ -214,7 +216,7 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException { var pluginsLoader = PluginsLoader.createPluginsLoader(nodeEnv.modulesFile(), nodeEnv.pluginsFile()); bootstrap.setPluginsLoader(pluginsLoader); - if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) { + if (bootstrap.useEntitlements()) { LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements"); List pluginData = Stream.concat( @@ -280,7 +282,11 @@ protected void validateNodeBeforeAcceptingRequests( final BoundTransportAddress boundTransportAddress, List checks ) throws NodeValidationException { - BootstrapChecks.check(context, boundTransportAddress, checks); + var additionalChecks = new ArrayList<>(checks); + if (bootstrap.useEntitlements() == false) { + additionalChecks.add(new BootstrapChecks.AllPermissionCheck()); + } + BootstrapChecks.check(context, boundTransportAddress, additionalChecks); } }; INSTANCE = new Elasticsearch(bootstrap.spawner(), node); From 83d62473d10e8473fa849760a780db0ffab2ffbe Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 8 Jan 2025 13:42:12 +0400 Subject: [PATCH 08/52] Add missing timeouts to rest-api-spec shutdown APIs (#118921) --- docs/changelog/118921.yaml | 5 +++++ .../rest-api-spec/api/shutdown.delete_node.json | 11 ++++++++++- .../rest-api-spec/api/shutdown.put_node.json | 11 ++++++++++- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/118921.yaml diff --git a/docs/changelog/118921.yaml b/docs/changelog/118921.yaml new file mode 100644 index 0000000000000..bd341616d8a14 --- /dev/null +++ b/docs/changelog/118921.yaml @@ -0,0 +1,5 @@ +pr: 118921 +summary: Add missing timeouts to rest-api-spec shutdown APIs +area: Infra/Node Lifecycle +type: bug +issues: [] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json index d990d1da1f144..6f1ec484e94d0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.delete_node.json @@ -26,6 +26,15 @@ } ] }, - "params":{} + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json index bf20cf3b70bac..90b19557f5fb2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/shutdown.put_node.json @@ -26,7 +26,16 @@ } ] }, - "params":{}, + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + } + }, "body":{ "description":"The shutdown type definition to register", "required": true From ee7848e73260ab8dd711c71e4815555241444aab Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 8 Jan 2025 13:42:31 +0400 Subject: [PATCH 09/52] Add missing parameter to xpack.info rest-api-spec (#118954) * Add missing parameter to xpack.info rest-api-spec * Update docs/changelog/118954.yaml --- docs/changelog/118954.yaml | 5 +++++ .../src/main/resources/rest-api-spec/api/xpack.info.json | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 docs/changelog/118954.yaml diff --git a/docs/changelog/118954.yaml b/docs/changelog/118954.yaml new file mode 100644 index 0000000000000..ab2f2cda5c11e --- /dev/null +++ b/docs/changelog/118954.yaml @@ -0,0 +1,5 @@ +pr: 118954 +summary: Add missing parameter to `xpack.info` rest-api-spec +area: Infra/REST API +type: bug +issues: [] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json index 68b2a5d2c2c8b..35895f0ddb581 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/xpack.info.json @@ -20,6 +20,12 @@ ] }, "params":{ + "human":{ + "type":"boolean", + "required":false, + "description":"Defines whether additional human-readable information is included in the response. In particular, it adds descriptions and a tag line. The default value is true.", + "default":true + }, "categories":{ "type":"list", "description":"Comma-separated list of info categories. Can be any of: build, license, features" From 38260aacd41d93b06f4c3aab80f6b63e37468dcd Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 8 Jan 2025 11:14:16 +0100 Subject: [PATCH 10/52] [Test] Fix `index.mapping.total_fields.limit` random update (#119728) It's stupid to pick a low value for `index.mapping.total_fields.limit` when at the same time you also add random runtime fields => pick a value large enough to avoid failing future random mapping updates. Closes #119646 Closes #119632 Closes #119631 --- .../lucene/AbstractIndexCompatibilityTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java index 8c9a42dc926e9..13c647983fad5 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java @@ -207,7 +207,7 @@ protected static void updateRandomIndexSettings(String indexName) throws IOExcep switch (i) { case 0 -> settings.putList(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), "field_" + randomInt(2)); case 1 -> settings.put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), randomIntBetween(1, 100)); - case 2 -> settings.put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), randomLongBetween(0L, 1000L)); + case 2 -> settings.put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), randomLongBetween(100L, 1000L)); case 3 -> settings.put(IndexSettings.MAX_SLICES_PER_SCROLL.getKey(), randomIntBetween(1, 1024)); default -> throw new IllegalStateException(); } From ab77144aa4973c9d90eee63e392ead7cdb11df4e Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 8 Jan 2025 11:38:05 +0100 Subject: [PATCH 11/52] Nullify sourceAsMap once a search hit is processed (#119734) he sourceAsMap can use a significant amount of memory, still it is only hold for caching. Therefore it might make sense to nullify it once a search hit is processed so we free that memory. --- .../src/main/java/org/elasticsearch/search/SearchHit.java | 8 ++++++++ .../search/fetch/FetchPhaseDocsIterator.java | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 701d451ac2c14..0d7b76c3c3997 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -491,6 +491,13 @@ public Map getSourceAsMap() { return sourceAsMap; } + /** + * Set the cache document as a map to {@code null}. + */ + public void resetSourceAsMap() { + sourceAsMap = null; + } + /** * The hit field matching the given field name. */ @@ -728,6 +735,7 @@ private void deallocate() { if (SearchHit.this.source instanceof RefCounted r) { r.decRef(); } + SearchHit.this.sourceAsMap = null; SearchHit.this.source = null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhaseDocsIterator.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhaseDocsIterator.java index 4a242f70e8d02..9093b08f9d6f2 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhaseDocsIterator.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhaseDocsIterator.java @@ -86,7 +86,10 @@ public final SearchHit[] iterate( } currentDoc = docs[i].docId; assert searchHits[docs[i].index] == null; - searchHits[docs[i].index] = nextDoc(docs[i].docId); + SearchHit searchHit = nextDoc(docs[i].docId); + // free some memory + searchHit.resetSourceAsMap(); + searchHits[docs[i].index] = searchHit; } catch (ContextIndexSearcher.TimeExceededException e) { if (allowPartialResults == false) { purgeSearchHits(searchHits); From afda404524d1944cc78a1d0de6ab0903c10c7326 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 8 Jan 2025 11:47:21 +0100 Subject: [PATCH 12/52] Optimized index sorting for OTel logs (#119504) --- docs/changelog/119504.yaml | 5 +++++ .../resources/component-templates/logs-otel@mappings.yaml | 1 + 2 files changed, 6 insertions(+) create mode 100644 docs/changelog/119504.yaml diff --git a/docs/changelog/119504.yaml b/docs/changelog/119504.yaml new file mode 100644 index 0000000000000..f63e422face10 --- /dev/null +++ b/docs/changelog/119504.yaml @@ -0,0 +1,5 @@ +pr: 119504 +summary: Optimized index sorting for OTel logs +area: Data streams +type: enhancement +issues: [] diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index 9f19e2e04d2ca..9c47cfad993e2 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -9,6 +9,7 @@ template: mode: logsdb sort: field: [ "resource.attributes.host.name", "@timestamp" ] + order: [ "asc", "desc" ] mappings: properties: attributes: From bf387193783b5f8f5033b1dec5f3f24a1a837ac8 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Date: Wed, 8 Jan 2025 12:08:45 +0100 Subject: [PATCH 13/52] [Inference API] Add Jina AI API to do inference for Embedding and Rerank models (#118652) --- docs/changelog/118652.yaml | 5 + .../org/elasticsearch/TransportVersions.java | 1 + .../xpack/inference/InferenceCrudIT.java | 20 +- .../InferenceNamedWriteablesProvider.java | 28 + .../xpack/inference/InferencePlugin.java | 2 + .../action/jinaai/JinaAIActionCreator.java | 58 + .../action/jinaai/JinaAIActionVisitor.java | 21 + .../JinaAIEmbeddingsRequestManager.java | 57 + .../http/sender/JinaAIRequestManager.java | 28 + .../sender/JinaAIRerankRequestManager.java | 56 + .../external/jinaai/JinaAIAccount.java | 32 + .../jinaai/JinaAIResponseHandler.java | 62 + .../jinaai/JinaAIEmbeddingsRequest.java | 84 + .../jinaai/JinaAIEmbeddingsRequestEntity.java | 67 + .../request/jinaai/JinaAIRequest.java | 26 + .../request/jinaai/JinaAIRerankRequest.java | 86 + .../jinaai/JinaAIRerankRequestEntity.java | 58 + .../external/request/jinaai/JinaAIUtils.java | 26 + .../JinaAIEmbeddingsResponseEntity.java | 110 + .../jinaai/JinaAIErrorResponseEntity.java | 46 + .../jinaai/JinaAIRerankResponseEntity.java | 158 ++ .../services/jinaai/JinaAIModel.java | 68 + .../JinaAIRateLimitServiceSettings.java | 15 + .../services/jinaai/JinaAIService.java | 358 +++ .../services/jinaai/JinaAIServiceFields.java | 13 + .../jinaai/JinaAIServiceSettings.java | 159 ++ .../embeddings/JinaAIEmbeddingsModel.java | 140 ++ .../JinaAIEmbeddingsServiceSettings.java | 162 ++ .../JinaAIEmbeddingsTaskSettings.java | 183 ++ .../jinaai/rerank/JinaAIRerankModel.java | 148 ++ .../rerank/JinaAIRerankServiceSettings.java | 113 + .../rerank/JinaAIRerankTaskSettings.java | 166 ++ .../jinaai/JinaAIResponseHandlerTests.java | 138 ++ .../JinaAIEmbeddingsRequestEntityTests.java | 54 + .../jinaai/JinaAIEmbeddingsRequestTests.java | 101 + .../request/jinaai/JinaAIRequestTests.java | 36 + .../JinaAIRerankRequestEntityTests.java | 140 ++ .../jinaai/JinaAIRerankRequestTests.java | 110 + .../request/jinaai/JinaAIUtilsTests.java | 23 + .../JinaAIEmbeddingsResponseEntityTests.java | 397 ++++ .../JinaAIErrorResponseEntityTests.java | 51 + .../JinaAIRerankResponseEntityTests.java | 180 ++ .../jinaai/JinaAIServiceSettingsTests.java | 174 ++ .../services/jinaai/JinaAIServiceTests.java | 2003 +++++++++++++++++ .../JinaAIEmbeddingsModelTests.java | 168 ++ .../JinaAIEmbeddingsServiceSettingsTests.java | 187 ++ .../JinaAIEmbeddingsTaskSettingsTests.java | 193 ++ .../jinaai/rerank/JinaAIRerankModelTests.java | 74 + .../JinaAIRerankServiceSettingsTests.java | 83 + .../rerank/JinaAIRerankTaskSettingsTests.java | 132 ++ 50 files changed, 6791 insertions(+), 9 deletions(-) create mode 100644 docs/changelog/118652.yaml create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionVisitor.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIEmbeddingsRequestManager.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRequestManager.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRerankRequestManager.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIAccount.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandler.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequest.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequest.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequest.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtils.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntity.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIModel.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIRateLimitServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceFields.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettings.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandlerTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequestTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtilsTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntityTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettingsTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModelTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettingsTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettingsTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModelTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettingsTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettingsTests.java diff --git a/docs/changelog/118652.yaml b/docs/changelog/118652.yaml new file mode 100644 index 0000000000000..0b08686230405 --- /dev/null +++ b/docs/changelog/118652.yaml @@ -0,0 +1,5 @@ +pr: 118652 +summary: Add Jina AI API to do inference for Embedding and Rerank models +area: Machine Learning +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 22f69edc3a5f8..0587ab8b46edc 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -151,6 +151,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_CCS_TELEMETRY_STATS = def(8_816_00_0); public static final TransportVersion TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID = def(8_817_00_0); public static final TransportVersion ESQL_ENABLE_NODE_LEVEL_REDUCTION = def(8_818_00_0); + public static final TransportVersion JINA_AI_INTEGRATION_ADDED = def(8_819_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 49fce930cd726..2f8cfc8f3e659 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -150,9 +150,9 @@ public void testGetServicesWithoutTaskType() throws IOException { List services = getAllServices(); if ((ElasticInferenceServiceFeature.DEPRECATED_ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled() || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) { - assertThat(services.size(), equalTo(18)); + assertThat(services.size(), equalTo(19)); } else { - assertThat(services.size(), equalTo(17)); + assertThat(services.size(), equalTo(18)); } String[] providers = new String[services.size()]; @@ -175,6 +175,7 @@ public void testGetServicesWithoutTaskType() throws IOException { "googleaistudio", "googlevertexai", "hugging_face", + "jinaai", "mistral", "openai", "streaming_completion_test_service", @@ -188,13 +189,13 @@ public void testGetServicesWithoutTaskType() throws IOException { || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) { providerList.add(6, "elastic"); } - assertArrayEquals(providers, providerList.toArray()); + assertArrayEquals(providerList.toArray(), providers); } @SuppressWarnings("unchecked") public void testGetServicesWithTextEmbeddingTaskType() throws IOException { List services = getServices(TaskType.TEXT_EMBEDDING); - assertThat(services.size(), equalTo(13)); + assertThat(services.size(), equalTo(14)); String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { @@ -204,7 +205,6 @@ public void testGetServicesWithTextEmbeddingTaskType() throws IOException { Arrays.sort(providers); assertArrayEquals( - providers, List.of( "alibabacloud-ai-search", "amazonbedrock", @@ -215,18 +215,20 @@ public void testGetServicesWithTextEmbeddingTaskType() throws IOException { "googleaistudio", "googlevertexai", "hugging_face", + "jinaai", "mistral", "openai", "text_embedding_test_service", "watsonxai" - ).toArray() + ).toArray(), + providers ); } @SuppressWarnings("unchecked") public void testGetServicesWithRerankTaskType() throws IOException { List services = getServices(TaskType.RERANK); - assertThat(services.size(), equalTo(5)); + assertThat(services.size(), equalTo(6)); String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { @@ -236,8 +238,8 @@ public void testGetServicesWithRerankTaskType() throws IOException { Arrays.sort(providers); assertArrayEquals( - providers, - List.of("alibabacloud-ai-search", "cohere", "elasticsearch", "googlevertexai", "test_reranking_service").toArray() + List.of("alibabacloud-ai-search", "cohere", "elasticsearch", "googlevertexai", "jinaai", "test_reranking_service").toArray(), + providers ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index 71fbcf6d8ef49..6fc9870034018 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -75,6 +75,11 @@ import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings; import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserServiceSettings; import org.elasticsearch.xpack.inference.services.ibmwatsonx.embeddings.IbmWatsonxEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankServiceSettings; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionServiceSettings; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionTaskSettings; @@ -132,6 +137,7 @@ public static List getNamedWriteables() { addAmazonBedrockNamedWriteables(namedWriteables); addEisNamedWriteables(namedWriteables); addAlibabaCloudSearchNamedWriteables(namedWriteables); + addJinaAINamedWriteables(namedWriteables); addUnifiedNamedWriteables(namedWriteables); @@ -569,6 +575,28 @@ private static void addAlibabaCloudSearchNamedWriteables(List namedWriteables) { + namedWriteables.add( + new NamedWriteableRegistry.Entry(ServiceSettings.class, JinaAIServiceSettings.NAME, JinaAIServiceSettings::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + JinaAIEmbeddingsServiceSettings.NAME, + JinaAIEmbeddingsServiceSettings::new + ) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(TaskSettings.class, JinaAIEmbeddingsTaskSettings.NAME, JinaAIEmbeddingsTaskSettings::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(ServiceSettings.class, JinaAIRerankServiceSettings.NAME, JinaAIRerankServiceSettings::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(TaskSettings.class, JinaAIRerankTaskSettings.NAME, JinaAIRerankTaskSettings::new) + ); + } + private static void addEisNamedWriteables(List namedWriteables) { namedWriteables.add( new NamedWriteableRegistry.Entry( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index ee7be8bf88c7d..b16c53a428d73 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -113,6 +113,7 @@ import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceService; import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserService; import org.elasticsearch.xpack.inference.services.ibmwatsonx.IbmWatsonxService; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIService; import org.elasticsearch.xpack.inference.services.mistral.MistralService; import org.elasticsearch.xpack.inference.services.openai.OpenAiService; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; @@ -320,6 +321,7 @@ public List getInferenceServiceFactories() { context -> new AmazonBedrockService(httpFactory.get(), amazonBedrockFactory.get(), serviceComponents.get()), context -> new AlibabaCloudSearchService(httpFactory.get(), serviceComponents.get()), context -> new IbmWatsonxService(httpFactory.get(), serviceComponents.get()), + context -> new JinaAIService(httpFactory.get(), serviceComponents.get()), ElasticsearchInternalService::new ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java new file mode 100644 index 0000000000000..4d5827a3bf266 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionCreator.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.jinaai; + +import org.elasticsearch.inference.InputType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.JinaAIEmbeddingsRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.JinaAIRerankRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel; + +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; + +/** + * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the jinaai model type. + */ +public class JinaAIActionCreator implements JinaAIActionVisitor { + private final Sender sender; + private final ServiceComponents serviceComponents; + + public JinaAIActionCreator(Sender sender, ServiceComponents serviceComponents) { + this.sender = Objects.requireNonNull(sender); + this.serviceComponents = Objects.requireNonNull(serviceComponents); + } + + @Override + public ExecutableAction create(JinaAIEmbeddingsModel model, Map taskSettings, InputType inputType) { + var overriddenModel = JinaAIEmbeddingsModel.of(model, taskSettings, inputType); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( + overriddenModel.getServiceSettings().getCommonSettings().uri(), + "JinaAI embeddings" + ); + var requestCreator = JinaAIEmbeddingsRequestManager.of(overriddenModel, serviceComponents.threadPool()); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); + } + + @Override + public ExecutableAction create(JinaAIRerankModel model, Map taskSettings) { + var overriddenModel = JinaAIRerankModel.of(model, taskSettings); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( + overriddenModel.getServiceSettings().getCommonSettings().uri(), + "JinaAI rerank" + ); + var requestCreator = JinaAIRerankRequestManager.of(overriddenModel, serviceComponents.threadPool()); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionVisitor.java new file mode 100644 index 0000000000000..c585e68e3d731 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/jinaai/JinaAIActionVisitor.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.jinaai; + +import org.elasticsearch.inference.InputType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel; + +import java.util.Map; + +public interface JinaAIActionVisitor { + ExecutableAction create(JinaAIEmbeddingsModel model, Map taskSettings, InputType inputType); + + ExecutableAction create(JinaAIRerankModel model, Map taskSettings); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIEmbeddingsRequestManager.java new file mode 100644 index 0000000000000..c0828224cd1a9 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIEmbeddingsRequestManager.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.xpack.inference.external.jinaai.JinaAIResponseHandler; +import org.elasticsearch.xpack.inference.external.request.jinaai.JinaAIEmbeddingsRequest; +import org.elasticsearch.xpack.inference.external.response.jinaai.JinaAIEmbeddingsResponseEntity; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel; + +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +public class JinaAIEmbeddingsRequestManager extends JinaAIRequestManager { + private static final Logger logger = LogManager.getLogger(JinaAIEmbeddingsRequestManager.class); + private static final ResponseHandler HANDLER = createEmbeddingsHandler(); + + private static ResponseHandler createEmbeddingsHandler() { + return new JinaAIResponseHandler("jinaai text embedding", JinaAIEmbeddingsResponseEntity::fromResponse); + } + + public static JinaAIEmbeddingsRequestManager of(JinaAIEmbeddingsModel model, ThreadPool threadPool) { + return new JinaAIEmbeddingsRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } + + private final JinaAIEmbeddingsModel model; + + private JinaAIEmbeddingsRequestManager(JinaAIEmbeddingsModel model, ThreadPool threadPool) { + super(threadPool, model); + this.model = Objects.requireNonNull(model); + } + + @Override + public void execute( + InferenceInputs inferenceInputs, + RequestSender requestSender, + Supplier hasRequestCompletedFunction, + ActionListener listener + ) { + List docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs(); + JinaAIEmbeddingsRequest request = new JinaAIEmbeddingsRequest(docsInput, model); + + execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRequestManager.java new file mode 100644 index 0000000000000..3a0d6e4e17f5b --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRequestManager.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIModel; + +import java.util.Objects; + +abstract class JinaAIRequestManager extends BaseRequestManager { + + protected JinaAIRequestManager(ThreadPool threadPool, JinaAIModel model) { + super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model), model.rateLimitServiceSettings().rateLimitSettings()); + } + + record RateLimitGrouping(int apiKeyHash) { + public static RateLimitGrouping of(JinaAIModel model) { + Objects.requireNonNull(model); + + return new RateLimitGrouping(model.apiKey().hashCode()); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRerankRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRerankRequestManager.java new file mode 100644 index 0000000000000..26f134873bca0 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/JinaAIRerankRequestManager.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.xpack.inference.external.jinaai.JinaAIResponseHandler; +import org.elasticsearch.xpack.inference.external.request.jinaai.JinaAIRerankRequest; +import org.elasticsearch.xpack.inference.external.response.jinaai.JinaAIRerankResponseEntity; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel; + +import java.util.Objects; +import java.util.function.Supplier; + +public class JinaAIRerankRequestManager extends JinaAIRequestManager { + private static final Logger logger = LogManager.getLogger(JinaAIRerankRequestManager.class); + private static final ResponseHandler HANDLER = createJinaAIResponseHandler(); + + private static ResponseHandler createJinaAIResponseHandler() { + return new JinaAIResponseHandler("jinaai rerank", (request, response) -> JinaAIRerankResponseEntity.fromResponse(response)); + } + + public static JinaAIRerankRequestManager of(JinaAIRerankModel model, ThreadPool threadPool) { + return new JinaAIRerankRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } + + private final JinaAIRerankModel model; + + private JinaAIRerankRequestManager(JinaAIRerankModel model, ThreadPool threadPool) { + super(threadPool, model); + this.model = model; + } + + @Override + public void execute( + InferenceInputs inferenceInputs, + RequestSender requestSender, + Supplier hasRequestCompletedFunction, + ActionListener listener + ) { + var rerankInput = QueryAndDocsInputs.of(inferenceInputs); + JinaAIRerankRequest request = new JinaAIRerankRequest(rerankInput.getQuery(), rerankInput.getChunks(), model); + + execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIAccount.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIAccount.java new file mode 100644 index 0000000000000..722a785db4795 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIAccount.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.jinaai; + +import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIModel; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri; + +public record JinaAIAccount(URI uri, SecureString apiKey) { + + public static JinaAIAccount of(JinaAIModel model, CheckedSupplier uriBuilder) { + var uri = buildUri(model.uri(), "JinaAI", uriBuilder); + + return new JinaAIAccount(uri, model.apiKey()); + } + + public JinaAIAccount { + Objects.requireNonNull(uri); + Objects.requireNonNull(apiKey); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandler.java new file mode 100644 index 0000000000000..66dc85b3bdb6a --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandler.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.jinaai; + +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; +import org.elasticsearch.xpack.inference.external.http.retry.RetryException; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.response.jinaai.JinaAIErrorResponseEntity; + +/** + * Defines how to handle various errors returned from the JinaAI integration. + * + */ +public class JinaAIResponseHandler extends BaseResponseHandler { + static final String VALIDATION_ERROR_MESSAGE = "Received an input validation error response"; + static final String PAYMENT_ERROR_MESSAGE = "Payment required"; + + public JinaAIResponseHandler(String requestType, ResponseParser parseFunction) { + super(requestType, parseFunction, JinaAIErrorResponseEntity::fromResponse); + } + + /** + * Validates the status code throws an RetryException if not in the range [200, 300). + * + * @param request The http request + * @param result The http response and body + * @throws RetryException Throws if status code is {@code >= 300 or < 200 } + */ + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + if (result.isSuccessfulResponse()) { + return; + } + + // handle error codes + int statusCode = result.response().getStatusLine().getStatusCode(); + if (statusCode == 500) { + throw new RetryException(true, buildError(SERVER_ERROR, request, result)); + } else if (statusCode > 500) { + throw new RetryException(false, buildError(SERVER_ERROR, request, result)); + } else if (statusCode == 429) { + throw new RetryException(true, buildError(RATE_LIMIT, request, result)); + } else if (statusCode == 400 || statusCode == 422) { + throw new RetryException(false, buildError(VALIDATION_ERROR_MESSAGE, request, result)); + } else if (statusCode == 401) { + throw new RetryException(false, buildError(AUTHENTICATION, request, result)); + } else if (statusCode == 402) { + throw new RetryException(false, buildError(PAYMENT_ERROR_MESSAGE, request, result)); + } else if (statusCode >= 300 && statusCode < 400) { + throw new RetryException(false, buildError(REDIRECTION, request, result)); + } else { + throw new RetryException(false, buildError(UNSUCCESSFUL, request, result)); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequest.java new file mode 100644 index 0000000000000..d99f15a7703ae --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequest.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ByteArrayEntity; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.inference.external.jinaai.JinaAIAccount; +import org.elasticsearch.xpack.inference.external.request.HttpRequest; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Objects; + +public class JinaAIEmbeddingsRequest extends JinaAIRequest { + + private final JinaAIAccount account; + private final List input; + private final JinaAIEmbeddingsTaskSettings taskSettings; + private final String model; + private final String inferenceEntityId; + + public JinaAIEmbeddingsRequest(List input, JinaAIEmbeddingsModel embeddingsModel) { + Objects.requireNonNull(embeddingsModel); + + account = JinaAIAccount.of(embeddingsModel, JinaAIEmbeddingsRequest::buildDefaultUri); + this.input = Objects.requireNonNull(input); + taskSettings = embeddingsModel.getTaskSettings(); + model = embeddingsModel.getServiceSettings().getCommonSettings().modelId(); + inferenceEntityId = embeddingsModel.getInferenceEntityId(); + } + + @Override + public HttpRequest createHttpRequest() { + HttpPost httpPost = new HttpPost(account.uri()); + + ByteArrayEntity byteEntity = new ByteArrayEntity( + Strings.toString(new JinaAIEmbeddingsRequestEntity(input, taskSettings, model)).getBytes(StandardCharsets.UTF_8) + ); + httpPost.setEntity(byteEntity); + + decorateWithAuthHeader(httpPost, account); + + return new HttpRequest(httpPost, getInferenceEntityId()); + } + + @Override + public String getInferenceEntityId() { + return inferenceEntityId; + } + + @Override + public URI getURI() { + return account.uri(); + } + + @Override + public Request truncate() { + return this; + } + + @Override + public boolean[] getTruncationInfo() { + return null; + } + + public static URI buildDefaultUri() throws URISyntaxException { + return new URIBuilder().setScheme("https") + .setHost(JinaAIUtils.HOST) + .setPathSegments(JinaAIUtils.VERSION_1, JinaAIUtils.EMBEDDINGS_PATH) + .build(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntity.java new file mode 100644 index 0000000000000..d4f98f1eb52ca --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntity.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings.invalidInputTypeMessage; + +public record JinaAIEmbeddingsRequestEntity(List input, JinaAIEmbeddingsTaskSettings taskSettings, @Nullable String model) + implements + ToXContentObject { + + private static final String SEARCH_DOCUMENT = "retrieval.passage"; + private static final String SEARCH_QUERY = "retrieval.query"; + private static final String CLUSTERING = "separation"; + private static final String CLASSIFICATION = "classification"; + private static final String INPUT_FIELD = "input"; + private static final String MODEL_FIELD = "model"; + public static final String TASK_TYPE_FIELD = "task"; + + public JinaAIEmbeddingsRequestEntity { + Objects.requireNonNull(input); + Objects.requireNonNull(taskSettings); + Objects.requireNonNull(model); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INPUT_FIELD, input); + builder.field(MODEL_FIELD, model); + + if (taskSettings.getInputType() != null) { + builder.field(TASK_TYPE_FIELD, convertToString(taskSettings.getInputType())); + } + + builder.endObject(); + return builder; + } + + // default for testing + static String convertToString(InputType inputType) { + return switch (inputType) { + case INGEST -> SEARCH_DOCUMENT; + case SEARCH -> SEARCH_QUERY; + case CLASSIFICATION -> CLASSIFICATION; + case CLUSTERING -> CLUSTERING; + default -> { + assert false : invalidInputTypeMessage(inputType); + yield null; + } + }; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequest.java new file mode 100644 index 0000000000000..8b1e26a36238b --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequest.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.jinaai.JinaAIAccount; +import org.elasticsearch.xpack.inference.external.request.Request; + +import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader; + +public abstract class JinaAIRequest implements Request { + + public static void decorateWithAuthHeader(HttpPost request, JinaAIAccount account) { + request.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()); + request.setHeader(createAuthBearerHeader(account.apiKey())); + request.setHeader(JinaAIUtils.createRequestSourceHeader()); + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequest.java new file mode 100644 index 0000000000000..93d4ab830c604 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequest.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ByteArrayEntity; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.inference.external.jinaai.JinaAIAccount; +import org.elasticsearch.xpack.inference.external.request.HttpRequest; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Objects; + +public class JinaAIRerankRequest extends JinaAIRequest { + + private final JinaAIAccount account; + private final String query; + private final List input; + private final JinaAIRerankTaskSettings taskSettings; + private final String model; + private final String inferenceEntityId; + + public JinaAIRerankRequest(String query, List input, JinaAIRerankModel model) { + Objects.requireNonNull(model); + + this.account = JinaAIAccount.of(model, JinaAIRerankRequest::buildDefaultUri); + this.input = Objects.requireNonNull(input); + this.query = Objects.requireNonNull(query); + taskSettings = model.getTaskSettings(); + this.model = model.getServiceSettings().modelId(); + inferenceEntityId = model.getInferenceEntityId(); + } + + @Override + public HttpRequest createHttpRequest() { + HttpPost httpPost = new HttpPost(account.uri()); + + ByteArrayEntity byteEntity = new ByteArrayEntity( + Strings.toString(new JinaAIRerankRequestEntity(query, input, taskSettings, model)).getBytes(StandardCharsets.UTF_8) + ); + httpPost.setEntity(byteEntity); + + decorateWithAuthHeader(httpPost, account); + + return new HttpRequest(httpPost, getInferenceEntityId()); + } + + @Override + public String getInferenceEntityId() { + return inferenceEntityId; + } + + @Override + public URI getURI() { + return account.uri(); + } + + @Override + public Request truncate() { + return this; + } + + @Override + public boolean[] getTruncationInfo() { + return null; + } + + public static URI buildDefaultUri() throws URISyntaxException { + return new URIBuilder().setScheme("https") + .setHost(JinaAIUtils.HOST) + .setPathSegments(JinaAIUtils.VERSION_1, JinaAIUtils.RERANK_PATH) + .build(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntity.java new file mode 100644 index 0000000000000..7f470d5fa91f5 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntity.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public record JinaAIRerankRequestEntity(String model, String query, List documents, JinaAIRerankTaskSettings taskSettings) + implements + ToXContentObject { + + private static final String DOCUMENTS_FIELD = "documents"; + private static final String QUERY_FIELD = "query"; + private static final String MODEL_FIELD = "model"; + + public JinaAIRerankRequestEntity { + Objects.requireNonNull(query); + Objects.requireNonNull(documents); + Objects.requireNonNull(model); + Objects.requireNonNull(taskSettings); + } + + public JinaAIRerankRequestEntity(String query, List input, JinaAIRerankTaskSettings taskSettings, String model) { + this(model, query, input, taskSettings != null ? taskSettings : JinaAIRerankTaskSettings.EMPTY_SETTINGS); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.field(MODEL_FIELD, model); + builder.field(QUERY_FIELD, query); + builder.field(DOCUMENTS_FIELD, documents); + + if (taskSettings.getTopNDocumentsOnly() != null) { + builder.field(JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, taskSettings.getTopNDocumentsOnly()); + } + + var return_documents = taskSettings.getDoesReturnDocuments(); + if (return_documents != null) { + builder.field(JinaAIRerankTaskSettings.RETURN_DOCUMENTS, return_documents); + } + + builder.endObject(); + return builder; + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtils.java new file mode 100644 index 0000000000000..fccbd1e230556 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtils.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; + +public class JinaAIUtils { + public static final String HOST = "api.jina.ai"; + public static final String VERSION_1 = "v1"; + public static final String EMBEDDINGS_PATH = "embeddings"; + public static final String RERANK_PATH = "rerank"; + public static final String REQUEST_SOURCE_HEADER = "Request-Source"; + public static final String ELASTIC_REQUEST_SOURCE = "unspecified:elasticsearch"; + + public static Header createRequestSourceHeader() { + return new BasicHeader(REQUEST_SOURCE_HEADER, ELASTIC_REQUEST_SOURCE); + } + + private JinaAIUtils() {} +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java new file mode 100644 index 0000000000000..26bde5f5f48ad --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a generative AI + */ + +package org.elasticsearch.xpack.inference.external.response.jinaai; + +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.response.XContentUtils; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.consumeUntilObjectEnd; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField; + +public class JinaAIEmbeddingsResponseEntity { + private static final String FAILED_TO_FIND_FIELD_TEMPLATE = "Failed to find required field [%s] in JinaAI embeddings response"; + + /** + * Parses the JinaAI json response. + * For a request like: + * + * 
+     *     
+     *        {
+     *            "inputs": ["hello this is my name", "I wish I was there!"]
+     *        }
+     *     
+     *
+ * + * The response would look like: + * + * 
+     * 
+     * {
+     *  "object": "list",
+     *  "data": [
+     *      {
+     *          "object": "embedding",
+     *          "embedding": [
+     *              -0.009327292,
+     *              -0.0028842222,
+     *          ],
+     *          "index": 0
+     *      },
+     *      {
+     *          "object": "embedding",
+     *          "embedding": [ ... ],
+     *          "index": 1
+     *      }
+     *  ],
+     *  "model": "jina-embeddings-v3",
+     *  "usage": {
+     *      "prompt_tokens": 8,
+     *      "total_tokens": 8
+     *  }
+     * }
+     * 
+     *
+ */ + public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { + var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); + + try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { + moveToFirstToken(jsonParser); + + XContentParser.Token token = jsonParser.currentToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + + positionParserAtTokenAfterField(jsonParser, "data", FAILED_TO_FIND_FIELD_TEMPLATE); + + List embeddingList = parseList( + jsonParser, + JinaAIEmbeddingsResponseEntity::parseEmbeddingObject + ); + + return new InferenceTextEmbeddingFloatResults(embeddingList); + } + } + + private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser) + throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + + positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE); + + List embeddingValuesList = parseList(parser, XContentUtils::parseFloat); + // parse and discard the rest of the object + consumeUntilObjectEnd(parser); + + return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + } + + private JinaAIEmbeddingsResponseEntity() {} +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntity.java new file mode 100644 index 0000000000000..99d29b26e4a04 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntity.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.jinaai; + +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; + +public class JinaAIErrorResponseEntity extends ErrorResponse { + + private JinaAIErrorResponseEntity(String errorMessage) { + super(errorMessage); + } + + /** + * Parse an HTTP response into a JinaAIErrorResponseEntity + * + * @param response The error response + * @return An error entity if the response is JSON with a `detail` field containing the error message + * or null if the response does not contain the message field + */ + public static ErrorResponse fromResponse(HttpResult response) { + try ( + XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, response.body()) + ) { + var responseMap = jsonParser.map(); + var message = (String) responseMap.get("detail"); + if (message != null) { + return new JinaAIErrorResponseEntity(message); + } + } catch (Exception e) { + // swallow the error + } + + return ErrorResponse.UNDEFINED_ERROR; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntity.java new file mode 100644 index 0000000000000..d22bc875041e0 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntity.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a generative AI + */ + +package org.elasticsearch.xpack.inference.external.response.jinaai; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList; +import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField; +import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField; + +public class JinaAIRerankResponseEntity { + + private static final Logger logger = LogManager.getLogger(JinaAIRerankResponseEntity.class); + + /** + * Parses the JinaAI ranked response. + * + * For a request like: + * "model": "jina-reranker-v2-base-multilingual", + * "query": "What is the capital of the United States?", + * "top_n": 3, + * "documents": ["Carson City is the capital city of the American state of Nevada.", + * "The Commonwealth of the Northern Mariana ... Its capital is Saipan.", + * "Washington, D.C. (also known as simply Washington or D.C., ... It is a federal district.", + * "Capital punishment (the death penalty) ... As of 2017, capital punishment is legal in 30 of the 50 states."] + *

+ * The response will look like (without whitespace): + * { + * "id": "1983d114-a6e8-4940-b121-eb4ac3f6f703", + * "results": [ + * { + * "document": { + * "text": "Washington, D.C. is the capital of the United States. It is a federal district." + * }, + * "index": 2, + * "relevance_score": 0.98005307 + * }, + * { + * "document": { + * "text": "Capital punishment (the death penalty) As of 2017, capital punishment is legal in 30 of the 50 states." + * }, + * "index": 3, + * "relevance_score": 0.27904198 + * }, + * { + * "document": { + * "text": "Carson City is the capital city of the American state of Nevada." + * }, + * "index": 0, + * "relevance_score": 0.10194652 + * } + * ], + * "usage": {"total_tokens": 15} + * } + * + * @param response the http response from JinaAI + * @return the parsed response + * @throws IOException if there is an error parsing the response + */ + public static InferenceServiceResults fromResponse(HttpResult response) throws IOException { + var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); + + try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { + moveToFirstToken(jsonParser); + + XContentParser.Token token = jsonParser.currentToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + + positionParserAtTokenAfterField(jsonParser, "results", FAILED_TO_FIND_FIELD_TEMPLATE); + + token = jsonParser.currentToken(); + if (token == XContentParser.Token.START_ARRAY) { + return new RankedDocsResults(parseList(jsonParser, JinaAIRerankResponseEntity::parseRankedDocObject)); + } else { + throwUnknownToken(token, jsonParser); + } + + // This should never be reached. The above code should either return successfully or hit the throwUnknownToken + // or throw a parsing exception + throw new IllegalStateException("Reached an invalid state while parsing the JinaAI response"); + } + } + + private static RankedDocsResults.RankedDoc parseRankedDocObject(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + int index = -1; + float relevanceScore = -1; + String documentText = null; + parser.nextToken(); + while (parser.currentToken() != XContentParser.Token.END_OBJECT) { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + switch (parser.currentName()) { + case "index": + parser.nextToken(); // move to VALUE_NUMBER + index = parser.intValue(); + parser.nextToken(); // move to next FIELD_NAME or END_OBJECT + break; + case "relevance_score": + parser.nextToken(); // move to VALUE_NUMBER + relevanceScore = parser.floatValue(); + parser.nextToken(); // move to next FIELD_NAME or END_OBJECT + break; + case "document": + parser.nextToken(); // move to START_OBJECT; document text is wrapped in an object + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + do { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME && parser.currentName().equals("text")) { + parser.nextToken(); // move to VALUE_STRING + documentText = parser.text(); + } + } while (parser.nextToken() != XContentParser.Token.END_OBJECT); + parser.nextToken();// move past END_OBJECT + // parser should now be at the next FIELD_NAME or END_OBJECT + break; + default: + throwUnknownField(parser.currentName(), parser); + } + } else { + parser.nextToken(); + } + } + + if (index == -1) { + logger.warn("Failed to find required field [index] in JinaAI rerank response"); + } + if (relevanceScore == -1) { + logger.warn("Failed to find required field [relevance_score] in JinaAI rerank response"); + } + // documentText may or may not be present depending on the request parameter + + return new RankedDocsResults.RankedDoc(index, relevanceScore, documentText); + } + + private JinaAIRerankResponseEntity() {} + + static String FAILED_TO_FIND_FIELD_TEMPLATE = "Failed to find required field [%s] in JinaAI rerank response"; +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIModel.java new file mode 100644 index 0000000000000..bfd8235e3da48 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIModel.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionVisitor; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets; + +import java.net.URI; +import java.util.Map; +import java.util.Objects; + +public abstract class JinaAIModel extends Model { + private final SecureString apiKey; + private final JinaAIRateLimitServiceSettings rateLimitServiceSettings; + + public JinaAIModel( + ModelConfigurations configurations, + ModelSecrets secrets, + @Nullable ApiKeySecrets apiKeySecrets, + JinaAIRateLimitServiceSettings rateLimitServiceSettings + ) { + super(configurations, secrets); + + this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings); + apiKey = ServiceUtils.apiKey(apiKeySecrets); + } + + protected JinaAIModel(JinaAIModel model, TaskSettings taskSettings) { + super(model, taskSettings); + + rateLimitServiceSettings = model.rateLimitServiceSettings(); + apiKey = model.apiKey(); + } + + protected JinaAIModel(JinaAIModel model, ServiceSettings serviceSettings) { + super(model, serviceSettings); + + rateLimitServiceSettings = model.rateLimitServiceSettings(); + apiKey = model.apiKey(); + } + + public SecureString apiKey() { + return apiKey; + } + + public JinaAIRateLimitServiceSettings rateLimitServiceSettings() { + return rateLimitServiceSettings; + } + + public abstract ExecutableAction accept(JinaAIActionVisitor creator, Map taskSettings, InputType inputType); + + public abstract URI uri(); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIRateLimitServiceSettings.java new file mode 100644 index 0000000000000..ac65ad1c9d714 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIRateLimitServiceSettings.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai; + +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; + +public interface JinaAIRateLimitServiceSettings { + RateLimitSettings rateLimitSettings(); + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java new file mode 100644 index 0000000000000..11a72f811e8d3 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java @@ -0,0 +1,358 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.LazyInitializable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInference; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.EmptySettingsConfiguration; +import org.elasticsearch.inference.InferenceServiceConfiguration; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.SettingsConfiguration; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.inference.TaskSettingsConfiguration; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; +import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; +import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionCreator; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.SenderService; +import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModel; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; +import static org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceFields.EMBEDDING_MAX_BATCH_SIZE; + +public class JinaAIService extends SenderService { + public static final String NAME = "jinaai"; + + private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.RERANK); + + public JinaAIService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { + super(factory, serviceComponents); + } + + @Override + public String name() { + return NAME; + } + + @Override + public void parseRequestConfig( + String inferenceEntityId, + TaskType taskType, + Map config, + ActionListener parsedModelListener + ) { + try { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + + ChunkingSettings chunkingSettings = null; + if (TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap( + removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS) + ); + } + JinaAIModel model = createModel( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + chunkingSettings, + serviceSettingsMap, + TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), + ConfigurationParseContext.REQUEST + ); + + throwIfNotEmptyMap(config, NAME); + throwIfNotEmptyMap(serviceSettingsMap, NAME); + throwIfNotEmptyMap(taskSettingsMap, NAME); + + parsedModelListener.onResponse(model); + } catch (Exception e) { + parsedModelListener.onFailure(e); + } + } + + private static JinaAIModel createModelFromPersistent( + String inferenceEntityId, + TaskType taskType, + Map serviceSettings, + Map taskSettings, + ChunkingSettings chunkingSettings, + @Nullable Map secretSettings, + String failureMessage + ) { + return createModel( + inferenceEntityId, + taskType, + serviceSettings, + taskSettings, + chunkingSettings, + secretSettings, + failureMessage, + ConfigurationParseContext.PERSISTENT + ); + } + + private static JinaAIModel createModel( + String inferenceEntityId, + TaskType taskType, + Map serviceSettings, + Map taskSettings, + ChunkingSettings chunkingSettings, + @Nullable Map secretSettings, + String failureMessage, + ConfigurationParseContext context + ) { + return switch (taskType) { + case TEXT_EMBEDDING -> new JinaAIEmbeddingsModel( + inferenceEntityId, + NAME, + serviceSettings, + taskSettings, + chunkingSettings, + secretSettings, + context + ); + case RERANK -> new JinaAIRerankModel(inferenceEntityId, NAME, serviceSettings, taskSettings, secretSettings, context); + default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST); + }; + } + + @Override + public JinaAIModel parsePersistedConfigWithSecrets( + String inferenceEntityId, + TaskType taskType, + Map config, + Map secrets + ) { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + Map secretSettingsMap = removeFromMapOrThrowIfNull(secrets, ModelSecrets.SECRET_SETTINGS); + + ChunkingSettings chunkingSettings = null; + if (TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMap(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + + return createModelFromPersistent( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + chunkingSettings, + secretSettingsMap, + parsePersistedConfigErrorMsg(inferenceEntityId, NAME) + ); + } + + @Override + public JinaAIModel parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map config) { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + + ChunkingSettings chunkingSettings = null; + if (TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMap(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + + return createModelFromPersistent( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + chunkingSettings, + null, + parsePersistedConfigErrorMsg(inferenceEntityId, NAME) + ); + } + + @Override + public InferenceServiceConfiguration getConfiguration() { + return Configuration.get(); + } + + @Override + public EnumSet supportedTaskTypes() { + return supportedTaskTypes; + } + + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + + @Override + public void doInfer( + Model model, + InferenceInputs inputs, + Map taskSettings, + InputType inputType, + TimeValue timeout, + ActionListener listener + ) { + if (model instanceof JinaAIModel == false) { + listener.onFailure(createInvalidModelException(model)); + return; + } + + JinaAIModel jinaaiModel = (JinaAIModel) model; + var actionCreator = new JinaAIActionCreator(getSender(), getServiceComponents()); + + var action = jinaaiModel.accept(actionCreator, taskSettings, inputType); + action.execute(inputs, timeout, listener); + } + + @Override + protected void doChunkedInfer( + Model model, + DocumentsOnlyInput inputs, + Map taskSettings, + InputType inputType, + TimeValue timeout, + ActionListener> listener + ) { + if (model instanceof JinaAIModel == false) { + listener.onFailure(createInvalidModelException(model)); + return; + } + + JinaAIModel jinaaiModel = (JinaAIModel) model; + var actionCreator = new JinaAIActionCreator(getSender(), getServiceComponents()); + + List batchedRequests = new EmbeddingRequestChunker( + inputs.getInputs(), + EMBEDDING_MAX_BATCH_SIZE, + EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(model.getServiceSettings().elementType()), + jinaaiModel.getConfigurations().getChunkingSettings() + ).batchRequestsWithListeners(listener); + + for (var request : batchedRequests) { + var action = jinaaiModel.accept(actionCreator, taskSettings, inputType); + action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); + } + } + + /** + * For text embedding models get the embedding size and + * update the service settings. + * + * @param model The new model + * @param listener The listener + */ + @Override + public void checkModelConfig(Model model, ActionListener listener) { + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); + } + + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof JinaAIEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? defaultSimilarity() : similarityFromModel; + var maxInputTokens = serviceSettings.maxInputTokens(); + + var updatedServiceSettings = new JinaAIEmbeddingsServiceSettings( + new JinaAIServiceSettings( + serviceSettings.getCommonSettings().uri(), + serviceSettings.getCommonSettings().modelId(), + serviceSettings.getCommonSettings().rateLimitSettings() + ), + similarityToUse, + embeddingSize, + maxInputTokens + ); + + return new JinaAIEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } + } + + /** + * Return the default similarity measure for the embedding type. + * JinaAI embeddings are normalized to unit vectors therefore Dot + * Product similarity can be used and is the default for all JinaAI + * models. + * + * @return The default similarity. + */ + static SimilarityMeasure defaultSimilarity() { + return SimilarityMeasure.DOT_PRODUCT; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.JINA_AI_INTEGRATION_ADDED; + } + + public static class Configuration { + public static InferenceServiceConfiguration get() { + return configuration.getOrCompute(); + } + + private static final LazyInitializable configuration = new LazyInitializable<>( + () -> { + var configurationMap = new HashMap(); + + configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); + configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); + + return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { + Map taskSettingsConfig; + switch (t) { + case TEXT_EMBEDDING -> taskSettingsConfig = JinaAIEmbeddingsModel.Configuration.get(); + case RERANK -> taskSettingsConfig = JinaAIRerankModel.Configuration.get(); + default -> taskSettingsConfig = EmptySettingsConfiguration.get(); + } + return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); + }).toList()).setConfiguration(configurationMap).build(); + } + ); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceFields.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceFields.java new file mode 100644 index 0000000000000..2df8f1440e471 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceFields.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai; + +public class JinaAIServiceFields { + + static final int EMBEDDING_MAX_BATCH_SIZE = 2048; +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettings.java new file mode 100644 index 0000000000000..66c6193f653f1 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettings.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceFields.URL; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.convertToUri; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.createOptionalUri; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString; + +public class JinaAIServiceSettings extends FilteredXContentObject implements ServiceSettings, JinaAIRateLimitServiceSettings { + + public static final String NAME = "jinaai_service_settings"; + public static final String MODEL_ID = "model_id"; + private static final Logger logger = LogManager.getLogger(JinaAIServiceSettings.class); + // See https://jina.ai/contact-sales/#rate-limit + public static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(2_000); + + public static JinaAIServiceSettings fromMap(Map map, ConfigurationParseContext context) { + ValidationException validationException = new ValidationException(); + + String url = extractOptionalString(map, URL, ModelConfigurations.SERVICE_SETTINGS, validationException); + URI uri = convertToUri(url, URL, ModelConfigurations.SERVICE_SETTINGS, validationException); + RateLimitSettings rateLimitSettings = RateLimitSettings.of( + map, + DEFAULT_RATE_LIMIT_SETTINGS, + validationException, + JinaAIService.NAME, + context + ); + + String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new JinaAIServiceSettings(uri, modelId, rateLimitSettings); + } + + private final URI uri; + private final String modelId; + private final RateLimitSettings rateLimitSettings; + + public JinaAIServiceSettings(@Nullable URI uri, String modelId, @Nullable RateLimitSettings rateLimitSettings) { + this.uri = uri; + this.modelId = Objects.requireNonNull(modelId); + this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS); + } + + public JinaAIServiceSettings(@Nullable String url, String modelId, @Nullable RateLimitSettings rateLimitSettings) { + this(createOptionalUri(url), modelId, rateLimitSettings); + } + + public JinaAIServiceSettings(StreamInput in) throws IOException { + uri = createOptionalUri(in.readOptionalString()); + modelId = in.readOptionalString(); + rateLimitSettings = new RateLimitSettings(in); + } + + @Override + public RateLimitSettings rateLimitSettings() { + return rateLimitSettings; + } + + public URI uri() { + return uri; + } + + @Override + public String modelId() { + return modelId; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + toXContentFragment(builder, params); + + builder.endObject(); + return builder; + } + + public XContentBuilder toXContentFragment(XContentBuilder builder, Params params) throws IOException { + return toXContentFragmentOfExposedFields(builder, params); + } + + @Override + public XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + if (uri != null) { + builder.field(URL, uri.toString()); + } + if (modelId != null) { + builder.field(MODEL_ID, modelId); + } + rateLimitSettings.toXContent(builder, params); + + return builder; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.JINA_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + var uriToWrite = uri != null ? uri.toString() : null; + out.writeOptionalString(uriToWrite); + out.writeOptionalString(modelId); + rateLimitSettings.writeTo(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JinaAIServiceSettings that = (JinaAIServiceSettings) o; + return Objects.equals(uri, that.uri) + && Objects.equals(modelId, that.modelId) + && Objects.equals(rateLimitSettings, that.rateLimitSettings); + } + + @Override + public int hashCode() { + return Objects.hash(uri, modelId, rateLimitSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java new file mode 100644 index 0000000000000..dd479802cdf13 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.embeddings; + +import org.elasticsearch.common.util.LazyInitializable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.SettingsConfiguration; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; +import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; +import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionVisitor; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIModel; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; + +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.inference.external.request.jinaai.JinaAIEmbeddingsRequestEntity.TASK_TYPE_FIELD; + +public class JinaAIEmbeddingsModel extends JinaAIModel { + public static JinaAIEmbeddingsModel of(JinaAIEmbeddingsModel model, Map taskSettings, InputType inputType) { + var requestTaskSettings = JinaAIEmbeddingsTaskSettings.fromMap(taskSettings); + return new JinaAIEmbeddingsModel(model, JinaAIEmbeddingsTaskSettings.of(model.getTaskSettings(), requestTaskSettings, inputType)); + } + + public JinaAIEmbeddingsModel( + String inferenceId, + String service, + Map serviceSettings, + Map taskSettings, + ChunkingSettings chunkingSettings, + @Nullable Map secrets, + ConfigurationParseContext context + ) { + this( + inferenceId, + service, + JinaAIEmbeddingsServiceSettings.fromMap(serviceSettings, context), + JinaAIEmbeddingsTaskSettings.fromMap(taskSettings), + chunkingSettings, + DefaultSecretSettings.fromMap(secrets) + ); + } + + // should only be used for testing + JinaAIEmbeddingsModel( + String modelId, + String service, + JinaAIEmbeddingsServiceSettings serviceSettings, + JinaAIEmbeddingsTaskSettings taskSettings, + ChunkingSettings chunkingSettings, + @Nullable DefaultSecretSettings secretSettings + ) { + super( + new ModelConfigurations(modelId, TaskType.TEXT_EMBEDDING, service, serviceSettings, taskSettings, chunkingSettings), + new ModelSecrets(secretSettings), + secretSettings, + serviceSettings.getCommonSettings() + ); + } + + private JinaAIEmbeddingsModel(JinaAIEmbeddingsModel model, JinaAIEmbeddingsTaskSettings taskSettings) { + super(model, taskSettings); + } + + public JinaAIEmbeddingsModel(JinaAIEmbeddingsModel model, JinaAIEmbeddingsServiceSettings serviceSettings) { + super(model, serviceSettings); + } + + @Override + public JinaAIEmbeddingsServiceSettings getServiceSettings() { + return (JinaAIEmbeddingsServiceSettings) super.getServiceSettings(); + } + + @Override + public JinaAIEmbeddingsTaskSettings getTaskSettings() { + return (JinaAIEmbeddingsTaskSettings) super.getTaskSettings(); + } + + @Override + public DefaultSecretSettings getSecretSettings() { + return (DefaultSecretSettings) super.getSecretSettings(); + } + + @Override + public ExecutableAction accept(JinaAIActionVisitor visitor, Map taskSettings, InputType inputType) { + return visitor.create(this, taskSettings, inputType); + } + + @Override + public URI uri() { + return getServiceSettings().getCommonSettings().uri(); + } + + public static class Configuration { + public static Map get() { + return configuration.getOrCompute(); + } + + private static final LazyInitializable, RuntimeException> configuration = + new LazyInitializable<>(() -> { + var configurationMap = new HashMap(); + + configurationMap.put( + TASK_TYPE_FIELD, + new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) + .setLabel("Task") + .setOrder(1) + .setRequired(false) + .setSensitive(false) + .setTooltip("Specifies the task type passed to the model.") + .setType(SettingsConfigurationFieldType.STRING) + .setOptions( + Stream.of("retrieval.query", "retrieval.passage", "classification", "separation") + .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) + .toList() + ) + .setValue("") + .build() + ); + + return Collections.unmodifiableMap(configurationMap); + }); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettings.java new file mode 100644 index 0000000000000..449da72674be4 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettings.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.embeddings; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings; +import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType; + +public class JinaAIEmbeddingsServiceSettings extends FilteredXContentObject implements ServiceSettings { + public static final String NAME = "jinaai_embeddings_service_settings"; + + public static JinaAIEmbeddingsServiceSettings fromMap(Map map, ConfigurationParseContext context) { + ValidationException validationException = new ValidationException(); + var commonServiceSettings = JinaAIServiceSettings.fromMap(map, context); + SimilarityMeasure similarity = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException); + Integer dims = removeAsType(map, DIMENSIONS, Integer.class); + Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new JinaAIEmbeddingsServiceSettings(commonServiceSettings, similarity, dims, maxInputTokens); + } + + private final JinaAIServiceSettings commonSettings; + private final SimilarityMeasure similarity; + private final Integer dimensions; + private final Integer maxInputTokens; + + public JinaAIEmbeddingsServiceSettings( + JinaAIServiceSettings commonSettings, + @Nullable SimilarityMeasure similarity, + @Nullable Integer dimensions, + @Nullable Integer maxInputTokens + ) { + this.commonSettings = commonSettings; + this.similarity = similarity; + this.dimensions = dimensions; + this.maxInputTokens = maxInputTokens; + } + + public JinaAIEmbeddingsServiceSettings(StreamInput in) throws IOException { + this.commonSettings = new JinaAIServiceSettings(in); + this.similarity = in.readOptionalEnum(SimilarityMeasure.class); + this.dimensions = in.readOptionalVInt(); + this.maxInputTokens = in.readOptionalVInt(); + } + + public JinaAIServiceSettings getCommonSettings() { + return commonSettings; + } + + @Override + public SimilarityMeasure similarity() { + return similarity; + } + + @Override + public Integer dimensions() { + return dimensions; + } + + public Integer maxInputTokens() { + return maxInputTokens; + } + + @Override + public String modelId() { + return commonSettings.modelId(); + } + + @Override + public DenseVectorFieldMapper.ElementType elementType() { + return DenseVectorFieldMapper.ElementType.FLOAT; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder = commonSettings.toXContentFragment(builder, params); + if (similarity != null) { + builder.field(SIMILARITY, similarity); + } + if (dimensions != null) { + builder.field(DIMENSIONS, dimensions); + } + if (maxInputTokens != null) { + builder.field(MAX_INPUT_TOKENS, maxInputTokens); + } + builder.endObject(); + return builder; + } + + @Override + protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + commonSettings.toXContentFragmentOfExposedFields(builder, params); + + return builder; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.JINA_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + commonSettings.writeTo(out); + out.writeOptionalEnum(SimilarityMeasure.translateSimilarity(similarity, out.getTransportVersion())); + out.writeOptionalVInt(dimensions); + out.writeOptionalVInt(maxInputTokens); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JinaAIEmbeddingsServiceSettings that = (JinaAIEmbeddingsServiceSettings) o; + return Objects.equals(commonSettings, that.commonSettings) + && Objects.equals(similarity, that.similarity) + && Objects.equals(dimensions, that.dimensions) + && Objects.equals(maxInputTokens, that.maxInputTokens); + } + + @Override + public int hashCode() { + return Objects.hash(commonSettings, similarity, dimensions, maxInputTokens); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettings.java new file mode 100644 index 0000000000000..77150b5097aa6 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettings.java @@ -0,0 +1,183 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.embeddings; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalEnum; + +/** + * Defines the task settings for the JinaAI text embeddings service. + * + */ +public class JinaAIEmbeddingsTaskSettings implements TaskSettings { + + public static final String NAME = "jinaai_embeddings_task_settings"; + public static final JinaAIEmbeddingsTaskSettings EMPTY_SETTINGS = new JinaAIEmbeddingsTaskSettings((InputType) null); + static final String INPUT_TYPE = "input_type"; + static final EnumSet VALID_REQUEST_VALUES = EnumSet.of( + InputType.INGEST, + InputType.SEARCH, + InputType.CLASSIFICATION, + InputType.CLUSTERING + ); + + public static JinaAIEmbeddingsTaskSettings fromMap(Map map) { + if (map == null || map.isEmpty()) { + return EMPTY_SETTINGS; + } + + ValidationException validationException = new ValidationException(); + + InputType inputType = extractOptionalEnum( + map, + INPUT_TYPE, + ModelConfigurations.TASK_SETTINGS, + InputType::fromString, + VALID_REQUEST_VALUES, + validationException + ); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new JinaAIEmbeddingsTaskSettings(inputType); + } + + /** + * Creates a new {@link JinaAIEmbeddingsTaskSettings} by preferring non-null fields from the provided parameters. + * For the input type, preference is given to requestInputType if it is not null and not UNSPECIFIED. + * Then preference is given to the requestTaskSettings and finally to originalSettings even if the value is null. + * + * Similarly, for the truncation field preference is given to requestTaskSettings if it is not null and then to + * originalSettings. + * @param originalSettings the settings stored as part of the inference entity configuration + * @param requestTaskSettings the settings passed in within the task_settings field of the request + * @param requestInputType the input type passed in the request parameters + * @return a constructed {@link JinaAIEmbeddingsTaskSettings} + */ + public static JinaAIEmbeddingsTaskSettings of( + JinaAIEmbeddingsTaskSettings originalSettings, + JinaAIEmbeddingsTaskSettings requestTaskSettings, + InputType requestInputType + ) { + var inputTypeToUse = getValidInputType(originalSettings, requestTaskSettings, requestInputType); + + return new JinaAIEmbeddingsTaskSettings(inputTypeToUse); + } + + private static InputType getValidInputType( + JinaAIEmbeddingsTaskSettings originalSettings, + JinaAIEmbeddingsTaskSettings requestTaskSettings, + InputType requestInputType + ) { + InputType inputTypeToUse = originalSettings.inputType; + + if (VALID_REQUEST_VALUES.contains(requestInputType)) { + inputTypeToUse = requestInputType; + } else if (requestTaskSettings.inputType != null) { + inputTypeToUse = requestTaskSettings.inputType; + } + + return inputTypeToUse; + } + + private final InputType inputType; + + public JinaAIEmbeddingsTaskSettings(StreamInput in) throws IOException { + this(in.readOptionalEnum(InputType.class)); + } + + public JinaAIEmbeddingsTaskSettings(@Nullable InputType inputType) { + validateInputType(inputType); + this.inputType = inputType; + } + + private static void validateInputType(InputType inputType) { + if (inputType == null) { + return; + } + + assert VALID_REQUEST_VALUES.contains(inputType) : invalidInputTypeMessage(inputType); + } + + @Override + public boolean isEmpty() { + return inputType == null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (inputType != null) { + builder.field(INPUT_TYPE, inputType); + } + + builder.endObject(); + return builder; + } + + public InputType getInputType() { + return inputType; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.JINA_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalEnum(inputType); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JinaAIEmbeddingsTaskSettings that = (JinaAIEmbeddingsTaskSettings) o; + return Objects.equals(inputType, that.inputType); + } + + @Override + public int hashCode() { + return Objects.hash(inputType); + } + + public static String invalidInputTypeMessage(InputType inputType) { + return Strings.format("received invalid input type value [%s]", inputType.toString()); + } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + JinaAIEmbeddingsTaskSettings updatedSettings = JinaAIEmbeddingsTaskSettings.fromMap(new HashMap<>(newSettings)); + return of(this, updatedSettings, updatedSettings.inputType != null ? updatedSettings.inputType : this.inputType); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java new file mode 100644 index 0000000000000..2fb9228d3b652 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.rerank; + +import org.elasticsearch.common.util.LazyInitializable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.SettingsConfiguration; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; +import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionVisitor; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIModel; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; + +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings.RETURN_DOCUMENTS; +import static org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY; + +public class JinaAIRerankModel extends JinaAIModel { + public static JinaAIRerankModel of(JinaAIRerankModel model, Map taskSettings) { + var requestTaskSettings = JinaAIRerankTaskSettings.fromMap(taskSettings); + return new JinaAIRerankModel(model, JinaAIRerankTaskSettings.of(model.getTaskSettings(), requestTaskSettings)); + } + + public JinaAIRerankModel( + String inferenceId, + String service, + Map serviceSettings, + Map taskSettings, + @Nullable Map secrets, + ConfigurationParseContext context + ) { + this( + inferenceId, + service, + JinaAIRerankServiceSettings.fromMap(serviceSettings, context), + JinaAIRerankTaskSettings.fromMap(taskSettings), + DefaultSecretSettings.fromMap(secrets) + ); + } + + // should only be used for testing + JinaAIRerankModel( + String modelId, + String service, + JinaAIRerankServiceSettings serviceSettings, + JinaAIRerankTaskSettings taskSettings, + @Nullable DefaultSecretSettings secretSettings + ) { + super( + new ModelConfigurations(modelId, TaskType.RERANK, service, serviceSettings, taskSettings), + new ModelSecrets(secretSettings), + secretSettings, + serviceSettings.getCommonSettings() + ); + } + + private JinaAIRerankModel(JinaAIRerankModel model, JinaAIRerankTaskSettings taskSettings) { + super(model, taskSettings); + } + + public JinaAIRerankModel(JinaAIRerankModel model, JinaAIRerankServiceSettings serviceSettings) { + super(model, serviceSettings); + } + + @Override + public JinaAIRerankServiceSettings getServiceSettings() { + return (JinaAIRerankServiceSettings) super.getServiceSettings(); + } + + @Override + public JinaAIRerankTaskSettings getTaskSettings() { + return (JinaAIRerankTaskSettings) super.getTaskSettings(); + } + + @Override + public DefaultSecretSettings getSecretSettings() { + return (DefaultSecretSettings) super.getSecretSettings(); + } + + /** + * Accepts a visitor to create an executable action. The returned action will not return documents in the response. + * @param visitor _ + * @param taskSettings _ + * @param inputType ignored for rerank task + * @return the rerank action + */ + @Override + public ExecutableAction accept(JinaAIActionVisitor visitor, Map taskSettings, InputType inputType) { + return visitor.create(this, taskSettings); + } + + @Override + public URI uri() { + return getServiceSettings().getCommonSettings().uri(); + } + + public static class Configuration { + public static Map get() { + return configuration.getOrCompute(); + } + + private static final LazyInitializable, RuntimeException> configuration = + new LazyInitializable<>(() -> { + var configurationMap = new HashMap(); + + configurationMap.put( + RETURN_DOCUMENTS, + new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE) + .setLabel("Return Documents") + .setOrder(1) + .setRequired(false) + .setSensitive(false) + .setTooltip("Specify whether to return doc text within the results.") + .setType(SettingsConfigurationFieldType.BOOLEAN) + .setValue(false) + .build() + ); + configurationMap.put( + TOP_N_DOCS_ONLY, + new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) + .setLabel("Top N") + .setOrder(2) + .setRequired(false) + .setSensitive(false) + .setTooltip("The number of most relevant documents to return, defaults to the number of the documents.") + .setType(SettingsConfigurationFieldType.INTEGER) + .build() + ); + + return Collections.unmodifiableMap(configurationMap); + }); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettings.java new file mode 100644 index 0000000000000..a9e492c2738c2 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettings.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.rerank; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIRateLimitServiceSettings; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings; +import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class JinaAIRerankServiceSettings extends FilteredXContentObject implements ServiceSettings, JinaAIRateLimitServiceSettings { + public static final String NAME = "jinaai_rerank_service_settings"; + + private static final Logger logger = LogManager.getLogger(JinaAIRerankServiceSettings.class); + + public static JinaAIRerankServiceSettings fromMap(Map map, ConfigurationParseContext context) { + ValidationException validationException = new ValidationException(); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + var commonServiceSettings = JinaAIServiceSettings.fromMap(map, context); + + return new JinaAIRerankServiceSettings(commonServiceSettings); + } + + private final JinaAIServiceSettings commonSettings; + + public JinaAIRerankServiceSettings(JinaAIServiceSettings commonSettings) { + this.commonSettings = commonSettings; + } + + public JinaAIRerankServiceSettings(StreamInput in) throws IOException { + this.commonSettings = new JinaAIServiceSettings(in); + } + + public JinaAIServiceSettings getCommonSettings() { + return commonSettings; + } + + @Override + public String modelId() { + return commonSettings.modelId(); + } + + @Override + public RateLimitSettings rateLimitSettings() { + return commonSettings.rateLimitSettings(); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder = commonSettings.toXContentFragment(builder, params); + + builder.endObject(); + return builder; + } + + @Override + protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + commonSettings.toXContentFragmentOfExposedFields(builder, params); + return builder; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.JINA_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + commonSettings.writeTo(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JinaAIRerankServiceSettings that = (JinaAIRerankServiceSettings) o; + return Objects.equals(commonSettings, that.commonSettings); + } + + @Override + public int hashCode() { + return Objects.hash(commonSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettings.java new file mode 100644 index 0000000000000..8dd93d89b6fb5 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettings.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.rerank; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalBoolean; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; + +/** + * Defines the task settings for the JinaAI rerank service. + * + */ +public class JinaAIRerankTaskSettings implements TaskSettings { + + public static final String NAME = "jinaai_rerank_task_settings"; + public static final String RETURN_DOCUMENTS = "return_documents"; + public static final String TOP_N_DOCS_ONLY = "top_n"; + + public static final JinaAIRerankTaskSettings EMPTY_SETTINGS = new JinaAIRerankTaskSettings(null, null); + + public static JinaAIRerankTaskSettings fromMap(Map map) { + ValidationException validationException = new ValidationException(); + + if (map == null || map.isEmpty()) { + return EMPTY_SETTINGS; + } + + Boolean returnDocuments = extractOptionalBoolean(map, RETURN_DOCUMENTS, validationException); + Integer topNDocumentsOnly = extractOptionalPositiveInteger( + map, + TOP_N_DOCS_ONLY, + ModelConfigurations.TASK_SETTINGS, + validationException + ); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return of(topNDocumentsOnly, returnDocuments); + } + + /** + * Creates a new {@link JinaAIRerankTaskSettings} by preferring non-null fields from the request settings over the original settings. + * + * @param originalSettings the settings stored as part of the inference entity configuration + * @param requestTaskSettings the settings passed in within the task_settings field of the request + * @return a constructed {@link JinaAIRerankTaskSettings} + */ + public static JinaAIRerankTaskSettings of(JinaAIRerankTaskSettings originalSettings, JinaAIRerankTaskSettings requestTaskSettings) { + return new JinaAIRerankTaskSettings( + requestTaskSettings.getTopNDocumentsOnly() != null + ? requestTaskSettings.getTopNDocumentsOnly() + : originalSettings.getTopNDocumentsOnly(), + requestTaskSettings.getReturnDocuments() != null + ? requestTaskSettings.getReturnDocuments() + : originalSettings.getReturnDocuments() + ); + } + + public static JinaAIRerankTaskSettings of(Integer topNDocumentsOnly, Boolean returnDocuments) { + return new JinaAIRerankTaskSettings(topNDocumentsOnly, returnDocuments); + } + + private final Integer topNDocumentsOnly; + private final Boolean returnDocuments; + + public JinaAIRerankTaskSettings(StreamInput in) throws IOException { + this(in.readOptionalInt(), in.readOptionalBoolean()); + } + + public JinaAIRerankTaskSettings(@Nullable Integer topNDocumentsOnly, @Nullable Boolean doReturnDocuments) { + this.topNDocumentsOnly = topNDocumentsOnly; + this.returnDocuments = doReturnDocuments; + } + + @Override + public boolean isEmpty() { + return topNDocumentsOnly == null && returnDocuments == null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (topNDocumentsOnly != null) { + builder.field(TOP_N_DOCS_ONLY, topNDocumentsOnly); + } + if (returnDocuments != null) { + builder.field(RETURN_DOCUMENTS, returnDocuments); + } + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.JINA_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalInt(topNDocumentsOnly); + out.writeOptionalBoolean(returnDocuments); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JinaAIRerankTaskSettings that = (JinaAIRerankTaskSettings) o; + return Objects.equals(returnDocuments, that.returnDocuments) && Objects.equals(topNDocumentsOnly, that.topNDocumentsOnly); + } + + @Override + public int hashCode() { + return Objects.hash(returnDocuments, topNDocumentsOnly); + } + + public static String invalidInputTypeMessage(InputType inputType) { + return Strings.format("received invalid input type value [%s]", inputType.toString()); + } + + public Boolean getDoesReturnDocuments() { + return returnDocuments; + } + + public Integer getTopNDocumentsOnly() { + return topNDocumentsOnly; + } + + public Boolean getReturnDocuments() { + return returnDocuments; + } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + JinaAIRerankTaskSettings updatedSettings = JinaAIRerankTaskSettings.fromMap(new HashMap<>(newSettings)); + return JinaAIRerankTaskSettings.of(this, updatedSettings); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandlerTests.java new file mode 100644 index 0000000000000..4c18915e0187b --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/jinaai/JinaAIResponseHandlerTests.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.jinaai; + +import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HttpResponse; +import org.apache.http.StatusLine; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.RetryException; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.hamcrest.MatcherAssert; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.core.Is.is; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class JinaAIResponseHandlerTests extends ESTestCase { + public void testCheckForFailureStatusCode_DoesNotThrowForStatusCodesBetween200And299() { + callCheckForFailureStatusCode(randomIntBetween(200, 299), "id"); + } + + public void testCheckForFailureStatusCode_ThrowsFor503() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(503, "id")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a server error status code for request from inference entity id [id] status [503]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor500_WithShouldRetryTrue() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(500, "id")); + assertTrue(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a server error status code for request from inference entity id [id] status [500]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor429_WithShouldRetryTrue() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(429, "id")); + assertTrue(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a rate limit status code for request from inference entity id [id] status [429]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.TOO_MANY_REQUESTS)); + } + + public void testCheckForFailureStatusCode_ThrowsFor400() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(400, "id")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received an input validation error response for request from inference entity id [id] status [400]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor400_InputsTooLarge() { + var exception = expectThrows( + RetryException.class, + () -> callCheckForFailureStatusCode(400, "\"input\" length 2049 is larger than the largest allowed size 2048", "id") + ); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received an input validation error response for request from inference entity id [id] status [400]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor401() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(401, "inferenceEntityId")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString( + "Received an authentication error status code for request from inference entity id [inferenceEntityId] status [401]" + ) + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.UNAUTHORIZED)); + } + + public void testCheckForFailureStatusCode_ThrowsFor402() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(402, "inferenceEntityId")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat(exception.getCause().getMessage(), containsString("Payment required")); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.PAYMENT_REQUIRED)); + } + + private static void callCheckForFailureStatusCode(int statusCode, String modelId) { + callCheckForFailureStatusCode(statusCode, null, modelId); + } + + private static void callCheckForFailureStatusCode(int statusCode, @Nullable String errorMessage, String modelId) { + var statusLine = mock(StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(statusCode); + + var httpResponse = mock(HttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + var header = mock(Header.class); + when(header.getElements()).thenReturn(new HeaderElement[] {}); + when(httpResponse.getFirstHeader(anyString())).thenReturn(header); + + String escapedErrorMessage = errorMessage != null ? errorMessage.replace("\\", "\\\\").replace("\"", "\\\"") : errorMessage; + + String responseJson = Strings.format(""" + { + "detail": "%s" + } + """, escapedErrorMessage); + + var mockRequest = mock(Request.class); + when(mockRequest.getInferenceEntityId()).thenReturn(modelId); + var httpResult = new HttpResult(httpResponse, errorMessage == null ? new byte[] {} : responseJson.getBytes(StandardCharsets.UTF_8)); + var handler = new JinaAIResponseHandler("", (request, result) -> null); + + handler.checkForFailureStatusCode(mockRequest, httpResult); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntityTests.java new file mode 100644 index 0000000000000..7f3f6e5cdeb82 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestEntityTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; + +public class JinaAIEmbeddingsRequestEntityTests extends ESTestCase { + public void testXContent_WritesAllFields_WhenTheyAreDefined() throws IOException { + var entity = new JinaAIEmbeddingsRequestEntity(List.of("abc"), new JinaAIEmbeddingsTaskSettings(InputType.INGEST), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + MatcherAssert.assertThat(xContentResult, is(""" + {"input":["abc"],"model":"model","task":"retrieval.passage"}""")); + } + + public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws IOException { + var entity = new JinaAIEmbeddingsRequestEntity(List.of("abc"), JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + MatcherAssert.assertThat(xContentResult, is(""" + {"input":["abc"],"model":"model"}""")); + } + + public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { + var thrownException = expectThrows( + AssertionError.class, + () -> JinaAIEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED) + ); + MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestTests.java new file mode 100644 index 0000000000000..05194ceb0de9e --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIEmbeddingsRequestTests.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModelTests; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + +public class JinaAIEmbeddingsRequestTests extends ESTestCase { + public void testCreateRequest_UrlDefined() throws IOException { + var request = createRequest( + List.of("abc"), + JinaAIEmbeddingsModelTests.createModel("url", "secret", JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, null, null, "model") + ); + + var httpRequest = request.createHttpRequest(); + MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + MatcherAssert.assertThat(httpPost.getURI().toString(), is("url")); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); + MatcherAssert.assertThat( + httpPost.getLastHeader(JinaAIUtils.REQUEST_SOURCE_HEADER).getValue(), + is(JinaAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "model"))); + } + + public void testCreateRequest_AllOptionsDefined() throws IOException { + var request = createRequest( + List.of("abc"), + JinaAIEmbeddingsModelTests.createModel("url", "secret", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model") + ); + + var httpRequest = request.createHttpRequest(); + MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + MatcherAssert.assertThat(httpPost.getURI().toString(), is("url")); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); + MatcherAssert.assertThat( + httpPost.getLastHeader(JinaAIUtils.REQUEST_SOURCE_HEADER).getValue(), + is(JinaAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "model", "task", "retrieval.passage"))); + } + + public void testCreateRequest_InputTypeSearch() throws IOException { + var request = createRequest( + List.of("abc"), + JinaAIEmbeddingsModelTests.createModel("url", "secret", new JinaAIEmbeddingsTaskSettings(InputType.SEARCH), null, null, "model") + ); + + var httpRequest = request.createHttpRequest(); + MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + MatcherAssert.assertThat(httpPost.getURI().toString(), is("url")); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); + MatcherAssert.assertThat( + httpPost.getLastHeader(JinaAIUtils.REQUEST_SOURCE_HEADER).getValue(), + is(JinaAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "model", "task", "retrieval.query"))); + } + + public static JinaAIEmbeddingsRequest createRequest(List input, JinaAIEmbeddingsModel model) { + return new JinaAIEmbeddingsRequest(input, model); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequestTests.java new file mode 100644 index 0000000000000..031b44225628c --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRequestTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.jinaai.JinaAIAccount; + +import java.net.URI; + +import static org.hamcrest.Matchers.is; + +public class JinaAIRequestTests extends ESTestCase { + + public void testDecorateWithAuthHeader() { + var request = new HttpPost("http://www.abc.com"); + + JinaAIRequest.decorateWithAuthHeader( + request, + new JinaAIAccount(URI.create("http://www.abc.com"), new SecureString(new char[] { 'a', 'b', 'c' })) + ); + + assertThat(request.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + assertThat(request.getFirstHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer abc")); + assertThat(request.getFirstHeader(JinaAIUtils.REQUEST_SOURCE_HEADER).getValue(), is(JinaAIUtils.ELASTIC_REQUEST_SOURCE)); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntityTests.java new file mode 100644 index 0000000000000..7fd738fa2a8e4 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestEntityTests.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString; +import static org.hamcrest.MatcherAssert.assertThat; + +public class JinaAIRerankRequestEntityTests extends ESTestCase { + public void testXContent_SingleRequest_WritesModelAndTopNIfDefined() throws IOException { + var entity = new JinaAIRerankRequestEntity("query", List.of("abc"), new JinaAIRerankTaskSettings(8, null), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ], + "top_n": 8 + } + """)); + } + + public void testXContent_SingleRequest_WritesModelAndTopNIfDefined_ReturnDocumentsTrue() throws IOException { + var entity = new JinaAIRerankRequestEntity("query", List.of("abc"), new JinaAIRerankTaskSettings(8, true), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ], + "top_n": 8, + "return_documents": true + } + """)); + } + + public void testXContent_SingleRequest_WritesModelAndTopNIfDefined_ReturnDocumentsFalse() throws IOException { + var entity = new JinaAIRerankRequestEntity("query", List.of("abc"), new JinaAIRerankTaskSettings(8, false), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ], + "top_n": 8, + "return_documents": false + } + """)); + } + + public void testXContent_SingleRequest_DoesNotWriteTopNIfNull() throws IOException { + var entity = new JinaAIRerankRequestEntity("query", List.of("abc"), null, "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ] + } + """)); + } + + public void testXContent_MultipleRequests_WritesModelAndTopNIfDefined() throws IOException { + var entity = new JinaAIRerankRequestEntity("query", List.of("abc", "def"), new JinaAIRerankTaskSettings(8, null), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc", + "def" + ], + "top_n": 8 + } + """)); + } + + public void testXContent_MultipleRequests_DoesNotWriteTopNIfNull() throws IOException { + var entity = new JinaAIRerankRequestEntity("query", List.of("abc", "def"), null, "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc", + "def" + ] + } + """)); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestTests.java new file mode 100644 index 0000000000000..819362d397ba5 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIRerankRequestTests.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModelTests; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; + +public class JinaAIRerankRequestTests extends ESTestCase { + + private static final String API_KEY = "foo"; + + public void testCreateRequest_WithoutModelSet_And_WithoutTopNSet() throws IOException { + var input = "input"; + var query = "query"; + var modelId = "model"; + + var request = createRequest(query, input, modelId, null); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY)); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + + assertThat(requestMap, aMapWithSize(3)); + assertThat(requestMap.get("documents"), is(List.of(input))); + assertThat(requestMap.get("query"), is(query)); + assertThat(requestMap.get("model"), is(modelId)); + } + + public void testCreateRequest_WithTopNSet() throws IOException { + var input = "input"; + var query = "query"; + var topN = 1; + var modelId = "model"; + + var request = createRequest(query, input, modelId, topN); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY)); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + + assertThat(requestMap, aMapWithSize(4)); + assertThat(requestMap.get("documents"), is(List.of(input))); + assertThat(requestMap.get("query"), is(query)); + assertThat(requestMap.get("top_n"), is(topN)); + assertThat(requestMap.get("model"), is(modelId)); + } + + public void testCreateRequest_WithModelSet() throws IOException { + var input = "input"; + var query = "query"; + var modelId = "model"; + + var request = createRequest(query, input, modelId, null); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY)); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + + assertThat(requestMap, aMapWithSize(3)); + assertThat(requestMap.get("documents"), is(List.of(input))); + assertThat(requestMap.get("query"), is(query)); + assertThat(requestMap.get("model"), is(modelId)); + } + + public void testTruncate_DoesNotTruncate() { + var request = createRequest("query", "input", "null", null); + var truncatedRequest = request.truncate(); + + assertThat(truncatedRequest, sameInstance(request)); + } + + private static JinaAIRerankRequest createRequest(String query, String input, @Nullable String modelId, @Nullable Integer topN) { + var rerankModel = JinaAIRerankModelTests.createModel(API_KEY, modelId, topN); + return new JinaAIRerankRequest(query, List.of(input), rerankModel); + + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtilsTests.java new file mode 100644 index 0000000000000..e3b4cfbed20ef --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/jinaai/JinaAIUtilsTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.jinaai; + +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.is; + +public class JinaAIUtilsTests extends ESTestCase { + + public void testCreateRequestSourceHeader() { + var requestSourceHeader = JinaAIUtils.createRequestSourceHeader(); + + assertThat(requestSourceHeader.getName(), is("Request-Source")); + assertThat(requestSourceHeader.getValue(), is("unspecified:elasticsearch")); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java new file mode 100644 index 0000000000000..7dbb9d5441a4a --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java @@ -0,0 +1,397 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.jinaai; + +import org.apache.http.HttpResponse; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.request.Request; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class JinaAIEmbeddingsResponseEntityTests extends ESTestCase { + public void testFromResponse_CreatesResultsForASingleItem() throws IOException { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.014539449, + -0.015288644 + ] + } + ], + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }))) + ); + } + + public void testFromResponse_CreatesResultsForMultipleItems() throws IOException { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.014539449, + -0.015288644 + ] + }, + { + "object": "embedding", + "index": 1, + "embedding": [ + 0.0123, + -0.0123 + ] + } + ], + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is( + List.of( + new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }), + new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123F, -0.0123F }) + ) + ) + ); + } + + public void testFromResponse_FailsWhenDataFieldIsNotPresent() { + String responseJson = """ + { + "object": "list", + "not_data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.014539449, + -0.015288644 + ] + } + ], + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + var thrownException = expectThrows( + IllegalStateException.class, + () -> JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat(thrownException.getMessage(), is("Failed to find required field [data] in JinaAI embeddings response")); + } + + public void testFromResponse_FailsWhenDataFieldNotAnArray() { + String responseJson = """ + { + "object": "list", + "data": { + "test": { + "object": "embedding", + "index": 0, + "embedding": [ + 0.014539449, + -0.015288644 + ] + } + }, + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + var thrownException = expectThrows( + ParsingException.class, + () -> JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat( + thrownException.getMessage(), + is("Failed to parse object: expecting token of type [START_ARRAY] but found [START_OBJECT]") + ); + } + + public void testFromResponse_FailsWhenEmbeddingsDoesNotExist() { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embeddingzzz": [ + 0.014539449, + -0.015288644 + ] + } + ], + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + var thrownException = expectThrows( + IllegalStateException.class, + () -> JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat(thrownException.getMessage(), is("Failed to find required field [embedding] in JinaAI embeddings response")); + } + + public void testFromResponse_FailsWhenEmbeddingValueIsAString() { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + "abc" + ] + } + ], + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + var thrownException = expectThrows( + ParsingException.class, + () -> JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat( + thrownException.getMessage(), + is("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [VALUE_STRING]") + ); + } + + public void testFromResponse_SucceedsWhenEmbeddingValueIsInt() throws IOException { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 1 + ] + } + ], + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 1.0F }))) + ); + } + + public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOException { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 40294967295 + ] + } + ], + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 4.0294965E10F }))) + ); + } + + public void testFromResponse_FailsWhenEmbeddingValueIsAnObject() { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + {} + ] + } + ], + "model": "jina-embeddings-v3", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + """; + + var thrownException = expectThrows( + ParsingException.class, + () -> JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat( + thrownException.getMessage(), + is("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [START_OBJECT]") + ); + } + + public void testFieldsInDifferentOrderServer() throws IOException { + // The fields of the objects in the data array are reordered + String response = """ + { + "object": "list", + "id": "6667830b-716b-4796-9a61-33b67b5cc81d", + "model": "jina-embeddings-v3", + "data": [ + { + "embedding": [ + -0.9, + 0.5, + 0.3 + ], + "index": 0, + "object": "embedding" + }, + { + "index": 0, + "embedding": [ + 0.1, + 0.5 + ], + "object": "embedding" + }, + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.5, + 0.5 + ] + } + ], + "usage": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0 + } + }"""; + + InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + mock(Request.class), + new HttpResult(mock(HttpResponse.class), response.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + parsedResults.embeddings(), + is( + List.of( + new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.9F, 0.5F, 0.3F }), + new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.1F, 0.5F }), + new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.5F, 0.5F }) + ) + ) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntityTests.java new file mode 100644 index 0000000000000..ce3bd10566cd8 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIErrorResponseEntityTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.jinaai; + +import org.apache.http.HttpResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; +import org.hamcrest.MatcherAssert; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class JinaAIErrorResponseEntityTests extends ESTestCase { + public void testFromResponse() { + String message = "\"input\" length 2049 is larger than the largest allowed size 2048"; + String escapedMessage = message.replace("\\", "\\\\").replace("\"", "\\\""); + String responseJson = Strings.format(""" + { + "detail": "%s" + } + """, escapedMessage); + + ErrorResponse errorResponse = JinaAIErrorResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + assertNotNull(errorResponse); + MatcherAssert.assertThat(errorResponse.getErrorMessage(), is(message)); + } + + public void testFromResponse_noMessage() { + String responseJson = """ + { + "error": "abc" + } + """; + + ErrorResponse errorResponse = JinaAIErrorResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + MatcherAssert.assertThat(errorResponse, is(ErrorResponse.UNDEFINED_ERROR)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntityTests.java new file mode 100644 index 0000000000000..33fe9819bd88a --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIRerankResponseEntityTests.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.jinaai; + +import org.apache.http.HttpResponse; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class JinaAIRerankResponseEntityTests extends ESTestCase { + + public void testResponseLiteral() throws IOException { + InferenceServiceResults parsedResults = JinaAIRerankResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseLiteral.getBytes(StandardCharsets.UTF_8)) + ); + + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + List expected = responseLiteralDocs(); + for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) { + assertEquals(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), expected.get(i).index()); + } + } + + public void testGeneratedResponse() throws IOException { + int numDocs = randomIntBetween(1, 10); + + List expected = new ArrayList<>(numDocs); + StringBuilder responseBuilder = new StringBuilder(); + + responseBuilder.append("{"); + responseBuilder.append("\"model\": \"model\","); + responseBuilder.append("\"index\":\"").append(randomAlphaOfLength(36)).append("\","); + responseBuilder.append("\"results\": ["); + List indices = linear(numDocs); + List scores = linearFloats(numDocs); + for (int i = 0; i < numDocs; i++) { + int index = indices.remove(randomInt(indices.size() - 1)); + + responseBuilder.append("{"); + responseBuilder.append("\"index\":").append(index).append(","); + responseBuilder.append("\"relevance_score\":").append(scores.get(i).toString()).append("}"); + expected.add(new RankedDocsResults.RankedDoc(index, scores.get(i), null)); + if (i < numDocs - 1) { + responseBuilder.append(","); + } + } + responseBuilder.append("],"); + responseBuilder.append("\"usage\": {"); + responseBuilder.append("\"total_tokens\": 15}"); + responseBuilder.append("}"); + + InferenceServiceResults parsedResults = JinaAIRerankResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseBuilder.toString().getBytes(StandardCharsets.UTF_8)) + ); + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) { + assertEquals(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), expected.get(i).index()); + } + } + + private ArrayList responseLiteralDocs() { + var list = new ArrayList(); + + list.add(new RankedDocsResults.RankedDoc(2, 0.98005307F, null)); + list.add(new RankedDocsResults.RankedDoc(3, 0.27904198F, null)); + list.add(new RankedDocsResults.RankedDoc(0, 0.10194652F, null)); + return list; + + }; + + private final String responseLiteral = """ + { + "model": "model", + "results": [ + { + "index": 2, + "relevance_score": 0.98005307 + }, + { + "index": 3, + "relevance_score": 0.27904198 + }, + { + "index": 0, + "relevance_score": 0.10194652 + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + + public void testResponseLiteralWithDocuments() throws IOException { + InferenceServiceResults parsedResults = JinaAIRerankResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseLiteralWithDocuments.getBytes(StandardCharsets.UTF_8)) + ); + + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + MatcherAssert.assertThat(((RankedDocsResults) parsedResults).getRankedDocs(), is(responseLiteralDocsWithText)); + } + + private final String responseLiteralWithDocuments = """ + { + "model": "model", + "results": [ + { + "document": { + "text": "Washington, D.C.." + }, + "index": 2, + "relevance_score": 0.98005307 + }, + { + "document": { + "text": "Capital punishment has existed in the United States since beforethe United States was a country. " + }, + "index": 3, + "relevance_score": 0.27904198 + }, + { + "document": { + "text": "Carson City is the capital city of the American state of Nevada." + }, + "index": 0, + "relevance_score": 0.10194652 + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + + private final List responseLiteralDocsWithText = List.of( + new RankedDocsResults.RankedDoc(2, 0.98005307F, "Washington, D.C.."), + new RankedDocsResults.RankedDoc( + 3, + 0.27904198F, + "Capital punishment has existed in the United States since beforethe United States was a country. " + ), + new RankedDocsResults.RankedDoc(0, 0.10194652F, "Carson City is the capital city of the American state of Nevada.") + ); + + private ArrayList linear(int n) { + ArrayList list = new ArrayList<>(); + for (int i = 0; i <= n; i++) { + list.add(i); + } + return list; + } + + // creates a list of doubles of monotonically decreasing magnitude + private ArrayList linearFloats(int n) { + ArrayList list = new ArrayList<>(); + float startValue = 1.0f; + float decrement = startValue / n + 1; + for (int i = 0; i <= n; i++) { + list.add(startValue - (i * decrement)); + } + return list; + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettingsTests.java new file mode 100644 index 0000000000000..4729e9e059d93 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceSettingsTests.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ServiceFields; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +public class JinaAIServiceSettingsTests extends AbstractWireSerializingTestCase { + + public static JinaAIServiceSettings createRandomWithNonNullUrl() { + return createRandom(randomAlphaOfLength(15)); + } + + /** + * The created settings can have a url set to null. + */ + public static JinaAIServiceSettings createRandom() { + var url = randomBoolean() ? randomAlphaOfLength(15) : null; + return createRandom(url); + } + + private static JinaAIServiceSettings createRandom(String url) { + var model = randomAlphaOfLength(15); + + return new JinaAIServiceSettings(ServiceUtils.createOptionalUri(url), model, RateLimitSettingsTests.createRandom()); + } + + public void testFromMap() { + var url = "https://www.abc.com"; + var model = "model"; + var serviceSettings = JinaAIServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.URL, url, JinaAIServiceSettings.MODEL_ID, model)), + ConfigurationParseContext.REQUEST + ); + + MatcherAssert.assertThat(serviceSettings, is(new JinaAIServiceSettings(ServiceUtils.createUri(url), model, null))); + } + + public void testFromMap_WithRateLimit() { + var url = "https://www.abc.com"; + var model = "model"; + var serviceSettings = JinaAIServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.URL, + url, + JinaAIServiceSettings.MODEL_ID, + model, + RateLimitSettings.FIELD_NAME, + new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, 3)) + ) + ), + ConfigurationParseContext.REQUEST + ); + + MatcherAssert.assertThat( + serviceSettings, + is(new JinaAIServiceSettings(ServiceUtils.createUri(url), model, new RateLimitSettings(3))) + ); + } + + public void testFromMap_WhenUsingModelId() { + var url = "https://www.abc.com"; + var model = "model"; + var serviceSettings = JinaAIServiceSettings.fromMap( + new HashMap<>(Map.of(ServiceFields.URL, url, JinaAIServiceSettings.MODEL_ID, model)), + ConfigurationParseContext.PERSISTENT + ); + + MatcherAssert.assertThat(serviceSettings, is(new JinaAIServiceSettings(ServiceUtils.createUri(url), model, null))); + } + + public void testFromMap_MissingUrl_DoesNotThrowException() { + var serviceSettings = JinaAIServiceSettings.fromMap( + new HashMap<>(Map.of(JinaAIServiceSettings.MODEL_ID, "model")), + ConfigurationParseContext.PERSISTENT + ); + assertNull(serviceSettings.uri()); + } + + public void testFromMap_EmptyUrl_ThrowsError() { + var thrownException = expectThrows( + ValidationException.class, + () -> JinaAIServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, "")), ConfigurationParseContext.PERSISTENT) + ); + + MatcherAssert.assertThat( + thrownException.getMessage(), + containsString( + Strings.format( + "Validation Failed: 1: [service_settings] Invalid value empty string. [%s] must be a non-empty string;", + ServiceFields.URL + ) + ) + ); + } + + public void testFromMap_InvalidUrl_ThrowsError() { + var url = "https://www.abc^.com"; + var thrownException = expectThrows( + ValidationException.class, + () -> JinaAIServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, url)), ConfigurationParseContext.PERSISTENT) + ); + + MatcherAssert.assertThat( + thrownException.getMessage(), + containsString( + Strings.format("Validation Failed: 1: [service_settings] Invalid url [%s] received for field [%s]", url, ServiceFields.URL) + ) + ); + } + + public void testXContent_WritesModelId() throws IOException { + var entity = new JinaAIServiceSettings((String) null, "model", new RateLimitSettings(1)); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, is(""" + {"model_id":"model","rate_limit":{"requests_per_minute":1}}""")); + } + + @Override + protected Writeable.Reader instanceReader() { + return JinaAIServiceSettings::new; + } + + @Override + protected JinaAIServiceSettings createTestInstance() { + return createRandomWithNonNullUrl(); + } + + @Override + protected JinaAIServiceSettings mutateInstance(JinaAIServiceSettings instance) throws IOException { + return randomValueOtherThan(instance, JinaAIServiceSettingsTests::createRandom); + } + + public static Map getServiceSettingsMap(@Nullable String url, String model) { + var map = new HashMap(); + + if (url != null) { + map.put(ServiceFields.URL, url); + } + + map.put(JinaAIServiceSettings.MODEL_ID, model); + + return map; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java new file mode 100644 index 0000000000000..5a1bf8ec383c1 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java @@ -0,0 +1,2003 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a generative AI + */ + +package org.elasticsearch.xpack.inference.services.jinaai; + +import org.apache.http.HttpHeaders; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInference; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.InferenceServiceConfiguration; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsModelTests; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings; +import org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettingsTests; +import org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankModelTests; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.elasticsearch.xpack.inference.Utils.getInvalidModel; +import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap; +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettings; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettingsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; +import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class JinaAIServiceTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + private HttpClientManager clientManager; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + public void testParseRequestConfig_CreatesAJinaAIEmbeddingsModel() throws IOException { + try (var service = createJinaAIService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParseRequestConfig_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException { + try (var service = createJinaAIService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST), + createRandomChunkingSettingsMap(), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParseRequestConfig_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException { + try (var service = createJinaAIService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParseRequestConfig_OptionalTaskSettings() throws IOException { + try (var service = createJinaAIService()) { + + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), equalTo(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParseRequestConfig_ThrowsUnsupportedTaskType() throws IOException { + try (var service = createJinaAIService()) { + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "The [jinaai] service does not support task type [sparse_embedding]" + ); + + service.parseRequestConfig( + "id", + TaskType.SPARSE_EMBEDDING, + getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ), + failureListener + ); + } + } + + private static ActionListener getModelListenerForException(Class exceptionClass, String expectedMessage) { + return ActionListener.wrap((model) -> fail("Model parsing should have failed"), e -> { + MatcherAssert.assertThat(e, instanceOf(exceptionClass)); + MatcherAssert.assertThat(e.getMessage(), is(expectedMessage)); + }); + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInConfig() throws IOException { + try (var service = createJinaAIService()) { + var config = getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ); + config.put("extra_key", "value"); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [jinaai] service" + ); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInServiceSettingsMap() throws IOException { + try (var service = createJinaAIService()) { + var serviceSettings = JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"); + serviceSettings.put("extra_key", "value"); + + var config = getRequestConfigMap( + serviceSettings, + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [jinaai] service" + ); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInTaskSettingsMap() throws IOException { + try (var service = createJinaAIService()) { + var taskSettingsMap = JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST); + taskSettingsMap.put("extra_key", "value"); + + var config = getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + taskSettingsMap, + getSecretSettingsMap("secret") + ); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [jinaai] service" + ); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInSecretSettingsMap() throws IOException { + try (var service = createJinaAIService()) { + var secretSettingsMap = getSecretSettingsMap("secret"); + secretSettingsMap.put("extra_key", "value"); + + var config = getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + secretSettingsMap + ); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [jinaai] service" + ); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + } + } + + public void testParseRequestConfig_CreatesAJinaAIEmbeddingsModelWithoutUrl() throws IOException { + try (var service = createJinaAIService()) { + var modelListener = ActionListener.wrap((model) -> { + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + assertNull(embeddingsModel.getServiceSettings().getCommonSettings().uri()); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, (e) -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap(null, "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAJinaAIEmbeddingsModel() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + createRandomChunkingSettingsMap(), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_ThrowsErrorTryingToParseInvalidModel() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "oldmodel"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ); + + var thrownException = expectThrows( + ElasticsearchStatusException.class, + () -> service.parsePersistedConfigWithSecrets( + "id", + TaskType.SPARSE_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ) + ); + + MatcherAssert.assertThat( + thrownException.getMessage(), + is("Failed to parse stored model [id] for [jinaai] service, please delete and add the service again") + ); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAJinaAIEmbeddingsModelWithoutUrl() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap(null, "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + assertNull(embeddingsModel.getServiceSettings().getCommonSettings().uri()); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST))); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH), + getSecretSettingsMap("secret") + ); + persistedConfig.config().put("extra_key", "value"); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.SEARCH))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInSecretsSettings() throws IOException { + try (var service = createJinaAIService()) { + var secretSettingsMap = getSecretSettingsMap("secret"); + secretSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + secretSettingsMap + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSecrets() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + persistedConfig.secrets().put("extra_key", "value"); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInServiceSettings() throws IOException { + try (var service = createJinaAIService()) { + var serviceSettingsMap = JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"); + serviceSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + serviceSettingsMap, + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInTaskSettings() throws IOException { + try (var service = createJinaAIService()) { + var taskSettingsMap = JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH); + taskSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + taskSettingsMap, + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.SEARCH))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfig_CreatesAJinaAIEmbeddingsModel() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + createRandomChunkingSettingsMap() + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_CreatesAJinaAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_ThrowsErrorTryingToParseInvalidModel() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model_old"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty() + ); + + var thrownException = expectThrows( + ElasticsearchStatusException.class, + () -> service.parsePersistedConfig("id", TaskType.SPARSE_EMBEDDING, persistedConfig.config()) + ); + + MatcherAssert.assertThat( + thrownException.getMessage(), + is("Failed to parse stored model [id] for [jinaai] service, please delete and add the service again") + ); + } + } + + public void testParsePersistedConfig_CreatesAJinaAIEmbeddingsModelWithoutUrl() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap(null, "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + assertNull(embeddingsModel.getServiceSettings().getCommonSettings().uri()); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException { + try (var service = createJinaAIService()) { + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty() + ); + persistedConfig.config().put("extra_key", "value"); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInServiceSettings() throws IOException { + try (var service = createJinaAIService()) { + var serviceSettingsMap = JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"); + serviceSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + serviceSettingsMap, + JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.SEARCH))); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInTaskSettings() throws IOException { + try (var service = createJinaAIService()) { + var taskSettingsMap = JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST); + taskSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + JinaAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model"), + taskSettingsMap + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + MatcherAssert.assertThat(model, instanceOf(JinaAIEmbeddingsModel.class)); + + var embeddingsModel = (JinaAIEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST))); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testInfer_ThrowsErrorWhenModelIsNotJinaAIModel() throws IOException { + var sender = mock(Sender.class); + + var factory = mock(HttpRequestSender.Factory.class); + when(factory.createSender()).thenReturn(sender); + + var mockModel = getInvalidModel("model_id", "service_name"); + + try (var service = new JinaAIService(factory, createWithEmptySettings(threadPool))) { + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + mockModel, + null, + List.of(""), + false, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); + MatcherAssert.assertThat( + thrownException.getMessage(), + is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.") + ); + + verify(factory, times(1)).createSender(); + verify(sender, times(1)).start(); + } + + verify(sender, times(1)).close(); + verifyNoMoreInteractions(factory); + verifyNoMoreInteractions(sender); + } + + public void testCheckModelConfig_UpdatesDimensions() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "jina-clip-v2", + "object": "list", + "usage": { + "total_tokens": 5, + "prompt_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 1, + "jina-clip-v2" + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.checkModelConfig(model, listener); + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat( + result, + // the dimension is set to 2 because there are 2 embeddings returned from the mock server + is( + JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 2, + "jina-clip-v2" + ) + ) + ); + } + } + + public void testCheckModelConfig_UpdatesSimilarityToDotProduct_WhenItIsNull() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "jina-clip-v2", + "object": "list", + "usage": { + "total_tokens": 5, + "prompt_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 1, + "jina-clip-v2", + null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.checkModelConfig(model, listener); + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat( + result, + // the dimension is set to 2 because there are 2 embeddings returned from the mock server + is( + JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 2, + "jina-clip-v2", + SimilarityMeasure.DOT_PRODUCT + ) + ) + ); + } + } + + public void testCheckModelConfig_DoesNotUpdateSimilarity_WhenItIsSpecifiedAsCosine() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "jina-clip-v2", + "object": "list", + "usage": { + "total_tokens": 5, + "prompt_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 1, + "jina-clip-v2", + SimilarityMeasure.COSINE + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.checkModelConfig(model, listener); + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat( + result, + // the dimension is set to 2 because there are 2 embeddings returned from the mock server + is( + JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 2, + "jina-clip-v2", + SimilarityMeasure.COSINE + ) + ) + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = JinaAIEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + randomNonNegativeInt(), + randomNonNegativeInt(), + randomAlphaOfLength(10), + similarityMeasure + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? JinaAIService.defaultSimilarity() : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + + public void testInfer_Embedding_UnauthorisedResponse() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "detail": "Unauthorized" + } + """; + webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "model", + null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var error = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + MatcherAssert.assertThat(error.getMessage(), containsString("Received an authentication error status code for request")); + MatcherAssert.assertThat(error.getMessage(), containsString("Error message: [Unauthorized]")); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + } + } + + public void testInfer_Rerank_UnauthorisedResponse() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "detail": "Unauthorized" + } + """; + webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson)); + + var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "model", 1024, false); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var error = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + MatcherAssert.assertThat(error.getMessage(), containsString("Received an authentication error status code for request")); + MatcherAssert.assertThat(error.getMessage(), containsString("Error message: [Unauthorized]")); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + } + } + + public void testInfer_Embedding_Get_Response_Ingest() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "jina-clip-v2", + "object": "list", + "usage": { + "total_tokens": 5, + "prompt_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "jina-clip-v2", + null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap()); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2", "task", "retrieval.passage"))); + } + } + + public void testInfer_Embedding_Get_Response_Search() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "jina-clip-v2", + "object": "list", + "usage": { + "total_tokens": 5, + "prompt_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "jina-clip-v2", + null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.SEARCH, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap()); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2", "task", "retrieval.query"))); + } + } + + public void testInfer_Embedding_Get_Response_clustering() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + {"model":"jina-clip-v2","object":"list","usage":{"total_tokens":5,"prompt_tokens":5}, + "data":[{"object":"embedding","index":0,"embedding":[0.123, -0.123]}]} + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "jina-clip-v2", + null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.CLUSTERING, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap()); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2", "task", "separation"))); + } + } + + public void testInfer_Embedding_Get_Response_NullInputType() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "jina-clip-v2", + "object": "list", + "usage": { + "total_tokens": 5, + "prompt_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "jina-clip-v2", + null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer(model, null, List.of("abc"), false, new HashMap<>(), null, InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap()); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2"))); + } + } + + public void testInfer_Rerank_Get_Response_NoReturnDocuments_NoTopN() throws IOException { + String responseJson = """ + { + "model": "model", + "results": [ + { + "index": 2, + "relevance_score": 0.98005307 + }, + { + "index": 1, + "relevance_score": 0.27904198 + }, + { + "index": 0, + "relevance_score": 0.10194652 + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", null, false); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2", "candidate3"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + var resultAsMap = result.asMap(); + assertThat( + resultAsMap, + is( + Map.of( + "rerank", + List.of( + Map.of("ranked_doc", Map.of("index", 2, "relevance_score", 0.98005307F)), + Map.of("ranked_doc", Map.of("index", 1, "relevance_score", 0.27904198F)), + Map.of("ranked_doc", Map.of("index", 0, "relevance_score", 0.10194652F)) + ) + ) + ) + ); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "query", + "query", + "documents", + List.of("candidate1", "candidate2", "candidate3"), + "model", + "model", + "return_documents", + false + ) + ) + ); + + } + } + + public void testInfer_Rerank_Get_Response_NoReturnDocuments_TopN() throws IOException { + String responseJson = """ + { + "model": "model", + "results": [ + { + "index": 2, + "relevance_score": 0.98005307 + }, + { + "index": 1, + "relevance_score": 0.27904198 + }, + { + "index": 0, + "relevance_score": 0.10194652 + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", 3, false); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2", "candidate3", "candidate4"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + var resultAsMap = result.asMap(); + assertThat( + resultAsMap, + is( + Map.of( + "rerank", + List.of( + Map.of("ranked_doc", Map.of("index", 2, "relevance_score", 0.98005307F)), + Map.of("ranked_doc", Map.of("index", 1, "relevance_score", 0.27904198F)), + Map.of("ranked_doc", Map.of("index", 0, "relevance_score", 0.10194652F)) + ) + ) + ) + ); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "query", + "query", + "documents", + List.of("candidate1", "candidate2", "candidate3", "candidate4"), + "model", + "model", + "return_documents", + false, + "top_n", + 3 + ) + ) + ); + + } + + } + + public void testInfer_Rerank_Get_Response_ReturnDocumentsNull_NoTopN() throws IOException { + String responseJson = """ + { + "model": "model", + "results": [ + { + "index": 2, + "relevance_score": 0.98005307, + "document": { + "text": "candidate3" + } + }, + { + "index": 1, + "relevance_score": 0.27904198, + "document": { + "text": "candidate2" + } + }, + { + "index": 0, + "relevance_score": 0.10194652, + "document": { + "text": "candidate1" + } + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", null, null); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2", "candidate3"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + var resultAsMap = result.asMap(); + assertThat( + resultAsMap, + is( + Map.of( + "rerank", + List.of( + Map.of("ranked_doc", Map.of("text", "candidate3", "index", 2, "relevance_score", 0.98005307F)), + Map.of("ranked_doc", Map.of("text", "candidate2", "index", 1, "relevance_score", 0.27904198F)), + Map.of("ranked_doc", Map.of("text", "candidate1", "index", 0, "relevance_score", 0.10194652F)) + ) + ) + ) + ); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat( + requestMap, + is(Map.of("query", "query", "documents", List.of("candidate1", "candidate2", "candidate3"), "model", "model")) + ); + + } + + } + + public void testInfer_Rerank_Get_Response_ReturnDocuments_TopN() throws IOException { + String responseJson = """ + { + "model": "model", + "results": [ + { + "index": 2, + "relevance_score": 0.98005307, + "document": { + "text": "candidate3" + } + }, + { + "index": 1, + "relevance_score": 0.27904198, + "document": { + "text": "candidate2" + } + }, + { + "index": 0, + "relevance_score": 0.10194652, + "document": { + "text": "candidate1" + } + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + var model = JinaAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", 3, true); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2", "candidate3", "candidate4"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + var resultAsMap = result.asMap(); + assertThat( + resultAsMap, + is( + Map.of( + "rerank", + List.of( + Map.of("ranked_doc", Map.of("text", "candidate3", "index", 2, "relevance_score", 0.98005307F)), + Map.of("ranked_doc", Map.of("text", "candidate2", "index", 1, "relevance_score", 0.27904198F)), + Map.of("ranked_doc", Map.of("text", "candidate1", "index", 0, "relevance_score", 0.10194652F)) + ) + ) + ) + ); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "query", + "query", + "documents", + List.of("candidate1", "candidate2", "candidate3", "candidate4"), + "model", + "model", + "return_documents", + true, + "top_n", + 3 + ) + ) + ); + + } + + } + + public void testInfer_Embedding_DoesNotSetInputType_WhenNotPresentInTaskSettings_AndUnspecifiedIsPassedInRequest() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "jina-clip-v2", + "object": "list", + "usage": { + "total_tokens": 5, + "prompt_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new JinaAIEmbeddingsTaskSettings((InputType) null), + 1024, + 1024, + "jina-clip-v2", + null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.UNSPECIFIED, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat(result.asMap(), Matchers.is(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })))); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "jina-clip-v2"))); + } + } + + public void test_Embedding_ChunkedInfer_BatchesCallsChunkingSettingsSet() throws IOException { + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new JinaAIEmbeddingsTaskSettings((InputType) null), + createRandomChunkingSettings(), + 1024, + 1024, + "jina-clip-v2" + ); + + test_Embedding_ChunkedInfer_BatchesCalls(model); + } + + public void test_Embedding_ChunkedInfer_ChunkingSettingsNotSet() throws IOException { + var model = JinaAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new JinaAIEmbeddingsTaskSettings((InputType) null), + null, + 1024, + 1024, + "jina-clip-v2" + ); + + test_Embedding_ChunkedInfer_BatchesCalls(model); + } + + private void test_Embedding_ChunkedInfer_BatchesCalls(JinaAIEmbeddingsModel model) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new JinaAIService(senderFactory, createWithEmptySettings(threadPool))) { + + // Batching will call the service with 2 input + String responseJson = """ + { + "model": "jina-clip-v2", + "object": "list", + "usage": { + "total_tokens": 5, + "prompt_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + }, + { + "object": "embedding", + "index": 1, + "embedding": [ + 0.223, + -0.223 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + PlainActionFuture> listener = new PlainActionFuture<>(); + // 2 input + service.chunkedInfer( + model, + null, + List.of("foo", "bar"), + new HashMap<>(), + InputType.UNSPECIFIED, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var results = listener.actionGet(TIMEOUT); + assertThat(results, hasSize(2)); + { + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); + var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("foo", floatResult.chunks().get(0).matchedText()); + assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + } + { + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); + var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("bar", floatResult.chunks().get(0).matchedText()); + assertArrayEquals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding(), 0.0f); + } + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("foo", "bar"), "model", "jina-clip-v2"))); + } + } + + public void testDefaultSimilarity() { + assertEquals(SimilarityMeasure.DOT_PRODUCT, JinaAIService.defaultSimilarity()); + } + + @SuppressWarnings("checkstyle:LineLength") + public void testGetConfiguration() throws Exception { + try (var service = createJinaAIService()) { + String content = XContentHelper.stripWhitespace( + """ + { + "provider": "jinaai", + "task_types": [ + { + "task_type": "text_embedding", + "configuration": { + "task": { + "default_value": null, + "depends_on": [], + "display": "dropdown", + "label": "Task", + "options": [ + { + "label": "retrieval.query", + "value": "retrieval.query" + }, + { + "label": "retrieval.passage", + "value": "retrieval.passage" + }, + { + "label": "classification", + "value": "classification" + }, + { + "label": "separation", + "value": "separation" + } + ], + "order": 1, + "required": false, + "sensitive": false, + "tooltip": "Specifies the task type passed to the model.", + "type": "str", + "ui_restrictions": [], + "validations": [], + "value": "" + } + } + }, + { + "task_type": "rerank", + "configuration": { + "top_n": { + "default_value": null, + "depends_on": [], + "display": "numeric", + "label": "Top N", + "order": 2, + "required": false, + "sensitive": false, + "tooltip": "The number of most relevant documents to return, defaults to the number of the documents.", + "type": "int", + "ui_restrictions": [], + "validations": [], + "value": null + }, + "return_documents": { + "default_value": null, + "depends_on": [], + "display": "toggle", + "label": "Return Documents", + "order": 1, + "required": false, + "sensitive": false, + "tooltip": "Specify whether to return doc text within the results.", + "type": "bool", + "ui_restrictions": [], + "validations": [], + "value": false + } + } + } + ], + "configuration": { + "api_key": { + "default_value": null, + "depends_on": [], + "display": "textbox", + "label": "API Key", + "order": 1, + "required": true, + "sensitive": true, + "tooltip": "API Key for the provider you're connecting to.", + "type": "str", + "ui_restrictions": [], + "validations": [], + "value": null + }, + "rate_limit.requests_per_minute": { + "default_value": null, + "depends_on": [], + "display": "numeric", + "label": "Rate Limit", + "order": 6, + "required": false, + "sensitive": false, + "tooltip": "Minimize the number of rate limit errors.", + "type": "int", + "ui_restrictions": [], + "validations": [], + "value": null + } + } + } + """ + ); + InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( + new BytesArray(content), + XContentType.JSON + ); + boolean humanReadable = true; + BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable); + InferenceServiceConfiguration serviceConfiguration = service.getConfiguration(); + assertToXContentEquivalent( + originalBytes, + toXContent(serviceConfiguration, XContentType.JSON, humanReadable), + XContentType.JSON + ); + } + } + + public void testDoesNotSupportsStreaming() throws IOException { + try (var service = new JinaAIService(mock(), createWithEmptySettings(mock()))) { + assertFalse(service.canStream(TaskType.COMPLETION)); + assertFalse(service.canStream(TaskType.ANY)); + } + } + + private Map getRequestConfigMap( + Map serviceSettings, + Map taskSettings, + Map chunkingSettings, + Map secretSettings + ) { + var requestConfigMap = getRequestConfigMap(serviceSettings, taskSettings, secretSettings); + requestConfigMap.put(ModelConfigurations.CHUNKING_SETTINGS, chunkingSettings); + + return requestConfigMap; + } + + private Map getRequestConfigMap( + Map serviceSettings, + Map taskSettings, + Map secretSettings + ) { + var builtServiceSettings = new HashMap<>(); + builtServiceSettings.putAll(serviceSettings); + builtServiceSettings.putAll(secretSettings); + + return new HashMap<>( + Map.of(ModelConfigurations.SERVICE_SETTINGS, builtServiceSettings, ModelConfigurations.TASK_SETTINGS, taskSettings) + ); + } + + private Map getRequestConfigMap(Map serviceSettings, Map secretSettings) { + var builtServiceSettings = new HashMap<>(); + builtServiceSettings.putAll(serviceSettings); + builtServiceSettings.putAll(secretSettings); + + return new HashMap<>(Map.of(ModelConfigurations.SERVICE_SETTINGS, builtServiceSettings)); + } + + private JinaAIService createJinaAIService() { + return new JinaAIService(mock(HttpRequestSender.Factory.class), createWithEmptySettings(threadPool)); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModelTests.java new file mode 100644 index 0000000000000..58455bb1f54ea --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModelTests.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.embeddings; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import org.hamcrest.MatcherAssert; + +import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettingsTests.getTaskSettingsMap; +import static org.hamcrest.Matchers.is; + +public class JinaAIEmbeddingsModelTests extends ESTestCase { + + public void testOverrideWith_DoesNotOverrideAndModelRemainsEqual_WhenSettingsAreEmpty_AndInputTypeIsInvalid() { + var model = createModel("url", "api_key", null, null, "model"); + + var overriddenModel = JinaAIEmbeddingsModel.of(model, Map.of(), InputType.UNSPECIFIED); + MatcherAssert.assertThat(overriddenModel, is(model)); + } + + public void testOverrideWith_DoesNotOverrideAndModelRemainsEqual_WhenSettingsAreNull_AndInputTypeIsInvalid() { + var model = createModel("url", "api_key", null, null, "model"); + + var overriddenModel = JinaAIEmbeddingsModel.of(model, null, InputType.UNSPECIFIED); + MatcherAssert.assertThat(overriddenModel, is(model)); + } + + public void testOverrideWith_SetsInputTypeToIngest_WhenTheFieldIsNullInModelTaskSettings_AndNullInRequestTaskSettings() { + var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings((InputType) null), null, null, "model"); + + var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.INGEST); + var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_SetsInputType_FromRequest_IfValid_OverridingStoredTaskSettings() { + var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model"); + + var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.SEARCH); + var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.SEARCH), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_SetsInputType_FromRequest_IfValid_OverridingRequestTaskSettings() { + var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings((InputType) null), null, null, "model"); + + var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(InputType.INGEST), InputType.SEARCH); + var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.SEARCH), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_OverridesInputType_WithRequestTaskSettingsSearch_WhenRequestInputTypeIsInvalid() { + var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model"); + + var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(InputType.SEARCH), InputType.UNSPECIFIED); + var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.SEARCH), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_DoesNotSetInputType_FromRequest_IfInputTypeIsInvalid() { + var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings((InputType) null), null, null, "model"); + + var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.UNSPECIFIED); + var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings((InputType) null), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_DoesNotSetInputType_WhenRequestTaskSettingsIsNull_AndRequestInputTypeIsInvalid() { + var model = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model"); + + var overriddenModel = JinaAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.UNSPECIFIED); + var expectedModel = createModel("url", "api_key", new JinaAIEmbeddingsTaskSettings(InputType.INGEST), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public static JinaAIEmbeddingsModel createModel(String url, String apiKey, @Nullable Integer tokenLimit, @Nullable String model) { + return createModel(url, apiKey, JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, tokenLimit, null, model); + } + + public static JinaAIEmbeddingsModel createModel( + String url, + String apiKey, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model + ) { + return createModel(url, apiKey, JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, tokenLimit, dimensions, model); + } + + public static JinaAIEmbeddingsModel createModel( + String url, + String apiKey, + JinaAIEmbeddingsTaskSettings taskSettings, + ChunkingSettings chunkingSettings, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model + ) { + return new JinaAIEmbeddingsModel( + "id", + "service", + new JinaAIEmbeddingsServiceSettings( + new JinaAIServiceSettings(url, model, null), + SimilarityMeasure.DOT_PRODUCT, + dimensions, + tokenLimit + ), + taskSettings, + chunkingSettings, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static JinaAIEmbeddingsModel createModel( + String url, + String apiKey, + JinaAIEmbeddingsTaskSettings taskSettings, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model + ) { + return new JinaAIEmbeddingsModel( + "id", + "service", + new JinaAIEmbeddingsServiceSettings( + new JinaAIServiceSettings(url, model, null), + SimilarityMeasure.DOT_PRODUCT, + dimensions, + tokenLimit + ), + taskSettings, + null, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static JinaAIEmbeddingsModel createModel( + String url, + String apiKey, + JinaAIEmbeddingsTaskSettings taskSettings, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model, + @Nullable SimilarityMeasure similarityMeasure + ) { + return new JinaAIEmbeddingsModel( + "id", + "service", + new JinaAIEmbeddingsServiceSettings(new JinaAIServiceSettings(url, model, null), similarityMeasure, dimensions, tokenLimit), + taskSettings, + null, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettingsTests.java new file mode 100644 index 0000000000000..6847d249a57a0 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsServiceSettingsTests.java @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.embeddings; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; +import org.elasticsearch.xpack.inference.InferenceNamedWriteablesProvider; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ServiceFields; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.is; + +public class JinaAIEmbeddingsServiceSettingsTests extends AbstractWireSerializingTestCase { + public static JinaAIEmbeddingsServiceSettings createRandom() { + SimilarityMeasure similarityMeasure = null; + Integer dims = null; + similarityMeasure = SimilarityMeasure.DOT_PRODUCT; + dims = 1024; + Integer maxInputTokens = randomBoolean() ? null : randomIntBetween(128, 256); + + var commonSettings = JinaAIServiceSettingsTests.createRandom(); + + return new JinaAIEmbeddingsServiceSettings(commonSettings, similarityMeasure, dims, maxInputTokens); + } + + public void testFromMap() { + var url = "https://www.abc.com"; + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var dims = 1536; + var maxInputTokens = 512; + var model = "model"; + var serviceSettings = JinaAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.URL, + url, + ServiceFields.SIMILARITY, + similarity, + ServiceFields.DIMENSIONS, + dims, + ServiceFields.MAX_INPUT_TOKENS, + maxInputTokens, + JinaAIServiceSettings.MODEL_ID, + model + ) + ), + ConfigurationParseContext.PERSISTENT + ); + + MatcherAssert.assertThat( + serviceSettings, + is( + new JinaAIEmbeddingsServiceSettings( + new JinaAIServiceSettings(ServiceUtils.createUri(url), model, null), + SimilarityMeasure.DOT_PRODUCT, + dims, + maxInputTokens + ) + ) + ); + } + + public void testFromMap_WithModelId() { + var url = "https://www.abc.com"; + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var dims = 1536; + var maxInputTokens = 512; + var model = "model"; + var serviceSettings = JinaAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.URL, + url, + ServiceFields.SIMILARITY, + similarity, + ServiceFields.DIMENSIONS, + dims, + ServiceFields.MAX_INPUT_TOKENS, + maxInputTokens, + JinaAIServiceSettings.MODEL_ID, + model + ) + ), + ConfigurationParseContext.REQUEST + ); + + MatcherAssert.assertThat( + serviceSettings, + is( + new JinaAIEmbeddingsServiceSettings( + new JinaAIServiceSettings(ServiceUtils.createUri(url), model, null), + SimilarityMeasure.DOT_PRODUCT, + dims, + maxInputTokens + ) + ) + ); + } + + public void testFromMap_InvalidSimilarity_ThrowsError() { + var similarity = "by_size"; + var thrownException = expectThrows( + ValidationException.class, + () -> JinaAIEmbeddingsServiceSettings.fromMap( + new HashMap<>(Map.of(JinaAIServiceSettings.MODEL_ID, "model", ServiceFields.SIMILARITY, similarity)), + ConfigurationParseContext.PERSISTENT + ) + ); + + MatcherAssert.assertThat( + thrownException.getMessage(), + is( + "Validation Failed: 1: [service_settings] Invalid value [by_size] received. [similarity] " + + "must be one of [cosine, dot_product, l2_norm];" + ) + ); + } + + public void testToXContent_WritesAllValues() throws IOException { + var serviceSettings = new JinaAIEmbeddingsServiceSettings( + new JinaAIServiceSettings("url", "model", new RateLimitSettings(3)), + SimilarityMeasure.COSINE, + 5, + 10 + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + serviceSettings.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + assertThat(xContentResult, is(""" + {"url":"url","model_id":"model",""" + """ + "rate_limit":{"requests_per_minute":3},"similarity":"cosine","dimensions":5,"max_input_tokens":10}""")); + } + + @Override + protected Writeable.Reader instanceReader() { + return JinaAIEmbeddingsServiceSettings::new; + } + + @Override + protected JinaAIEmbeddingsServiceSettings createTestInstance() { + return createRandom(); + } + + @Override + protected JinaAIEmbeddingsServiceSettings mutateInstance(JinaAIEmbeddingsServiceSettings instance) throws IOException { + return randomValueOtherThan(instance, JinaAIEmbeddingsServiceSettingsTests::createRandom); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + List entries = new ArrayList<>(); + entries.addAll(new MlInferenceNamedXContentProvider().getNamedWriteables()); + entries.addAll(InferenceNamedWriteablesProvider.getNamedWriteables()); + return new NamedWriteableRegistry(entries); + } + + public static Map getServiceSettingsMap(@Nullable String url, String model) { + var map = new HashMap<>(JinaAIServiceSettingsTests.getServiceSettingsMap(url, model)); + return map; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettingsTests.java new file mode 100644 index 0000000000000..8535381b43adc --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsTaskSettingsTests.java @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.embeddings; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithoutUnspecified; +import static org.elasticsearch.xpack.inference.services.jinaai.embeddings.JinaAIEmbeddingsTaskSettings.VALID_REQUEST_VALUES; +import static org.hamcrest.Matchers.is; + +public class JinaAIEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase { + + public static JinaAIEmbeddingsTaskSettings createRandom() { + var inputType = randomBoolean() ? randomWithoutUnspecified() : null; + + return new JinaAIEmbeddingsTaskSettings(inputType); + } + + public void testIsEmpty() { + var randomSettings = createRandom(); + var stringRep = Strings.toString(randomSettings); + assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); + } + + public void testUpdatedTaskSettings_NotUpdated_UseInitialSettings() { + var initialSettings = createRandom(); + var newSettings = new JinaAIEmbeddingsTaskSettings((InputType) null); + Map newSettingsMap = new HashMap<>(); + JinaAIEmbeddingsTaskSettings updatedSettings = (JinaAIEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + assertEquals(initialSettings.getInputType(), updatedSettings.getInputType()); + } + + public void testUpdatedTaskSettings_Updated_UseNewSettings() { + var initialSettings = createRandom(); + var newSettings = new JinaAIEmbeddingsTaskSettings(randomWithoutUnspecified()); + Map newSettingsMap = new HashMap<>(); + newSettingsMap.put(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, newSettings.getInputType().toString()); + JinaAIEmbeddingsTaskSettings updatedSettings = (JinaAIEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + assertEquals(newSettings.getInputType(), updatedSettings.getInputType()); + } + + public void testFromMap_CreatesEmptySettings_WhenAllFieldsAreNull() { + MatcherAssert.assertThat( + JinaAIEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of())), + is(new JinaAIEmbeddingsTaskSettings((InputType) null)) + ); + } + + public void testFromMap_CreatesEmptySettings_WhenMapIsNull() { + MatcherAssert.assertThat(JinaAIEmbeddingsTaskSettings.fromMap(null), is(new JinaAIEmbeddingsTaskSettings((InputType) null))); + } + + public void testFromMap_CreatesSettings_WhenAllFieldsOfSettingsArePresent() { + MatcherAssert.assertThat( + JinaAIEmbeddingsTaskSettings.fromMap( + new HashMap<>(Map.of(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, InputType.INGEST.toString())) + ), + is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST)) + ); + } + + public void testFromMap_ReturnsFailure_WhenInputTypeIsInvalid() { + var exception = expectThrows( + ValidationException.class, + () -> JinaAIEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, "abc"))) + ); + + MatcherAssert.assertThat( + exception.getMessage(), + is( + Strings.format( + "Validation Failed: 1: [task_settings] Invalid value [abc] received. [input_type] must be one of [%s];", + getValidValuesSortedAndCombined(VALID_REQUEST_VALUES) + ) + ) + ); + } + + public void testFromMap_ReturnsFailure_WhenInputTypeIsUnspecified() { + var exception = expectThrows( + ValidationException.class, + () -> JinaAIEmbeddingsTaskSettings.fromMap( + new HashMap<>(Map.of(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, InputType.UNSPECIFIED.toString())) + ) + ); + + MatcherAssert.assertThat( + exception.getMessage(), + is( + Strings.format( + "Validation Failed: 1: [task_settings] Invalid value [unspecified] received. [input_type] must be one of [%s];", + getValidValuesSortedAndCombined(VALID_REQUEST_VALUES) + ) + ) + ); + } + + private static > String getValidValuesSortedAndCombined(EnumSet validValues) { + var validValuesAsStrings = validValues.stream().map(value -> value.toString().toLowerCase(Locale.ROOT)).toArray(String[]::new); + Arrays.sort(validValuesAsStrings); + + return String.join(", ", validValuesAsStrings); + } + + public void testXContent_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { + var thrownException = expectThrows(AssertionError.class, () -> new JinaAIEmbeddingsTaskSettings(InputType.UNSPECIFIED)); + MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); + } + + public void testOf_KeepsOriginalValuesWhenRequestSettingsAreNull_AndRequestInputTypeIsInvalid() { + var taskSettings = new JinaAIEmbeddingsTaskSettings(InputType.INGEST); + var overriddenTaskSettings = JinaAIEmbeddingsTaskSettings.of( + taskSettings, + JinaAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + InputType.UNSPECIFIED + ); + MatcherAssert.assertThat(overriddenTaskSettings, is(taskSettings)); + } + + public void testOf_UsesRequestTaskSettings() { + var taskSettings = new JinaAIEmbeddingsTaskSettings((InputType) null); + var overriddenTaskSettings = JinaAIEmbeddingsTaskSettings.of( + taskSettings, + new JinaAIEmbeddingsTaskSettings(InputType.INGEST), + InputType.UNSPECIFIED + ); + + MatcherAssert.assertThat(overriddenTaskSettings, is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST))); + } + + public void testOf_UsesRequestTaskSettings_AndRequestInputType() { + var taskSettings = new JinaAIEmbeddingsTaskSettings(InputType.SEARCH); + var overriddenTaskSettings = JinaAIEmbeddingsTaskSettings.of( + taskSettings, + new JinaAIEmbeddingsTaskSettings((InputType) null), + InputType.INGEST + ); + + MatcherAssert.assertThat(overriddenTaskSettings, is(new JinaAIEmbeddingsTaskSettings(InputType.INGEST))); + } + + @Override + protected Writeable.Reader instanceReader() { + return JinaAIEmbeddingsTaskSettings::new; + } + + @Override + protected JinaAIEmbeddingsTaskSettings createTestInstance() { + return createRandom(); + } + + @Override + protected JinaAIEmbeddingsTaskSettings mutateInstance(JinaAIEmbeddingsTaskSettings instance) throws IOException { + return randomValueOtherThan(instance, JinaAIEmbeddingsTaskSettingsTests::createRandom); + } + + public static Map getTaskSettingsMapEmpty() { + return new HashMap<>(); + } + + public static Map getTaskSettingsMap(@Nullable InputType inputType) { + var map = new HashMap(); + + if (inputType != null) { + map.put(JinaAIEmbeddingsTaskSettings.INPUT_TYPE, inputType.toString()); + } + + return map; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModelTests.java new file mode 100644 index 0000000000000..d6b3df5fd3717 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModelTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.rerank; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; + +public class JinaAIRerankModelTests { + + public static JinaAIRerankModel createModel(String apiKey, String modelId, @Nullable Integer topN) { + return new JinaAIRerankModel( + "id", + "service", + new JinaAIRerankServiceSettings(new JinaAIServiceSettings(ESTestCase.randomAlphaOfLength(10), modelId, null)), + new JinaAIRerankTaskSettings(topN, null), + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static JinaAIRerankModel createModel(String modelId, @Nullable Integer topN) { + return new JinaAIRerankModel( + "id", + "service", + new JinaAIRerankServiceSettings(new JinaAIServiceSettings(ESTestCase.randomAlphaOfLength(10), modelId, null)), + new JinaAIRerankTaskSettings(topN, null), + new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8)) + ); + } + + public static JinaAIRerankModel createModel(String modelId, @Nullable Integer topN, Boolean returnDocuments) { + return new JinaAIRerankModel( + "id", + "service", + new JinaAIRerankServiceSettings(new JinaAIServiceSettings(ESTestCase.randomAlphaOfLength(10), modelId, null)), + new JinaAIRerankTaskSettings(topN, returnDocuments), + new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8)) + ); + } + + public static JinaAIRerankModel createModel(String url, String modelId, @Nullable Integer topN, Boolean returnDocuments) { + return new JinaAIRerankModel( + "id", + "service", + new JinaAIRerankServiceSettings(new JinaAIServiceSettings(url, modelId, null)), + new JinaAIRerankTaskSettings(topN, returnDocuments), + new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8)) + ); + } + + public static JinaAIRerankModel createModel( + String url, + String apiKey, + String modelId, + @Nullable Integer topN, + Boolean returnDocuments + ) { + return new JinaAIRerankModel( + "id", + "service", + new JinaAIRerankServiceSettings(new JinaAIServiceSettings(url, modelId, null)), + new JinaAIRerankTaskSettings(topN, returnDocuments), + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettingsTests.java new file mode 100644 index 0000000000000..47f67bd8cefb8 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankServiceSettingsTests.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.rerank; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettings; +import org.elasticsearch.xpack.inference.services.jinaai.JinaAIServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString; + +public class JinaAIRerankServiceSettingsTests extends AbstractBWCWireSerializationTestCase { + public static JinaAIRerankServiceSettings createRandom() { + return new JinaAIRerankServiceSettings( + new JinaAIServiceSettings( + randomFrom(new String[] { null, Strings.format("http://%s.com", randomAlphaOfLength(8)) }), + randomAlphaOfLength(10), + RateLimitSettingsTests.createRandom() + ) + ); + } + + public void testToXContent_WritesAllValues() throws IOException { + var url = "http://www.abc.com"; + var model = "model"; + + var serviceSettings = new JinaAIRerankServiceSettings(new JinaAIServiceSettings(url, model, null)); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + serviceSettings.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "url":"http://www.abc.com", + "model_id":"model", + "rate_limit": { + "requests_per_minute": 2000 + } + } + """)); + } + + @Override + protected Writeable.Reader instanceReader() { + return JinaAIRerankServiceSettings::new; + } + + @Override + protected JinaAIRerankServiceSettings createTestInstance() { + return createRandom(); + } + + @Override + protected JinaAIRerankServiceSettings mutateInstance(JinaAIRerankServiceSettings instance) throws IOException { + return randomValueOtherThan(instance, JinaAIRerankServiceSettingsTests::createRandom); + } + + @Override + protected JinaAIRerankServiceSettings mutateInstanceForVersion(JinaAIRerankServiceSettings instance, TransportVersion version) { + return instance; + } + + public static Map getServiceSettingsMap(@Nullable String url, @Nullable String model) { + return new HashMap<>(JinaAIServiceSettingsTests.getServiceSettingsMap(url, model)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettingsTests.java new file mode 100644 index 0000000000000..fa70248d01513 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankTaskSettingsTests.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.jinaai.rerank; + +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; + +public class JinaAIRerankTaskSettingsTests extends AbstractWireSerializingTestCase { + + public static JinaAIRerankTaskSettings createRandom() { + var returnDocuments = randomBoolean() ? randomBoolean() : null; + var topNDocsOnly = randomBoolean() ? randomIntBetween(1, 10) : null; + + return new JinaAIRerankTaskSettings(topNDocsOnly, returnDocuments); + } + + public void testFromMap_WithValidValues_ReturnsSettings() { + Map taskMap = Map.of(JinaAIRerankTaskSettings.RETURN_DOCUMENTS, true, JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, 5); + var settings = JinaAIRerankTaskSettings.fromMap(new HashMap<>(taskMap)); + assertTrue(settings.getReturnDocuments()); + assertEquals(5, settings.getTopNDocumentsOnly().intValue()); + } + + public void testFromMap_WithNullValues_ReturnsSettingsWithNulls() { + var settings = JinaAIRerankTaskSettings.fromMap(Map.of()); + assertNull(settings.getReturnDocuments()); + assertNull(settings.getTopNDocumentsOnly()); + } + + public void testFromMap_WithInvalidReturnDocuments_ThrowsValidationException() { + Map taskMap = Map.of( + JinaAIRerankTaskSettings.RETURN_DOCUMENTS, + "invalid", + JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, + 5 + ); + var thrownException = expectThrows(ValidationException.class, () -> JinaAIRerankTaskSettings.fromMap(new HashMap<>(taskMap))); + assertThat(thrownException.getMessage(), containsString("field [return_documents] is not of the expected type")); + } + + public void testFromMap_WithInvalidTopNDocsOnly_ThrowsValidationException() { + Map taskMap = Map.of( + JinaAIRerankTaskSettings.RETURN_DOCUMENTS, + true, + JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, + "invalid" + ); + var thrownException = expectThrows(ValidationException.class, () -> JinaAIRerankTaskSettings.fromMap(new HashMap<>(taskMap))); + assertThat(thrownException.getMessage(), containsString("field [top_n] is not of the expected type")); + } + + public void testUpdatedTaskSettings_WithEmptyMap_ReturnsSameSettings() { + var initialSettings = new JinaAIRerankTaskSettings(5, true); + JinaAIRerankTaskSettings updatedSettings = (JinaAIRerankTaskSettings) initialSettings.updatedTaskSettings(Map.of()); + assertEquals(initialSettings, updatedSettings); + } + + public void testUpdatedTaskSettings_WithNewReturnDocuments_ReturnsUpdatedSettings() { + var initialSettings = new JinaAIRerankTaskSettings(5, true); + Map newSettings = Map.of(JinaAIRerankTaskSettings.RETURN_DOCUMENTS, false); + JinaAIRerankTaskSettings updatedSettings = (JinaAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertFalse(updatedSettings.getReturnDocuments()); + assertEquals(initialSettings.getTopNDocumentsOnly(), updatedSettings.getTopNDocumentsOnly()); + } + + public void testUpdatedTaskSettings_WithNewTopNDocsOnly_ReturnsUpdatedSettings() { + var initialSettings = new JinaAIRerankTaskSettings(5, true); + Map newSettings = Map.of(JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, 7); + JinaAIRerankTaskSettings updatedSettings = (JinaAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertEquals(7, updatedSettings.getTopNDocumentsOnly().intValue()); + assertEquals(initialSettings.getReturnDocuments(), updatedSettings.getReturnDocuments()); + } + + public void testUpdatedTaskSettings_WithMultipleNewValues_ReturnsUpdatedSettings() { + var initialSettings = new JinaAIRerankTaskSettings(5, true); + Map newSettings = Map.of( + JinaAIRerankTaskSettings.RETURN_DOCUMENTS, + false, + JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, + 7 + ); + JinaAIRerankTaskSettings updatedSettings = (JinaAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertFalse(updatedSettings.getReturnDocuments()); + assertEquals(7, updatedSettings.getTopNDocumentsOnly().intValue()); + } + + @Override + protected Writeable.Reader instanceReader() { + return JinaAIRerankTaskSettings::new; + } + + @Override + protected JinaAIRerankTaskSettings createTestInstance() { + return createRandom(); + } + + @Override + protected JinaAIRerankTaskSettings mutateInstance(JinaAIRerankTaskSettings instance) throws IOException { + return randomValueOtherThan(instance, JinaAIRerankTaskSettingsTests::createRandom); + } + + public static Map getTaskSettingsMapEmpty() { + return new HashMap<>(); + } + + public static Map getTaskSettingsMap(@Nullable Integer topNDocumentsOnly, Boolean returnDocuments) { + var map = new HashMap(); + + if (topNDocumentsOnly != null) { + map.put(JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY, topNDocumentsOnly.toString()); + } + + if (returnDocuments != null) { + map.put(JinaAIRerankTaskSettings.RETURN_DOCUMENTS, returnDocuments.toString()); + } + + return map; + } +} From 447dcaad0e9de7a650f26ba0763effdea833b01d Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 8 Jan 2025 12:37:29 +0100 Subject: [PATCH 14/52] Disable N-2 BWC tests for non-snapshot builds (#119583) The tests added in #119468 do not work when executed in the `release-test` pipeline: at some point a 9.0 non-snapshot node tries to join the 8.18.0-SNAPSHOT cluster which is impossible due to a missing `esql.metrics_syntax` feature which is only declared in snapshot builds. Additionaly, the tests rely on unrelease code in 8.x so they won't work until 8.18.0 is released. I don't know how we deal with such issues today, but I'd like to unmute the tests so that they are executed on CI while being disabled in `release-test`. Releates #119550 --- muted-tests.yml | 24 ---------------------- qa/lucene-index-compatibility/build.gradle | 5 +++-- 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 4993c66d14716..d38a3a7abd730 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -236,38 +236,14 @@ tests: - class: org.elasticsearch.smoketest.MlWithSecurityIT method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config} issue: https://github.com/elastic/elasticsearch/issues/119548 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 8.18.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119549 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 8.18.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119550 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119551 - class: org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshotTests method: testSkipNonRootOfNestedDocuments issue: https://github.com/elastic/elasticsearch/issues/119553 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119560 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 9.0.0]} - issue: https://github.com/elastic/elasticsearch/issues/119561 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 9.0.0]} - issue: https://github.com/elastic/elasticsearch/issues/119562 - class: org.elasticsearch.xpack.ml.integration.ForecastIT method: testOverflowToDisk issue: https://github.com/elastic/elasticsearch/issues/117740 - class: org.elasticsearch.xpack.security.authc.ldap.MultiGroupMappingIT issue: https://github.com/elastic/elasticsearch/issues/119599 -- class: org.elasticsearch.lucene.FullClusterRestartSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=8.18.0} - issue: https://github.com/elastic/elasticsearch/issues/119631 -- class: org.elasticsearch.lucene.FullClusterRestartSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=9.0.0} - issue: https://github.com/elastic/elasticsearch/issues/119632 - class: org.elasticsearch.search.profile.dfs.DfsProfilerIT method: testProfileDfs issue: https://github.com/elastic/elasticsearch/issues/119711 diff --git a/qa/lucene-index-compatibility/build.gradle b/qa/lucene-index-compatibility/build.gradle index 37e5eea85a08b..3b2e69ec9859f 100644 --- a/qa/lucene-index-compatibility/build.gradle +++ b/qa/lucene-index-compatibility/build.gradle @@ -14,7 +14,9 @@ buildParams.bwcVersions.withLatestReadOnlyIndexCompatible { bwcVersion -> tasks.named("javaRestTest").configure { systemProperty("tests.minimum.index.compatible", bwcVersion) usesBwcDistribution(bwcVersion) - enabled = true + + // Tests rely on unreleased code in 8.18 branch + enabled = buildParams.isSnapshotBuild() } } @@ -22,4 +24,3 @@ tasks.withType(Test).configureEach { // CI doesn't like it when there's multiple clusters running at once maxParallelForks = 1 } - From 12eb1cfda1e42f60945caea6b639b8dda47b8ed5 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Wed, 8 Jan 2025 14:34:37 +0200 Subject: [PATCH 15/52] Metrics for indexing failures due to version conflicts (#119067) This exposes new OTel node and index based metrics for indexing failures due to version conflicts. In addition, the /_cat/shards, /_cat/indices and /_cat/nodes APIs also expose the same metric, under the newly added column iifvc. Relates: #107601 --- docs/changelog/119067.yaml | 5 + docs/reference/cat/nodes.asciidoc | 3 + docs/reference/cat/shards.asciidoc | 3 + .../datastreams/DataStreamAutoshardingIT.java | 2 +- .../test/cat.shards/10_basic.yml | 1 + .../monitor/metrics/IndicesMetricsIT.java | 39 +++- .../metrics/NodeIndexingMetricsIT.java | 211 +++++++++++++++++- .../org/elasticsearch/TransportVersions.java | 1 + .../index/shard/IndexingStats.java | 22 +- .../index/shard/InternalIndexingStats.java | 7 + .../monitor/metrics/IndicesMetrics.java | 10 +- .../monitor/metrics/NodeMetrics.java | 16 ++ .../rest/action/cat/RestIndicesAction.java | 16 ++ .../rest/action/cat/RestNodesAction.java | 6 + .../rest/action/cat/RestShardsAction.java | 6 + .../cluster/node/stats/NodeStatsTests.java | 1 + .../metadata/IndexMetadataStatsTests.java | 16 +- .../index/shard/IndexShardTests.java | 81 +++++++ .../action/cat/RestShardsActionTests.java | 160 +++++++++---- .../index/shard/IndexShardTestCase.java | 19 +- .../indices/IndexStatsMonitoringDocTests.java | 16 +- .../IndicesStatsMonitoringDocTests.java | 2 +- .../node/NodeStatsMonitoringDocTests.java | 1 + 23 files changed, 578 insertions(+), 66 deletions(-) create mode 100644 docs/changelog/119067.yaml diff --git a/docs/changelog/119067.yaml b/docs/changelog/119067.yaml new file mode 100644 index 0000000000000..c7ddd570bea18 --- /dev/null +++ b/docs/changelog/119067.yaml @@ -0,0 +1,5 @@ +pr: 119067 +summary: Metrics for indexing failures due to version conflicts +area: CRUD +type: feature +issues: [] diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index c52315423f87e..a5a813e8d37d5 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -239,6 +239,9 @@ Number of indexing operations, such as `1`. `indexing.index_failed`, `iif`, `indexingIndexFailed`:: Number of failed indexing operations, such as `0`. +`indexing.index_failed_due_to_version_conflict`, `iifvc`, `indexingIndexFailedDueToVersionConflict`:: +Number of failed indexing operations due to version conflict, such as `0`. + `merges.current`, `mc`, `mergesCurrent`:: Number of current merge operations, such as `0`. diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index f73ac6e263cd2..2d3859e74c87e 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -162,6 +162,9 @@ Number of indexing operations, such as `1`. `indexing.index_failed`, `iif`, `indexingIndexFailed`:: Number of failed indexing operations, such as `0`. +`indexing.index_failed_due_to_version_conflict`, `iifvc`, `indexingIndexFailedDueToVersionConflict`:: +Number of failed indexing operations due to version conflict, such as `0`. + `merges.current`, `mc`, `mergesCurrent`:: Number of current merge operations, such as `0`. diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java index ac73385a97d70..91f18ad3573fd 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java @@ -493,7 +493,7 @@ private static ShardStats getShardStats(IndexMetadata indexMeta, int shardIndex, CommonStats stats = new CommonStats(); stats.docs = new DocsStats(100, 0, randomByteSizeValue().getBytes()); stats.store = new StoreStats(); - stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1)); + stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1)); return new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0); } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml index 03d8b2068d23e..45f381eab80b1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -45,6 +45,7 @@ indexing.index_time .+ \n indexing.index_total .+ \n indexing.index_failed .+ \n + indexing.index_failed_due_to_version_conflict .+ \n merges.current .+ \n merges.current_docs .+ \n merges.current_size .+ \n diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java index 31c7720ffec1c..fee2c0494365e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -76,6 +77,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { static final String STANDARD_INDEXING_COUNT = "es.indices.standard.indexing.total"; static final String STANDARD_INDEXING_TIME = "es.indices.standard.indexing.time"; static final String STANDARD_INDEXING_FAILURE = "es.indices.standard.indexing.failure.total"; + static final String STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT = "es.indices.standard.indexing.failure.version_conflict.total"; static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total"; static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.size"; @@ -89,6 +91,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { static final String TIME_SERIES_INDEXING_COUNT = "es.indices.time_series.indexing.total"; static final String TIME_SERIES_INDEXING_TIME = "es.indices.time_series.indexing.time"; static final String TIME_SERIES_INDEXING_FAILURE = "es.indices.time_series.indexing.failure.total"; + static final String TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT = + "es.indices.time_series.indexing.failure.version_conflict.total"; static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total"; static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.size"; @@ -102,6 +106,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { static final String LOGSDB_INDEXING_COUNT = "es.indices.logsdb.indexing.total"; static final String LOGSDB_INDEXING_TIME = "es.indices.logsdb.indexing.time"; static final String LOGSDB_INDEXING_FAILURE = "es.indices.logsdb.indexing.failure.total"; + static final String LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT = "es.indices.logsdb.indexing.failure.version_conflict.total"; public void testIndicesMetrics() { String indexNode = internalCluster().startNode(); @@ -132,7 +137,9 @@ public void testIndicesMetrics() { STANDARD_INDEXING_TIME, greaterThanOrEqualTo(0L), STANDARD_INDEXING_FAILURE, - equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexCount()) + equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexFailedCount()), + STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT, + equalTo(indexing1.getIndexFailedDueToVersionConflictCount() - indexing0.getIndexFailedDueToVersionConflictCount()) ) ); @@ -155,7 +162,9 @@ public void testIndicesMetrics() { TIME_SERIES_INDEXING_TIME, greaterThanOrEqualTo(0L), TIME_SERIES_INDEXING_FAILURE, - equalTo(indexing2.getIndexFailedCount() - indexing1.getIndexFailedCount()) + equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexFailedCount()), + TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT, + equalTo(indexing1.getIndexFailedDueToVersionConflictCount() - indexing0.getIndexFailedDueToVersionConflictCount()) ) ); @@ -177,13 +186,14 @@ public void testIndicesMetrics() { LOGSDB_INDEXING_TIME, greaterThanOrEqualTo(0L), LOGSDB_INDEXING_FAILURE, - equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()) + equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()), + LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT, + equalTo(indexing3.getIndexFailedDueToVersionConflictCount() - indexing2.getIndexFailedDueToVersionConflictCount()) ) ); // already collected indexing stats - collectThenAssertMetrics( - telemetry, - 4, + Map> zeroMatchers = new HashMap<>(); + zeroMatchers.putAll( Map.of( STANDARD_INDEXING_COUNT, equalTo(0L), @@ -191,22 +201,35 @@ public void testIndicesMetrics() { equalTo(0L), STANDARD_INDEXING_FAILURE, equalTo(0L), - + STANDARD_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT, + equalTo(0L) + ) + ); + zeroMatchers.putAll( + Map.of( TIME_SERIES_INDEXING_COUNT, equalTo(0L), TIME_SERIES_INDEXING_TIME, equalTo(0L), TIME_SERIES_INDEXING_FAILURE, equalTo(0L), - + TIME_SERIES_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT, + equalTo(0L) + ) + ); + zeroMatchers.putAll( + Map.of( LOGSDB_INDEXING_COUNT, equalTo(0L), LOGSDB_INDEXING_TIME, equalTo(0L), LOGSDB_INDEXING_FAILURE, + equalTo(0L), + LOGSDB_INDEXING_FAILURE_DUE_TO_VERSION_CONFLICT, equalTo(0L) ) ); + collectThenAssertMetrics(telemetry, 4, zeroMatchers); String searchNode = internalCluster().startDataOnlyNode(); indicesService = internalCluster().getInstance(IndicesService.class, searchNode); telemetry = internalCluster().getInstance(PluginsService.class, searchNode) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java index 1635a08e1768b..04130d176b9e5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java @@ -9,14 +9,17 @@ package org.elasticsearch.monitor.metrics; +import org.apache.lucene.analysis.TokenStream; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -25,6 +28,13 @@ import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexingPressure; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.engine.VersionConflictEngineException; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestStatus; @@ -43,13 +53,16 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; +import static java.util.Collections.singletonMap; import static org.elasticsearch.index.IndexingPressure.MAX_COORDINATING_BYTES; import static org.elasticsearch.index.IndexingPressure.MAX_PRIMARY_BYTES; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) @@ -66,7 +79,7 @@ public List> getSettings() { @Override protected Collection> nodePlugins() { - return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class); + return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class, TestAnalysisPlugin.class); } @Override @@ -77,6 +90,197 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .build(); } + public void testZeroMetricsForVersionConflictsForNonIndexingOperations() { + final String dataNode = internalCluster().startNode(); + ensureStableCluster(1); + + final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + plugin.resetMeter(); + + assertAcked(prepareCreate("index_no_refresh", Settings.builder().put("index.refresh_interval", "-1"))); + assertAcked(prepareCreate("index_with_default_refresh")); + + for (String indexName : List.of("index_no_refresh", "index_with_default_refresh")) { + String docId = randomUUID(); + client(dataNode).index(new IndexRequest(indexName).id(docId).source(Map.of())).actionGet(); + // test version conflicts are counted when getting from the translog + if (randomBoolean()) { + // this get has the side effect of tracking translog location in the live version map, + // which potentially influences the engine conflict exception path + client(dataNode).get(new GetRequest(indexName, docId).realtime(randomBoolean())).actionGet(); + } + { + var e = expectThrows( + VersionConflictEngineException.class, + () -> client(dataNode).get( + new GetRequest(indexName, docId).version(10).versionType(randomFrom(VersionType.EXTERNAL, VersionType.EXTERNAL_GTE)) + ).actionGet() + ); + assertThat(e.getMessage(), containsString("version conflict")); + assertThat(e.status(), is(RestStatus.CONFLICT)); + } + if (randomBoolean()) { + client(dataNode).get(new GetRequest(indexName, docId).realtime(false)).actionGet(); + } + client(dataNode).admin().indices().prepareRefresh(indexName).get(); + { + var e = expectThrows( + VersionConflictEngineException.class, + () -> client(dataNode).get( + new GetRequest(indexName, docId).version(5) + .versionType(randomFrom(VersionType.EXTERNAL, VersionType.EXTERNAL_GTE)) + .realtime(false) + ).actionGet() + ); + assertThat(e.getMessage(), containsString("version conflict")); + assertThat(e.status(), is(RestStatus.CONFLICT)); + } + // updates + { + var e = expectThrows( + VersionConflictEngineException.class, + () -> client(dataNode).update( + new UpdateRequest(indexName, docId).setIfPrimaryTerm(1) + .setIfSeqNo(randomIntBetween(2, 5)) + .doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))) + ).actionGet() + ); + assertThat(e.getMessage(), containsString("version conflict")); + assertThat(e.status(), is(RestStatus.CONFLICT)); + } + // deletes + { + var e = expectThrows( + VersionConflictEngineException.class, + () -> client(dataNode).delete( + new DeleteRequest(indexName, docId).setIfPrimaryTerm(randomIntBetween(2, 5)).setIfSeqNo(0) + ).actionGet() + ); + assertThat(e.getMessage(), containsString("version conflict")); + assertThat(e.status(), is(RestStatus.CONFLICT)); + } + } + + // simulate async apm `polling` call for metrics + plugin.collect(); + + // there are no indexing (version conflict) failures reported because only gets/updates/deletes generated the conflicts + // and those are not "indexing" operations + var indexingFailedTotal = getSingleRecordedMetric(plugin::getLongAsyncCounterMeasurement, "es.indexing.indexing.failed.total"); + assertThat(indexingFailedTotal.getLong(), equalTo(0L)); + var indexingFailedDueToVersionConflictTotal = getSingleRecordedMetric( + plugin::getLongAsyncCounterMeasurement, + "es.indexing.indexing.failed.version_conflict.total" + ); + assertThat(indexingFailedDueToVersionConflictTotal.getLong(), equalTo(0L)); + } + + public void testMetricsForIndexingVersionConflicts() { + final String dataNode = internalCluster().startNode(); + ensureStableCluster(1); + + final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + plugin.resetMeter(); + + assertAcked( + prepareCreate( + "test", + Settings.builder() + .put("index.refresh_interval", "-1") + .put("index.analysis.analyzer.test_analyzer.type", "custom") + .put("index.analysis.analyzer.test_analyzer.tokenizer", "standard") + .putList("index.analysis.analyzer.test_analyzer.filter", "test_token_filter") + ).setMapping(Map.of("properties", Map.of("test_field", Map.of("type", "text", "analyzer", "test_analyzer")))).get() + ); + + String docId = randomUUID(); + // successful index (with version) + client(dataNode).index( + new IndexRequest("test").id(docId) + .version(10) + .versionType(randomFrom(VersionType.EXTERNAL, VersionType.EXTERNAL_GTE)) + .source(Map.of()) + ).actionGet(); + // if_primary_term conflict + { + var e = expectThrows( + VersionConflictEngineException.class, + () -> client(dataNode).index(new IndexRequest("test").id(docId).source(Map.of()).setIfSeqNo(0).setIfPrimaryTerm(2)) + .actionGet() + ); + assertThat(e.getMessage(), containsString("version conflict")); + assertThat(e.status(), is(RestStatus.CONFLICT)); + } + // if_seq_no conflict + { + var e = expectThrows( + VersionConflictEngineException.class, + () -> client(dataNode).index(new IndexRequest("test").id(docId).source(Map.of()).setIfSeqNo(1).setIfPrimaryTerm(1)) + .actionGet() + ); + assertThat(e.getMessage(), containsString("version conflict")); + assertThat(e.status(), is(RestStatus.CONFLICT)); + } + // version conflict + { + var e = expectThrows( + VersionConflictEngineException.class, + () -> client(dataNode).index( + new IndexRequest("test").id(docId) + .source(Map.of()) + .version(3) + .versionType(randomFrom(VersionType.EXTERNAL, VersionType.EXTERNAL_GTE)) + ).actionGet() + ); + assertThat(e.getMessage(), containsString("version conflict")); + assertThat(e.status(), is(RestStatus.CONFLICT)); + } + // indexing failure that is NOT a version conflict + PluginsService pluginService = internalCluster().getInstance(PluginsService.class, dataNode); + pluginService.filterPlugins(TestAnalysisPlugin.class).forEach(p -> p.throwParsingError.set(true)); + { + var e = expectThrows( + MapperParsingException.class, + () -> client(dataNode).index(new IndexRequest("test").id(docId + "other").source(Map.of("test_field", "this will error"))) + .actionGet() + ); + assertThat(e.status(), is(RestStatus.BAD_REQUEST)); + } + + plugin.collect(); + + var indexingFailedTotal = getSingleRecordedMetric(plugin::getLongAsyncCounterMeasurement, "es.indexing.indexing.failed.total"); + assertThat(indexingFailedTotal.getLong(), equalTo(4L)); + var indexingFailedDueToVersionConflictTotal = getSingleRecordedMetric( + plugin::getLongAsyncCounterMeasurement, + "es.indexing.indexing.failed.version_conflict.total" + ); + assertThat(indexingFailedDueToVersionConflictTotal.getLong(), equalTo(3L)); + } + + public static final class TestAnalysisPlugin extends Plugin implements AnalysisPlugin { + final AtomicBoolean throwParsingError = new AtomicBoolean(false); + + @Override + public Map> getTokenFilters() { + return singletonMap("test_token_filter", (indexSettings, environment, name, settings) -> new AbstractTokenFilterFactory(name) { + @Override + public TokenStream create(TokenStream tokenStream) { + if (throwParsingError.get()) { + throw new MapperParsingException("simulate mapping parsing error"); + } + return tokenStream; + } + }); + } + } + public void testNodeIndexingMetricsArePublishing() { final String dataNode = internalCluster().startNode(); @@ -116,6 +320,11 @@ public void testNodeIndexingMetricsArePublishing() { var indexingFailedTotal = getSingleRecordedMetric(plugin::getLongAsyncCounterMeasurement, "es.indexing.indexing.failed.total"); assertThat(indexingFailedTotal.getLong(), equalTo(0L)); + var indexingFailedDueToVersionConflictTotal = getSingleRecordedMetric( + plugin::getLongAsyncCounterMeasurement, + "es.indexing.indexing.failed.version_conflict.total" + ); + assertThat(indexingFailedDueToVersionConflictTotal.getLong(), equalTo(0L)); var deletionTotal = getSingleRecordedMetric(plugin::getLongAsyncCounterMeasurement, "es.indexing.deletion.docs.total"); assertThat(deletionTotal.getLong(), equalTo((long) deletesCount)); diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 0587ab8b46edc..39f9a1f34af3c 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -152,6 +152,7 @@ static TransportVersion def(int id) { public static final TransportVersion TEXT_EMBEDDING_QUERY_VECTOR_BUILDER_INFER_MODEL_ID = def(8_817_00_0); public static final TransportVersion ESQL_ENABLE_NODE_LEVEL_REDUCTION = def(8_818_00_0); public static final TransportVersion JINA_AI_INTEGRATION_ADDED = def(8_819_00_0); + public static final TransportVersion TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC = def(8_820_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java index b0a4d333ba77f..62e456d95f467 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java @@ -34,6 +34,7 @@ public static class Stats implements Writeable, ToXContentFragment { private long indexTimeInMillis; private long indexCurrent; private long indexFailedCount; + private long indexFailedDueToVersionConflictCount; private long deleteCount; private long deleteTimeInMillis; private long deleteCurrent; @@ -50,6 +51,9 @@ public Stats(StreamInput in) throws IOException { indexTimeInMillis = in.readVLong(); indexCurrent = in.readVLong(); indexFailedCount = in.readVLong(); + if (in.getTransportVersion().onOrAfter(TransportVersions.TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC)) { + indexFailedDueToVersionConflictCount = in.readVLong(); + } deleteCount = in.readVLong(); deleteTimeInMillis = in.readVLong(); deleteCurrent = in.readVLong(); @@ -67,6 +71,7 @@ public Stats( long indexTimeInMillis, long indexCurrent, long indexFailedCount, + long indexFailedDueToVersionConflictCount, long deleteCount, long deleteTimeInMillis, long deleteCurrent, @@ -80,6 +85,7 @@ public Stats( this.indexTimeInMillis = indexTimeInMillis; this.indexCurrent = indexCurrent; this.indexFailedCount = indexFailedCount; + this.indexFailedDueToVersionConflictCount = indexFailedDueToVersionConflictCount; this.deleteCount = deleteCount; this.deleteTimeInMillis = deleteTimeInMillis; this.deleteCurrent = deleteCurrent; @@ -96,6 +102,7 @@ public void add(Stats stats) { indexTimeInMillis += stats.indexTimeInMillis; indexCurrent += stats.indexCurrent; indexFailedCount += stats.indexFailedCount; + indexFailedDueToVersionConflictCount += stats.indexFailedDueToVersionConflictCount; deleteCount += stats.deleteCount; deleteTimeInMillis += stats.deleteTimeInMillis; @@ -124,6 +131,13 @@ public long getIndexFailedCount() { return indexFailedCount; } + /** + * The number of indexing operations that failed because of a version conflict (a subset of all index failed operations) + */ + public long getIndexFailedDueToVersionConflictCount() { + return indexFailedDueToVersionConflictCount; + } + /** * The total amount of time spend on executing index operations. */ @@ -191,6 +205,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(indexTimeInMillis); out.writeVLong(indexCurrent); out.writeVLong(indexFailedCount); + if (out.getTransportVersion().onOrAfter(TransportVersions.TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC)) { + out.writeVLong(indexFailedDueToVersionConflictCount); + } out.writeVLong(deleteCount); out.writeVLong(deleteTimeInMillis); out.writeVLong(deleteCurrent); @@ -209,6 +226,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.humanReadableField(Fields.INDEX_TIME_IN_MILLIS, Fields.INDEX_TIME, getIndexTime()); builder.field(Fields.INDEX_CURRENT, indexCurrent); builder.field(Fields.INDEX_FAILED, indexFailedCount); + builder.field(Fields.INDEX_FAILED_DUE_TO_VERSION_CONFLICT, indexFailedDueToVersionConflictCount); builder.field(Fields.DELETE_TOTAL, deleteCount); builder.humanReadableField(Fields.DELETE_TIME_IN_MILLIS, Fields.DELETE_TIME, getDeleteTime()); @@ -232,6 +250,7 @@ public boolean equals(Object o) { && indexTimeInMillis == that.indexTimeInMillis && indexCurrent == that.indexCurrent && indexFailedCount == that.indexFailedCount + && indexFailedDueToVersionConflictCount == that.indexFailedDueToVersionConflictCount && deleteCount == that.deleteCount && deleteTimeInMillis == that.deleteTimeInMillis && deleteCurrent == that.deleteCurrent @@ -249,6 +268,7 @@ public int hashCode() { indexTimeInMillis, indexCurrent, indexFailedCount, + indexFailedDueToVersionConflictCount, deleteCount, deleteTimeInMillis, deleteCurrent, @@ -323,12 +343,12 @@ public int hashCode() { static final class Fields { static final String INDEXING = "indexing"; - static final String TYPES = "types"; static final String INDEX_TOTAL = "index_total"; static final String INDEX_TIME = "index_time"; static final String INDEX_TIME_IN_MILLIS = "index_time_in_millis"; static final String INDEX_CURRENT = "index_current"; static final String INDEX_FAILED = "index_failed"; + static final String INDEX_FAILED_DUE_TO_VERSION_CONFLICT = "index_failed_due_to_version_conflict"; static final String DELETE_TOTAL = "delete_total"; static final String DELETE_TIME = "delete_time"; static final String DELETE_TIME_IN_MILLIS = "delete_time_in_millis"; diff --git a/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java index d0bb7629c3969..13d270ba36786 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java @@ -9,9 +9,11 @@ package org.elasticsearch.index.shard; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.VersionConflictEngineException; import java.util.concurrent.TimeUnit; @@ -78,6 +80,9 @@ public void postIndex(ShardId shardId, Engine.Index index, Exception ex) { if (index.origin().isRecovery() == false) { totalStats.indexCurrent.dec(); totalStats.indexFailed.inc(); + if (ExceptionsHelper.unwrapCause(ex) instanceof VersionConflictEngineException) { + totalStats.indexFailedDueToVersionConflicts.inc(); + } } } @@ -124,6 +129,7 @@ static class StatsHolder { private final MeanMetric deleteMetric = new MeanMetric(); private final CounterMetric indexCurrent = new CounterMetric(); private final CounterMetric indexFailed = new CounterMetric(); + private final CounterMetric indexFailedDueToVersionConflicts = new CounterMetric(); private final CounterMetric deleteCurrent = new CounterMetric(); private final CounterMetric noopUpdates = new CounterMetric(); @@ -140,6 +146,7 @@ IndexingStats.Stats stats( TimeUnit.NANOSECONDS.toMillis(totalIndexingTimeInNanos), indexCurrent.count(), indexFailed.count(), + indexFailedDueToVersionConflicts.count(), deleteMetric.count(), TimeUnit.NANOSECONDS.toMillis(deleteMetric.sum()), deleteCurrent.count(), diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java index 99011d101d342..dfaf6535e4d85 100644 --- a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java +++ b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java @@ -53,7 +53,7 @@ public IndicesMetrics(MeterRegistry meterRegistry, IndicesService indicesService } private static List registerAsyncMetrics(MeterRegistry registry, IndicesStatsCache cache) { - final int TOTAL_METRICS = 48; + final int TOTAL_METRICS = 52; List metrics = new ArrayList<>(TOTAL_METRICS); for (IndexMode indexMode : IndexMode.values()) { String name = indexMode.getName(); @@ -156,6 +156,14 @@ private static List registerAsyncMetrics(MeterRegistry registry, diffGauge(() -> cache.getOrRefresh().get(indexMode).indexing.getIndexFailedCount()) ) ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".indexing.failure.version_conflict.total", + "current indexing failures due to version conflict of " + name + " indices", + "unit", + diffGauge(() -> cache.getOrRefresh().get(indexMode).indexing.getIndexFailedDueToVersionConflictCount()) + ) + ); } assert metrics.size() == TOTAL_METRICS : "total number of metrics has changed"; return metrics; diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java index 94395193622e0..2fabd03970f05 100644 --- a/server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java +++ b/server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java @@ -365,6 +365,22 @@ private void registerAsyncMetrics(MeterRegistry registry) { ) ); + metrics.add( + registry.registerLongAsyncCounter( + "es.indexing.indexing.failed.version_conflict.total", + "Total number of failed indexing operations due to version conflict", + "operations", + () -> new LongWithAttributes( + Optional.ofNullable(stats.getOrRefresh()) + .map(o -> o.getIndices()) + .map(o -> o.getIndexing()) + .map(o -> o.getTotal()) + .map(o -> o.getIndexFailedDueToVersionConflictCount()) + .orElse(0L) + ) + ) + ); + metrics.add( registry.registerLongAsyncCounter( "es.indexing.deletion.docs.total", diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index a968ea4520f40..703f1f2c18408 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -303,6 +303,15 @@ protected Table getTableWithHeader(final RestRequest request) { "sibling:pri;alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops" ); table.addCell("pri.indexing.index_failed", "default:false;text-align:right;desc:number of failed indexing ops"); + table.addCell( + "indexing.index_failed_due_to_version_conflict", + "sibling:pri;alias:iifvc,indexingIndexFailedDueToVersionConflict;default:false;text-align:right;" + + "desc:number of failed indexing ops due to version conflict" + ); + table.addCell( + "pri.indexing.index_failed_due_to_version_conflict", + "default:false;text-align:right;desc:number of failed indexing ops due to version conflict" + ); table.addCell("merges.current", "sibling:pri;alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges"); table.addCell("pri.merges.current", "default:false;text-align:right;desc:number of current merges"); @@ -670,6 +679,13 @@ Table buildTable( table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell( + totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedDueToVersionConflictCount() + ); + table.addCell( + primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedDueToVersionConflictCount() + ); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index 39e679f2c0ad0..5560bd6dc49c8 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -230,6 +230,11 @@ protected Table getTableWithHeader(final RestRequest request) { "indexing.index_failed", "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops" ); + table.addCell( + "indexing.index_failed_due_to_version_conflict", + "alias:iifvc,indexingIndexFailedDueToVersionConflict;default:false;text-align:right;" + + "desc:number of failed indexing ops due to version conflict" + ); table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges"); table.addCell( @@ -466,6 +471,7 @@ Table buildTable( table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexTime()); table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCount()); table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexFailedCount()); + table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexFailedDueToVersionConflictCount()); MergeStats mergeStats = indicesStats == null ? null : indicesStats.getMerge(); table.addCell(mergeStats == null ? null : mergeStats.getCurrent()); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java index 8b5e21de2d741..717e6e2d8f35b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java @@ -169,6 +169,11 @@ protected Table getTableWithHeader(final RestRequest request) { "indexing.index_failed", "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops" ); + table.addCell( + "indexing.index_failed_due_to_version_conflict", + "alias:iifvc,indexingIndexFailedDueToVersionConflict;default:false;text-align:right;" + + "desc:number of failed indexing ops due to version conflict" + ); table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges"); table.addCell( @@ -373,6 +378,7 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexTime())); table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexCount())); table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedCount())); + table.addCell(getOrNull(commonStats, CommonStats::getIndexing, i -> i.getTotal().getIndexFailedDueToVersionConflictCount())); table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrent)); table.addCell(getOrNull(commonStats, CommonStats::getMerge, MergeStats::getCurrentNumDocs)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index dccdbee23c775..0e80b70c4a0fd 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -587,6 +587,7 @@ private static CommonStats createShardLevelCommonStats() { ++iota, ++iota, ++iota, + ++iota, false, ++iota, ++iota, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java index 2885673ba5539..1709e2d4d372f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java @@ -113,7 +113,21 @@ private ShardStats createShardStats( commonStats.getIndexing() .getTotal() .add( - new IndexingStats.Stats(0, 0, 0, 0, 0, 0, 0, 0, false, 0, totalIndexingTimeSinceShardStartedInNanos, totalActiveTimeInNanos) + new IndexingStats.Stats( + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + false, + 0, + totalIndexingTimeSinceShardStartedInNanos, + totalActiveTimeInNanos + ) ); return new ShardStats(shardRouting, commonStats, null, null, null, null, null, false, false, 0); } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index eacb4cf35a422..7d436ab5d8d22 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -84,6 +84,7 @@ import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.index.engine.ReadOnlyEngine; import org.elasticsearch.index.engine.Segment; +import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.FieldDataStats; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -1729,6 +1730,86 @@ public String[] listAll() throws IOException { } } + public void testIndexingErrors() throws IOException { + AtomicBoolean throwOnIndex = new AtomicBoolean(); + IndexShard shard = newStartedShard(randomBoolean(), Settings.EMPTY, config -> new InternalEngine(config) { + @Override + public IndexResult index(Index index) throws IOException { + if (throwOnIndex.get()) { + throw new IOException("test indexing errors"); + } else { + return super.index(index); + } + } + }); + long nbIndexedDocs = randomIntBetween(1, 10); + AtomicLong nbFailed = new AtomicLong(); + for (int id = 0; id < nbIndexedDocs; id++) { + throwOnIndex.set(randomBoolean()); + if (throwOnIndex.get()) { + nbFailed.incrementAndGet(); + int finalId = id; + expectThrows(IOException.class, () -> indexDoc(shard, "_doc", "test" + finalId)); + } else { + Engine.IndexResult indexResult = indexDoc(shard, "_doc", "test" + id); + assertThat(indexResult.isCreated(), is(true)); + } + } + assertThat(shard.indexingStats().getTotal().getIndexFailedCount(), equalTo(nbFailed.get())); + assertThat(shard.indexingStats().getTotal().getIndexFailedDueToVersionConflictCount(), equalTo(0L)); + assertThat(shard.indexingStats().getTotal().getIndexCount(), equalTo(nbIndexedDocs - nbFailed.get())); + closeShards(shard); + } + + public void testIndexingErrorsDueToVersionConflict() throws IOException { + AtomicBoolean throwOnIndex = new AtomicBoolean(); + IndexShard shard = newStartedShard(true, Settings.EMPTY, config -> new InternalEngine(config) { + @Override + public IndexResult index(Index index) throws IOException { + if (throwOnIndex.get()) { + throw new IOException("test indexing errors"); + } else { + return super.index(index); + } + } + }); + long nbIndexedDocs = randomIntBetween(1, 10); + AtomicLong indexingFailedCount = new AtomicLong(); + AtomicLong indexingFailedWithVersionConflictCount = new AtomicLong(); + AtomicLong indexingSuccessCount = new AtomicLong(); + for (int id = 0; id < nbIndexedDocs; id++) { + if (randomBoolean()) { + // version conflict + throwOnIndex.set(false); + indexingFailedWithVersionConflictCount.incrementAndGet(); + indexingFailedCount.incrementAndGet(); + Engine.IndexResult indexResult = indexDoc(shard, "test" + id, 10L, "{}", XContentType.JSON, null); + assertThat(indexResult.isCreated(), is(false)); + assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); + } else { + throwOnIndex.set(randomBoolean()); + int finalId = id; + if (throwOnIndex.get()) { + // indexing failure + indexingFailedCount.incrementAndGet(); + expectThrows(IOException.class, () -> indexDoc(shard, "_doc", "test" + finalId)); + } else { + // indexing successful + indexingSuccessCount.incrementAndGet(); + Engine.IndexResult indexResult = indexDoc(shard, "_doc", "test" + id); + assertThat(indexResult.isCreated(), is(true)); + } + } + } + assertThat(shard.indexingStats().getTotal().getIndexCount(), equalTo(indexingSuccessCount.get())); + assertThat(shard.indexingStats().getTotal().getIndexFailedCount(), equalTo(indexingFailedCount.get())); + assertThat( + shard.indexingStats().getTotal().getIndexFailedDueToVersionConflictCount(), + equalTo(indexingFailedWithVersionConflictCount.get()) + ); + closeShards(shard); + } + public void testRefreshMetric() throws IOException { IndexShard shard = newStartedShard(); // refresh on: finalize and end of recovery diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java index 29857ef4a519f..b1da067e2f7e6 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.cluster.ClusterState; @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.Table; +import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -40,11 +41,91 @@ public class RestShardsActionTests extends ESTestCase { + private DiscoveryNode localNode; + private List shardRoutings; + private ClusterStateResponse clusterStateResponse; + private IndicesStatsResponse indicesStatsResponse; + public void testBuildTable() { - final int numShards = randomIntBetween(1, 5); - DiscoveryNode localNode = DiscoveryNodeUtils.create("local"); + mockShardStats(randomBoolean()); + + final RestShardsAction action = new RestShardsAction(); + final Table table = action.buildTable(new FakeRestRequest(), clusterStateResponse, indicesStatsResponse); + + // now, verify the table is correct + List headers = table.getHeaders(); + assertThat(headers.get(0).value, equalTo("index")); + assertThat(headers.get(1).value, equalTo("shard")); + assertThat(headers.get(2).value, equalTo("prirep")); + assertThat(headers.get(3).value, equalTo("state")); + assertThat(headers.get(4).value, equalTo("docs")); + assertThat(headers.get(5).value, equalTo("store")); + assertThat(headers.get(6).value, equalTo("dataset")); + assertThat(headers.get(7).value, equalTo("ip")); + assertThat(headers.get(8).value, equalTo("id")); + assertThat(headers.get(9).value, equalTo("node")); + assertThat(headers.get(10).value, equalTo("unassigned.reason")); + + final List> rows = table.getRows(); + assertThat(rows.size(), equalTo(shardRoutings.size())); + + Iterator shardRoutingsIt = shardRoutings.iterator(); + for (final List row : rows) { + ShardRouting shardRouting = shardRoutingsIt.next(); + ShardStats shardStats = indicesStatsResponse.asMap().get(shardRouting); + assertThat(row.get(0).value, equalTo(shardRouting.getIndexName())); + assertThat(row.get(1).value, equalTo(shardRouting.getId())); + assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r")); + assertThat(row.get(3).value, equalTo(shardRouting.state())); + assertThat(row.get(7).value, equalTo(localNode.getHostAddress())); + assertThat(row.get(8).value, equalTo(localNode.getId())); + assertThat(row.get(70).value, equalTo(shardStats.getDataPath())); + assertThat(row.get(71).value, equalTo(shardStats.getStatePath())); + } + } + + public void testShardStatsForIndexing() { + mockShardStats(true); + + final RestShardsAction action = new RestShardsAction(); + final Table table = action.buildTable(new FakeRestRequest(), clusterStateResponse, indicesStatsResponse); + + // now, verify the table is correct + List headers = table.getHeaders(); + assertThat(headers.get(29).value, equalTo("indexing.delete_current")); + assertThat(headers.get(30).value, equalTo("indexing.delete_time")); + assertThat(headers.get(31).value, equalTo("indexing.delete_total")); + assertThat(headers.get(32).value, equalTo("indexing.index_current")); + assertThat(headers.get(33).value, equalTo("indexing.index_time")); + assertThat(headers.get(34).value, equalTo("indexing.index_total")); + assertThat(headers.get(35).value, equalTo("indexing.index_failed")); + assertThat(headers.get(36).value, equalTo("indexing.index_failed_due_to_version_conflict")); + + final List> rows = table.getRows(); + assertThat(rows.size(), equalTo(shardRoutings.size())); - List shardRoutings = new ArrayList<>(numShards); + Iterator shardRoutingsIt = shardRoutings.iterator(); + for (final List row : rows) { + ShardRouting shardRouting = shardRoutingsIt.next(); + ShardStats shardStats = indicesStatsResponse.asMap().get(shardRouting); + assertThat(row.get(29).value, equalTo(shardStats.getStats().getIndexing().getTotal().getDeleteCurrent())); + assertThat(row.get(30).value, equalTo(shardStats.getStats().getIndexing().getTotal().getDeleteTime())); + assertThat(row.get(31).value, equalTo(shardStats.getStats().getIndexing().getTotal().getDeleteCount())); + assertThat(row.get(32).value, equalTo(shardStats.getStats().getIndexing().getTotal().getIndexCurrent())); + assertThat(row.get(33).value, equalTo(shardStats.getStats().getIndexing().getTotal().getIndexTime())); + assertThat(row.get(34).value, equalTo(shardStats.getStats().getIndexing().getTotal().getIndexCount())); + assertThat(row.get(35).value, equalTo(shardStats.getStats().getIndexing().getTotal().getIndexFailedCount())); + assertThat( + row.get(36).value, + equalTo(shardStats.getStats().getIndexing().getTotal().getIndexFailedDueToVersionConflictCount()) + ); + } + } + + private void mockShardStats(boolean includeCommonStats) { + final int numShards = randomIntBetween(1, 5); + this.localNode = DiscoveryNodeUtils.create("local"); + this.shardRoutings = new ArrayList<>(numShards); Map shardStatsMap = new HashMap<>(); String index = "index"; for (int i = 0; i < numShards; i++) { @@ -53,10 +134,33 @@ public void testBuildTable() { Path path = createTempDir().resolve("indices") .resolve(shardRouting.shardId().getIndex().getUUID()) .resolve(String.valueOf(shardRouting.shardId().id())); + CommonStats commonStats = null; + if (includeCommonStats) { + commonStats = new CommonStats(randomFrom(CommonStatsFlags.ALL, new CommonStatsFlags(CommonStatsFlags.Flag.Indexing))); + commonStats.indexing.add( + new IndexingStats( + new IndexingStats.Stats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomBoolean(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong() + ) + ) + ); + } ShardStats shardStats = new ShardStats( shardRouting, new ShardPath(false, path, path, shardRouting.shardId()), - null, + commonStats, null, null, null, @@ -67,56 +171,18 @@ public void testBuildTable() { shardRoutings.add(shardRouting); } - IndexStats indexStats = mock(IndexStats.class); - when(indexStats.getPrimaries()).thenReturn(new CommonStats()); - when(indexStats.getTotal()).thenReturn(new CommonStats()); - - IndicesStatsResponse stats = mock(IndicesStatsResponse.class); - when(stats.asMap()).thenReturn(shardStatsMap); + this.indicesStatsResponse = mock(IndicesStatsResponse.class); + when(this.indicesStatsResponse.asMap()).thenReturn(shardStatsMap); DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); when(discoveryNodes.get(localNode.getId())).thenReturn(localNode); - ClusterStateResponse state = mock(ClusterStateResponse.class); + this.clusterStateResponse = mock(ClusterStateResponse.class); RoutingTable routingTable = mock(RoutingTable.class); when(routingTable.allShardsIterator()).thenReturn(shardRoutings); ClusterState clusterState = mock(ClusterState.class); when(clusterState.routingTable()).thenReturn(routingTable); when(clusterState.nodes()).thenReturn(discoveryNodes); - when(state.getState()).thenReturn(clusterState); - - final RestShardsAction action = new RestShardsAction(); - final Table table = action.buildTable(new FakeRestRequest(), state, stats); - - // now, verify the table is correct - List headers = table.getHeaders(); - assertThat(headers.get(0).value, equalTo("index")); - assertThat(headers.get(1).value, equalTo("shard")); - assertThat(headers.get(2).value, equalTo("prirep")); - assertThat(headers.get(3).value, equalTo("state")); - assertThat(headers.get(4).value, equalTo("docs")); - assertThat(headers.get(5).value, equalTo("store")); - assertThat(headers.get(6).value, equalTo("dataset")); - assertThat(headers.get(7).value, equalTo("ip")); - assertThat(headers.get(8).value, equalTo("id")); - assertThat(headers.get(9).value, equalTo("node")); - assertThat(headers.get(10).value, equalTo("unassigned.reason")); - - final List> rows = table.getRows(); - assertThat(rows.size(), equalTo(numShards)); - - Iterator shardRoutingsIt = shardRoutings.iterator(); - for (final List row : rows) { - ShardRouting shardRouting = shardRoutingsIt.next(); - ShardStats shardStats = shardStatsMap.get(shardRouting); - assertThat(row.get(0).value, equalTo(shardRouting.getIndexName())); - assertThat(row.get(1).value, equalTo(shardRouting.getId())); - assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r")); - assertThat(row.get(3).value, equalTo(shardRouting.state())); - assertThat(row.get(7).value, equalTo(localNode.getHostAddress())); - assertThat(row.get(8).value, equalTo(localNode.getId())); - assertThat(row.get(69).value, equalTo(shardStats.getDataPath())); - assertThat(row.get(70).value, equalTo(shardStats.getStatePath())); - } + when(clusterStateResponse.getState()).thenReturn(clusterState); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 5b9a9e7a4efee..2ae4bb0343101 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -993,12 +993,23 @@ protected Engine.IndexResult indexDoc(IndexShard shard, String type, String id) } protected Engine.IndexResult indexDoc(IndexShard shard, String type, String id, String source) throws IOException { - return indexDoc(shard, id, source, XContentType.JSON, null); + return indexDoc(shard, id, Versions.MATCH_ANY, source, XContentType.JSON, null); } - // Uses an auto-generated ID if `id` is null/empty protected Engine.IndexResult indexDoc(IndexShard shard, String id, String source, XContentType xContentType, String routing) throws IOException { + return indexDoc(shard, id, Versions.MATCH_ANY, source, xContentType, routing); + } + + // Uses an auto-generated ID if `id` is null/empty + protected Engine.IndexResult indexDoc( + IndexShard shard, + String id, + long version, + String source, + XContentType xContentType, + String routing + ) throws IOException { long autoGeneratedTimestamp = IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP; if (Strings.isEmpty(id)) { id = UUIDs.base64UUID(); @@ -1008,7 +1019,7 @@ protected Engine.IndexResult indexDoc(IndexShard shard, String id, String source Engine.IndexResult result; if (shard.routingEntry().primary()) { result = shard.applyIndexOperationOnPrimary( - Versions.MATCH_ANY, + version, VersionType.INTERNAL, sourceToParse, SequenceNumbers.UNASSIGNED_SEQ_NO, @@ -1024,7 +1035,7 @@ protected Engine.IndexResult indexDoc(IndexShard shard, String id, String source .build() ); result = shard.applyIndexOperationOnPrimary( - Versions.MATCH_ANY, + version, VersionType.INTERNAL, sourceToParse, SequenceNumbers.UNASSIGNED_SEQ_NO, diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java index b6c059b7a0dcc..4811d65e6ed85 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java @@ -389,7 +389,21 @@ private static CommonStats mockCommonStats() { commonStats.getStore().add(new StoreStats(++iota, no, no)); commonStats.getRefresh().add(new RefreshStats(no, ++iota, no, ++iota, (int) no)); - final IndexingStats.Stats indexingStats = new IndexingStats.Stats(++iota, ++iota, no, no, no, no, no, no, false, ++iota, no, no); + final IndexingStats.Stats indexingStats = new IndexingStats.Stats( + ++iota, + ++iota, + no, + no, + no, + no, + no, + no, + no, + false, + ++iota, + no, + no + ); commonStats.getIndexing().add(new IndexingStats(indexingStats)); final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no, no, no); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java index 6822f54633bdc..ca7651ce84497 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java @@ -183,7 +183,7 @@ private CommonStats mockCommonStats() { commonStats.getDocs().add(new DocsStats(1L, 0L, randomNonNegativeLong() >> 8)); // >> 8 to avoid overflow - we add these things up commonStats.getStore().add(new StoreStats(2L, 0L, 0L)); - final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, 0L, 0L, 0L, 0L, 0L, 0L, true, 5L, 0, 0); + final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, true, 5L, 0, 0); commonStats.getIndexing().add(new IndexingStats(indexingStats)); final SearchStats.Stats searchStats = new SearchStats.Stats(6L, 7L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java index 3d7f843358646..be87b92479c21 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java @@ -344,6 +344,7 @@ private static NodeStats mockNodeStats() { no, no, no, + no, false, ++iota, no, From 80547f899866706ceb8b8d2832021cf29e6e28c9 Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Wed, 8 Jan 2025 08:49:58 -0500 Subject: [PATCH 16/52] [Inference API] Fix unique ID message for inference ID matches trained model ID (#119543) * fix unique ID message for inference ID matches trained model ID * Update docs/changelog/119543.yaml * update test to match fix --- docs/changelog/119543.yaml | 7 +++++++ .../elasticsearch/xpack/core/ml/job/messages/Messages.java | 2 ++ .../inference/action/TransportPutInferenceModelAction.java | 2 +- .../xpack/ml/integration/ModelIdUniquenessIT.java | 5 +++-- 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/119543.yaml diff --git a/docs/changelog/119543.yaml b/docs/changelog/119543.yaml new file mode 100644 index 0000000000000..7027ea2a49672 --- /dev/null +++ b/docs/changelog/119543.yaml @@ -0,0 +1,7 @@ +pr: 119543 +summary: "[Inference API] Fix unique ID message for inference ID matches trained model\ + \ ID" +area: Machine Learning +type: bug +issues: + - 111312 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index 9f9def6a0678d..f01ac08f922c4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -281,6 +281,8 @@ public final class Messages { public static final String FIELD_CANNOT_BE_NULL = "Field [{0}] cannot be null"; public static final String MODEL_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT = "Model IDs must be unique. Requested model ID [{}] matches existing model IDs but must not."; + public static final String INFERENCE_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT = + "Inference endpoint IDs must be unique. Requested inference endpoint ID [{}] matches existing trained model ID(s) but must not."; public static final String MODEL_ID_DOES_NOT_MATCH_EXISTING_MODEL_IDS_BUT_MUST_FOR_IN_CLUSTER_SERVICE = "Requested model ID [{}] does not have a matching trained model and thus cannot be updated."; public static final String INFERENCE_ENTITY_NON_EXISTANT_NO_UPDATE = "The inference endpoint [{}] does not exist and cannot be updated"; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java index 2baee7f8afd66..7c2a139672e8f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java @@ -153,7 +153,7 @@ protected void masterOperation( if ((assignments == null || assignments.isEmpty()) == false) { listener.onFailure( ExceptionsHelper.badRequestException( - Messages.MODEL_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT, + Messages.INFERENCE_ID_MATCHES_EXISTING_MODEL_IDS_BUT_MUST_NOT, request.getInferenceEntityId() ) ); diff --git a/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelIdUniquenessIT.java b/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelIdUniquenessIT.java index 9904cfb752de5..f6f5a1299c105 100644 --- a/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelIdUniquenessIT.java +++ b/x-pack/plugin/ml/qa/ml-inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ModelIdUniquenessIT.java @@ -29,9 +29,10 @@ public void testPutInferenceModelFailsWhenTrainedModelWithIdAlreadyExists() thro assertThat( e.getMessage(), Matchers.containsString( - "Model IDs must be unique. Requested model ID [" + modelId + "] matches existing model IDs but must not." + "Inference endpoint IDs must be unique. Requested inference endpoint ID [" + + modelId + + "] matches existing trained model ID(s) but must not." ) - ); } From 28ce53f0aab69d733ebd1092ea915e3b87ee98f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Wed, 8 Jan 2025 15:17:41 +0100 Subject: [PATCH 17/52] [Entitlements] Fix "No SecurityManager when entitlements are enabled" (#119742) --- .../java/org/elasticsearch/server/cli/SystemJvmOptions.java | 2 +- .../org/elasticsearch/example/ExampleSecurityExtension.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 928a8ba43cae1..8b3977fe66428 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -141,7 +141,7 @@ private static Stream maybeWorkaroundG1Bug() { @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) private static Stream maybeAllowSecurityManager(boolean useEntitlements) { - if (useEntitlements == false && RuntimeVersionFeature.isSecurityManagerAvailable()) { + if (RuntimeVersionFeature.isSecurityManagerAvailable()) { // Will become conditional on useEntitlements once entitlements can run without SM return Stream.of("-Djava.security.manager=allow"); } diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java index 8400e7df54f4d..5d8684bf32f89 100644 --- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java +++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java @@ -36,7 +36,8 @@ public class ExampleSecurityExtension implements SecurityExtension { static { - if (RuntimeVersionFeature.isSecurityManagerAvailable()) { + final boolean useEntitlements = Boolean.parseBoolean(System.getProperty("es.entitlements.enabled")); + if (useEntitlements == false && RuntimeVersionFeature.isSecurityManagerAvailable()) { // check that the extension's policy works. AccessController.doPrivileged((PrivilegedAction) () -> { System.getSecurityManager().checkPropertyAccess("myproperty"); From f0996583e9daf91bd5a2f03d355fce2ae5fc6698 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 8 Jan 2025 09:25:22 -0500 Subject: [PATCH 18/52] ESQL: Speed type error testing (#119678) This shaves a few minutes off of the ESQL build: ``` 14m 50s -> 12m 38s ``` It does so by moving the type error testing from parameterized tests to a single, stand alone test per scalar that checks the errors for all unsupported types. It gets the list from the parameterized tests the same way as we were doing. But it's *fast*. AND, this will let us test a huge number of combinations without nearly as much overhead as we had before. In the worse case, unary math functions, this doesn't save any time. Maybe .1 second per function. For binary math functions it saves a *little* time. About a second per function. But for non-math, multivalued functions: wow. IpPrefix is ternary and it's test goes from 56.8 seconds to 2.6 seconds! Here are a few examples. | name | before | after | before| after | | -----------------: | -----: | -----------: | ----: | ----: | | Sin | 2.6s | 2.5s | 400 | 291 | | ATan2 | 17.4s | 16.1s | 8270 | 5961 | | IpPrefix | 56.8s | :tada: 2.6s | 40650 | 191 | | Equals | 69.9s | 50.6s | 30130 | 28131 | | NotEquals | 67.1s | 46.8s | 30100 | 28101 | | GreaterThan | 63.7s | 57.8s | 29940 | 27791 | | GreaterThanOrEqual | 61.1s | 61.6s | 29940 | 27791 | | LessThan | 63.7s | 61.3s | 29940 | 27791 | | LessThanOrEqual | 61.1s | 59.8s | 29940 | 27791 | | Case | 115.3s | :tada: 45.1s | 63756 | 13236 | | DateDiff | 3.4s | 4.0s?| 507 | 271 | | DateExtract | 12.1s | 3.4s | 3406 | 156 | | DateFormat | 8.1s | 2.4s | 2849 | 100 | | DateParse | 10.6s | 2.8s | 2992 | 276 | | DateTrunc | 10.9s | 3.4s | 3320 | 790 | | ByteLength | 5.7s | 4.0s | 520 | 391 | | EndsWith | 13.7s | 7.2s | 3880 | 1411 | | Hash | 30.7s | 17.4s | 3980 | 1511 | | LTrim | 27.1s | 29.0s?| 2840 | 2711| | Locate | 85.3s | :tada: 10.3s | 44310 | 1461 | | Replace | 96.5s | :tada: 10.1s | 42010 | 1711 | | RTrim | 15.6s | 20.0s?| 2840 | 2711 | | Split | 6.6s | 4.0s | 3360 | 397 | | StartsWith | 5.5s | :tada: 0.7s | 2800 | 330 | | Substring | 115.2s | :tada: 2.7s | 85386 | 483 | | Trim | 17.4s | 17.8s | 2840 | 2710 | Gradle Enterprise is also not happy with the raw *number* of tests ESQL runs. So lowering the overall number is important. See the table above. This strategy is *super* effective for that. It takes us ``` 769459 -> 470429 ``` --- .../function/AbstractFunctionTestCase.java | 29 +-- .../AbstractScalarFunctionTestCase.java | 54 +++--- ...ErrorsForCasesWithoutExamplesTestCase.java | 167 ++++++++++++++++++ .../grouping/CategorizeErrorTests.java | 37 ++++ .../function/grouping/CategorizeTests.java | 2 +- .../scalar/conditional/CaseErrorTests.java | 67 +++++++ .../scalar/conditional/CaseTests.java | 33 ---- .../scalar/date/DateDiffErrorTests.java | 42 +++++ .../function/scalar/date/DateDiffTests.java | 42 ----- .../scalar/date/DateExtractErrorTests.java | 42 +++++ .../scalar/date/DateExtractTests.java | 9 +- .../scalar/date/DateFormatErrorTests.java | 42 +++++ .../function/scalar/date/DateFormatTests.java | 9 +- .../scalar/date/DateParseErrorTests.java | 37 ++++ .../function/scalar/date/DateParseTests.java | 5 +- .../scalar/date/DateTruncErrorTests.java | 41 +++++ .../function/scalar/date/DateTruncTests.java | 6 +- .../scalar/ip/CIDRMatchErrorTests.java | 41 +++++ .../function/scalar/ip/CIDRMatchTests.java | 6 +- .../scalar/ip/IpPrefixErrorTests.java | 41 +++++ .../function/scalar/ip/IpPrefixTests.java | 6 +- .../function/scalar/math/Atan2ErrorTests.java | 37 ++++ .../function/scalar/math/Atan2Tests.java | 2 +- .../function/scalar/math/SinErrorTests.java | 37 ++++ .../function/scalar/math/SinTests.java | 2 +- .../scalar/string/AbstractTrimTests.java | 2 +- .../scalar/string/BitLengthErrorTests.java | 37 ++++ .../scalar/string/BitLengthTests.java | 2 +- .../scalar/string/ByteLengthErrorTests.java | 37 ++++ .../scalar/string/ByteLengthTests.java | 2 +- .../scalar/string/EndsWithErrorTests.java | 37 ++++ .../function/scalar/string/EndsWithTests.java | 2 +- .../scalar/string/HashErrorTests.java | 37 ++++ .../function/scalar/string/HashTests.java | 2 +- .../scalar/string/LTrimErrorTests.java | 37 ++++ .../scalar/string/LeftErrorTests.java | 41 +++++ .../function/scalar/string/LeftTests.java | 6 +- .../scalar/string/LengthErrorTests.java | 37 ++++ .../function/scalar/string/LengthTests.java | 7 +- .../scalar/string/LocateErrorTests.java | 42 +++++ .../function/scalar/string/LocateTests.java | 7 +- .../scalar/string/RTrimErrorTests.java | 37 ++++ .../scalar/string/ReplaceErrorTests.java | 37 ++++ .../function/scalar/string/ReplaceTests.java | 2 +- .../scalar/string/RightErrorTests.java | 41 +++++ .../function/scalar/string/RightTests.java | 6 +- .../scalar/string/SpaceErrorTests.java | 37 ++++ .../function/scalar/string/SpaceTests.java | 2 +- .../scalar/string/SplitErrorTests.java | 37 ++++ .../function/scalar/string/SplitTests.java | 2 +- .../scalar/string/StartsWithErrorTests.java | 37 ++++ .../scalar/string/StartsWithTests.java | 2 +- .../scalar/string/SubstringErrorTests.java | 41 +++++ .../scalar/string/SubstringTests.java | 9 +- .../scalar/string/ToLowerErrorTests.java | 38 ++++ .../function/scalar/string/ToLowerTests.java | 4 +- .../scalar/string/ToUpperErrorTests.java | 38 ++++ .../function/scalar/string/ToUpperTests.java | 4 +- .../scalar/string/TrimErrorTests.java | 37 ++++ .../operator/arithmetic/NegErrorTests.java | 37 ++++ .../operator/arithmetic/NegTests.java | 2 +- .../operator/comparison/EqualsErrorTests.java | 41 +++++ .../operator/comparison/EqualsTests.java | 11 +- .../comparison/GreaterThanErrorTests.java | 43 +++++ .../GreaterThanOrEqualErrorTests.java | 43 +++++ .../comparison/GreaterThanOrEqualTests.java | 12 +- .../operator/comparison/GreaterThanTests.java | 12 +- .../comparison/LessThanErrorTests.java | 43 +++++ .../comparison/LessThanOrEqualErrorTests.java | 43 +++++ .../comparison/LessThanOrEqualTests.java | 12 +- .../operator/comparison/LessThanTests.java | 12 +- .../comparison/NotEqualsErrorTests.java | 41 +++++ .../operator/comparison/NotEqualsTests.java | 11 +- 73 files changed, 1673 insertions(+), 272 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index c609eb3a7ad41..03b9dba298951 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -191,7 +191,6 @@ protected static List anyNullIsNull( ExpectedType expectedType, ExpectedEvaluatorToString evaluatorToString ) { - typesRequired(testCaseSuppliers); List suppliers = new ArrayList<>(testCaseSuppliers.size()); suppliers.addAll(testCaseSuppliers); @@ -274,7 +273,7 @@ protected static List anyNullIsNull( } @FunctionalInterface - protected interface PositionalErrorMessageSupplier { + public interface PositionalErrorMessageSupplier { /** * This interface defines functions to supply error messages for incorrect types in specific positions. Functions which have * the same type requirements for all positions can simplify this with a lambda returning a string constant. @@ -291,7 +290,9 @@ protected interface PositionalErrorMessageSupplier { /** * Adds test cases containing unsupported parameter types that assert * that they throw type errors. + * @deprecated make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase} instead */ + @Deprecated protected static List errorsForCasesWithoutExamples( List testCaseSuppliers, PositionalErrorMessageSupplier positionalErrorMessageSupplier @@ -331,11 +332,14 @@ protected interface TypeErrorMessageSupplier { String apply(boolean includeOrdinal, List> validPerPosition, List types); } + /** + * @deprecated make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase} instead + */ + @Deprecated protected static List errorsForCasesWithoutExamples( List testCaseSuppliers, TypeErrorMessageSupplier typeErrorMessageSupplier ) { - typesRequired(testCaseSuppliers); List suppliers = new ArrayList<>(testCaseSuppliers.size()); suppliers.addAll(testCaseSuppliers); @@ -346,7 +350,7 @@ protected static List errorsForCasesWithoutExamples( .map(s -> s.types().size()) .collect(Collectors.toSet()) .stream() - .flatMap(count -> allPermutations(count)) + .flatMap(AbstractFunctionTestCase::allPermutations) .filter(types -> valid.contains(types) == false) /* * Skip any cases with more than one null. Our tests don't generate @@ -366,10 +370,6 @@ private static List append(List orig, DataType extra) { return longer; } - protected static Stream representable() { - return DataType.types().stream().filter(DataType::isRepresentable); - } - protected static TestCaseSupplier typeErrorSupplier( boolean includeOrdinal, List> validPerPosition, @@ -398,7 +398,7 @@ protected static TestCaseSupplier typeErrorSupplier( ); } - private static List> validPerPosition(Set> valid) { + static List> validPerPosition(Set> valid) { int max = valid.stream().mapToInt(List::size).max().getAsInt(); List> result = new ArrayList<>(max); for (int i = 0; i < max; i++) { @@ -1327,17 +1327,6 @@ public void allMemoryReleased() { } } - /** - * Validate that we know the types for all the test cases already created - * @param suppliers - list of suppliers before adding in the illegal type combinations - */ - protected static void typesRequired(List suppliers) { - String bad = suppliers.stream().filter(s -> s.types() == null).map(s -> s.name()).collect(Collectors.joining("\n")); - if (bad.equals("") == false) { - throw new IllegalArgumentException("types required but not found for these tests:\n" + bad); - } - } - /** * Returns true if the current test case is for an aggregation function. *

diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java index 958b706a3f260..65b9c447170f4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java @@ -58,7 +58,11 @@ public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTes *

* * @param entirelyNullPreservesType See {@link #anyNullIsNull(boolean, List)} + * @deprecated use {@link #parameterSuppliersFromTypedDataWithDefaultChecksNoErrors} + * and make a subclass of {@link ErrorsForCasesWithoutExamplesTestCase}. + * It's a long faster. */ + @Deprecated protected static Iterable parameterSuppliersFromTypedDataWithDefaultChecks( boolean entirelyNullPreservesType, List suppliers, @@ -72,6 +76,23 @@ protected static Iterable parameterSuppliersFromTypedDataWithDefaultCh ); } + /** + * Converts a list of test cases into a list of parameter suppliers. + * Also, adds a default set of extra test cases. + *

+ * Use if possible, as this method may get updated with new checks in the future. + *

+ * + * @param entirelyNullPreservesType See {@link #anyNullIsNull(boolean, List)} + */ + protected static Iterable parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( + // TODO remove after removing parameterSuppliersFromTypedDataWithDefaultChecks rename this to that. + boolean entirelyNullPreservesType, + List suppliers + ) { + return parameterSuppliersFromTypedData(anyNullIsNull(entirelyNullPreservesType, randomizeBytesRefsOffset(suppliers))); + } + /** * Converts a list of test cases into a list of parameter suppliers. * Also, adds a default set of extra test cases. @@ -364,43 +385,10 @@ public void testFold() { } } - public static String errorMessageStringForBinaryOperators( - boolean includeOrdinal, - List> validPerPosition, - List types, - PositionalErrorMessageSupplier positionalErrorMessageSupplier - ) { - try { - return typeErrorMessage(includeOrdinal, validPerPosition, types, positionalErrorMessageSupplier); - } catch (IllegalStateException e) { - // This means all the positional args were okay, so the expected error is from the combination - if (types.get(0).equals(DataType.UNSIGNED_LONG)) { - return "first argument of [] is [unsigned_long] and second is [" - + types.get(1).typeName() - + "]. [unsigned_long] can only be operated on together with another [unsigned_long]"; - - } - if (types.get(1).equals(DataType.UNSIGNED_LONG)) { - return "first argument of [] is [" - + types.get(0).typeName() - + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]"; - } - return "first argument of [] is [" - + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) - + "] so second argument must also be [" - + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) - + "] but was [" - + types.get(1).typeName() - + "]"; - - } - } - /** * Adds test cases containing unsupported parameter types that immediately fail. */ protected static List failureForCasesWithoutExamples(List testCaseSuppliers) { - typesRequired(testCaseSuppliers); List suppliers = new ArrayList<>(testCaseSuppliers.size()); suppliers.addAll(testCaseSuppliers); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java new file mode 100644 index 0000000000000..e2ad9e81939d6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.hamcrest.Matcher; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; +import static org.hamcrest.Matchers.greaterThan; + +public abstract class ErrorsForCasesWithoutExamplesTestCase extends ESTestCase { + protected abstract List cases(); + + /** + * Build the expression being tested, for the given source and list of arguments. Test classes need to implement this + * to have something to test. + * + * @param source the source + * @param args arg list from the test case, should match the length expected + * @return an expression for evaluating the function being tested on the given arguments + */ + protected abstract Expression build(Source source, List args); + + protected abstract Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature); + + protected final List paramsToSuppliers(Iterable cases) { + List result = new ArrayList<>(); + for (Object[] c : cases) { + if (c.length != 1) { + throw new IllegalArgumentException("weird layout for test cases"); + } + TestCaseSupplier supplier = (TestCaseSupplier) c[0]; + result.add(supplier); + } + return result; + } + + public final void test() { + int checked = 0; + List cases = cases(); + Set> valid = cases.stream().map(TestCaseSupplier::types).collect(Collectors.toSet()); + List> validPerPosition = AbstractFunctionTestCase.validPerPosition(valid); + Iterable> missingSignatures = missingSignatures(cases, valid)::iterator; + for (List signature : missingSignatures) { + logger.debug("checking {}", signature); + List args = new ArrayList<>(signature.size()); + for (DataType type : signature) { + args.add(randomLiteral(type)); + } + Expression expression = build(Source.synthetic(sourceForSignature(signature)), args); + assertTrue("expected unresolved " + expression, expression.typeResolved().unresolved()); + assertThat(expression.typeResolved().message(), expectedTypeErrorMatcher(validPerPosition, signature)); + checked++; + } + logger.info("checked {} signatures", checked); + assertThat("didn't check any signatures", checked, greaterThan(0)); + } + + private Stream> missingSignatures(List cases, Set> valid) { + return cases.stream() + .map(s -> s.types().size()) + .collect(Collectors.toSet()) + .stream() + .flatMap(AbstractFunctionTestCase::allPermutations) + .filter(types -> valid.contains(types) == false) + /* + * Skip any cases with more than one null. Our tests don't generate + * the full combinatorial explosions of all nulls - just a single null. + * Hopefully , cases will function the same as , + * cases. + */ + .filter(types -> types.stream().filter(t -> t == DataType.NULL).count() <= 1); + } + + protected static String sourceForSignature(List signature) { + StringBuilder source = new StringBuilder(); + for (DataType type : signature) { + if (false == source.isEmpty()) { + source.append(", "); + } + source.append(type.typeName()); + } + return source.toString(); + } + + /** + * Build the expected error message for an invalid type signature. + */ + protected static String typeErrorMessage( + boolean includeOrdinal, + List> validPerPosition, + List signature, + AbstractFunctionTestCase.PositionalErrorMessageSupplier expectedTypeSupplier + ) { + int badArgPosition = -1; + for (int i = 0; i < signature.size(); i++) { + if (validPerPosition.get(i).contains(signature.get(i)) == false) { + badArgPosition = i; + break; + } + } + if (badArgPosition == -1) { + throw new IllegalStateException( + "Can't generate error message for these types, you probably need a custom error message function" + ); + } + String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : ""; + String source = sourceForSignature(signature); + String expectedTypeString = expectedTypeSupplier.apply(validPerPosition.get(badArgPosition), badArgPosition); + String name = signature.get(badArgPosition).typeName(); + return ordinal + "argument of [" + source + "] must be [" + expectedTypeString + "], found value [] type [" + name + "]"; + } + + protected static String errorMessageStringForBinaryOperators( + List> validPerPosition, + List signature, + AbstractFunctionTestCase.PositionalErrorMessageSupplier positionalErrorMessageSupplier + ) { + try { + return typeErrorMessage(true, validPerPosition, signature, positionalErrorMessageSupplier); + } catch (IllegalStateException e) { + String source = sourceForSignature(signature); + // This means all the positional args were okay, so the expected error is from the combination + if (signature.get(0).equals(DataType.UNSIGNED_LONG)) { + return "first argument of [" + + source + + "] is [unsigned_long] and second is [" + + signature.get(1).typeName() + + "]. [unsigned_long] can only be operated on together with another [unsigned_long]"; + + } + if (signature.get(1).equals(DataType.UNSIGNED_LONG)) { + return "first argument of [" + + source + + "] is [" + + signature.get(0).typeName() + + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]"; + } + return "first argument of [" + + source + + "] is [" + + (signature.get(0).isNumeric() ? "numeric" : signature.get(0).typeName()) + + "] so second argument must also be [" + + (signature.get(0).isNumeric() ? "numeric" : signature.get(0).typeName()) + + "] but was [" + + signature.get(1).typeName() + + "]"; + + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java new file mode 100644 index 0000000000000..f674f9b2c3d72 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.grouping; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class CategorizeErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(CategorizeTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Categorize(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java index 68bafd2af9960..dfdfd82d08afe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/CategorizeTests.java @@ -51,7 +51,7 @@ public static Iterable parameters() { ) ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java new file mode 100644 index 0000000000000..28e1b846660c0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseErrorTests.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class CaseErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(CaseTests.parameters()).stream() + // Take only the shorter signatures because we don't have error generation for longer cases + // TODO handle longer signatures + .filter(c -> c.types().size() < 4) + .toList(); + } + + @Override + protected Expression build(Source source, List args) { + return new Case(source, args.get(0), args.subList(1, args.size())); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + if (signature.get(0) != DataType.BOOLEAN && signature.get(0) != DataType.NULL) { + return typeErrorMessage(signature, 0, "boolean"); + } + DataType mainType = signature.get(1).noText(); + for (int i = 2; i < signature.size(); i++) { + if (i % 2 == 0 && i != signature.size() - 1) { + // condition + if (signature.get(i) != DataType.BOOLEAN && signature.get(i) != DataType.NULL) { + return typeErrorMessage(signature, i, "boolean"); + } + } else { + // value + if (signature.get(i).noText() != mainType) { + return typeErrorMessage(signature, i, mainType.typeName()); + } + } + } + throw new IllegalStateException("can't find bad arg for " + signature); + } + + private static Matcher typeErrorMessage(List signature, int badArgPosition, String expectedTypeString) { + String ordinal = TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT); + String sig = sourceForSignature(signature); + String name = signature.get(badArgPosition).typeName(); + return equalTo(ordinal + " argument of [" + sig + "] must be [" + expectedTypeString + "], found value [] type [" + name + "]"); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index 93322ab0e35a7..05923246520fc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; @@ -25,7 +24,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -90,10 +88,6 @@ public static Iterable parameters() { ) ); } - suppliers = errorsForCasesWithoutExamples( - suppliers, - (includeOrdinal, validPerPosition, types) -> typeErrorMessage(includeOrdinal, types) - ); for (DataType type : TYPES) { fourAndFiveArgs(suppliers, true, randomSingleValuedCondition(), 0, type, List.of()); @@ -852,31 +846,4 @@ protected Matcher allNullsMatcher() { } return super.allNullsMatcher(); } - - private static String typeErrorMessage(boolean includeOrdinal, List types) { - if (types.get(0) != DataType.BOOLEAN && types.get(0) != DataType.NULL) { - return typeErrorMessage(includeOrdinal, types, 0, "boolean"); - } - DataType mainType = types.get(1).noText(); - for (int i = 2; i < types.size(); i++) { - if (i % 2 == 0 && i != types.size() - 1) { - // condition - if (types.get(i) != DataType.BOOLEAN && types.get(i) != DataType.NULL) { - return typeErrorMessage(includeOrdinal, types, i, "boolean"); - } - } else { - // value - if (types.get(i).noText() != mainType) { - return typeErrorMessage(includeOrdinal, types, i, mainType.typeName()); - } - } - } - throw new IllegalStateException("can't find bad arg for " + types); - } - - private static String typeErrorMessage(boolean includeOrdinal, List types, int badArgPosition, String expectedTypeString) { - String ordinal = includeOrdinal ? TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " " : ""; - String name = types.get(badArgPosition).typeName(); - return ordinal + "argument of [] must be [" + expectedTypeString + "], found value [" + name + "] type [" + name + "]"; - } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java new file mode 100644 index 0000000000000..a3a808de277d7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateDiffErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateDiffTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateDiff(source, args.get(0), args.get(1), args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> { + if (i == 0) { + return "string"; + } + return "datetime"; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java index 4dbdfd1e56854..da069e3c37cc4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java @@ -71,48 +71,6 @@ public static Iterable parameters() { ) ) ); - suppliers.add( - new TestCaseSupplier( - "Date Diff Error Type unit", - List.of(DataType.INTEGER, DataType.DATETIME, DataType.DATETIME), - () -> TestCaseSupplier.TestCase.typeError( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.INTEGER, "unit"), - new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), - new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") - ), - "first argument of [] must be [string], found value [unit] type [integer]" - ) - ) - ); - suppliers.add( - new TestCaseSupplier( - "Date Diff Error Type startTimestamp", - List.of(DataType.TEXT, DataType.INTEGER, DataType.DATETIME), - () -> TestCaseSupplier.TestCase.typeError( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"), - new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.INTEGER, "startTimestamp"), - new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") - ), - "second argument of [] must be [datetime], found value [startTimestamp] type [integer]" - ) - ) - ); - suppliers.add( - new TestCaseSupplier( - "Date Diff Error Type endTimestamp", - List.of(DataType.TEXT, DataType.DATETIME, DataType.INTEGER), - () -> TestCaseSupplier.TestCase.typeError( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"), - new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), - new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.INTEGER, "endTimestamp") - ), - "third argument of [] must be [datetime], found value [endTimestamp] type [integer]" - ) - ) - ); suppliers.add(new TestCaseSupplier("Date Diff In Year - 1", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> { ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01Z"); ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java new file mode 100644 index 0000000000000..d5b9a06c8738e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateExtractErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateExtractTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateExtract(source, args.get(0), args.get(1), EsqlTestUtils.TEST_CFG); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "datetime"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java index db375171b7224..be978eda06758 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java @@ -39,7 +39,7 @@ public DateExtractTests(@Name("TestCase") Supplier te @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecks( + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( true, List.of( new TestCaseSupplier( @@ -84,12 +84,7 @@ public static Iterable parameters() { ) .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit") ) - ), - (v, p) -> switch (p) { - case 0 -> "string"; - case 1 -> "datetime"; - default -> ""; - } + ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java new file mode 100644 index 0000000000000..985f1144fbcf2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateFormatErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateFormatTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateFormat(source, args.get(0), args.get(1), EsqlTestUtils.TEST_CFG); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "datetime"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java index d132c2ef2b365..8dfdd1ba486c7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java @@ -31,7 +31,7 @@ public DateFormatTests(@Name("TestCase") Supplier tes @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecks( + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( true, List.of( new TestCaseSupplier( @@ -58,12 +58,7 @@ public static Iterable parameters() { equalTo(BytesRefs.toBytesRef("2023")) ) ) - ), - (v, p) -> switch (p) { - case 0 -> "string"; - case 1 -> "datetime"; - default -> ""; - } + ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java new file mode 100644 index 0000000000000..21d9b5fb00537 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateParseErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateParseTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java index 04683ecb65467..b51866c0cc8a7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java @@ -36,7 +36,7 @@ public DateParseTests(@Name("TestCase") Supplier test @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecks( + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( true, List.of( new TestCaseSupplier( @@ -143,8 +143,7 @@ public static Iterable parameters() { + "failed to parse date field [not a date] with format [yyyy-MM-dd]" ) ) - ), - (v, p) -> "string" + ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java new file mode 100644 index 0000000000000..9494888f9314f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class DateTruncErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(DateTruncTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new DateTrunc(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "dateperiod or timeduration"; + case 1 -> "date_nanos or datetime"; + default -> null; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java index 93970f64df845..50de64ff8b173 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java @@ -56,11 +56,7 @@ public static Iterable parameters() { suppliers.addAll(ofDuration(Duration.ofSeconds(30), ts, "2023-02-17T10:25:30.00Z")); suppliers.add(randomSecond()); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "dateperiod or timeduration"; - case 1 -> "date_nanos or datetime"; - default -> null; - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static List ofDatePeriod(Period period, long value, String expectedDate) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java new file mode 100644 index 0000000000000..319adbc00569e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class CIDRMatchErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(CIDRMatchTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new CIDRMatch(source, args.get(0), List.of(args.get(1))); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "ip"; + case 1 -> "string"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java index e777a0ce587e0..5047a3a4ab118 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java @@ -84,11 +84,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "ip"; - case 1 -> "string"; - default -> ""; - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java new file mode 100644 index 0000000000000..1db58c225475e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class IpPrefixErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(IpPrefixTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new IpPrefix(source, args.get(0), args.get(1), args.size() == 3 ? args.get(2) : null); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "ip"; + case 1, 2 -> "integer"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java index 5209d042b6408..08b517b54553d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java @@ -106,11 +106,7 @@ public static Iterable parameters() { }) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "ip"; - case 1, 2 -> "integer"; - default -> ""; - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java new file mode 100644 index 0000000000000..75fa802067931 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2ErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Atan2ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(Atan2Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Atan2(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, i) -> "numeric")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java index c475a75699da7..d7463be25e886 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Tests.java @@ -36,7 +36,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java new file mode 100644 index 0000000000000..4637f8a85096a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.math; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class SinErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(SinTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Sin(source, args.getFirst()); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, i) -> "numeric")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java index 990356f8df6de..fba95289e3e8e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinTests.java @@ -33,7 +33,7 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java index d069f7ffe2298..b1271b3a5b45e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractTrimTests.java @@ -67,7 +67,7 @@ static Iterable parameters(String name, boolean trimLeading, boolean t })); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers); } private static TestCaseSupplier.TestCase testCase(String name, DataType type, String data, String expected) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java new file mode 100644 index 0000000000000..23411d486901c --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class BitLengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(BitLengthTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new BitLength(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java index bce4328a08abf..6f7c81a102996 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java @@ -40,7 +40,7 @@ public static Iterable parameters() { } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java new file mode 100644 index 0000000000000..629639485f784 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ByteLengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ByteLengthTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ByteLength(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java index 866b8e0cd8da3..dec96a6fb554e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ByteLengthTests.java @@ -42,7 +42,7 @@ public static Iterable parameters() { cases.addAll(makeTestCases("3 bytes, 1 code point", () -> "☕", 3)); cases.addAll(makeTestCases("6 bytes, 2 code points", () -> "❗️", 6)); cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 100)); - return parameterSuppliersFromTypedDataWithDefaultChecks(ENTIRELY_NULL_PRESERVES_TYPE, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(ENTIRELY_NULL_PRESERVES_TYPE, cases); } private static List makeTestCases(String title, Supplier text, int expectedByteLength) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java new file mode 100644 index 0000000000000..12067fa88b86b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class EndsWithErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(EndsWithTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new EndsWith(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java index 1b2e9c41cb25c..c41b1e14257ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java @@ -73,7 +73,7 @@ public static Iterable parameters() { ); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (valid, position) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static TestCaseSupplier.TestCase testCase( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java new file mode 100644 index 0000000000000..63ef07d333eb8 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class HashErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(HashTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Hash(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java index c5cdf97eccd17..c25270474959b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java @@ -57,7 +57,7 @@ public static Iterable parameters() { .withWarning("Line -1:-1: java.security.NoSuchAlgorithmException: invalid MessageDigest not available") .withFoldingException(InvalidArgumentException.class, "invalid algorithm for []: invalid MessageDigest not available"); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } private static List createTestCases(String algorithm) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java new file mode 100644 index 0000000000000..8b3a20606eb93 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LTrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LTrimTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new LTrim(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java new file mode 100644 index 0000000000000..44b9c13e1baa6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LeftErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LeftTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Left(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "integer"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java index e4e54a9e0935f..2a194a39d34e5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java @@ -181,11 +181,7 @@ public static Iterable parameters() { ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "string"; - case 1 -> "integer"; - default -> ""; - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static String unicodeLeftSubstring(String str, int length) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java new file mode 100644 index 0000000000000..ca8f2cb208209 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LengthErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LengthTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Length(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java index ba4c8c8ce1ea4..e4d8292fc2cb7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.hamcrest.Matcher; import java.util.ArrayList; import java.util.List; @@ -49,7 +48,7 @@ public static Iterable parameters() { cases.addAll(makeTestCases("6 bytes, 2 code points", () -> "❗️", 2)); cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 100)); cases.addAll(makeTestCases("100 random code points", () -> randomUnicodeOfCodepointLength(100), 100)); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } private static List makeTestCases(String title, Supplier text, int expectedLength) { @@ -87,10 +86,6 @@ private static List makeTestCases(String title, Supplier resultsMatcher(List typedData) { - return equalTo(UnicodeUtil.codePointCount((BytesRef) typedData.get(0).data())); - } - @Override protected Expression build(Source source, List args) { return new Length(source, args.get(0)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java new file mode 100644 index 0000000000000..a9ee74a029374 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateErrorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LocateErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LocateTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Locate(source, args.get(0), args.get(1), args.size() < 3 ? null : args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> { + if (p == 0 || p == 1) { + return "string"; + } + return "integer"; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java index a10f97c45aa04..ac060b9454564 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java @@ -76,12 +76,7 @@ public static Iterable parameters() { } } - suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> { - if (p == 0 || p == 1) { - return "string"; - } - return "integer"; - }); + suppliers = anyNullIsNull(true, suppliers); // Here follows some non-randomized examples that we want to cover on every run suppliers.add(supplier("a tiger", "a t", null, 1)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java new file mode 100644 index 0000000000000..38ab08345428b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class RTrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(RTrimTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new RTrim(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java new file mode 100644 index 0000000000000..ed1cbda5c3de0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ReplaceErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ReplaceTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Replace(source, args.get(0), args.get(1), args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java index bf1325854f1a2..dfc4db228f8e6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java @@ -103,7 +103,7 @@ public static Iterable parameters() { "Unclosed character class near index 0\n[\n^".replaceAll("\n", System.lineSeparator()) ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers); } private static TestCaseSupplier fixedCase(String name, String str, String oldStr, String newStr, String result) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java new file mode 100644 index 0000000000000..3b90a26fb908f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class RightErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(RightTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Right(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1 -> "integer"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java index bf93ef42ed6ad..1a65407e538c2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java @@ -179,11 +179,7 @@ public static Iterable parameters() { equalTo(new BytesRef(unicodeRightSubstring(text, length))) ); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { - case 0 -> "string"; - case 1 -> "integer"; - default -> throw new IllegalStateException("bad parameter number"); - }); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static String unicodeRightSubstring(String str, int length) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java new file mode 100644 index 0000000000000..bee85a645743b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class SpaceErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(SpaceTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Space(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "integer")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java index 308ce2c9d932f..40c9b33609a31 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SpaceTests.java @@ -71,7 +71,7 @@ public static Iterable parameters() { .withFoldingException(IllegalArgumentException.class, "Creating strings longer than [" + max + "] bytes is not supported"); })); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "integer"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java new file mode 100644 index 0000000000000..c113f52a1824a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class SplitErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(SplitTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Split(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java index 098be8e1fda37..c66812ccdd1df 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java @@ -63,7 +63,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java new file mode 100644 index 0000000000000..9ca061997e762 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class StartsWithErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(StartsWithTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new StartsWith(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java index 60ed3b05ad642..789059fb7b6ba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java @@ -51,7 +51,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (valid, position) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java new file mode 100644 index 0000000000000..245f4e2254e50 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class SubstringErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(SubstringTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Substring(source, args.get(0), args.get(1), args.size() < 3 ? null : args.get(2)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { + case 0 -> "string"; + case 1, 2 -> "integer"; + default -> ""; + })); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java index ae8a2a1840dfb..289cc0129c4e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java @@ -36,7 +36,7 @@ public SubstringTests(@Name("TestCase") Supplier test @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecks( + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( true, List.of(new TestCaseSupplier("Substring basic test", List.of(DataType.KEYWORD, DataType.INTEGER, DataType.INTEGER), () -> { int start = between(1, 8); @@ -105,12 +105,7 @@ public static Iterable parameters() { equalTo(new BytesRef("")) ); }) - ), - (v, p) -> switch (p) { - case 0 -> "string"; - case 1, 2 -> "integer"; - default -> ""; - } + ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java new file mode 100644 index 0000000000000..5744f340627c6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToLowerErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToLowerTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToLower(source, args.get(0), EsqlTestUtils.TEST_CFG); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java index 026d190c06e3f..b355feb6130a3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java @@ -47,9 +47,7 @@ public static Iterable parameters() { suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); - - // add null as parameter - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } public void testRandomLocale() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java new file mode 100644 index 0000000000000..94c6dc6e4a29a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToUpperErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToUpperTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToUpper(source, args.get(0), EsqlTestUtils.TEST_CFG); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java index 027ac54d15875..fdae4f953a0fa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java @@ -47,9 +47,7 @@ public static Iterable parameters() { suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); - - // add null as parameter - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } public void testRandomLocale() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java new file mode 100644 index 0000000000000..dd6de5ba9f404 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class TrimErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(TrimTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Trim(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java new file mode 100644 index 0000000000000..fcb877f15e41c --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class NegErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(NegTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Neg(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "numeric, date_period or time_duration")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java index 5bfaccfbd9347..a8c7b5b5a83fd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java @@ -110,7 +110,7 @@ public static Iterable parameters() { equalTo(arg.negated()) ).withoutEvaluator(); }))); - return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "numeric, date_period or time_duration"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java new file mode 100644 index 0000000000000..cecb6a2987fd4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class EqualsErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(EqualsTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Equals(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(errorMessageStringForBinaryOperators(validPerPosition, signature, (l, p) -> TYPE_ERROR)); + } + + private static final String TYPE_ERROR = + "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long," + + " semantic_text, text, unsigned_long or version"; +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 6666eb8adab61..8e46f7d28540d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -236,18 +236,9 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> typeErrorString) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } - private static String typeErrorString = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long," - + " semantic_text, text, unsigned_long or version"; - @Override protected Expression build(Source source, List args) { return new Equals(source, args.get(0), args.get(1)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java new file mode 100644 index 0000000000000..5b56642bb6241 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanErrorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class GreaterThanErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(GreaterThanTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new GreaterThan(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + errorMessageStringForBinaryOperators( + validPerPosition, + signature, + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java new file mode 100644 index 0000000000000..904affebd3b5e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualErrorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class GreaterThanOrEqualErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(GreaterThanOrEqualTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new GreaterThanOrEqual(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + errorMessageStringForBinaryOperators( + validPerPosition, + signature, + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index 0fbd49abd885b..f00e8a19f9dac 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -158,17 +158,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( - o, - v, - t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" - ) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index ccc66df60fb3f..169f1886af47f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -172,17 +172,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( - o, - v, - t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" - ) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java new file mode 100644 index 0000000000000..94e3be44ba2f2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanErrorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LessThanErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LessThanTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new LessThan(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + errorMessageStringForBinaryOperators( + validPerPosition, + signature, + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java new file mode 100644 index 0000000000000..6074cc7c63db2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualErrorTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class LessThanOrEqualErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(LessThanOrEqualTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new LessThanOrEqual(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + errorMessageStringForBinaryOperators( + validPerPosition, + signature, + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 1e91a65e04c0e..7eccc2c8cc86b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -158,17 +158,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( - o, - v, - t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" - ) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 69dc59bac6456..a17be8aa97978 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -172,17 +172,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators( - o, - v, - t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" - ) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java new file mode 100644 index 0000000000000..ed0a477ec613c --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class NotEqualsErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(NotEqualsTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new NotEquals(source, args.get(0), args.get(1)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(errorMessageStringForBinaryOperators(validPerPosition, signature, (l, p) -> TYPE_ERROR)); + } + + private static final String TYPE_ERROR = + "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, " + + "unsigned_long or version"; +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index 7b57b97dfe28e..296b4b37cfa73 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -232,18 +232,9 @@ public static Iterable parameters() { false ) ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples( - anyNullIsNull(true, suppliers), - (o, v, t) -> AbstractScalarFunctionTestCase.errorMessageStringForBinaryOperators(o, v, t, (l, p) -> typeErrorString) - ) - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } - private static String typeErrorString = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, " - + "unsigned_long or version"; - @Override protected Expression build(Source source, List args) { return new NotEquals(source, args.get(0), args.get(1)); From 8b35fdd08f663d72d784f38798e48780d4e90383 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 9 Jan 2025 01:34:08 +1100 Subject: [PATCH 19/52] Mute org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests testSingleMatchFunctionFilterPushdownWithStringValues {default} #119720 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index d38a3a7abd730..2b10565aa0365 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -250,6 +250,9 @@ tests: - class: org.elasticsearch.upgrades.DataStreamsUpgradeIT method: testUpgradeDataStream issue: https://github.com/elastic/elasticsearch/issues/119717 +- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests + method: testSingleMatchFunctionFilterPushdownWithStringValues {default} + issue: https://github.com/elastic/elasticsearch/issues/119720 # Examples: # From 7ecf022fae44d493ce2df639f0bb3277dd804f5c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 9 Jan 2025 01:34:31 +1100 Subject: [PATCH 20/52] Mute org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests testSingleMatchFunctionPushdownWithCasting {default} #119722 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 2b10565aa0365..d742540e6d498 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -253,6 +253,9 @@ tests: - class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests method: testSingleMatchFunctionFilterPushdownWithStringValues {default} issue: https://github.com/elastic/elasticsearch/issues/119720 +- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests + method: testSingleMatchFunctionPushdownWithCasting {default} + issue: https://github.com/elastic/elasticsearch/issues/119722 # Examples: # From 59e9391a8211477dbc21888f7d0323368aba67d4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 9 Jan 2025 01:34:48 +1100 Subject: [PATCH 21/52] Mute org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests testSingleMatchOperatorFilterPushdownWithStringValues {default} #119721 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index d742540e6d498..05a25a529b7cc 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -256,6 +256,9 @@ tests: - class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests method: testSingleMatchFunctionPushdownWithCasting {default} issue: https://github.com/elastic/elasticsearch/issues/119722 +- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests + method: testSingleMatchOperatorFilterPushdownWithStringValues {default} + issue: https://github.com/elastic/elasticsearch/issues/119721 # Examples: # From 8edcdd45639d5be83b3eced086cd6b87e8a03194 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Wed, 8 Jan 2025 09:38:23 -0500 Subject: [PATCH 22/52] Fix bbq_hnsw merge file cleanup on random IO exceptions (#119691) Since we open two temporary files during quantized vector merging, its possible that the second file fails to be created. In that case, we should ensure the previously created temporary files are removed. closes https://github.com/elastic/elasticsearch/issues/119392 --- docs/changelog/119691.yaml | 6 +++ .../ES818BinaryQuantizedVectorsWriter.java | 41 +++++++++++-------- 2 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 docs/changelog/119691.yaml diff --git a/docs/changelog/119691.yaml b/docs/changelog/119691.yaml new file mode 100644 index 0000000000000..186944394908d --- /dev/null +++ b/docs/changelog/119691.yaml @@ -0,0 +1,6 @@ +pr: 119691 +summary: Fix `bbq_hnsw` merge file cleanup on random IO exceptions +area: Vector Search +type: bug +issues: + - 119392 diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java index 02dda6a4a9da1..932925ea423ba 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsWriter.java @@ -443,21 +443,27 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( float cDotC ) throws IOException { long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); - final IndexOutput tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( - binarizedVectorData.getName(), - "temp", - segmentWriteState.context - ); - final IndexOutput tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( - binarizedVectorData.getName(), - "score_temp", - segmentWriteState.context - ); IndexInput binarizedDataInput = null; IndexInput binarizedScoreDataInput = null; + IndexOutput tempQuantizedVectorData = null; + IndexOutput tempScoreQuantizedVectorData = null; boolean success = false; OptimizedScalarQuantizer quantizer = new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction()); try { + // Since we are opening two files, it's possible that one or the other fails to open + // we open them within the try to ensure they are cleaned + tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "temp", + segmentWriteState.context + ); + tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "score_temp", + segmentWriteState.context + ); + final String tempQuantizedVectorDataName = tempQuantizedVectorData.getName(); + final String tempScoreQuantizedVectorDataName = tempScoreQuantizedVectorData.getName(); FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); if (fieldInfo.getVectorSimilarityFunction() == COSINE) { floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues); @@ -516,8 +522,8 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( IOUtils.close(finalBinarizedDataInput, finalBinarizedScoreDataInput); IOUtils.deleteFilesIgnoringExceptions( segmentWriteState.directory, - tempQuantizedVectorData.getName(), - tempScoreQuantizedVectorData.getName() + tempQuantizedVectorDataName, + tempScoreQuantizedVectorDataName ); }); } finally { @@ -528,11 +534,12 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( binarizedDataInput, binarizedScoreDataInput ); - IOUtils.deleteFilesIgnoringExceptions( - segmentWriteState.directory, - tempQuantizedVectorData.getName(), - tempScoreQuantizedVectorData.getName() - ); + if (tempQuantizedVectorData != null) { + IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempQuantizedVectorData.getName()); + } + if (tempScoreQuantizedVectorData != null) { + IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempScoreQuantizedVectorData.getName()); + } } } } From 6f43d898fb5eafd317fabe53a1b6d402b56815f9 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 8 Jan 2025 16:23:42 +0100 Subject: [PATCH 23/52] Revert "Nullify sourceAsMap once a search hit is processed (#119734)" (#119751) This reverts commit ab77144aa4973c9d90eee63e392ead7cdb11df4e. --- .../src/main/java/org/elasticsearch/search/SearchHit.java | 8 -------- .../search/fetch/FetchPhaseDocsIterator.java | 5 +---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 0d7b76c3c3997..701d451ac2c14 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -491,13 +491,6 @@ public Map getSourceAsMap() { return sourceAsMap; } - /** - * Set the cache document as a map to {@code null}. - */ - public void resetSourceAsMap() { - sourceAsMap = null; - } - /** * The hit field matching the given field name. */ @@ -735,7 +728,6 @@ private void deallocate() { if (SearchHit.this.source instanceof RefCounted r) { r.decRef(); } - SearchHit.this.sourceAsMap = null; SearchHit.this.source = null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhaseDocsIterator.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhaseDocsIterator.java index 9093b08f9d6f2..4a242f70e8d02 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhaseDocsIterator.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhaseDocsIterator.java @@ -86,10 +86,7 @@ public final SearchHit[] iterate( } currentDoc = docs[i].docId; assert searchHits[docs[i].index] == null; - SearchHit searchHit = nextDoc(docs[i].docId); - // free some memory - searchHit.resetSourceAsMap(); - searchHits[docs[i].index] = searchHit; + searchHits[docs[i].index] = nextDoc(docs[i].docId); } catch (ContextIndexSearcher.TimeExceededException e) { if (allowPartialResults == false) { purgeSearchHits(searchHits); From deeeadc01f86d08f887cbfcb0dc40caf3b7374f4 Mon Sep 17 00:00:00 2001 From: kanoshiou <73424326+kanoshiou@users.noreply.github.com> Date: Wed, 8 Jan 2025 23:43:11 +0800 Subject: [PATCH 24/52] ESQL: Allow the data type of `null` in filters (#118324) * Allow the data type of `null` in filters --- docs/changelog/118324.yaml | 6 +++++ .../src/main/resources/null.csv-spec | 12 +++++++++ .../xpack/esql/analysis/Verifier.java | 14 +++++----- .../xpack/esql/analysis/VerifierTests.java | 27 +++++++++++++++++++ .../optimizer/LogicalPlanOptimizerTests.java | 13 +++++++++ 5 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 docs/changelog/118324.yaml diff --git a/docs/changelog/118324.yaml b/docs/changelog/118324.yaml new file mode 100644 index 0000000000000..729ff56f6a253 --- /dev/null +++ b/docs/changelog/118324.yaml @@ -0,0 +1,6 @@ +pr: 118324 +summary: Allow the data type of `null` in filters +area: ES|QL +type: bug +issues: + - 116351 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec index 9914d073a589d..7bf3bc7613e01 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec @@ -191,3 +191,15 @@ emp_no:integer | languages:integer | height:double | x:double | y:double | z:dou 10020 | null | 1.41 | 1.41 | 1.41 | 40031.0 | 40031 10021 | null | 1.47 | 1.47 | 1.47 | 60408.0 | 60408 ; + +whereNull +FROM employees +| WHERE NULL and emp_no <= 10021 +| SORT first_name, last_name +| EVAL fullname = CONCAT(first_name, " ", last_name) +| KEEP fullname, job_positions, salary, salary_change +| limit 5 +; + +fullname:keyword | job_positions:keyword | salary:integer | salary_change:double +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index e146b517ad1c8..8245d09ed69c2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -237,9 +237,13 @@ private void checkSort(LogicalPlan p, Set failures) { private static void checkFilterConditionType(LogicalPlan p, Set localFailures) { if (p instanceof Filter f) { Expression condition = f.condition(); - if (condition.dataType() != BOOLEAN) { - localFailures.add(fail(condition, "Condition expression needs to be boolean, found [{}]", condition.dataType())); - } + checkConditionExpressionDataType(condition, localFailures); + } + } + + private static void checkConditionExpressionDataType(Expression expression, Set localFailures) { + if (expression.dataType() != NULL && expression.dataType() != BOOLEAN) { + localFailures.add(fail(expression, "Condition expression needs to be boolean, found [{}]", expression.dataType())); } } @@ -432,9 +436,7 @@ private static void checkInvalidNamedExpressionUsage( } Expression f = fe.filter(); // check the filter has to be a boolean term, similar as checkFilterConditionType - if (f.dataType() != NULL && f.dataType() != BOOLEAN) { - failures.add(fail(f, "Condition expression needs to be boolean, found [{}]", f.dataType())); - } + checkConditionExpressionDataType(f, failures); // but that the filter doesn't use grouping or aggregate functions fe.filter().forEachDown(c -> { if (c instanceof AggregateFunction af) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index fe6d1e00e5d24..b2362b5c2aa68 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -945,6 +945,33 @@ public void testPeriodAndDurationInEval() { public void testFilterNonBoolField() { assertEquals("1:19: Condition expression needs to be boolean, found [INTEGER]", error("from test | where emp_no")); + + assertEquals( + "1:19: Condition expression needs to be boolean, found [KEYWORD]", + error("from test | where concat(first_name, \"foobar\")") + ); + } + + public void testFilterNullField() { + // `where null` should return empty result set + query("from test | where null"); + + // Value null of type `BOOLEAN` + query("from test | where null::boolean"); + + // Provide `NULL` type in `EVAL` + query("from t | EVAL x = null | where x"); + + // `to_string(null)` is of `KEYWORD` type null, resulting in `to_string(null) == "abc"` being of `BOOLEAN` + query("from t | where to_string(null) == \"abc\""); + + // Other DataTypes can contain null values + assertEquals("1:19: Condition expression needs to be boolean, found [KEYWORD]", error("from test | where null::string")); + assertEquals("1:19: Condition expression needs to be boolean, found [INTEGER]", error("from test | where null::integer")); + assertEquals( + "1:45: Condition expression needs to be boolean, found [DATETIME]", + error("from test | EVAL x = null::datetime | where x") + ); } public void testFilterDateConstant() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index d46572b7c8561..4d175dea05071 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -6808,4 +6808,17 @@ public void testMatchFunctionIsNotNullable() { containsString("[MATCH] function cannot operate on [text::keyword], which is not a field from an index mapping") ); } + + public void testWhereNull() { + var plan = plan(""" + from test + | sort salary + | rename emp_no as e, first_name as f + | keep salary, e, f + | where null + | LIMIT 12 + """); + var local = as(plan, LocalRelation.class); + assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY)); + } } From fd1be8ce6fe54b26f35ea700e2cfc1369e08e989 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Wed, 8 Jan 2025 16:44:15 +0100 Subject: [PATCH 25/52] Hash functions (#118938) This change adds md5, sha1 and sha256 hash functions. --- docs/changelog/118938.yaml | 5 ++ .../esql/functions/description/md5.asciidoc | 5 ++ .../esql/functions/description/sha1.asciidoc | 5 ++ .../functions/description/sha256.asciidoc | 5 ++ .../esql/functions/examples/hash.asciidoc | 13 ++++ .../esql/functions/examples/md5.asciidoc | 13 ++++ .../esql/functions/examples/sha1.asciidoc | 13 ++++ .../esql/functions/examples/sha256.asciidoc | 13 ++++ .../functions/kibana/definition/hash.json | 3 + .../esql/functions/kibana/definition/md5.json | 37 ++++++++++ .../functions/kibana/definition/sha1.json | 37 ++++++++++ .../functions/kibana/definition/sha256.json | 37 ++++++++++ .../esql/functions/kibana/docs/hash.md | 6 ++ .../esql/functions/kibana/docs/md5.md | 13 ++++ .../esql/functions/kibana/docs/sha1.md | 13 ++++ .../esql/functions/kibana/docs/sha256.md | 13 ++++ .../esql/functions/layout/hash.asciidoc | 1 + .../esql/functions/layout/md5.asciidoc | 15 ++++ .../esql/functions/layout/sha1.asciidoc | 15 ++++ .../esql/functions/layout/sha256.asciidoc | 15 ++++ .../esql/functions/parameters/md5.asciidoc | 6 ++ .../esql/functions/parameters/sha1.asciidoc | 6 ++ .../esql/functions/parameters/sha256.asciidoc | 6 ++ .../esql/functions/signature/md5.svg | 1 + .../esql/functions/signature/sha1.svg | 1 + .../esql/functions/signature/sha256.svg | 1 + .../esql/functions/string-functions.asciidoc | 6 ++ .../esql/functions/types/md5.asciidoc | 10 +++ .../esql/functions/types/sha1.asciidoc | 10 +++ .../esql/functions/types/sha256.asciidoc | 10 +++ .../src/main/resources/hash.csv-spec | 73 ++++++++++++++++++- .../xpack/esql/action/EsqlCapabilities.java | 4 + .../function/EsqlFunctionRegistry.java | 6 ++ .../scalar/ScalarFunctionWritables.java | 6 ++ .../scalar/string/AbstractHashFunction.java | 70 ++++++++++++++++++ .../function/scalar/string/Hash.java | 16 +++- .../function/scalar/string/Md5.java | 61 ++++++++++++++++ .../function/scalar/string/Sha1.java | 60 +++++++++++++++ .../function/scalar/string/Sha256.java | 60 +++++++++++++++ .../function/scalar/string/HashTests.java | 12 ++- .../function/scalar/string/Md5ErrorTests.java | 38 ++++++++++ .../scalar/string/Md5SerializationTests.java | 25 +++++++ .../function/scalar/string/Md5Tests.java | 39 ++++++++++ .../scalar/string/Sha1ErrorTests.java | 38 ++++++++++ .../scalar/string/Sha1SerializationTests.java | 25 +++++++ .../function/scalar/string/Sha1Tests.java | 39 ++++++++++ .../scalar/string/Sha256ErrorTests.java | 38 ++++++++++ .../string/Sha256SerializationTests.java | 25 +++++++ .../function/scalar/string/Sha256Tests.java | 39 ++++++++++ .../rest-api-spec/test/esql/60_usage.yml | 4 +- 50 files changed, 1003 insertions(+), 9 deletions(-) create mode 100644 docs/changelog/118938.yaml create mode 100644 docs/reference/esql/functions/description/md5.asciidoc create mode 100644 docs/reference/esql/functions/description/sha1.asciidoc create mode 100644 docs/reference/esql/functions/description/sha256.asciidoc create mode 100644 docs/reference/esql/functions/examples/hash.asciidoc create mode 100644 docs/reference/esql/functions/examples/md5.asciidoc create mode 100644 docs/reference/esql/functions/examples/sha1.asciidoc create mode 100644 docs/reference/esql/functions/examples/sha256.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/md5.json create mode 100644 docs/reference/esql/functions/kibana/definition/sha1.json create mode 100644 docs/reference/esql/functions/kibana/definition/sha256.json create mode 100644 docs/reference/esql/functions/kibana/docs/md5.md create mode 100644 docs/reference/esql/functions/kibana/docs/sha1.md create mode 100644 docs/reference/esql/functions/kibana/docs/sha256.md create mode 100644 docs/reference/esql/functions/layout/md5.asciidoc create mode 100644 docs/reference/esql/functions/layout/sha1.asciidoc create mode 100644 docs/reference/esql/functions/layout/sha256.asciidoc create mode 100644 docs/reference/esql/functions/parameters/md5.asciidoc create mode 100644 docs/reference/esql/functions/parameters/sha1.asciidoc create mode 100644 docs/reference/esql/functions/parameters/sha256.asciidoc create mode 100644 docs/reference/esql/functions/signature/md5.svg create mode 100644 docs/reference/esql/functions/signature/sha1.svg create mode 100644 docs/reference/esql/functions/signature/sha256.svg create mode 100644 docs/reference/esql/functions/types/md5.asciidoc create mode 100644 docs/reference/esql/functions/types/sha1.asciidoc create mode 100644 docs/reference/esql/functions/types/sha256.asciidoc create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java diff --git a/docs/changelog/118938.yaml b/docs/changelog/118938.yaml new file mode 100644 index 0000000000000..395da7912fd4b --- /dev/null +++ b/docs/changelog/118938.yaml @@ -0,0 +1,5 @@ +pr: 118938 +summary: Hash functions +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/reference/esql/functions/description/md5.asciidoc b/docs/reference/esql/functions/description/md5.asciidoc new file mode 100644 index 0000000000000..2ad847c0ce0e3 --- /dev/null +++ b/docs/reference/esql/functions/description/md5.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the MD5 hash of the input. diff --git a/docs/reference/esql/functions/description/sha1.asciidoc b/docs/reference/esql/functions/description/sha1.asciidoc new file mode 100644 index 0000000000000..5bc29f86cc591 --- /dev/null +++ b/docs/reference/esql/functions/description/sha1.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the SHA1 hash of the input. diff --git a/docs/reference/esql/functions/description/sha256.asciidoc b/docs/reference/esql/functions/description/sha256.asciidoc new file mode 100644 index 0000000000000..b2a7ef01e1069 --- /dev/null +++ b/docs/reference/esql/functions/description/sha256.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the SHA256 hash of the input. diff --git a/docs/reference/esql/functions/examples/hash.asciidoc b/docs/reference/esql/functions/examples/hash.asciidoc new file mode 100644 index 0000000000000..492e466eb395e --- /dev/null +++ b/docs/reference/esql/functions/examples/hash.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=hash] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=hash-result] +|=== + diff --git a/docs/reference/esql/functions/examples/md5.asciidoc b/docs/reference/esql/functions/examples/md5.asciidoc new file mode 100644 index 0000000000000..0b43bc5b791c9 --- /dev/null +++ b/docs/reference/esql/functions/examples/md5.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=md5] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=md5-result] +|=== + diff --git a/docs/reference/esql/functions/examples/sha1.asciidoc b/docs/reference/esql/functions/examples/sha1.asciidoc new file mode 100644 index 0000000000000..77786431a738a --- /dev/null +++ b/docs/reference/esql/functions/examples/sha1.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=sha1] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=sha1-result] +|=== + diff --git a/docs/reference/esql/functions/examples/sha256.asciidoc b/docs/reference/esql/functions/examples/sha256.asciidoc new file mode 100644 index 0000000000000..801c36d8effc8 --- /dev/null +++ b/docs/reference/esql/functions/examples/sha256.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=sha256] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=sha256-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/hash.json b/docs/reference/esql/functions/kibana/definition/hash.json index 17a60cf45acfe..dbf4a2542afc5 100644 --- a/docs/reference/esql/functions/kibana/definition/hash.json +++ b/docs/reference/esql/functions/kibana/definition/hash.json @@ -77,6 +77,9 @@ "returnType" : "keyword" } ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL md5 = hash(\"md5\", message), sha256 = hash(\"sha256\", message) \n| KEEP message, md5, sha256;" + ], "preview" : false, "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/md5.json b/docs/reference/esql/functions/kibana/definition/md5.json new file mode 100644 index 0000000000000..4d3a88e123ff4 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/md5.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "md5", + "description" : "Computes the MD5 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL md5 = md5(message)\n| KEEP message, md5;" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/sha1.json b/docs/reference/esql/functions/kibana/definition/sha1.json new file mode 100644 index 0000000000000..a6abb31368bb3 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/sha1.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "sha1", + "description" : "Computes the SHA1 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL sha1 = sha1(message)\n| KEEP message, sha1;" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/sha256.json b/docs/reference/esql/functions/kibana/definition/sha256.json new file mode 100644 index 0000000000000..700425d485b61 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/sha256.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "sha256", + "description" : "Computes the SHA256 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL sha256 = sha256(message)\n| KEEP message, sha256;" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/docs/hash.md b/docs/reference/esql/functions/kibana/docs/hash.md index 9826e80ec5bec..4e937778ba67a 100644 --- a/docs/reference/esql/functions/kibana/docs/hash.md +++ b/docs/reference/esql/functions/kibana/docs/hash.md @@ -5,3 +5,9 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### HASH Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512. +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) +| KEEP message, md5, sha256; +``` diff --git a/docs/reference/esql/functions/kibana/docs/md5.md b/docs/reference/esql/functions/kibana/docs/md5.md new file mode 100644 index 0000000000000..aacb8a3960165 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/md5.md @@ -0,0 +1,13 @@ + + +### MD5 +Computes the MD5 hash of the input. + +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = md5(message) +| KEEP message, md5; +``` diff --git a/docs/reference/esql/functions/kibana/docs/sha1.md b/docs/reference/esql/functions/kibana/docs/sha1.md new file mode 100644 index 0000000000000..a940aa133f06e --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/sha1.md @@ -0,0 +1,13 @@ + + +### SHA1 +Computes the SHA1 hash of the input. + +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha1 = sha1(message) +| KEEP message, sha1; +``` diff --git a/docs/reference/esql/functions/kibana/docs/sha256.md b/docs/reference/esql/functions/kibana/docs/sha256.md new file mode 100644 index 0000000000000..fbe576c7c20d6 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/sha256.md @@ -0,0 +1,13 @@ + + +### SHA256 +Computes the SHA256 hash of the input. + +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha256 = sha256(message) +| KEEP message, sha256; +``` diff --git a/docs/reference/esql/functions/layout/hash.asciidoc b/docs/reference/esql/functions/layout/hash.asciidoc index 27c55ada6319b..daf7fbf1170b2 100644 --- a/docs/reference/esql/functions/layout/hash.asciidoc +++ b/docs/reference/esql/functions/layout/hash.asciidoc @@ -12,3 +12,4 @@ image::esql/functions/signature/hash.svg[Embedded,opts=inline] include::../parameters/hash.asciidoc[] include::../description/hash.asciidoc[] include::../types/hash.asciidoc[] +include::../examples/hash.asciidoc[] diff --git a/docs/reference/esql/functions/layout/md5.asciidoc b/docs/reference/esql/functions/layout/md5.asciidoc new file mode 100644 index 0000000000000..82d3031d6bdfd --- /dev/null +++ b/docs/reference/esql/functions/layout/md5.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-md5]] +=== `MD5` + +*Syntax* + +[.text-center] +image::esql/functions/signature/md5.svg[Embedded,opts=inline] + +include::../parameters/md5.asciidoc[] +include::../description/md5.asciidoc[] +include::../types/md5.asciidoc[] +include::../examples/md5.asciidoc[] diff --git a/docs/reference/esql/functions/layout/sha1.asciidoc b/docs/reference/esql/functions/layout/sha1.asciidoc new file mode 100644 index 0000000000000..23e1e0e9ac2ab --- /dev/null +++ b/docs/reference/esql/functions/layout/sha1.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-sha1]] +=== `SHA1` + +*Syntax* + +[.text-center] +image::esql/functions/signature/sha1.svg[Embedded,opts=inline] + +include::../parameters/sha1.asciidoc[] +include::../description/sha1.asciidoc[] +include::../types/sha1.asciidoc[] +include::../examples/sha1.asciidoc[] diff --git a/docs/reference/esql/functions/layout/sha256.asciidoc b/docs/reference/esql/functions/layout/sha256.asciidoc new file mode 100644 index 0000000000000..d36a1345271f5 --- /dev/null +++ b/docs/reference/esql/functions/layout/sha256.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-sha256]] +=== `SHA256` + +*Syntax* + +[.text-center] +image::esql/functions/signature/sha256.svg[Embedded,opts=inline] + +include::../parameters/sha256.asciidoc[] +include::../description/sha256.asciidoc[] +include::../types/sha256.asciidoc[] +include::../examples/sha256.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/md5.asciidoc b/docs/reference/esql/functions/parameters/md5.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/md5.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/parameters/sha1.asciidoc b/docs/reference/esql/functions/parameters/sha1.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/sha1.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/parameters/sha256.asciidoc b/docs/reference/esql/functions/parameters/sha256.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/sha256.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/signature/md5.svg b/docs/reference/esql/functions/signature/md5.svg new file mode 100644 index 0000000000000..419af764a212e --- /dev/null +++ b/docs/reference/esql/functions/signature/md5.svg @@ -0,0 +1 @@ +MD5(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/sha1.svg b/docs/reference/esql/functions/signature/sha1.svg new file mode 100644 index 0000000000000..bab03a7eb88c8 --- /dev/null +++ b/docs/reference/esql/functions/signature/sha1.svg @@ -0,0 +1 @@ +SHA1(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/sha256.svg b/docs/reference/esql/functions/signature/sha256.svg new file mode 100644 index 0000000000000..b77126bbefbd8 --- /dev/null +++ b/docs/reference/esql/functions/signature/sha256.svg @@ -0,0 +1 @@ +SHA256(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index da9580a55151a..dd10e4c77581e 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -18,11 +18,14 @@ * <> * <> * <> +* <> * <> * <> * <> * <> * <> +* <> +* <> * <> * <> * <> @@ -43,11 +46,14 @@ include::layout/left.asciidoc[] include::layout/length.asciidoc[] include::layout/locate.asciidoc[] include::layout/ltrim.asciidoc[] +include::layout/md5.asciidoc[] include::layout/repeat.asciidoc[] include::layout/replace.asciidoc[] include::layout/reverse.asciidoc[] include::layout/right.asciidoc[] include::layout/rtrim.asciidoc[] +include::layout/sha1.asciidoc[] +include::layout/sha256.asciidoc[] include::layout/space.asciidoc[] include::layout/split.asciidoc[] include::layout/starts_with.asciidoc[] diff --git a/docs/reference/esql/functions/types/md5.asciidoc b/docs/reference/esql/functions/types/md5.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/md5.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/docs/reference/esql/functions/types/sha1.asciidoc b/docs/reference/esql/functions/types/sha1.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/sha1.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/docs/reference/esql/functions/types/sha256.asciidoc b/docs/reference/esql/functions/types/sha256.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/sha256.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec index fcac1e1859c6d..2614ff09fed06 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec @@ -1,18 +1,22 @@ hash required_capability: hash_function +// tag::hash[] FROM sample_data | WHERE message != "Connection error" | EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) | KEEP message, md5, sha256; +// end::hash[] ignoreOrder:true +// tag::hash-result[] message:keyword | md5:keyword | sha256:keyword Connected to 10.1.0.1 | abd7d1ce2bb636842a29246b3512dcae | 6d8372129ad78770f7185554dd39864749a62690216460752d6c075fa38ad85c Connected to 10.1.0.2 | 8f8f1cb60832d153f5b9ec6dc828b93f | b0db24720f15857091b3c99f4c4833586d0ea3229911b8777efb8d917cf27e9a Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78 | 75b0480188db8acc4d5cc666a51227eb2bc5b989cd8ca912609f33e0846eff57 Disconnected | ef70e46fd3bbc21e3e1f0b6815e750c0 | 04dfac3671b494ad53fcd152f7a14511bfb35747278aad8ce254a0d6e4ba4718 ; +// end::hash-result[] hashOfConvertedType @@ -94,12 +98,75 @@ input:integer | md5:keyword | sha256:keyword hashWithStats required_capability: hash_function +required_capability: hash_function_aliases_v1 FROM sample_data | EVAL md5="md5" -| STATS count = count(*) by hash(md5, message) +| STATS count = count(*) by hash(md5, message), md5(message), sha1(message), sha256(message) | WHERE count > 1; -count:long | hash(md5, message):keyword -3 | 2e92ae79ff32b37fee4368a594792183 +count:long | hash(md5, message):keyword | md5(message):keyword | sha1(message):keyword | sha256(message):keyword +3 | 2e92ae79ff32b37fee4368a594792183 | 2e92ae79ff32b37fee4368a594792183 | 1dbb3521876a899f82d6b0ff10eb32a01e03aba8 | 8d137af9c64fba09bbb003aba93f0b029898fe19e7927cd696f4c3e2b69f538d ; + + +md5Hash +required_capability: hash_function_aliases_v1 + +// tag::md5[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = md5(message) +| KEEP message, md5; +// end::md5[] +ignoreOrder:true + +// tag::md5-result[] +message:keyword | md5:keyword +Connected to 10.1.0.1 | abd7d1ce2bb636842a29246b3512dcae +Connected to 10.1.0.2 | 8f8f1cb60832d153f5b9ec6dc828b93f +Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78 +Disconnected | ef70e46fd3bbc21e3e1f0b6815e750c0 +; +// end::md5-result[] + + +sha1Hash +required_capability: hash_function_aliases_v1 + +// tag::sha1[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha1 = sha1(message) +| KEEP message, sha1; +// end::sha1[] +ignoreOrder:true + +// tag::sha1-result[] +message:keyword | sha1:keyword +Connected to 10.1.0.1 | 42b85531a79088036a17759db7d2de292b92f57f +Connected to 10.1.0.2 | d30db445da2e9237c9718d0c7e4fb7cbbe9c2cb4 +Connected to 10.1.0.3 | 2733848d943809f0b10cad3e980763e88afb9853 +Disconnected | 771e05f27b99fd59f638f41a7a4e977b1d4691fe +; +// end::sha1-result[] + +sha256Hash +required_capability: hash_function_aliases_v1 + +// tag::sha256[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha256 = sha256(message) +| KEEP message, sha256; +// end::sha256[] +ignoreOrder:true + +// tag::sha256-result[] +message:keyword | sha256:keyword +Connected to 10.1.0.1 | 6d8372129ad78770f7185554dd39864749a62690216460752d6c075fa38ad85c +Connected to 10.1.0.2 | b0db24720f15857091b3c99f4c4833586d0ea3229911b8777efb8d917cf27e9a +Connected to 10.1.0.3 | 75b0480188db8acc4d5cc666a51227eb2bc5b989cd8ca912609f33e0846eff57 +Disconnected | 04dfac3671b494ad53fcd152f7a14511bfb35747278aad8ce254a0d6e4ba4718 +; +// end::sha256-result[] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 5c259caa9c940..387e48702e708 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -458,6 +458,10 @@ public enum Cap { * Hash function */ HASH_FUNCTION, + /** + * Hash function aliases such as MD5 + */ + HASH_FUNCTION_ALIASES_V1, /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 908c9c5f197a8..d310973d6dbe7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -134,11 +134,14 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha1; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha256; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; @@ -333,11 +336,14 @@ private static FunctionDefinition[][] functions() { def(Left.class, Left::new, "left"), def(Length.class, Length::new, "length"), def(Locate.class, Locate::new, "locate"), + def(Md5.class, Md5::new, "md5"), def(RTrim.class, RTrim::new, "rtrim"), def(Repeat.class, Repeat::new, "repeat"), def(Replace.class, Replace::new, "replace"), def(Reverse.class, Reverse::new, "reverse"), def(Right.class, Right::new, "right"), + def(Sha1.class, Sha1::new, "sha1"), + def(Sha256.class, Sha256::new, "sha256"), def(Space.class, Space::new, "space"), def(StartsWith.class, StartsWith::new, "starts_with"), def(Substring.class, Substring::new, "substring"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java index 3cf0eef9074ad..c4b9f6885e617 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java @@ -37,10 +37,13 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha1; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha256; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; @@ -79,6 +82,7 @@ public static List getNamedWriteables() { entries.add(Left.ENTRY); entries.add(Locate.ENTRY); entries.add(Log.ENTRY); + entries.add(Md5.ENTRY); entries.add(Now.ENTRY); entries.add(Or.ENTRY); entries.add(Pi.ENTRY); @@ -88,6 +92,8 @@ public static List getNamedWriteables() { entries.add(Replace.ENTRY); entries.add(Reverse.ENTRY); entries.add(Round.ENTRY); + entries.add(Sha1.ENTRY); + entries.add(Sha256.ENTRY); entries.add(Split.ENTRY); entries.add(Substring.ENTRY); entries.add(StartsWith.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java new file mode 100644 index 0000000000000..a39f0a4f32db2 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction; + +import java.io.IOException; +import java.util.function.Function; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +public abstract class AbstractHashFunction extends UnaryScalarFunction { + + protected AbstractHashFunction(Source source, Expression field) { + super(source, field); + } + + protected AbstractHashFunction(StreamInput in) throws IOException { + super(in); + } + + protected abstract HashFunction getHashFunction(); + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + return isString(field, sourceText(), DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + return new HashConstantEvaluator.Factory( + source(), + context -> new BreakingBytesRefBuilder(context.breaker(), "hash"), + new Function<>() { + @Override + public HashFunction apply(DriverContext context) { + return getHashFunction().copy(); + } + + @Override + public String toString() { + return getHashFunction().toString(); + } + }, + toEvaluator.apply(field) + ); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java index 99c5908699ec2..52d33c0fc9d3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; @@ -46,7 +47,8 @@ public class Hash extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", - description = "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512." + description = "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512.", + examples = { @Example(file = "hash", tag = "hash") } ) public Hash( Source source, @@ -186,10 +188,18 @@ protected NodeInfo info() { public record HashFunction(String algorithm, MessageDigest digest) { + public static HashFunction create(String algorithm) { + try { + return new HashFunction(algorithm, MessageDigest.getInstance(algorithm)); + } catch (NoSuchAlgorithmException e) { + assert false : "Expected to create a valid hashing algorithm"; + throw new IllegalStateException(e); + } + } + public static HashFunction create(BytesRef literal) throws NoSuchAlgorithmException { var algorithm = literal.utf8ToString(); - var digest = MessageDigest.getInstance(algorithm); - return new HashFunction(algorithm, digest); + return new HashFunction(algorithm, MessageDigest.getInstance(algorithm)); } public HashFunction copy() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java new file mode 100644 index 0000000000000..b42ec1036cb5b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction; + +import java.io.IOException; +import java.util.List; + +public class Md5 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MD5", Md5::new); + + private static final HashFunction MD5 = HashFunction.create("MD5"); + + @FunctionInfo( + returnType = "keyword", + description = "Computes the MD5 hash of the input.", + examples = { @Example(file = "hash", tag = "md5") } + ) + public Md5(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + private Md5(StreamInput in) throws IOException { + super(in); + } + + @Override + protected HashFunction getHashFunction() { + return MD5; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Md5(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Md5::new, field); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java new file mode 100644 index 0000000000000..ba1f62562c03a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; + +public class Sha1 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "SHA1", Sha1::new); + + private static final Hash.HashFunction SHA1 = Hash.HashFunction.create("SHA1"); + + @FunctionInfo( + returnType = "keyword", + description = "Computes the SHA1 hash of the input.", + examples = { @Example(file = "hash", tag = "sha1") } + ) + public Sha1(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + private Sha1(StreamInput in) throws IOException { + super(in); + } + + @Override + protected Hash.HashFunction getHashFunction() { + return SHA1; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Sha1(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Sha1::new, field); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java new file mode 100644 index 0000000000000..b16767a3f7948 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; + +public class Sha256 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "SHA256", Sha256::new); + + private static final Hash.HashFunction SHA256 = Hash.HashFunction.create("SHA256"); + + @FunctionInfo( + returnType = "keyword", + description = "Computes the SHA256 hash of the input.", + examples = { @Example(file = "hash", tag = "sha256") } + ) + public Sha256(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + private Sha256(StreamInput in) throws IOException { + super(in); + } + + @Override + protected Hash.HashFunction getHashFunction() { + return SHA256; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Sha256(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Sha256::new, field); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java index c25270474959b..fe62b6d7b4f1b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java @@ -87,12 +87,22 @@ private static TestCaseSupplier createTestCase(String algorithm, boolean forceLi }); } + static void addHashFunctionTestCases(List cases, String algorithm) { + TestCaseSupplier.forUnaryStrings( + cases, + "HashConstantEvaluator[algorithm=" + algorithm + ", input=Attribute[channel=0]]", + DataType.KEYWORD, + input -> new BytesRef(HashTests.hash(algorithm, BytesRefs.toString(input))), + List.of() + ); + } + private static TestCaseSupplier.TypedData createTypedData(String value, boolean forceLiteral, DataType type, String name) { var data = new TestCaseSupplier.TypedData(new BytesRef(value), type, name); return forceLiteral ? data.forceLiteral() : data; } - private static String hash(String algorithm, String input) { + static String hash(String algorithm, String input) { try { return HexFormat.of().formatHex(MessageDigest.getInstance(algorithm).digest(input.getBytes(StandardCharsets.UTF_8))); } catch (NoSuchAlgorithmException e) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java new file mode 100644 index 0000000000000..701cef748c932 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Md5ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Md5Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Md5(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java new file mode 100644 index 0000000000000..666e5a3ea5d58 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Md5SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Md5 createTestInstance() { + return new Md5(randomSource(), randomChild()); + } + + @Override + protected Md5 mutateInstance(Md5 instance) throws IOException { + return new Md5(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java new file mode 100644 index 0000000000000..3c0a2067a81b0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Md5Tests extends AbstractScalarFunctionTestCase { + + public Md5Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + HashTests.addHashFunctionTestCases(cases, "MD5"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); + } + + @Override + protected Expression build(Source source, List args) { + return new Md5(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java new file mode 100644 index 0000000000000..613aabe658ceb --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Sha1ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Sha1Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha1(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java new file mode 100644 index 0000000000000..12c9cc7580de9 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Sha1SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Sha1 createTestInstance() { + return new Sha1(randomSource(), randomChild()); + } + + @Override + protected Sha1 mutateInstance(Sha1 instance) throws IOException { + return new Sha1(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java new file mode 100644 index 0000000000000..06c23f186d4c8 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Sha1Tests extends AbstractScalarFunctionTestCase { + + public Sha1Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + HashTests.addHashFunctionTestCases(cases, "SHA1"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha1(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java new file mode 100644 index 0000000000000..a9678346f8508 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Sha256ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Sha256Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha256(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java new file mode 100644 index 0000000000000..2f209fef25d36 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Sha256SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Sha256 createTestInstance() { + return new Sha256(randomSource(), randomChild()); + } + + @Override + protected Sha256 mutateInstance(Sha256 instance) throws IOException { + return new Sha256(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java new file mode 100644 index 0000000000000..0a136e3214082 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Sha256Tests extends AbstractScalarFunctionTestCase { + + public Sha256Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + HashTests.addHashFunctionTestCases(cases, "SHA256"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha256(source, args.get(0)); + } +} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index b6d75048591e5..35e5d66e91d97 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 130} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 133} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": @@ -163,4 +163,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 126} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 129} # check the "sister" test above for a likely update to the same esql.functions length check From 82b1f2a205d8c124be289c11674c55927bf25d14 Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Wed, 8 Jan 2025 10:59:36 -0500 Subject: [PATCH 26/52] Updating inference services API response (#118491) * Updating inference services API response * Update docs/changelog/118491.yaml * Delete docs/changelog/118491.yaml * Skip writing null default value config * Sorting output by service name * Updating Jina AI * [CI] Auto commit changes from spotless --------- Co-authored-by: elasticsearchmachine --- .../inference/EmptySettingsConfiguration.java | 19 - .../InferenceServiceConfiguration.java | 116 +++--- .../inference/SettingsConfiguration.java | 351 +++--------------- .../inference/TaskSettingsConfiguration.java | 154 -------- .../SettingsConfigurationDependency.java | 140 ------- .../SettingsConfigurationDisplayType.java | 35 -- .../SettingsConfigurationSelectOption.java | 132 ------- .../SettingsConfigurationValidation.java | 154 -------- .../SettingsConfigurationValidationType.java | 34 -- ...nferenceServiceConfigurationTestUtils.java | 14 +- .../InferenceServiceConfigurationTests.java | 138 +------ .../SettingsConfigurationTestUtils.java | 45 +-- .../inference/SettingsConfigurationTests.java | 192 +--------- .../TaskSettingsConfigurationTestUtils.java | 40 -- .../TaskSettingsConfigurationTests.java | 94 ----- ...SettingsConfigurationDisplayTypeTests.java | 30 -- ...tingsConfigurationValidationTypeTests.java | 29 -- .../xpack/inference/InferenceCrudIT.java | 17 +- .../TestDenseInferenceServiceExtension.java | 18 +- .../mock/TestRerankingServiceExtension.java | 18 +- .../TestSparseInferenceServiceExtension.java | 22 +- ...stStreamingCompletionServiceExtension.java | 18 +- .../TransportGetInferenceServicesAction.java | 10 +- .../AlibabaCloudSearchService.java | 66 +--- .../AlibabaCloudSearchEmbeddingsModel.java | 40 -- .../sparse/AlibabaCloudSearchSparseModel.java | 55 --- .../AmazonBedrockSecretSettings.java | 11 +- .../amazonbedrock/AmazonBedrockService.java | 44 +-- .../AmazonBedrockChatCompletionModel.java | 69 ---- .../services/anthropic/AnthropicService.java | 24 +- .../AnthropicChatCompletionModel.java | 69 ---- .../azureaistudio/AzureAiStudioService.java | 47 +-- .../AzureAiStudioChatCompletionModel.java | 33 -- .../AzureAiStudioEmbeddingsModel.java | 71 ---- .../AzureOpenAiSecretSettings.java | 11 +- .../azureopenai/AzureOpenAiService.java | 35 +- .../AzureOpenAiCompletionModel.java | 34 -- .../AzureOpenAiEmbeddingsModel.java | 34 -- .../services/cohere/CohereService.java | 18 +- .../embeddings/CohereEmbeddingsModel.java | 59 --- .../cohere/rerank/CohereRerankModel.java | 46 --- .../elastic/ElasticInferenceService.java | 27 +- .../elasticsearch/CustomElandRerankModel.java | 36 -- .../ElasticsearchInternalService.java | 51 +-- .../googleaistudio/GoogleAiStudioService.java | 22 +- .../GoogleVertexAiSecretSettings.java | 6 +- .../googlevertexai/GoogleVertexAiService.java | 45 +-- .../GoogleVertexAiEmbeddingsModel.java | 57 --- .../rerank/GoogleVertexAiRerankModel.java | 33 -- .../huggingface/HuggingFaceService.java | 25 +- .../elser/HuggingFaceElserService.java | 22 +- .../ibmwatsonx/IbmWatsonxService.java | 42 +-- .../services/jinaai/JinaAIService.java | 17 +- .../embeddings/JinaAIEmbeddingsModel.java | 41 -- .../jinaai/rerank/JinaAIRerankModel.java | 46 --- .../services/mistral/MistralService.java | 29 +- .../services/openai/OpenAiService.java | 45 +-- .../completion/OpenAiChatCompletionModel.java | 34 -- .../embeddings/OpenAiEmbeddingsModel.java | 34 -- .../settings/DefaultSecretSettings.java | 10 +- .../services/settings/RateLimitSettings.java | 10 +- .../services/SenderServiceTests.java | 17 +- .../AlibabaCloudSearchServiceTests.java | 196 ++-------- .../AmazonBedrockServiceTests.java | 239 +++--------- .../anthropic/AnthropicServiceTests.java | 105 +----- .../AzureAiStudioServiceTests.java | 242 +++--------- .../azureopenai/AzureOpenAiServiceTests.java | 118 ++---- .../services/cohere/CohereServiceTests.java | 162 ++------ .../elastic/ElasticInferenceServiceTests.java | 48 +-- .../ElasticsearchInternalServiceTests.java | 90 +---- .../GoogleAiStudioServiceTests.java | 52 +-- .../GoogleVertexAiServiceTests.java | 138 +------ .../HuggingFaceElserServiceTests.java | 48 +-- .../huggingface/HuggingFaceServiceTests.java | 51 +-- .../ibmwatsonx/IbmWatsonxServiceTests.java | 72 +--- .../services/jinaai/JinaAIServiceTests.java | 130 ++----- .../services/mistral/MistralServiceTests.java | 60 +-- .../services/openai/OpenAiServiceTests.java | 105 +----- 78 files changed, 740 insertions(+), 4351 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/inference/EmptySettingsConfiguration.java delete mode 100644 server/src/main/java/org/elasticsearch/inference/TaskSettingsConfiguration.java delete mode 100644 server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDependency.java delete mode 100644 server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayType.java delete mode 100644 server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationSelectOption.java delete mode 100644 server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidation.java delete mode 100644 server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationType.java delete mode 100644 server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTestUtils.java delete mode 100644 server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTests.java delete mode 100644 server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayTypeTests.java delete mode 100644 server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationTypeTests.java diff --git a/server/src/main/java/org/elasticsearch/inference/EmptySettingsConfiguration.java b/server/src/main/java/org/elasticsearch/inference/EmptySettingsConfiguration.java deleted file mode 100644 index 8a3f96750f2ea..0000000000000 --- a/server/src/main/java/org/elasticsearch/inference/EmptySettingsConfiguration.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference; - -import java.util.Collections; -import java.util.Map; - -public class EmptySettingsConfiguration { - public static Map get() { - return Collections.emptyMap(); - } -} diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceConfiguration.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceConfiguration.java index c8bd4f2e27e8b..41cf339c751d1 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceConfiguration.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceConfiguration.java @@ -25,10 +25,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; @@ -37,75 +39,88 @@ */ public class InferenceServiceConfiguration implements Writeable, ToXContentObject { - private final String provider; - private final List taskTypes; - private final Map configuration; + private final String service; + private final String name; + private final EnumSet taskTypes; + private final Map configurations; /** * Constructs a new {@link InferenceServiceConfiguration} instance with specified properties. * - * @param provider The name of the service provider. - * @param taskTypes A list of {@link TaskSettingsConfiguration} supported by the service provider. - * @param configuration The configuration of the service provider, defined by {@link SettingsConfiguration}. + * @param service The name of the service provider. + * @param name The user-friendly name of the service provider. + * @param taskTypes A list of {@link TaskType} supported by the service provider. + * @param configurations The configuration of the service provider, defined by {@link SettingsConfiguration}. */ private InferenceServiceConfiguration( - String provider, - List taskTypes, - Map configuration + String service, + String name, + EnumSet taskTypes, + Map configurations ) { - this.provider = provider; + this.service = service; + this.name = name; this.taskTypes = taskTypes; - this.configuration = configuration; + this.configurations = configurations; } public InferenceServiceConfiguration(StreamInput in) throws IOException { - this.provider = in.readString(); - this.taskTypes = in.readCollectionAsList(TaskSettingsConfiguration::new); - this.configuration = in.readMap(SettingsConfiguration::new); + this.service = in.readString(); + this.name = in.readString(); + this.taskTypes = in.readEnumSet(TaskType.class); + this.configurations = in.readMap(SettingsConfiguration::new); } - static final ParseField PROVIDER_FIELD = new ParseField("provider"); + static final ParseField SERVICE_FIELD = new ParseField("service"); + static final ParseField NAME_FIELD = new ParseField("name"); static final ParseField TASK_TYPES_FIELD = new ParseField("task_types"); - static final ParseField CONFIGURATION_FIELD = new ParseField("configuration"); + static final ParseField CONFIGURATIONS_FIELD = new ParseField("configurations"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "inference_service_configuration", true, args -> { - List taskTypes = (ArrayList) args[1]; - return new InferenceServiceConfiguration.Builder().setProvider((String) args[0]) - .setTaskTypes((List) args[1]) - .setConfiguration((Map) args[2]) + List taskTypes = (ArrayList) args[2]; + return new InferenceServiceConfiguration.Builder().setService((String) args[0]) + .setName((String) args[1]) + .setTaskTypes(EnumSet.copyOf(taskTypes.stream().map(TaskType::fromString).collect(Collectors.toList()))) + .setConfigurations((Map) args[3]) .build(); } ); static { - PARSER.declareString(constructorArg(), PROVIDER_FIELD); - PARSER.declareObjectArray(constructorArg(), (p, c) -> TaskSettingsConfiguration.fromXContent(p), TASK_TYPES_FIELD); - PARSER.declareObject(constructorArg(), (p, c) -> p.map(), CONFIGURATION_FIELD); + PARSER.declareString(constructorArg(), SERVICE_FIELD); + PARSER.declareString(constructorArg(), NAME_FIELD); + PARSER.declareStringArray(constructorArg(), TASK_TYPES_FIELD); + PARSER.declareObject(constructorArg(), (p, c) -> p.map(), CONFIGURATIONS_FIELD); } - public String getProvider() { - return provider; + public String getService() { + return service; } - public List getTaskTypes() { + public String getName() { + return name; + } + + public EnumSet getTaskTypes() { return taskTypes; } - public Map getConfiguration() { - return configuration; + public Map getConfigurations() { + return configurations; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { - builder.field(PROVIDER_FIELD.getPreferredName(), provider); + builder.field(SERVICE_FIELD.getPreferredName(), service); + builder.field(NAME_FIELD.getPreferredName(), name); builder.field(TASK_TYPES_FIELD.getPreferredName(), taskTypes); - builder.field(CONFIGURATION_FIELD.getPreferredName(), configuration); + builder.field(CONFIGURATIONS_FIELD.getPreferredName(), configurations); } builder.endObject(); return builder; @@ -125,17 +140,19 @@ public static InferenceServiceConfiguration fromXContentBytes(BytesReference sou @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(provider); + out.writeString(service); + out.writeString(name); out.writeCollection(taskTypes); - out.writeMapValues(configuration); + out.writeMapValues(configurations); } public Map toMap() { Map map = new HashMap<>(); - map.put(PROVIDER_FIELD.getPreferredName(), provider); + map.put(SERVICE_FIELD.getPreferredName(), service); + map.put(NAME_FIELD.getPreferredName(), name); map.put(TASK_TYPES_FIELD.getPreferredName(), taskTypes); - map.put(CONFIGURATION_FIELD.getPreferredName(), configuration); + map.put(CONFIGURATIONS_FIELD.getPreferredName(), configurations); return map; } @@ -145,39 +162,46 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; InferenceServiceConfiguration that = (InferenceServiceConfiguration) o; - return provider.equals(that.provider) + return service.equals(that.service) + && name.equals(that.name) && Objects.equals(taskTypes, that.taskTypes) - && Objects.equals(configuration, that.configuration); + && Objects.equals(configurations, that.configurations); } @Override public int hashCode() { - return Objects.hash(provider, taskTypes, configuration); + return Objects.hash(service, name, taskTypes, configurations); } public static class Builder { - private String provider; - private List taskTypes; - private Map configuration; + private String service; + private String name; + private EnumSet taskTypes; + private Map configurations; + + public Builder setService(String service) { + this.service = service; + return this; + } - public Builder setProvider(String provider) { - this.provider = provider; + public Builder setName(String name) { + this.name = name; return this; } - public Builder setTaskTypes(List taskTypes) { + public Builder setTaskTypes(EnumSet taskTypes) { this.taskTypes = taskTypes; return this; } - public Builder setConfiguration(Map configuration) { - this.configuration = configuration; + public Builder setConfigurations(Map configurations) { + this.configurations = configurations; return this; } public InferenceServiceConfiguration build() { - return new InferenceServiceConfiguration(provider, taskTypes, configuration); + return new InferenceServiceConfiguration(service, name, taskTypes, configurations); } } } diff --git a/server/src/main/java/org/elasticsearch/inference/SettingsConfiguration.java b/server/src/main/java/org/elasticsearch/inference/SettingsConfiguration.java index fb97e62f01b19..188b8a7e82b57 100644 --- a/server/src/main/java/org/elasticsearch/inference/SettingsConfiguration.java +++ b/server/src/main/java/org/elasticsearch/inference/SettingsConfiguration.java @@ -16,11 +16,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.inference.configuration.SettingsConfigurationDependency; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; -import org.elasticsearch.inference.configuration.SettingsConfigurationValidation; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -32,9 +28,7 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -47,120 +41,62 @@ */ public class SettingsConfiguration implements Writeable, ToXContentObject { - @Nullable - private final String category; @Nullable private final Object defaultValue; @Nullable - private final List dependsOn; - @Nullable - private final SettingsConfigurationDisplayType display; + private final String description; private final String label; - @Nullable - private final List options; - @Nullable - private final Integer order; - @Nullable - private final String placeholder; private final boolean required; private final boolean sensitive; - @Nullable - private final String tooltip; - @Nullable + private final boolean updatable; private final SettingsConfigurationFieldType type; - @Nullable - private final List uiRestrictions; - @Nullable - private final List validations; - @Nullable - private final Object value; /** * Constructs a new {@link SettingsConfiguration} instance with specified properties. * - * @param category The category of the configuration field. * @param defaultValue The default value for the configuration. - * @param dependsOn A list of {@link SettingsConfigurationDependency} indicating dependencies on other configurations. - * @param display The display type, defined by {@link SettingsConfigurationDisplayType}. + * @param description A description of the configuration. * @param label The display label associated with the config field. - * @param options A list of {@link SettingsConfigurationSelectOption} for selectable options. - * @param order The order in which this configuration appears. - * @param placeholder A placeholder text for the configuration field. * @param required A boolean indicating whether the configuration is required. * @param sensitive A boolean indicating whether the configuration contains sensitive information. - * @param tooltip A tooltip text providing additional information about the configuration. + * @param updatable A boolean indicating whether the configuration can be updated. * @param type The type of the configuration field, defined by {@link SettingsConfigurationFieldType}. - * @param uiRestrictions A list of UI restrictions in string format. - * @param validations A list of {@link SettingsConfigurationValidation} for validating the configuration. - * @param value The current value of the configuration. */ private SettingsConfiguration( - String category, Object defaultValue, - List dependsOn, - SettingsConfigurationDisplayType display, + String description, String label, - List options, - Integer order, - String placeholder, boolean required, boolean sensitive, - String tooltip, - SettingsConfigurationFieldType type, - List uiRestrictions, - List validations, - Object value + boolean updatable, + SettingsConfigurationFieldType type ) { - this.category = category; this.defaultValue = defaultValue; - this.dependsOn = dependsOn; - this.display = display; + this.description = description; this.label = label; - this.options = options; - this.order = order; - this.placeholder = placeholder; this.required = required; this.sensitive = sensitive; - this.tooltip = tooltip; + this.updatable = updatable; this.type = type; - this.uiRestrictions = uiRestrictions; - this.validations = validations; - this.value = value; } public SettingsConfiguration(StreamInput in) throws IOException { - this.category = in.readString(); this.defaultValue = in.readGenericValue(); - this.dependsOn = in.readOptionalCollectionAsList(SettingsConfigurationDependency::new); - this.display = in.readEnum(SettingsConfigurationDisplayType.class); + this.description = in.readOptionalString(); this.label = in.readString(); - this.options = in.readOptionalCollectionAsList(SettingsConfigurationSelectOption::new); - this.order = in.readOptionalInt(); - this.placeholder = in.readOptionalString(); this.required = in.readBoolean(); this.sensitive = in.readBoolean(); - this.tooltip = in.readOptionalString(); + this.updatable = in.readBoolean(); this.type = in.readEnum(SettingsConfigurationFieldType.class); - this.uiRestrictions = in.readOptionalStringCollectionAsList(); - this.validations = in.readOptionalCollectionAsList(SettingsConfigurationValidation::new); - this.value = in.readGenericValue(); } - static final ParseField CATEGORY_FIELD = new ParseField("category"); static final ParseField DEFAULT_VALUE_FIELD = new ParseField("default_value"); - static final ParseField DEPENDS_ON_FIELD = new ParseField("depends_on"); - static final ParseField DISPLAY_FIELD = new ParseField("display"); + static final ParseField DESCRIPTION_FIELD = new ParseField("description"); static final ParseField LABEL_FIELD = new ParseField("label"); - static final ParseField OPTIONS_FIELD = new ParseField("options"); - static final ParseField ORDER_FIELD = new ParseField("order"); - static final ParseField PLACEHOLDER_FIELD = new ParseField("placeholder"); static final ParseField REQUIRED_FIELD = new ParseField("required"); static final ParseField SENSITIVE_FIELD = new ParseField("sensitive"); - static final ParseField TOOLTIP_FIELD = new ParseField("tooltip"); + static final ParseField UPDATABLE_FIELD = new ParseField("updatable"); static final ParseField TYPE_FIELD = new ParseField("type"); - static final ParseField UI_RESTRICTIONS_FIELD = new ParseField("ui_restrictions"); - static final ParseField VALIDATIONS_FIELD = new ParseField("validations"); - static final ParseField VALUE_FIELD = new ParseField("value"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -168,27 +104,18 @@ public SettingsConfiguration(StreamInput in) throws IOException { true, args -> { int i = 0; - return new SettingsConfiguration.Builder().setCategory((String) args[i++]) - .setDefaultValue(args[i++]) - .setDependsOn((List) args[i++]) - .setDisplay((SettingsConfigurationDisplayType) args[i++]) + return new SettingsConfiguration.Builder().setDefaultValue(args[i++]) + .setDescription((String) args[i++]) .setLabel((String) args[i++]) - .setOptions((List) args[i++]) - .setOrder((Integer) args[i++]) - .setPlaceholder((String) args[i++]) .setRequired((Boolean) args[i++]) .setSensitive((Boolean) args[i++]) - .setTooltip((String) args[i++]) + .setUpdatable((Boolean) args[i++]) .setType((SettingsConfigurationFieldType) args[i++]) - .setUiRestrictions((List) args[i++]) - .setValidations((List) args[i++]) - .setValue(args[i]) .build(); } ); static { - PARSER.declareString(optionalConstructorArg(), CATEGORY_FIELD); PARSER.declareField(optionalConstructorArg(), (p, c) -> { if (p.currentToken() == XContentParser.Token.VALUE_STRING) { return p.text(); @@ -201,68 +128,31 @@ public SettingsConfiguration(StreamInput in) throws IOException { } throw new XContentParseException("Unsupported token [" + p.currentToken() + "]"); }, DEFAULT_VALUE_FIELD, ObjectParser.ValueType.VALUE); - PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> SettingsConfigurationDependency.fromXContent(p), DEPENDS_ON_FIELD); - PARSER.declareField( - optionalConstructorArg(), - (p, c) -> SettingsConfigurationDisplayType.displayType(p.text()), - DISPLAY_FIELD, - ObjectParser.ValueType.STRING_OR_NULL - ); + PARSER.declareStringOrNull(optionalConstructorArg(), DESCRIPTION_FIELD); PARSER.declareString(constructorArg(), LABEL_FIELD); - PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> SettingsConfigurationSelectOption.fromXContent(p), OPTIONS_FIELD); - PARSER.declareInt(optionalConstructorArg(), ORDER_FIELD); - PARSER.declareStringOrNull(optionalConstructorArg(), PLACEHOLDER_FIELD); PARSER.declareBoolean(optionalConstructorArg(), REQUIRED_FIELD); PARSER.declareBoolean(optionalConstructorArg(), SENSITIVE_FIELD); - PARSER.declareStringOrNull(optionalConstructorArg(), TOOLTIP_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), UPDATABLE_FIELD); PARSER.declareField( optionalConstructorArg(), (p, c) -> p.currentToken() == XContentParser.Token.VALUE_NULL ? null : SettingsConfigurationFieldType.fieldType(p.text()), TYPE_FIELD, ObjectParser.ValueType.STRING_OR_NULL ); - PARSER.declareStringArray(optionalConstructorArg(), UI_RESTRICTIONS_FIELD); - PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> SettingsConfigurationValidation.fromXContent(p), VALIDATIONS_FIELD); - PARSER.declareField( - optionalConstructorArg(), - (p, c) -> parseConfigurationValue(p), - VALUE_FIELD, - ObjectParser.ValueType.VALUE_OBJECT_ARRAY - ); - } - - public String getCategory() { - return category; } public Object getDefaultValue() { return defaultValue; } - public List getDependsOn() { - return dependsOn; - } - - public SettingsConfigurationDisplayType getDisplay() { - return display; + public String getDescription() { + return description; } public String getLabel() { return label; } - public List getOptions() { - return options; - } - - public Integer getOrder() { - return order; - } - - public String getPlaceholder() { - return placeholder; - } - public boolean isRequired() { return required; } @@ -271,26 +161,14 @@ public boolean isSensitive() { return sensitive; } - public String getTooltip() { - return tooltip; + public boolean isUpdatable() { + return updatable; } public SettingsConfigurationFieldType getType() { return type; } - public List getUiRestrictions() { - return uiRestrictions; - } - - public List getValidations() { - return validations; - } - - public Object getValue() { - return value; - } - /** * Parses a configuration value from a parser context. * This method can parse strings, numbers, booleans, objects, and null values, matching the types commonly @@ -319,47 +197,20 @@ public static Object parseConfigurationValue(XContentParser p) throws IOExceptio public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { - if (category != null) { - builder.field(CATEGORY_FIELD.getPreferredName(), category); - } - builder.field(DEFAULT_VALUE_FIELD.getPreferredName(), defaultValue); - if (dependsOn != null) { - builder.xContentList(DEPENDS_ON_FIELD.getPreferredName(), dependsOn); - } else { - builder.xContentList(DEPENDS_ON_FIELD.getPreferredName(), new ArrayList<>()); + if (defaultValue != null) { + builder.field(DEFAULT_VALUE_FIELD.getPreferredName(), defaultValue); } - if (display != null) { - builder.field(DISPLAY_FIELD.getPreferredName(), display.toString()); + if (description != null) { + builder.field(DESCRIPTION_FIELD.getPreferredName(), description); } builder.field(LABEL_FIELD.getPreferredName(), label); - if (options != null) { - builder.xContentList(OPTIONS_FIELD.getPreferredName(), options); - } - if (order != null) { - builder.field(ORDER_FIELD.getPreferredName(), order); - } - if (placeholder != null) { - builder.field(PLACEHOLDER_FIELD.getPreferredName(), placeholder); - } builder.field(REQUIRED_FIELD.getPreferredName(), required); builder.field(SENSITIVE_FIELD.getPreferredName(), sensitive); - if (tooltip != null) { - builder.field(TOOLTIP_FIELD.getPreferredName(), tooltip); - } + builder.field(UPDATABLE_FIELD.getPreferredName(), updatable); + if (type != null) { builder.field(TYPE_FIELD.getPreferredName(), type.toString()); } - if (uiRestrictions != null) { - builder.stringListField(UI_RESTRICTIONS_FIELD.getPreferredName(), uiRestrictions); - } else { - builder.stringListField(UI_RESTRICTIONS_FIELD.getPreferredName(), new ArrayList<>()); - } - if (validations != null) { - builder.xContentList(VALIDATIONS_FIELD.getPreferredName(), validations); - } else { - builder.xContentList(VALIDATIONS_FIELD.getPreferredName(), new ArrayList<>()); - } - builder.field(VALUE_FIELD.getPreferredName(), value); } builder.endObject(); return builder; @@ -379,57 +230,29 @@ public static SettingsConfiguration fromXContentBytes(BytesReference source, XCo @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(category); out.writeGenericValue(defaultValue); - out.writeOptionalCollection(dependsOn); - out.writeEnum(display); + out.writeOptionalString(description); out.writeString(label); - out.writeOptionalCollection(options); - out.writeOptionalInt(order); - out.writeOptionalString(placeholder); out.writeBoolean(required); out.writeBoolean(sensitive); - out.writeOptionalString(tooltip); + out.writeBoolean(updatable); out.writeEnum(type); - out.writeOptionalStringCollection(uiRestrictions); - out.writeOptionalCollection(validations); - out.writeGenericValue(value); } public Map toMap() { Map map = new HashMap<>(); - Optional.ofNullable(category).ifPresent(c -> map.put(CATEGORY_FIELD.getPreferredName(), c)); map.put(DEFAULT_VALUE_FIELD.getPreferredName(), defaultValue); - - Optional.ofNullable(dependsOn) - .ifPresent(d -> map.put(DEPENDS_ON_FIELD.getPreferredName(), d.stream().map(SettingsConfigurationDependency::toMap).toList())); - - Optional.ofNullable(display).ifPresent(d -> map.put(DISPLAY_FIELD.getPreferredName(), d.toString())); + Optional.ofNullable(description).ifPresent(t -> map.put(DESCRIPTION_FIELD.getPreferredName(), t)); map.put(LABEL_FIELD.getPreferredName(), label); - Optional.ofNullable(options) - .ifPresent(o -> map.put(OPTIONS_FIELD.getPreferredName(), o.stream().map(SettingsConfigurationSelectOption::toMap).toList())); - - Optional.ofNullable(order).ifPresent(o -> map.put(ORDER_FIELD.getPreferredName(), o)); - - Optional.ofNullable(placeholder).ifPresent(p -> map.put(PLACEHOLDER_FIELD.getPreferredName(), p)); - map.put(REQUIRED_FIELD.getPreferredName(), required); map.put(SENSITIVE_FIELD.getPreferredName(), sensitive); - - Optional.ofNullable(tooltip).ifPresent(t -> map.put(TOOLTIP_FIELD.getPreferredName(), t)); + map.put(UPDATABLE_FIELD.getPreferredName(), updatable); Optional.ofNullable(type).ifPresent(t -> map.put(TYPE_FIELD.getPreferredName(), t.toString())); - Optional.ofNullable(uiRestrictions).ifPresent(u -> map.put(UI_RESTRICTIONS_FIELD.getPreferredName(), u)); - - Optional.ofNullable(validations) - .ifPresent(v -> map.put(VALIDATIONS_FIELD.getPreferredName(), v.stream().map(SettingsConfigurationValidation::toMap).toList())); - - map.put(VALUE_FIELD.getPreferredName(), value); - return map; } @@ -440,77 +263,35 @@ public boolean equals(Object o) { SettingsConfiguration that = (SettingsConfiguration) o; return required == that.required && sensitive == that.sensitive - && Objects.equals(category, that.category) + && updatable == that.updatable && Objects.equals(defaultValue, that.defaultValue) - && Objects.equals(dependsOn, that.dependsOn) - && display == that.display + && Objects.equals(description, that.description) && Objects.equals(label, that.label) - && Objects.equals(options, that.options) - && Objects.equals(order, that.order) - && Objects.equals(placeholder, that.placeholder) - && Objects.equals(tooltip, that.tooltip) - && type == that.type - && Objects.equals(uiRestrictions, that.uiRestrictions) - && Objects.equals(validations, that.validations) - && Objects.equals(value, that.value); + && type == that.type; } @Override public int hashCode() { - return Objects.hash( - category, - defaultValue, - dependsOn, - display, - label, - options, - order, - placeholder, - required, - sensitive, - tooltip, - type, - uiRestrictions, - validations, - value - ); + return Objects.hash(defaultValue, description, label, required, sensitive, updatable, type); } public static class Builder { - private String category; private Object defaultValue; - private List dependsOn; - private SettingsConfigurationDisplayType display; + private String description; private String label; - private List options; - private Integer order; - private String placeholder; private boolean required; private boolean sensitive; - private String tooltip; + private boolean updatable; private SettingsConfigurationFieldType type; - private List uiRestrictions; - private List validations; - private Object value; - - public Builder setCategory(String category) { - this.category = category; - return this; - } public Builder setDefaultValue(Object defaultValue) { this.defaultValue = defaultValue; return this; } - public Builder setDependsOn(List dependsOn) { - this.dependsOn = dependsOn; - return this; - } - - public Builder setDisplay(SettingsConfigurationDisplayType display) { - this.display = display; + public Builder setDescription(String description) { + this.description = description; return this; } @@ -519,21 +300,6 @@ public Builder setLabel(String label) { return this; } - public Builder setOptions(List options) { - this.options = options; - return this; - } - - public Builder setOrder(Integer order) { - this.order = order; - return this; - } - - public Builder setPlaceholder(String placeholder) { - this.placeholder = placeholder; - return this; - } - public Builder setRequired(Boolean required) { this.required = Objects.requireNonNullElse(required, false); return this; @@ -544,8 +310,8 @@ public Builder setSensitive(Boolean sensitive) { return this; } - public Builder setTooltip(String tooltip) { - this.tooltip = tooltip; + public Builder setUpdatable(Boolean updatable) { + this.updatable = Objects.requireNonNullElse(updatable, false); return this; } @@ -554,39 +320,8 @@ public Builder setType(SettingsConfigurationFieldType type) { return this; } - public Builder setUiRestrictions(List uiRestrictions) { - this.uiRestrictions = uiRestrictions; - return this; - } - - public Builder setValidations(List validations) { - this.validations = validations; - return this; - } - - public Builder setValue(Object value) { - this.value = value; - return this; - } - public SettingsConfiguration build() { - return new SettingsConfiguration( - category, - defaultValue, - dependsOn, - display, - label, - options, - order, - placeholder, - required, - sensitive, - tooltip, - type, - uiRestrictions, - validations, - value - ); + return new SettingsConfiguration(defaultValue, description, label, required, sensitive, updatable, type); } } } diff --git a/server/src/main/java/org/elasticsearch/inference/TaskSettingsConfiguration.java b/server/src/main/java/org/elasticsearch/inference/TaskSettingsConfiguration.java deleted file mode 100644 index 150532f138e8d..0000000000000 --- a/server/src/main/java/org/elasticsearch/inference/TaskSettingsConfiguration.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference; - -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; - -/** - * Represents the configuration field settings for a specific task type inference provider. - */ -public class TaskSettingsConfiguration implements Writeable, ToXContentObject { - - private final TaskType taskType; - private final Map configuration; - - /** - * Constructs a new {@link TaskSettingsConfiguration} instance with specified properties. - * - * @param taskType The {@link TaskType} this configuration describes. - * @param configuration The configuration of the task, defined by {@link SettingsConfiguration}. - */ - private TaskSettingsConfiguration(TaskType taskType, Map configuration) { - this.taskType = taskType; - this.configuration = configuration; - } - - public TaskSettingsConfiguration(StreamInput in) throws IOException { - this.taskType = in.readEnum(TaskType.class); - this.configuration = in.readMap(SettingsConfiguration::new); - } - - static final ParseField TASK_TYPE_FIELD = new ParseField("task_type"); - static final ParseField CONFIGURATION_FIELD = new ParseField("configuration"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "task_configuration", - true, - args -> { - return new TaskSettingsConfiguration.Builder().setTaskType(TaskType.fromString((String) args[0])) - .setConfiguration((Map) args[1]) - .build(); - } - ); - - static { - PARSER.declareString(constructorArg(), TASK_TYPE_FIELD); - PARSER.declareObject(constructorArg(), (p, c) -> p.map(), CONFIGURATION_FIELD); - } - - public TaskType getTaskType() { - return taskType; - } - - public Map getConfiguration() { - return configuration; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - { - builder.field(TASK_TYPE_FIELD.getPreferredName(), taskType); - builder.field(CONFIGURATION_FIELD.getPreferredName(), configuration); - } - builder.endObject(); - return builder; - } - - public static TaskSettingsConfiguration fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - - public static TaskSettingsConfiguration fromXContentBytes(BytesReference source, XContentType xContentType) { - try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { - return TaskSettingsConfiguration.fromXContent(parser); - } catch (IOException e) { - throw new ElasticsearchParseException("failed to parse task configuration", e); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeEnum(taskType); - out.writeMapValues(configuration); - } - - public Map toMap() { - Map map = new HashMap<>(); - - map.put(TASK_TYPE_FIELD.getPreferredName(), taskType); - map.put(CONFIGURATION_FIELD.getPreferredName(), configuration); - - return map; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TaskSettingsConfiguration that = (TaskSettingsConfiguration) o; - return Objects.equals(taskType, that.taskType) && Objects.equals(configuration, that.configuration); - } - - @Override - public int hashCode() { - return Objects.hash(taskType, configuration); - } - - public static class Builder { - - private TaskType taskType; - private Map configuration; - - public Builder setTaskType(TaskType taskType) { - this.taskType = taskType; - return this; - } - - public Builder setConfiguration(Map configuration) { - this.configuration = configuration; - return this; - } - - public TaskSettingsConfiguration build() { - return new TaskSettingsConfiguration(taskType, configuration); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDependency.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDependency.java deleted file mode 100644 index d319d1a395f85..0000000000000 --- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDependency.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference.configuration; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ObjectParser; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParseException; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; - -/** - * Represents a dependency within a connector configuration, defining a specific field and its associated value. - * This class is used to encapsulate configuration dependencies in a structured format. - */ -public class SettingsConfigurationDependency implements Writeable, ToXContentObject { - - private final String field; - private final Object value; - - /** - * Constructs a new instance of SettingsConfigurationDependency. - * - * @param field The name of the field in the service dependency. - * @param value The value associated with the field. - */ - public SettingsConfigurationDependency(String field, Object value) { - this.field = field; - this.value = value; - } - - public SettingsConfigurationDependency(StreamInput in) throws IOException { - this.field = in.readString(); - this.value = in.readGenericValue(); - } - - private static final ParseField FIELD_FIELD = new ParseField("field"); - private static final ParseField VALUE_FIELD = new ParseField("value"); - - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "service_configuration_dependency", - true, - args -> new SettingsConfigurationDependency.Builder().setField((String) args[0]).setValue(args[1]).build() - ); - - static { - PARSER.declareString(constructorArg(), FIELD_FIELD); - PARSER.declareField(constructorArg(), (p, c) -> { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return p.text(); - } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) { - return p.numberValue(); - } else if (p.currentToken() == XContentParser.Token.VALUE_BOOLEAN) { - return p.booleanValue(); - } else if (p.currentToken() == XContentParser.Token.VALUE_NULL) { - return null; - } - throw new XContentParseException("Unsupported token [" + p.currentToken() + "]"); - }, VALUE_FIELD, ObjectParser.ValueType.VALUE); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - { - builder.field(FIELD_FIELD.getPreferredName(), field); - builder.field(VALUE_FIELD.getPreferredName(), value); - } - builder.endObject(); - return builder; - } - - public Map toMap() { - Map map = new HashMap<>(); - map.put(FIELD_FIELD.getPreferredName(), field); - map.put(VALUE_FIELD.getPreferredName(), value); - return map; - } - - public static SettingsConfigurationDependency fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(field); - out.writeGenericValue(value); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SettingsConfigurationDependency that = (SettingsConfigurationDependency) o; - return Objects.equals(field, that.field) && Objects.equals(value, that.value); - } - - @Override - public int hashCode() { - return Objects.hash(field, value); - } - - public static class Builder { - - private String field; - private Object value; - - public Builder setField(String field) { - this.field = field; - return this; - } - - public Builder setValue(Object value) { - this.value = value; - return this; - } - - public SettingsConfigurationDependency build() { - return new SettingsConfigurationDependency(field, value); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayType.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayType.java deleted file mode 100644 index e072238a52d01..0000000000000 --- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayType.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference.configuration; - -import java.util.Locale; - -public enum SettingsConfigurationDisplayType { - TEXT, - TEXTBOX, - TEXTAREA, - NUMERIC, - TOGGLE, - DROPDOWN; - - @Override - public String toString() { - return name().toLowerCase(Locale.ROOT); - } - - public static SettingsConfigurationDisplayType displayType(String type) { - for (SettingsConfigurationDisplayType displayType : SettingsConfigurationDisplayType.values()) { - if (displayType.name().equalsIgnoreCase(type)) { - return displayType; - } - } - throw new IllegalArgumentException("Unknown " + SettingsConfigurationDisplayType.class.getSimpleName() + " [" + type + "]."); - } -} diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationSelectOption.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationSelectOption.java deleted file mode 100644 index 8ad8d561da58e..0000000000000 --- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationSelectOption.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference.configuration; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ObjectParser; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParseException; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; - -public class SettingsConfigurationSelectOption implements Writeable, ToXContentObject { - private final String label; - private final Object value; - - private SettingsConfigurationSelectOption(String label, Object value) { - this.label = label; - this.value = value; - } - - public SettingsConfigurationSelectOption(StreamInput in) throws IOException { - this.label = in.readString(); - this.value = in.readGenericValue(); - } - - private static final ParseField LABEL_FIELD = new ParseField("label"); - private static final ParseField VALUE_FIELD = new ParseField("value"); - - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "service_configuration_select_option", - true, - args -> new SettingsConfigurationSelectOption.Builder().setLabel((String) args[0]).setValue(args[1]).build() - ); - - static { - PARSER.declareString(constructorArg(), LABEL_FIELD); - PARSER.declareField(constructorArg(), (p, c) -> { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return p.text(); - } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) { - return p.numberValue(); - } - throw new XContentParseException("Unsupported token [" + p.currentToken() + "]"); - }, VALUE_FIELD, ObjectParser.ValueType.VALUE); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - { - builder.field(LABEL_FIELD.getPreferredName(), label); - builder.field(VALUE_FIELD.getPreferredName(), value); - } - builder.endObject(); - return builder; - } - - public Map toMap() { - Map map = new HashMap<>(); - map.put(LABEL_FIELD.getPreferredName(), label); - map.put(VALUE_FIELD.getPreferredName(), value); - return map; - } - - public static SettingsConfigurationSelectOption fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(label); - out.writeGenericValue(value); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SettingsConfigurationSelectOption that = (SettingsConfigurationSelectOption) o; - return Objects.equals(label, that.label) && Objects.equals(value, that.value); - } - - @Override - public int hashCode() { - return Objects.hash(label, value); - } - - public static class Builder { - - private String label; - private Object value; - - public Builder setLabel(String label) { - this.label = label; - return this; - } - - public Builder setValue(Object value) { - this.value = value; - return this; - } - - public Builder setLabelAndValue(String labelAndValue) { - this.label = labelAndValue; - this.value = labelAndValue; - return this; - } - - public SettingsConfigurationSelectOption build() { - return new SettingsConfigurationSelectOption(label, value); - } - } - -} diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidation.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidation.java deleted file mode 100644 index f106442d6d4ac..0000000000000 --- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidation.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference.configuration; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ObjectParser; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParseException; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; - -/** - * Represents a configuration validation entity, encapsulating a validation constraint and its corresponding type. - * This class is used to define and handle specific validation rules or requirements within a configuration context. - */ -public class SettingsConfigurationValidation implements Writeable, ToXContentObject { - - private final Object constraint; - private final SettingsConfigurationValidationType type; - - /** - * Constructs a new SettingsConfigurationValidation instance with specified constraint and type. - * This constructor initializes the object with a given validation constraint and its associated validation type. - * - * @param constraint The validation constraint (string, number or list), represented as generic Object type. - * @param type The type of configuration validation, specified as an instance of {@link SettingsConfigurationValidationType}. - */ - private SettingsConfigurationValidation(Object constraint, SettingsConfigurationValidationType type) { - this.constraint = constraint; - this.type = type; - } - - public SettingsConfigurationValidation(StreamInput in) throws IOException { - this.constraint = in.readGenericValue(); - this.type = in.readEnum(SettingsConfigurationValidationType.class); - } - - private static final ParseField CONSTRAINT_FIELD = new ParseField("constraint"); - private static final ParseField TYPE_FIELD = new ParseField("type"); - - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "service_configuration_validation", - true, - args -> new SettingsConfigurationValidation.Builder().setConstraint(args[0]) - .setType((SettingsConfigurationValidationType) args[1]) - .build() - ); - - static { - PARSER.declareField( - constructorArg(), - (p, c) -> parseConstraintValue(p), - CONSTRAINT_FIELD, - ObjectParser.ValueType.VALUE_OBJECT_ARRAY - ); - PARSER.declareField( - constructorArg(), - (p, c) -> SettingsConfigurationValidationType.validationType(p.text()), - TYPE_FIELD, - ObjectParser.ValueType.STRING - ); - } - - /** - * Parses the value of a constraint from the XContentParser stream. - * This method is designed to handle various types of constraint values as per the connector's protocol original specification. - * The constraints can be of type string, number, or list of values. - */ - private static Object parseConstraintValue(XContentParser p) throws IOException { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return p.text(); - } else if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) { - return p.numberValue(); - } else if (p.currentToken() == XContentParser.Token.START_ARRAY) { - return p.list(); - } - throw new XContentParseException("Unsupported token [" + p.currentToken() + "]"); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - { - builder.field(CONSTRAINT_FIELD.getPreferredName(), constraint); - builder.field(TYPE_FIELD.getPreferredName(), type.toString()); - } - builder.endObject(); - return builder; - } - - public Map toMap() { - return Map.of(CONSTRAINT_FIELD.getPreferredName(), constraint, TYPE_FIELD.getPreferredName(), type.toString()); - } - - public static SettingsConfigurationValidation fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeGenericValue(constraint); - out.writeEnum(type); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SettingsConfigurationValidation that = (SettingsConfigurationValidation) o; - return Objects.equals(constraint, that.constraint) && type == that.type; - } - - @Override - public int hashCode() { - return Objects.hash(constraint, type); - } - - public static class Builder { - - private Object constraint; - private SettingsConfigurationValidationType type; - - public Builder setConstraint(Object constraint) { - this.constraint = constraint; - return this; - } - - public Builder setType(SettingsConfigurationValidationType type) { - this.type = type; - return this; - } - - public SettingsConfigurationValidation build() { - return new SettingsConfigurationValidation(constraint, type); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationType.java b/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationType.java deleted file mode 100644 index 6fb07d38d7db5..0000000000000 --- a/server/src/main/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationType.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference.configuration; - -import java.util.Locale; - -public enum SettingsConfigurationValidationType { - LESS_THAN, - GREATER_THAN, - LIST_TYPE, - INCLUDED_IN, - REGEX; - - @Override - public String toString() { - return name().toLowerCase(Locale.ROOT); - } - - public static SettingsConfigurationValidationType validationType(String type) { - for (SettingsConfigurationValidationType displayType : SettingsConfigurationValidationType.values()) { - if (displayType.name().equalsIgnoreCase(type)) { - return displayType; - } - } - throw new IllegalArgumentException("Unknown " + SettingsConfigurationValidationType.class.getSimpleName() + " [" + type + "]."); - } -} diff --git a/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTestUtils.java b/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTestUtils.java index 8d145202f7165..2d7fe0dcb5558 100644 --- a/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTestUtils.java +++ b/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTestUtils.java @@ -9,8 +9,8 @@ package org.elasticsearch.inference; +import java.util.EnumSet; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; @@ -19,14 +19,16 @@ public class InferenceServiceConfigurationTestUtils { public static InferenceServiceConfiguration getRandomServiceConfigurationField() { - return new InferenceServiceConfiguration.Builder().setProvider(randomAlphaOfLength(10)) - .setTaskTypes(getRandomTaskTypeConfiguration()) - .setConfiguration(getRandomServiceConfiguration(10)) + return new InferenceServiceConfiguration.Builder().setService(randomAlphaOfLength(10)) + .setName(randomAlphaOfLength(6)) + .setTaskTypes(getRandomTaskTypes()) + .setConfigurations(getRandomServiceConfiguration(10)) .build(); } - private static List getRandomTaskTypeConfiguration() { - return List.of(TaskSettingsConfigurationTestUtils.getRandomTaskSettingsConfigurationField()); + private static EnumSet getRandomTaskTypes() { + TaskType[] values = TaskType.values(); + return EnumSet.of(values[randomInt(values.length - 1)]); } private static Map getRandomServiceConfiguration(int numFields) { diff --git a/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTests.java b/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTests.java index 7d97f85360c57..490ed68ab3e66 100644 --- a/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/inference/InferenceServiceConfigurationTests.java @@ -28,139 +28,26 @@ public class InferenceServiceConfigurationTests extends ESTestCase { public void testToXContent() throws IOException { String content = XContentHelper.stripWhitespace(""" { - "provider": "some_provider", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": { + "service": "some_provider", + "name": "Some Provider", + "task_types": ["text_embedding", "completion"], + "configurations": { "text_field_configuration": { - "default_value": null, - "depends_on": [ - { - "field": "some_field", - "value": true - } - ], - "display": "textbox", + "description": "Wow, this tooltip is useful.", "label": "Very important field", - "options": [], - "order": 4, "required": true, "sensitive": true, - "tooltip": "Wow, this tooltip is useful.", - "type": "str", - "ui_restrictions": [], - "validations": null, - "value": "" + "updatable": false, + "type": "str" }, "numeric_field_configuration": { "default_value": 3, - "depends_on": null, - "display": "numeric", + "description": "Wow, this tooltip is useful.", "label": "Very important numeric field", - "options": [], - "order": 2, "required": true, "sensitive": false, - "tooltip": "Wow, this tooltip is useful.", - "type": "int", - "ui_restrictions": [], - "validations": [ - { - "constraint": 0, - "type": "greater_than" - } - ], - "value": "" - } - } - }, - { - "task_type": "completion", - "configuration": { - "text_field_configuration": { - "default_value": null, - "depends_on": [ - { - "field": "some_field", - "value": true - } - ], - "display": "textbox", - "label": "Very important field", - "options": [], - "order": 4, - "required": true, - "sensitive": true, - "tooltip": "Wow, this tooltip is useful.", - "type": "str", - "ui_restrictions": [], - "validations": null, - "value": "" - }, - "numeric_field_configuration": { - "default_value": 3, - "depends_on": null, - "display": "numeric", - "label": "Very important numeric field", - "options": [], - "order": 2, - "required": true, - "sensitive": false, - "tooltip": "Wow, this tooltip is useful.", - "type": "int", - "ui_restrictions": [], - "validations": [ - { - "constraint": 0, - "type": "greater_than" - } - ], - "value": "" - } - } - } - ], - "configuration": { - "text_field_configuration": { - "default_value": null, - "depends_on": [ - { - "field": "some_field", - "value": true - } - ], - "display": "textbox", - "label": "Very important field", - "options": [], - "order": 4, - "required": true, - "sensitive": true, - "tooltip": "Wow, this tooltip is useful.", - "type": "str", - "ui_restrictions": [], - "validations": null, - "value": "" - }, - "numeric_field_configuration": { - "default_value": 3, - "depends_on": null, - "display": "numeric", - "label": "Very important numeric field", - "options": [], - "order": 2, - "required": true, - "sensitive": false, - "tooltip": "Wow, this tooltip is useful.", - "type": "int", - "ui_restrictions": [], - "validations": [ - { - "constraint": 0, - "type": "greater_than" - } - ], - "value": "" + "updatable": true, + "type": "int" } } } @@ -183,8 +70,9 @@ public void testToMap() { InferenceServiceConfiguration configField = InferenceServiceConfigurationTestUtils.getRandomServiceConfigurationField(); Map configFieldAsMap = configField.toMap(); - assertThat(configFieldAsMap.get("provider"), equalTo(configField.getProvider())); + assertThat(configFieldAsMap.get("service"), equalTo(configField.getService())); + assertThat(configFieldAsMap.get("name"), equalTo(configField.getName())); assertThat(configFieldAsMap.get("task_types"), equalTo(configField.getTaskTypes())); - assertThat(configFieldAsMap.get("configuration"), equalTo(configField.getConfiguration())); + assertThat(configFieldAsMap.get("configurations"), equalTo(configField.getConfigurations())); } } diff --git a/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTestUtils.java b/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTestUtils.java index 728dafc5383c1..ed78baeb9abe6 100644 --- a/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTestUtils.java +++ b/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTestUtils.java @@ -9,14 +9,7 @@ package org.elasticsearch.inference; -import org.elasticsearch.inference.configuration.SettingsConfigurationDependency; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; -import org.elasticsearch.inference.configuration.SettingsConfigurationValidation; -import org.elasticsearch.inference.configuration.SettingsConfigurationValidationType; - -import java.util.List; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomBoolean; @@ -25,50 +18,18 @@ public class SettingsConfigurationTestUtils { public static SettingsConfiguration getRandomSettingsConfigurationField() { - return new SettingsConfiguration.Builder().setCategory(randomAlphaOfLength(10)) - .setDefaultValue(randomAlphaOfLength(10)) - .setDependsOn(List.of(getRandomSettingsConfigurationDependency())) - .setDisplay(getRandomSettingsConfigurationDisplayType()) + return new SettingsConfiguration.Builder().setDefaultValue(randomAlphaOfLength(10)) + .setDescription(randomAlphaOfLength(10)) .setLabel(randomAlphaOfLength(10)) - .setOptions(List.of(getRandomSettingsConfigurationSelectOption(), getRandomSettingsConfigurationSelectOption())) - .setOrder(randomInt()) - .setPlaceholder(randomAlphaOfLength(10)) .setRequired(randomBoolean()) .setSensitive(randomBoolean()) - .setTooltip(randomAlphaOfLength(10)) + .setUpdatable(randomBoolean()) .setType(getRandomConfigurationFieldType()) - .setUiRestrictions(List.of(randomAlphaOfLength(10), randomAlphaOfLength(10))) - .setValidations(List.of(getRandomSettingsConfigurationValidation())) - .setValue(randomAlphaOfLength(10)) - .build(); - } - - private static SettingsConfigurationDependency getRandomSettingsConfigurationDependency() { - return new SettingsConfigurationDependency.Builder().setField(randomAlphaOfLength(10)).setValue(randomAlphaOfLength(10)).build(); - } - - private static SettingsConfigurationSelectOption getRandomSettingsConfigurationSelectOption() { - return new SettingsConfigurationSelectOption.Builder().setLabel(randomAlphaOfLength(10)).setValue(randomAlphaOfLength(10)).build(); - } - - private static SettingsConfigurationValidation getRandomSettingsConfigurationValidation() { - return new SettingsConfigurationValidation.Builder().setConstraint(randomAlphaOfLength(10)) - .setType(getRandomConfigurationValidationType()) .build(); } - public static SettingsConfigurationDisplayType getRandomSettingsConfigurationDisplayType() { - SettingsConfigurationDisplayType[] values = SettingsConfigurationDisplayType.values(); - return values[randomInt(values.length - 1)]; - } - public static SettingsConfigurationFieldType getRandomConfigurationFieldType() { SettingsConfigurationFieldType[] values = SettingsConfigurationFieldType.values(); return values[randomInt(values.length - 1)]; } - - public static SettingsConfigurationValidationType getRandomConfigurationValidationType() { - SettingsConfigurationValidationType[] values = SettingsConfigurationValidationType.values(); - return values[randomInt(values.length - 1)]; - } } diff --git a/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTests.java b/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTests.java index e1293366a1152..551a25fe52f18 100644 --- a/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/inference/SettingsConfigurationTests.java @@ -12,16 +12,12 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.inference.configuration.SettingsConfigurationDependency; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; -import org.elasticsearch.inference.configuration.SettingsConfigurationValidation; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; @@ -33,29 +29,12 @@ public class SettingsConfigurationTests extends ESTestCase { public void testToXContent() throws IOException { String content = XContentHelper.stripWhitespace(""" { - "default_value": null, - "depends_on": [ - { - "field": "some_field", - "value": true - } - ], - "display": "textbox", + "description": "Wow, this tooltip is useful.", "label": "Very important field", - "options": [], - "order": 4, "required": true, "sensitive": false, - "tooltip": "Wow, this tooltip is useful.", - "type": "str", - "ui_restrictions": [], - "validations": [ - { - "constraint": 0, - "type": "greater_than" - } - ], - "value": "" + "updatable": true, + "type": "str" } """); @@ -72,38 +51,12 @@ public void testToXContent() throws IOException { public void testToXContent_WithNumericSelectOptions() throws IOException { String content = XContentHelper.stripWhitespace(""" { - "default_value": null, - "depends_on": [ - { - "field": "some_field", - "value": true - } - ], - "display": "textbox", + "description": "Wow, this tooltip is useful.", "label": "Very important field", - "options": [ - { - "label": "five", - "value": 5 - }, - { - "label": "ten", - "value": 10 - } - ], - "order": 4, "required": true, "sensitive": false, - "tooltip": "Wow, this tooltip is useful.", - "type": "str", - "ui_restrictions": [], - "validations": [ - { - "constraint": 0, - "type": "greater_than" - } - ], - "value": "" + "updatable": true, + "type": "str" } """); @@ -135,153 +88,28 @@ public void testToXContentCrawlerConfig_WithNullValue() throws IOException { assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); } - public void testToXContentWithMultipleConstraintTypes() throws IOException { - String content = XContentHelper.stripWhitespace(""" - { - "default_value": null, - "depends_on": [ - { - "field": "some_field", - "value": true - } - ], - "display": "textbox", - "label": "Very important field", - "options": [], - "order": 4, - "required": true, - "sensitive": false, - "tooltip": "Wow, this tooltip is useful.", - "type": "str", - "ui_restrictions": [], - "validations": [ - { - "constraint": 32, - "type": "less_than" - }, - { - "constraint": "^\\\\\\\\d{4}-\\\\\\\\d{2}-\\\\\\\\d{2}$", - "type": "regex" - }, - { - "constraint": "int", - "type": "list_type" - }, - { - "constraint": [ - 1, - 2, - 3 - ], - "type": "included_in" - }, - { - "constraint": [ - "string_1", - "string_2", - "string_3" - ], - "type": "included_in" - } - ], - "value": "" - } - """); - - SettingsConfiguration configuration = SettingsConfiguration.fromXContentBytes(new BytesArray(content), XContentType.JSON); - boolean humanReadable = true; - BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable); - SettingsConfiguration parsed; - try (XContentParser parser = createParser(XContentType.JSON.xContent(), originalBytes)) { - parsed = SettingsConfiguration.fromXContent(parser); - } - assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); - } - public void testToMap() { SettingsConfiguration configField = SettingsConfigurationTestUtils.getRandomSettingsConfigurationField(); Map configFieldAsMap = configField.toMap(); - if (configField.getCategory() != null) { - assertThat(configFieldAsMap.get("category"), equalTo(configField.getCategory())); - } else { - assertFalse(configFieldAsMap.containsKey("category")); - } - assertThat(configFieldAsMap.get("default_value"), equalTo(configField.getDefaultValue())); - if (configField.getDependsOn() != null) { - List> dependsOnAsList = configField.getDependsOn() - .stream() - .map(SettingsConfigurationDependency::toMap) - .toList(); - assertThat(configFieldAsMap.get("depends_on"), equalTo(dependsOnAsList)); - } else { - assertFalse(configFieldAsMap.containsKey("depends_on")); - } - - if (configField.getDisplay() != null) { - assertThat(configFieldAsMap.get("display"), equalTo(configField.getDisplay().toString())); + if (configField.getDescription() != null) { + assertThat(configFieldAsMap.get("description"), equalTo(configField.getDescription())); } else { - assertFalse(configFieldAsMap.containsKey("display")); + assertFalse(configFieldAsMap.containsKey("description")); } assertThat(configFieldAsMap.get("label"), equalTo(configField.getLabel())); - if (configField.getOptions() != null) { - List> optionsAsList = configField.getOptions() - .stream() - .map(SettingsConfigurationSelectOption::toMap) - .toList(); - assertThat(configFieldAsMap.get("options"), equalTo(optionsAsList)); - } else { - assertFalse(configFieldAsMap.containsKey("options")); - } - - if (configField.getOrder() != null) { - assertThat(configFieldAsMap.get("order"), equalTo(configField.getOrder())); - } else { - assertFalse(configFieldAsMap.containsKey("order")); - } - - if (configField.getPlaceholder() != null) { - assertThat(configFieldAsMap.get("placeholder"), equalTo(configField.getPlaceholder())); - } else { - assertFalse(configFieldAsMap.containsKey("placeholder")); - } - assertThat(configFieldAsMap.get("required"), equalTo(configField.isRequired())); assertThat(configFieldAsMap.get("sensitive"), equalTo(configField.isSensitive())); - - if (configField.getTooltip() != null) { - assertThat(configFieldAsMap.get("tooltip"), equalTo(configField.getTooltip())); - } else { - assertFalse(configFieldAsMap.containsKey("tooltip")); - } + assertThat(configFieldAsMap.get("updatable"), equalTo(configField.isUpdatable())); if (configField.getType() != null) { assertThat(configFieldAsMap.get("type"), equalTo(configField.getType().toString())); } else { assertFalse(configFieldAsMap.containsKey("type")); } - - if (configField.getUiRestrictions() != null) { - assertThat(configFieldAsMap.get("ui_restrictions"), equalTo(configField.getUiRestrictions())); - } else { - assertFalse(configFieldAsMap.containsKey("ui_restrictions")); - } - - if (configField.getValidations() != null) { - List> validationsAsList = configField.getValidations() - .stream() - .map(SettingsConfigurationValidation::toMap) - .toList(); - assertThat(configFieldAsMap.get("validations"), equalTo(validationsAsList)); - } else { - assertFalse(configFieldAsMap.containsKey("validations")); - } - - assertThat(configFieldAsMap.get("value"), equalTo(configField.getValue())); - } } diff --git a/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTestUtils.java b/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTestUtils.java deleted file mode 100644 index 81abeaefd9f1a..0000000000000 --- a/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTestUtils.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference; - -import java.util.HashMap; -import java.util.Map; - -import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; -import static org.elasticsearch.test.ESTestCase.randomInt; - -public class TaskSettingsConfigurationTestUtils { - - public static TaskSettingsConfiguration getRandomTaskSettingsConfigurationField() { - return new TaskSettingsConfiguration.Builder().setTaskType(getRandomTaskType()) - .setConfiguration(getRandomServiceConfiguration(10)) - .build(); - } - - private static TaskType getRandomTaskType() { - TaskType[] values = TaskType.values(); - return values[randomInt(values.length - 1)]; - } - - private static Map getRandomServiceConfiguration(int numFields) { - var numConfigFields = randomInt(numFields); - Map configuration = new HashMap<>(); - for (int i = 0; i < numConfigFields; i++) { - configuration.put(randomAlphaOfLength(10), SettingsConfigurationTestUtils.getRandomSettingsConfigurationField()); - } - - return configuration; - } -} diff --git a/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTests.java b/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTests.java deleted file mode 100644 index d37fffc78ebd6..0000000000000 --- a/server/src/test/java/org/elasticsearch/inference/TaskSettingsConfigurationTests.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference; - -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; - -import java.io.IOException; -import java.util.Map; - -import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; -import static org.hamcrest.CoreMatchers.equalTo; - -public class TaskSettingsConfigurationTests extends ESTestCase { - public void testToXContent() throws IOException { - String content = XContentHelper.stripWhitespace(""" - { - "task_type": "text_embedding", - "configuration": { - "text_field_configuration": { - "default_value": null, - "depends_on": [ - { - "field": "some_field", - "value": true - } - ], - "display": "textbox", - "label": "Very important field", - "options": [], - "order": 4, - "required": true, - "sensitive": true, - "tooltip": "Wow, this tooltip is useful.", - "type": "str", - "ui_restrictions": [], - "validations": null, - "value": "" - }, - "numeric_field_configuration": { - "default_value": 3, - "depends_on": null, - "display": "numeric", - "label": "Very important numeric field", - "options": [], - "order": 2, - "required": true, - "sensitive": false, - "tooltip": "Wow, this tooltip is useful.", - "type": "int", - "ui_restrictions": [], - "validations": [ - { - "constraint": 0, - "type": "greater_than" - } - ], - "value": "" - } - } - } - """); - - TaskSettingsConfiguration configuration = TaskSettingsConfiguration.fromXContentBytes(new BytesArray(content), XContentType.JSON); - boolean humanReadable = true; - BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable); - TaskSettingsConfiguration parsed; - try (XContentParser parser = createParser(XContentType.JSON.xContent(), originalBytes)) { - parsed = TaskSettingsConfiguration.fromXContent(parser); - } - assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); - } - - public void testToMap() { - TaskSettingsConfiguration configField = TaskSettingsConfigurationTestUtils.getRandomTaskSettingsConfigurationField(); - Map configFieldAsMap = configField.toMap(); - - assertThat(configFieldAsMap.get("task_type"), equalTo(configField.getTaskType())); - assertThat(configFieldAsMap.get("configuration"), equalTo(configField.getConfiguration())); - } -} diff --git a/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayTypeTests.java b/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayTypeTests.java deleted file mode 100644 index 603ea9480783c..0000000000000 --- a/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationDisplayTypeTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference.configuration; - -import org.elasticsearch.inference.SettingsConfigurationTestUtils; -import org.elasticsearch.test.ESTestCase; - -import static org.hamcrest.Matchers.equalTo; - -public class SettingsConfigurationDisplayTypeTests extends ESTestCase { - - public void testDisplayType_WithValidConfigurationDisplayTypeString() { - SettingsConfigurationDisplayType displayType = SettingsConfigurationTestUtils.getRandomSettingsConfigurationDisplayType(); - assertThat(SettingsConfigurationDisplayType.displayType(displayType.toString()), equalTo(displayType)); - } - - public void testDisplayType_WithInvalidConfigurationDisplayTypeString_ExpectIllegalArgumentException() { - expectThrows( - IllegalArgumentException.class, - () -> SettingsConfigurationDisplayType.displayType("invalid configuration display type") - ); - } -} diff --git a/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationTypeTests.java b/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationTypeTests.java deleted file mode 100644 index d35968004ea0d..0000000000000 --- a/server/src/test/java/org/elasticsearch/inference/configuration/SettingsConfigurationValidationTypeTests.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.inference.configuration; - -import org.elasticsearch.inference.SettingsConfigurationTestUtils; -import org.elasticsearch.test.ESTestCase; - -import static org.hamcrest.Matchers.equalTo; - -public class SettingsConfigurationValidationTypeTests extends ESTestCase { - - public void testValidationType_WithValidConfigurationValidationTypeString() { - SettingsConfigurationValidationType validationType = SettingsConfigurationTestUtils.getRandomConfigurationValidationType(); - - assertThat(SettingsConfigurationValidationType.validationType(validationType.toString()), equalTo(validationType)); - } - - public void testValidationType_WithInvalidConfigurationValidationTypeString_ExpectIllegalArgumentException() { - assertThrows(IllegalArgumentException.class, () -> SettingsConfigurationValidationType.validationType("invalid validation type")); - } - -} diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 2f8cfc8f3e659..58d870ceed6f2 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -158,11 +158,9 @@ public void testGetServicesWithoutTaskType() throws IOException { String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { Map serviceConfig = (Map) services.get(i); - providers[i] = (String) serviceConfig.get("provider"); + providers[i] = (String) serviceConfig.get("service"); } - Arrays.sort(providers); - var providerList = new ArrayList<>( Arrays.asList( "alibabacloud-ai-search", @@ -200,10 +198,9 @@ public void testGetServicesWithTextEmbeddingTaskType() throws IOException { String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { Map serviceConfig = (Map) services.get(i); - providers[i] = (String) serviceConfig.get("provider"); + providers[i] = (String) serviceConfig.get("service"); } - Arrays.sort(providers); assertArrayEquals( List.of( "alibabacloud-ai-search", @@ -233,10 +230,9 @@ public void testGetServicesWithRerankTaskType() throws IOException { String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { Map serviceConfig = (Map) services.get(i); - providers[i] = (String) serviceConfig.get("provider"); + providers[i] = (String) serviceConfig.get("service"); } - Arrays.sort(providers); assertArrayEquals( List.of("alibabacloud-ai-search", "cohere", "elasticsearch", "googlevertexai", "jinaai", "test_reranking_service").toArray(), providers @@ -251,10 +247,9 @@ public void testGetServicesWithCompletionTaskType() throws IOException { String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { Map serviceConfig = (Map) services.get(i); - providers[i] = (String) serviceConfig.get("provider"); + providers[i] = (String) serviceConfig.get("service"); } - Arrays.sort(providers); assertArrayEquals( providers, List.of( @@ -285,11 +280,9 @@ public void testGetServicesWithSparseEmbeddingTaskType() throws IOException { String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { Map serviceConfig = (Map) services.get(i); - providers[i] = (String) serviceConfig.get("provider"); + providers[i] = (String) serviceConfig.get("service"); } - Arrays.sort(providers); - var providerList = new ArrayList<>(Arrays.asList("alibabacloud-ai-search", "elasticsearch", "hugging_face", "test_service")); if ((ElasticInferenceServiceFeature.DEPRECATED_ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled() || ElasticInferenceServiceFeature.ELASTIC_INFERENCE_SERVICE_FEATURE_FLAG.isEnabled())) { diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java index a6888f28159f4..89c79dd148598 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java @@ -18,7 +18,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceResults; @@ -29,10 +28,8 @@ import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnifiedCompletionRequest; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; @@ -260,23 +257,18 @@ public static InferenceServiceConfiguration get() { configurationMap.put( "model", - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("") .setLabel("Model") - .setOrder(1) .setRequired(true) .setSensitive(true) - .setTooltip("") .setType(SettingsConfigurationFieldType.STRING) .build() ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java index bbb773aa5129a..77c762a38baaf 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java @@ -17,7 +17,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceResults; @@ -27,10 +26,8 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnifiedCompletionRequest; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; @@ -174,23 +171,18 @@ public static InferenceServiceConfiguration get() { configurationMap.put( "model", - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("") .setLabel("Model") - .setOrder(1) .setRequired(true) .setSensitive(true) - .setTooltip("") .setType(SettingsConfigurationFieldType.STRING) .build() ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java index eea64304f503a..bef0b1812beda 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java @@ -17,7 +17,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceResults; @@ -27,10 +26,8 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnifiedCompletionRequest; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; @@ -208,35 +205,28 @@ public static InferenceServiceConfiguration get() { configurationMap.put( "model", - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("") .setLabel("Model") - .setOrder(1) .setRequired(true) .setSensitive(false) - .setTooltip("") .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( "hidden_field", - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("") .setLabel("Hidden Field") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("") .setType(SettingsConfigurationFieldType.STRING) .build() ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java index a39be00d9e7fa..e071b704c233e 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceResults; @@ -28,10 +27,8 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnifiedCompletionRequest; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; @@ -260,23 +257,18 @@ public static InferenceServiceConfiguration get() { configurationMap.put( "model_id", - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("") .setLabel("Model ID") - .setOrder(1) .setRequired(true) .setSensitive(true) - .setTooltip("") .setType(SettingsConfigurationFieldType.STRING) .build() ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceServicesAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceServicesAction.java index 002b2b0fe93b0..0d8d7e81019a6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceServicesAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceServicesAction.java @@ -21,9 +21,9 @@ import org.elasticsearch.xpack.core.inference.action.GetInferenceServicesAction; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; public class TransportGetInferenceServicesAction extends HandledTransportAction< @@ -72,7 +72,8 @@ private void getServiceConfigurationsForTaskType( service -> service.getValue().hideFromConfigurationApi() == false && service.getValue().supportedTaskTypes().contains(requestedTaskType) ) - .collect(Collectors.toSet()); + .sorted(Comparator.comparing(service -> service.getValue().name())) + .collect(Collectors.toCollection(ArrayList::new)); getServiceConfigurationsForServices(filteredServices, listener.delegateFailureAndWrap((delegate, configurations) -> { delegate.onResponse(new GetInferenceServicesAction.Response(configurations)); @@ -84,14 +85,15 @@ private void getAllServiceConfigurations(ActionListener service.getValue().hideFromConfigurationApi() == false) - .collect(Collectors.toSet()); + .sorted(Comparator.comparing(service -> service.getValue().name())) + .collect(Collectors.toCollection(ArrayList::new)); getServiceConfigurationsForServices(availableServices, listener.delegateFailureAndWrap((delegate, configurations) -> { delegate.onResponse(new GetInferenceServicesAction.Response(configurations)); })); } private void getServiceConfigurationsForServices( - Set> services, + ArrayList> services, ActionListener> listener ) { try { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java index 42a276c6ee838..24f7fa182b7c2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -25,11 +24,8 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; @@ -56,7 +52,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Stream; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; @@ -73,6 +68,7 @@ public class AlibabaCloudSearchService extends SenderService { public static final String NAME = AlibabaCloudSearchUtils.SERVICE_NAME; + private static final String SERVICE_NAME = "AlibabaCloud AI Search"; private static final EnumSet supportedTaskTypes = EnumSet.of( TaskType.TEXT_EMBEDDING, @@ -383,86 +379,62 @@ public static InferenceServiceConfiguration get() { configurationMap.put( SERVICE_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) + new SettingsConfiguration.Builder().setDescription("The name of the model service to use for the {infer} task.") .setLabel("Project ID") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("The name of the model service to use for the {infer} task.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of( - "ops-text-embedding-001", - "ops-text-embedding-zh-001", - "ops-text-embedding-en-001", - "ops-text-embedding-002", - "ops-text-sparse-embedding-001", - "ops-bge-reranker-larger" - ).map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()).toList() - ) .build() ); configurationMap.put( HOST, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription( + "The name of the host address used for the {infer} task. You can find the host address at " + + "https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] " + + "of the documentation." + ) .setLabel("Host") - .setOrder(3) .setRequired(true) .setSensitive(false) - .setTooltip( - "The name of the host address used for the {infer} task. You can find the host address at " - + "https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] " - + "of the documentation." - ) + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( HTTP_SCHEMA_NAME, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) + new SettingsConfiguration.Builder().setDescription("") .setLabel("HTTP Schema") - .setOrder(4) .setRequired(true) .setSensitive(false) - .setTooltip("") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("https", "http") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) .build() ); configurationMap.put( WORKSPACE_NAME, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The name of the workspace used for the {infer} task.") .setLabel("Workspace") - .setOrder(5) .setRequired(true) .setSensitive(false) - .setTooltip("The name of the workspace used for the {infer} task.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.putAll( - DefaultSecretSettings.toSettingsConfigurationWithTooltip("A valid API key for the AlibabaCloud AI Search API.") + DefaultSecretSettings.toSettingsConfigurationWithDescription("A valid API key for the AlibabaCloud AI Search API.") ); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case TEXT_EMBEDDING -> taskSettingsConfig = AlibabaCloudSearchEmbeddingsModel.Configuration.get(); - case SPARSE_EMBEDDING -> taskSettingsConfig = AlibabaCloudSearchSparseModel.Configuration.get(); - // COMPLETION, RERANK task types have no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java index 1bcc802ab18ea..2654ee4d22ce6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/embeddings/AlibabaCloudSearchEmbeddingsModel.java @@ -7,28 +7,19 @@ package org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor; -import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchEmbeddingsRequestEntity; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.stream.Stream; public class AlibabaCloudSearchEmbeddingsModel extends AlibabaCloudSearchModel { public static AlibabaCloudSearchEmbeddingsModel of( @@ -114,35 +105,4 @@ public DefaultSecretSettings getSecretSettings() { public ExecutableAction accept(AlibabaCloudSearchActionVisitor visitor, Map taskSettings, InputType inputType) { return visitor.create(this, taskSettings, inputType); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - AlibabaCloudSearchEmbeddingsRequestEntity.INPUT_TYPE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) - .setLabel("Input Type") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the type of input passed to the model.") - .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("ingest", "search") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) - .setValue("") - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java index 95bf500434c5a..0155d8fbc1f08 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/sparse/AlibabaCloudSearchSparseModel.java @@ -7,28 +7,19 @@ package org.elasticsearch.xpack.inference.services.alibabacloudsearch.sparse; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor; -import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchSparseRequestEntity; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.stream.Stream; public class AlibabaCloudSearchSparseModel extends AlibabaCloudSearchModel { public static AlibabaCloudSearchSparseModel of( @@ -108,50 +99,4 @@ public DefaultSecretSettings getSecretSettings() { public ExecutableAction accept(AlibabaCloudSearchActionVisitor visitor, Map taskSettings, InputType inputType) { return visitor.create(this, taskSettings, inputType); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - AlibabaCloudSearchSparseRequestEntity.INPUT_TYPE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) - .setLabel("Input Type") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the type of input passed to the model.") - .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("ingest", "search") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) - .setValue("") - .build() - ); - configurationMap.put( - AlibabaCloudSearchSparseRequestEntity.RETURN_TOKEN_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE) - .setLabel("Return Token") - .setOrder(2) - .setRequired(false) - .setSensitive(false) - .setTooltip( - "If `true`, the token name will be returned in the response. Defaults to `false` which means only the " - + "token ID will be returned in the response." - ) - .setType(SettingsConfigurationFieldType.BOOLEAN) - .setValue(true) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java index b5818d7e4a287..2105da235babe 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockSecretSettings.java @@ -18,7 +18,6 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SecretSettings; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xcontent.XContentBuilder; @@ -129,23 +128,21 @@ public static Map get() { var configurationMap = new HashMap(); configurationMap.put( ACCESS_KEY_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("A valid AWS access key that has permissions to use Amazon Bedrock.") .setLabel("Access Key") - .setOrder(1) .setRequired(true) .setSensitive(true) - .setTooltip("A valid AWS access key that has permissions to use Amazon Bedrock.") + .setUpdatable(true) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( SECRET_KEY_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("A valid AWS secret key that is paired with the access_key.") .setLabel("Secret Key") - .setOrder(2) .setRequired(true) .setSensitive(true) - .setTooltip("A valid AWS secret key that is paired with the access_key.") + .setUpdatable(true) .setType(SettingsConfigurationFieldType.STRING) .build() ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java index a88881220f933..07c5e91776192 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java @@ -18,7 +18,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -26,11 +25,8 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; @@ -57,7 +53,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Stream; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; @@ -77,6 +72,7 @@ public class AmazonBedrockService extends SenderService { public static final String NAME = "amazonbedrock"; + private static final String SERVICE_NAME = "Amazon Bedrock"; private final Sender amazonBedrockSender; @@ -382,61 +378,51 @@ public static InferenceServiceConfiguration get() { configurationMap.put( PROVIDER_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) + new SettingsConfiguration.Builder().setDescription("The model provider for your deployment.") .setLabel("Provider") - .setOrder(3) .setRequired(true) .setSensitive(false) - .setTooltip("The model provider for your deployment.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("amazontitan", "anthropic", "ai21labs", "cohere", "meta", "mistral") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) .build() ); configurationMap.put( MODEL_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription( + "The base model ID or an ARN to a custom model based on a foundational model." + ) .setLabel("Model") - .setOrder(4) .setRequired(true) .setSensitive(false) - .setTooltip("The base model ID or an ARN to a custom model based on a foundational model.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( REGION_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The region that your model or ARN is deployed in.") .setLabel("Region") - .setOrder(5) .setRequired(true) .setSensitive(false) - .setTooltip("The region that your model or ARN is deployed in.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.putAll(AmazonBedrockSecretSettings.Configuration.get()); configurationMap.putAll( - RateLimitSettings.toSettingsConfigurationWithTooltip( + RateLimitSettings.toSettingsConfigurationWithDescription( "By default, the amazonbedrock service sets the number of requests allowed per minute to 240." ) ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case COMPLETION -> taskSettingsConfig = AmazonBedrockChatCompletionModel.Configuration.get(); - // TEXT_EMBEDDING task type has no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionModel.java index 9339a8a05dc81..27dc607d671aa 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/completion/AmazonBedrockChatCompletionModel.java @@ -7,30 +7,19 @@ package org.elasticsearch.xpack.inference.services.amazonbedrock.completion; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.amazonbedrock.AmazonBedrockActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockModel; import org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockSecretSettings; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.MAX_NEW_TOKENS_FIELD; -import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.TEMPERATURE_FIELD; -import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.TOP_K_FIELD; -import static org.elasticsearch.xpack.inference.services.amazonbedrock.AmazonBedrockConstants.TOP_P_FIELD; - public class AmazonBedrockChatCompletionModel extends AmazonBedrockModel { public static AmazonBedrockChatCompletionModel of(AmazonBedrockChatCompletionModel completionModel, Map taskSettings) { @@ -91,62 +80,4 @@ public AmazonBedrockChatCompletionServiceSettings getServiceSettings() { public AmazonBedrockChatCompletionTaskSettings getTaskSettings() { return (AmazonBedrockChatCompletionTaskSettings) super.getTaskSettings(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - MAX_NEW_TOKENS_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Max New Tokens") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Sets the maximum number for the output tokens to be generated.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - configurationMap.put( - TEMPERATURE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Temperature") - .setOrder(2) - .setRequired(false) - .setSensitive(false) - .setTooltip("A number between 0.0 and 1.0 that controls the apparent creativity of the results.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - configurationMap.put( - TOP_P_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Top P") - .setOrder(3) - .setRequired(false) - .setSensitive(false) - .setTooltip("Alternative to temperature. A number in the range of 0.0 to 1.0, to eliminate low-probability tokens.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - configurationMap.put( - TOP_K_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Top K") - .setOrder(4) - .setRequired(false) - .setSensitive(false) - .setTooltip("Only available for anthropic, cohere, and mistral providers. Alternative to temperature.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java index 41852e4758a8c..9dbfb0732f463 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicService.java @@ -15,7 +15,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -23,9 +22,7 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.external.action.anthropic.AnthropicActionCreator; @@ -57,6 +54,7 @@ public class AnthropicService extends SenderService { public static final String NAME = "anthropic"; + private static final String SERVICE_NAME = "Anthropic"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.COMPLETION); @@ -258,31 +256,27 @@ public static InferenceServiceConfiguration get() { configurationMap.put( MODEL_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The name of the model to use for the inference task.") .setLabel("Model ID") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("The name of the model to use for the inference task.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); configurationMap.putAll( - RateLimitSettings.toSettingsConfigurationWithTooltip( + RateLimitSettings.toSettingsConfigurationWithDescription( "By default, the anthropic service sets the number of requests allowed per minute to 50." ) ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case COMPLETION -> taskSettingsConfig = AnthropicChatCompletionModel.Configuration.get(); - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionModel.java index df54ee4ec97c4..942cae8960daf 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/anthropic/completion/AnthropicChatCompletionModel.java @@ -8,14 +8,10 @@ package org.elasticsearch.xpack.inference.services.anthropic.completion; import org.apache.http.client.utils.URIBuilder; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.anthropic.AnthropicActionVisitor; import org.elasticsearch.xpack.inference.external.request.anthropic.AnthropicRequestUtils; @@ -26,15 +22,8 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.anthropic.AnthropicServiceFields.MAX_TOKENS; -import static org.elasticsearch.xpack.inference.services.anthropic.AnthropicServiceFields.TEMPERATURE_FIELD; -import static org.elasticsearch.xpack.inference.services.anthropic.AnthropicServiceFields.TOP_K_FIELD; -import static org.elasticsearch.xpack.inference.services.anthropic.AnthropicServiceFields.TOP_P_FIELD; - public class AnthropicChatCompletionModel extends AnthropicModel { public static AnthropicChatCompletionModel of(AnthropicChatCompletionModel model, Map taskSettings) { @@ -134,62 +123,4 @@ private static URI buildDefaultUri() throws URISyntaxException { .setPathSegments(AnthropicRequestUtils.API_VERSION_1, AnthropicRequestUtils.MESSAGES_PATH) .build(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - MAX_TOKENS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Max Tokens") - .setOrder(1) - .setRequired(true) - .setSensitive(false) - .setTooltip("The maximum number of tokens to generate before stopping.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - configurationMap.put( - TEMPERATURE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) - .setLabel("Temperature") - .setOrder(2) - .setRequired(false) - .setSensitive(false) - .setTooltip("The amount of randomness injected into the response.") - .setType(SettingsConfigurationFieldType.STRING) - .build() - ); - configurationMap.put( - TOP_K_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Top K") - .setOrder(3) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies to only sample from the top K options for each subsequent token.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - configurationMap.put( - TOP_P_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Top P") - .setOrder(4) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies to use Anthropic’s nucleus sampling.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java index bcd4a1abfbf00..649540f7efc5c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java @@ -17,7 +17,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -26,11 +25,8 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; @@ -56,7 +52,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Stream; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; @@ -77,6 +72,7 @@ public class AzureAiStudioService extends SenderService { static final String NAME = "azureaistudio"; + private static final String SERVICE_NAME = "Azure AI Studio"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION); public AzureAiStudioService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -410,62 +406,47 @@ public static InferenceServiceConfiguration get() { configurationMap.put( TARGET_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The target URL of your Azure AI Studio model deployment.") .setLabel("Target") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("The target URL of your Azure AI Studio model deployment.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( ENDPOINT_TYPE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) + new SettingsConfiguration.Builder().setDescription( + "Specifies the type of endpoint that is used in your model deployment." + ) .setLabel("Endpoint Type") - .setOrder(3) .setRequired(true) .setSensitive(false) - .setTooltip("Specifies the type of endpoint that is used in your model deployment.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("token", "realtime") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) .build() ); configurationMap.put( PROVIDER_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) + new SettingsConfiguration.Builder().setDescription("The model provider for your deployment.") .setLabel("Provider") - .setOrder(3) .setRequired(true) .setSensitive(false) - .setTooltip("The model provider for your deployment.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("cohere", "meta", "microsoft_phi", "mistral", "openai", "databricks") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) .build() ); configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case TEXT_EMBEDDING -> taskSettingsConfig = AzureAiStudioEmbeddingsModel.Configuration.get(); - case COMPLETION -> taskSettingsConfig = AzureAiStudioChatCompletionModel.Configuration.get(); - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionModel.java index 0492788c2adcd..5afb3aaed61ff 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionModel.java @@ -7,14 +7,10 @@ package org.elasticsearch.xpack.inference.services.azureaistudio.completion; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.azureaistudio.AzureAiStudioActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; @@ -25,12 +21,9 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.COMPLETIONS_URI_PATH; -import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.USER_FIELD; public class AzureAiStudioChatCompletionModel extends AzureAiStudioModel { @@ -109,30 +102,4 @@ protected URI getEndpointUri() throws URISyntaxException { public ExecutableAction accept(AzureAiStudioActionVisitor creator, Map taskSettings) { return creator.create(this, taskSettings); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - USER_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) - .setLabel("User") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the user issuing the request.") - .setType(SettingsConfigurationFieldType.STRING) - .setValue("") - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsModel.java index 8b0b52c69b82c..edbefe07cff02 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsModel.java @@ -7,15 +7,11 @@ package org.elasticsearch.xpack.inference.services.azureaistudio.embeddings; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.azureaistudio.AzureAiStudioActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; @@ -26,15 +22,9 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.DO_SAMPLE_FIELD; import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.EMBEDDINGS_URI_PATH; -import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.MAX_NEW_TOKENS_FIELD; -import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.TEMPERATURE_FIELD; -import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.TOP_P_FIELD; public class AzureAiStudioEmbeddingsModel extends AzureAiStudioModel { @@ -116,65 +106,4 @@ protected URI getEndpointUri() throws URISyntaxException { public ExecutableAction accept(AzureAiStudioActionVisitor creator, Map taskSettings) { return creator.create(this, taskSettings); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - DO_SAMPLE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Do Sample") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Instructs the inference process to perform sampling or not.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - configurationMap.put( - MAX_NEW_TOKENS_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Max New Tokens") - .setOrder(2) - .setRequired(false) - .setSensitive(false) - .setTooltip("Provides a hint for the maximum number of output tokens to be generated.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - configurationMap.put( - TEMPERATURE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Temperature") - .setOrder(3) - .setRequired(false) - .setSensitive(false) - .setTooltip("A number in the range of 0.0 to 2.0 that specifies the sampling temperature.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - configurationMap.put( - TOP_P_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Top P") - .setOrder(4) - .setRequired(false) - .setSensitive(false) - .setTooltip( - "A number in the range of 0.0 to 2.0 that is an alternative value to temperature. Should not be used " - + "if temperature is specified." - ) - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java index 70a29b28a607c..0601daf562ce9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiSecretSettings.java @@ -18,7 +18,6 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SecretSettings; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xcontent.XContentBuilder; @@ -147,23 +146,21 @@ public static Map get() { var configurationMap = new HashMap(); configurationMap.put( API_KEY, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("You must provide either an API key or an Entra ID.") .setLabel("API Key") - .setOrder(1) .setRequired(false) .setSensitive(true) - .setTooltip("You must provide either an API key or an Entra ID.") + .setUpdatable(true) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( ENTRA_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("You must provide either an API key or an Entra ID.") .setLabel("Entra ID") - .setOrder(2) .setRequired(false) .setSensitive(true) - .setTooltip("You must provide either an API key or an Entra ID.") + .setUpdatable(true) .setType(SettingsConfigurationFieldType.STRING) .build() ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java index 0ed69604c258a..4fca5a460a12a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -25,9 +24,7 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; @@ -68,6 +65,7 @@ public class AzureOpenAiService extends SenderService { public static final String NAME = "azureopenai"; + private static final String SERVICE_NAME = "Azure OpenAI"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION); public AzureOpenAiService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -353,56 +351,49 @@ public static InferenceServiceConfiguration get() { configurationMap.put( RESOURCE_NAME, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The name of your Azure OpenAI resource.") .setLabel("Resource Name") - .setOrder(3) .setRequired(true) .setSensitive(false) - .setTooltip("The name of your Azure OpenAI resource.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( API_VERSION, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The Azure API version ID to use.") .setLabel("API Version") - .setOrder(4) .setRequired(true) .setSensitive(false) - .setTooltip("The Azure API version ID to use.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( DEPLOYMENT_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The deployment name of your deployed models.") .setLabel("Deployment ID") - .setOrder(5) .setRequired(true) .setSensitive(false) - .setTooltip("The deployment name of your deployed models.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.putAll(AzureOpenAiSecretSettings.Configuration.get()); configurationMap.putAll( - RateLimitSettings.toSettingsConfigurationWithTooltip( + RateLimitSettings.toSettingsConfigurationWithDescription( "The azureopenai service sets a default number of requests allowed per minute depending on the task type." ) ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case TEXT_EMBEDDING -> taskSettingsConfig = AzureOpenAiEmbeddingsModel.Configuration.get(); - case COMPLETION -> taskSettingsConfig = AzureOpenAiCompletionModel.Configuration.get(); - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java index 8b2846fd9ced7..ed57187adfe64 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java @@ -7,14 +7,10 @@ package org.elasticsearch.xpack.inference.services.azureopenai.completion; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionVisitor; import org.elasticsearch.xpack.inference.external.request.azureopenai.AzureOpenAiUtils; @@ -23,12 +19,8 @@ import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiSecretSettings; import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.USER; - public class AzureOpenAiCompletionModel extends AzureOpenAiModel { public static AzureOpenAiCompletionModel of(AzureOpenAiCompletionModel model, Map taskSettings) { @@ -127,30 +119,4 @@ public String apiVersion() { public String[] operationPathSegments() { return new String[] { AzureOpenAiUtils.CHAT_PATH, AzureOpenAiUtils.COMPLETIONS_PATH }; } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - USER, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) - .setLabel("User") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the user issuing the request.") - .setType(SettingsConfigurationFieldType.STRING) - .setValue("") - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java index 0316804664510..dea5a8ca9e50a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java @@ -7,15 +7,11 @@ package org.elasticsearch.xpack.inference.services.azureopenai.embeddings; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionVisitor; import org.elasticsearch.xpack.inference.external.request.azureopenai.AzureOpenAiUtils; @@ -24,12 +20,8 @@ import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiSecretSettings; import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields.USER; - public class AzureOpenAiEmbeddingsModel extends AzureOpenAiModel { public static AzureOpenAiEmbeddingsModel of(AzureOpenAiEmbeddingsModel model, Map taskSettings) { @@ -131,30 +123,4 @@ public String apiVersion() { public String[] operationPathSegments() { return new String[] { AzureOpenAiUtils.EMBEDDINGS_PATH }; } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - USER, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) - .setLabel("User") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the user issuing the request.") - .setType(SettingsConfigurationFieldType.STRING) - .setValue("") - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index a7d17192bfa92..60ab8ca68d5d9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -25,7 +24,6 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; @@ -65,6 +63,7 @@ public class CohereService extends SenderService { public static final String NAME = "cohere"; + private static final String SERVICE_NAME = "Cohere"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION, TaskType.RERANK); // TODO Batching - We'll instantiate a batching class within the services that want to support it and pass it through to @@ -367,16 +366,11 @@ public static InferenceServiceConfiguration get() { configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case TEXT_EMBEDDING -> taskSettingsConfig = CohereEmbeddingsModel.Configuration.get(); - case RERANK -> taskSettingsConfig = CohereRerankModel.Configuration.get(); - // COMPLETION task type has no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java index 43a7bc0a5e678..0f62ab51145f4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java @@ -7,17 +7,12 @@ package org.elasticsearch.xpack.inference.services.cohere.embeddings; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.cohere.CohereActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; @@ -25,13 +20,7 @@ import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.stream.Stream; - -import static org.elasticsearch.xpack.inference.external.request.cohere.CohereEmbeddingsRequestEntity.INPUT_TYPE_FIELD; -import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceFields.TRUNCATE; public class CohereEmbeddingsModel extends CohereModel { public static CohereEmbeddingsModel of(CohereEmbeddingsModel model, Map taskSettings, InputType inputType) { @@ -110,52 +99,4 @@ public ExecutableAction accept(CohereActionVisitor visitor, Map public URI uri() { return getServiceSettings().getCommonSettings().uri(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - INPUT_TYPE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) - .setLabel("Input Type") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the type of input passed to the model.") - .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("classification", "clusterning", "ingest", "search") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) - .setValue("") - .build() - ); - configurationMap.put( - TRUNCATE, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) - .setLabel("Truncate") - .setOrder(2) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies how the API handles inputs longer than the maximum token length.") - .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("NONE", "START", "END") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) - .setValue("") - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java index cfcfb8a3d5dae..b84b98973bbe5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java @@ -7,15 +7,11 @@ package org.elasticsearch.xpack.inference.services.cohere.rerank; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.cohere.CohereActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; @@ -23,13 +19,8 @@ import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings.RETURN_DOCUMENTS; -import static org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings.TOP_N_DOCS_ONLY; - public class CohereRerankModel extends CohereModel { public static CohereRerankModel of(CohereRerankModel model, Map taskSettings) { var requestTaskSettings = CohereRerankTaskSettings.fromMap(taskSettings); @@ -111,41 +102,4 @@ public ExecutableAction accept(CohereActionVisitor visitor, Map public URI uri() { return getServiceSettings().uri(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - RETURN_DOCUMENTS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE) - .setLabel("Return Documents") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specify whether to return doc text within the results.") - .setType(SettingsConfigurationFieldType.BOOLEAN) - .setValue(false) - .build() - ); - configurationMap.put( - TOP_N_DOCS_ONLY, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Top N") - .setOrder(2) - .setRequired(false) - .setSensitive(false) - .setTooltip("The number of most relevant documents to return, defaults to the number of the documents.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java index 27abc03cb6e82..68782488099a1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -24,9 +23,7 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -67,6 +64,7 @@ public class ElasticInferenceService extends SenderService { private final ElasticInferenceServiceComponents elasticInferenceServiceComponents; + private static final String SERVICE_NAME = "Elastic"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.SPARSE_EMBEDDING); public ElasticInferenceService( @@ -317,38 +315,33 @@ public static InferenceServiceConfiguration get() { configurationMap.put( MODEL_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The name of the model to use for the inference task.") .setLabel("Model ID") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("The name of the model to use for the inference task.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( MAX_INPUT_TOKENS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) + new SettingsConfiguration.Builder().setDescription("Allows you to specify the maximum number of tokens per input.") .setLabel("Maximum Input Tokens") - .setOrder(3) .setRequired(false) .setSensitive(false) - .setTooltip("Allows you to specify the maximum number of tokens per input.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.INTEGER) .build() ); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - // SPARSE_EMBEDDING task type has no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java index 6388bb33bb78d..b4d579b699cdf 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandRerankModel.java @@ -7,17 +7,7 @@ package org.elasticsearch.xpack.inference.services.elasticsearch; -import org.elasticsearch.common.util.LazyInitializable; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.elasticsearch.xpack.inference.services.elasticsearch.RerankTaskSettings.RETURN_DOCUMENTS; public class CustomElandRerankModel extends CustomElandModel { @@ -35,30 +25,4 @@ public CustomElandRerankModel( public CustomElandInternalServiceSettings getServiceSettings() { return (CustomElandInternalServiceSettings) super.getServiceSettings(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - RETURN_DOCUMENTS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE) - .setLabel("Return Documents") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Returns the document instead of only the index.") - .setType(SettingsConfigurationFieldType.BOOLEAN) - .setValue(true) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index cb3eb5ca8e826..39ad729b1699e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -20,7 +20,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceExtension; @@ -29,12 +28,9 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnifiedCompletionRequest; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; @@ -71,7 +67,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; -import java.util.stream.Stream; import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; @@ -82,7 +77,6 @@ import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.MODEL_ID; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_THREADS; -import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V1_MODEL; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL_LINUX_X86; @@ -105,6 +99,7 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi public static final String DEFAULT_E5_ID = ".multilingual-e5-small-elasticsearch"; public static final String DEFAULT_RERANK_ID = ".rerank-v1-elasticsearch"; + private static final String SERVICE_NAME = "Elasticsearch"; private static final EnumSet supportedTaskTypes = EnumSet.of( TaskType.RERANK, TaskType.TEXT_EMBEDDING, @@ -1152,61 +1147,45 @@ public static InferenceServiceConfiguration get() { configurationMap.put( MODEL_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) + new SettingsConfiguration.Builder().setDefaultValue(MULTILINGUAL_E5_SMALL_MODEL_ID) + .setDescription("The name of the model to use for the inference task.") .setLabel("Model ID") - .setOrder(1) .setRequired(true) .setSensitive(false) - .setTooltip("The name of the model to use for the inference task.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of( - ELSER_V1_MODEL, - ELSER_V2_MODEL, - ELSER_V2_MODEL_LINUX_X86, - MULTILINGUAL_E5_SMALL_MODEL_ID, - MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 - ).map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()).toList() - ) - .setDefaultValue(MULTILINGUAL_E5_SMALL_MODEL_ID) .build() ); configurationMap.put( NUM_ALLOCATIONS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) + new SettingsConfiguration.Builder().setDefaultValue(1) + .setDescription("The total number of allocations this model is assigned across machine learning nodes.") .setLabel("Number Allocations") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("The total number of allocations this model is assigned across machine learning nodes.") + .setUpdatable(true) .setType(SettingsConfigurationFieldType.INTEGER) - .setDefaultValue(1) .build() ); configurationMap.put( NUM_THREADS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) + new SettingsConfiguration.Builder().setDefaultValue(2) + .setDescription("Sets the number of threads used by each model allocation during inference.") .setLabel("Number Threads") - .setOrder(3) .setRequired(true) .setSensitive(false) - .setTooltip("Sets the number of threads used by each model allocation during inference.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.INTEGER) - .setDefaultValue(2) .build() ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case RERANK -> taskSettingsConfig = CustomElandRerankModel.Configuration.get(); - // SPARSE_EMBEDDING, TEXT_EMBEDDING task types have no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java index 837a001d1f8f9..1dbf2ca3e2dad 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -25,9 +24,7 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; @@ -72,6 +69,7 @@ public class GoogleAiStudioService extends SenderService { public static final String NAME = "googleaistudio"; + private static final String SERVICE_NAME = "Google AI Studio"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION); public GoogleAiStudioService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -353,12 +351,11 @@ public static InferenceServiceConfiguration get() { configurationMap.put( MODEL_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("ID of the LLM you're using.") .setLabel("Model ID") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("ID of the LLM you're using.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); @@ -366,14 +363,11 @@ public static InferenceServiceConfiguration get() { configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - // COMPLETION, TEXT_EMBEDDING task types have no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java index 272bc9eaa9a62..b185800ed75f4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiSecretSettings.java @@ -18,7 +18,6 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SecretSettings; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xcontent.XContentBuilder; @@ -123,12 +122,11 @@ public static Map get() { var configurationMap = new HashMap(); configurationMap.put( SERVICE_ACCOUNT_JSON, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("API Key for the provider you're connecting to.") .setLabel("Credentials JSON") - .setOrder(1) .setRequired(true) .setSensitive(true) - .setTooltip("API Key for the provider you're connecting to.") + .setUpdatable(true) .setType(SettingsConfigurationFieldType.STRING) .build() ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java index b412f20289880..8fe9f29c73747 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -24,9 +23,7 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; @@ -67,6 +64,7 @@ public class GoogleVertexAiService extends SenderService { public static final String NAME = "googlevertexai"; + private static final String SERVICE_NAME = "Google Vertex AI"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.RERANK); public GoogleVertexAiService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -331,42 +329,39 @@ public static InferenceServiceConfiguration get() { configurationMap.put( MODEL_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("ID of the LLM you're using.") .setLabel("Model ID") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("ID of the LLM you're using.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( LOCATION, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription( + "Please provide the GCP region where the Vertex AI API(s) is enabled. " + + "For more information, refer to the {geminiVertexAIDocs}." + ) .setLabel("GCP Region") - .setOrder(3) .setRequired(true) .setSensitive(false) - .setTooltip( - "Please provide the GCP region where the Vertex AI API(s) is enabled. " - + "For more information, refer to the {geminiVertexAIDocs}." - ) + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( PROJECT_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription( + "The GCP Project ID which has Vertex AI API(s) enabled. For more information " + + "on the URL, refer to the {geminiVertexAIDocs}." + ) .setLabel("GCP Project") - .setOrder(4) .setRequired(true) .setSensitive(false) - .setTooltip( - "The GCP Project ID which has Vertex AI API(s) enabled. For more information " - + "on the URL, refer to the {geminiVertexAIDocs}." - ) + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); @@ -374,15 +369,11 @@ public static InferenceServiceConfiguration get() { configurationMap.putAll(GoogleVertexAiSecretSettings.Configuration.get()); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case TEXT_EMBEDDING -> taskSettingsConfig = GoogleVertexAiEmbeddingsModel.Configuration.get(); - case RERANK -> taskSettingsConfig = GoogleVertexAiRerankModel.Configuration.get(); - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModel.java index a5acbb80b76ec..98640623ee21c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/embeddings/GoogleVertexAiEmbeddingsModel.java @@ -8,17 +8,12 @@ package org.elasticsearch.xpack.inference.services.googlevertexai.embeddings; import org.apache.http.client.utils.URIBuilder; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.googlevertexai.GoogleVertexAiActionVisitor; import org.elasticsearch.xpack.inference.external.request.googlevertexai.GoogleVertexAiUtils; @@ -28,14 +23,9 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.stream.Stream; import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsTaskSettings.AUTO_TRUNCATE; -import static org.elasticsearch.xpack.inference.services.googlevertexai.embeddings.GoogleVertexAiEmbeddingsTaskSettings.INPUT_TYPE; public class GoogleVertexAiEmbeddingsModel extends GoogleVertexAiModel { @@ -165,51 +155,4 @@ public static URI buildUri(String location, String projectId, String modelId) th ) .build(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - INPUT_TYPE, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) - .setLabel("Input Type") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the type of input passed to the model.") - .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of( - InputType.CLASSIFICATION.toString(), - InputType.CLUSTERING.toString(), - InputType.INGEST.toString(), - InputType.SEARCH.toString() - ).map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()).toList() - ) - .setValue("") - .build() - ); - - configurationMap.put( - AUTO_TRUNCATE, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE) - .setLabel("Auto Truncate") - .setOrder(2) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies if the API truncates inputs longer than the maximum token length automatically.") - .setType(SettingsConfigurationFieldType.BOOLEAN) - .setValue(false) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankModel.java index e73d8d2e2613a..f522897b0d716 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/rerank/GoogleVertexAiRerankModel.java @@ -8,15 +8,11 @@ package org.elasticsearch.xpack.inference.services.googlevertexai.rerank; import org.apache.http.client.utils.URIBuilder; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.googlevertexai.GoogleVertexAiActionVisitor; import org.elasticsearch.xpack.inference.external.request.googlevertexai.GoogleVertexAiUtils; @@ -26,12 +22,9 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.inference.services.googlevertexai.rerank.GoogleVertexAiRerankTaskSettings.TOP_N; public class GoogleVertexAiRerankModel extends GoogleVertexAiModel { @@ -140,30 +133,4 @@ public static URI buildUri(String projectId) throws URISyntaxException { ) .build(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - TOP_N, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE) - .setLabel("Top N") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the number of the top n documents, which should be returned.") - .setType(SettingsConfigurationFieldType.BOOLEAN) - .setValue(false) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java index acb082cd2de8d..ef6beb8ec2627 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java @@ -16,16 +16,13 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; @@ -54,6 +51,7 @@ public class HuggingFaceService extends HuggingFaceBaseService { public static final String NAME = "hugging_face"; + private static final String SERVICE_NAME = "Hugging Face"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.SPARSE_EMBEDDING); public HuggingFaceService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -183,29 +181,24 @@ public static InferenceServiceConfiguration get() { configurationMap.put( URL, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDefaultValue("https://api.openai.com/v1/embeddings") + .setDescription("The URL endpoint to use for the requests.") .setLabel("URL") - .setOrder(1) .setRequired(true) .setSensitive(false) - .setTooltip("The URL endpoint to use for the requests.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) - .setValue("https://api.openai.com/v1/embeddings") - .setDefaultValue("https://api.openai.com/v1/embeddings") .build() ); configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - // SPARSE_EMBEDDING, TEXT_EMBEDDING task types have no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java index 01d2a75ebff5d..52d42f570a413 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java @@ -17,15 +17,12 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; @@ -58,6 +55,7 @@ public class HuggingFaceElserService extends HuggingFaceBaseService { public static final String NAME = "hugging_face_elser"; + private static final String SERVICE_NAME = "Hugging Face ELSER"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.SPARSE_EMBEDDING); public HuggingFaceElserService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -178,12 +176,11 @@ public static InferenceServiceConfiguration get() { configurationMap.put( URL, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The URL endpoint to use for the requests.") .setLabel("URL") - .setOrder(1) .setRequired(true) .setSensitive(false) - .setTooltip("The URL endpoint to use for the requests.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); @@ -191,14 +188,11 @@ public static InferenceServiceConfiguration get() { configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - // SPARSE_EMBEDDING task type has no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java index 482554e060a47..dd368f88a993c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -25,9 +24,7 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; @@ -68,6 +65,7 @@ public class IbmWatsonxService extends SenderService { public static final String NAME = "watsonxai"; + private static final String SERVICE_NAME = "IBM Watsonx"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING); public IbmWatsonxService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -326,72 +324,64 @@ public static InferenceServiceConfiguration get() { configurationMap.put( API_VERSION, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The IBM Watsonx API version ID to use.") .setLabel("API Version") - .setOrder(1) .setRequired(true) .setSensitive(false) - .setTooltip("The IBM Watsonx API version ID to use.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( PROJECT_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("") .setLabel("Project ID") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( MODEL_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The name of the model to use for the inference task.") .setLabel("Model ID") - .setOrder(3) .setRequired(true) .setSensitive(false) - .setTooltip("The name of the model to use for the inference task.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( URL, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("") .setLabel("URL") - .setOrder(4) .setRequired(true) .setSensitive(false) - .setTooltip("") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( MAX_INPUT_TOKENS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) + new SettingsConfiguration.Builder().setDescription("Allows you to specify the maximum number of tokens per input.") .setLabel("Maximum Input Tokens") - .setOrder(5) .setRequired(false) .setSensitive(false) - .setTooltip("Allows you to specify the maximum number of tokens per input.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.INTEGER) .build() ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - // TEXT_EMBEDDING task type has no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java index 11a72f811e8d3..ed76df5875562 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -25,7 +24,6 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; @@ -63,6 +61,7 @@ public class JinaAIService extends SenderService { public static final String NAME = "jinaai"; + private static final String SERVICE_NAME = "Jina AI"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.RERANK); public JinaAIService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -343,15 +342,11 @@ public static InferenceServiceConfiguration get() { configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case TEXT_EMBEDDING -> taskSettingsConfig = JinaAIEmbeddingsModel.Configuration.get(); - case RERANK -> taskSettingsConfig = JinaAIRerankModel.Configuration.get(); - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java index dd479802cdf13..2c2a24a75d098 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/embeddings/JinaAIEmbeddingsModel.java @@ -7,17 +7,12 @@ package org.elasticsearch.xpack.inference.services.jinaai.embeddings; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; -import org.elasticsearch.inference.configuration.SettingsConfigurationSelectOption; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; @@ -25,12 +20,7 @@ import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.stream.Stream; - -import static org.elasticsearch.xpack.inference.external.request.jinaai.JinaAIEmbeddingsRequestEntity.TASK_TYPE_FIELD; public class JinaAIEmbeddingsModel extends JinaAIModel { public static JinaAIEmbeddingsModel of(JinaAIEmbeddingsModel model, Map taskSettings, InputType inputType) { @@ -106,35 +96,4 @@ public ExecutableAction accept(JinaAIActionVisitor visitor, Map public URI uri() { return getServiceSettings().getCommonSettings().uri(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - TASK_TYPE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.DROPDOWN) - .setLabel("Task") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the task type passed to the model.") - .setType(SettingsConfigurationFieldType.STRING) - .setOptions( - Stream.of("retrieval.query", "retrieval.passage", "classification", "separation") - .map(v -> new SettingsConfigurationSelectOption.Builder().setLabelAndValue(v).build()) - .toList() - ) - .setValue("") - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java index 2fb9228d3b652..b072d0508c8ee 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/rerank/JinaAIRerankModel.java @@ -7,15 +7,11 @@ package org.elasticsearch.xpack.inference.services.jinaai.rerank; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.jinaai.JinaAIActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; @@ -23,13 +19,8 @@ import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings.RETURN_DOCUMENTS; -import static org.elasticsearch.xpack.inference.services.jinaai.rerank.JinaAIRerankTaskSettings.TOP_N_DOCS_ONLY; - public class JinaAIRerankModel extends JinaAIModel { public static JinaAIRerankModel of(JinaAIRerankModel model, Map taskSettings) { var requestTaskSettings = JinaAIRerankTaskSettings.fromMap(taskSettings); @@ -108,41 +99,4 @@ public ExecutableAction accept(JinaAIActionVisitor visitor, Map public URI uri() { return getServiceSettings().getCommonSettings().uri(); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - RETURN_DOCUMENTS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TOGGLE) - .setLabel("Return Documents") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specify whether to return doc text within the results.") - .setType(SettingsConfigurationFieldType.BOOLEAN) - .setValue(false) - .build() - ); - configurationMap.put( - TOP_N_DOCS_ONLY, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) - .setLabel("Top N") - .setOrder(2) - .setRequired(false) - .setSensitive(false) - .setTooltip("The number of most relevant documents to return, defaults to the number of the documents.") - .setType(SettingsConfigurationFieldType.INTEGER) - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java index dc0576651aeba..129d9023a1ebc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -25,9 +24,7 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; @@ -65,6 +62,7 @@ public class MistralService extends SenderService { public static final String NAME = "mistral"; + private static final String SERVICE_NAME = "Mistral"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING); public MistralService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -320,24 +318,24 @@ public static InferenceServiceConfiguration get() { configurationMap.put( MODEL_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription( + "Refer to the Mistral models documentation for the list of available text embedding models." + ) .setLabel("Model") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("Refer to the Mistral models documentation for the list of available text embedding models.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( MAX_INPUT_TOKENS, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) + new SettingsConfiguration.Builder().setDescription("Allows you to specify the maximum number of tokens per input.") .setLabel("Maximum Input Tokens") - .setOrder(3) .setRequired(false) .setSensitive(false) - .setTooltip("Allows you to specify the maximum number of tokens per input.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.INTEGER) .build() ); @@ -345,14 +343,11 @@ public static InferenceServiceConfiguration get() { configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration()); configurationMap.putAll(RateLimitSettings.toSettingsConfiguration()); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - // TEXT_EMBEDDING task type has no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index ed7a01e829dd2..ba9dea8ace8ee 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; @@ -25,9 +24,7 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.SimilarityMeasure; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; @@ -72,6 +69,7 @@ public class OpenAiService extends SenderService { public static final String NAME = "openai"; + private static final String SERVICE_NAME = "OpenAI"; private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.COMPLETION); public OpenAiService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { @@ -397,65 +395,58 @@ public static InferenceServiceConfiguration get() { configurationMap.put( MODEL_ID, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The name of the model to use for the inference task.") .setLabel("Model ID") - .setOrder(2) .setRequired(true) .setSensitive(false) - .setTooltip("The name of the model to use for the inference task.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( ORGANIZATION, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription("The unique identifier of your organization.") .setLabel("Organization ID") - .setOrder(3) .setRequired(false) .setSensitive(false) - .setTooltip("The unique identifier of your organization.") + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) .build() ); configurationMap.put( URL, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) - .setLabel("URL") - .setOrder(4) - .setRequired(true) - .setSensitive(false) - .setTooltip( + new SettingsConfiguration.Builder().setDefaultValue("https://api.openai.com/v1/chat/completions") + .setDescription( "The OpenAI API endpoint URL. For more information on the URL, refer to the " + "https://platform.openai.com/docs/api-reference." ) + .setLabel("URL") + .setRequired(true) + .setSensitive(false) + .setUpdatable(false) .setType(SettingsConfigurationFieldType.STRING) - .setDefaultValue("https://api.openai.com/v1/chat/completions") .build() ); configurationMap.putAll( - DefaultSecretSettings.toSettingsConfigurationWithTooltip( + DefaultSecretSettings.toSettingsConfigurationWithDescription( "The OpenAI API authentication key. For more details about generating OpenAI API keys, " + "refer to the https://platform.openai.com/account/api-keys." ) ); configurationMap.putAll( - RateLimitSettings.toSettingsConfigurationWithTooltip( + RateLimitSettings.toSettingsConfigurationWithDescription( "Default number of requests allowed per minute. For text_embedding is 3000. For completion is 500." ) ); - return new InferenceServiceConfiguration.Builder().setProvider(NAME).setTaskTypes(supportedTaskTypes.stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - case TEXT_EMBEDDING -> taskSettingsConfig = OpenAiEmbeddingsModel.Configuration.get(); - case COMPLETION -> taskSettingsConfig = OpenAiChatCompletionModel.Configuration.get(); - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()).setConfiguration(configurationMap).build(); + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); } ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java index 7d79d64b3a771..dea703b9ce243 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java @@ -7,28 +7,20 @@ package org.elasticsearch.xpack.inference.services.openai.completion; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnifiedCompletionRequest; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.openai.OpenAiModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Objects; -import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.USER; - public class OpenAiChatCompletionModel extends OpenAiModel { public static OpenAiChatCompletionModel of(OpenAiChatCompletionModel model, Map taskSettings) { @@ -118,30 +110,4 @@ public DefaultSecretSettings getSecretSettings() { public ExecutableAction accept(OpenAiActionVisitor creator, Map taskSettings) { return creator.create(this, taskSettings); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - USER, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) - .setLabel("User") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the user issuing the request.") - .setType(SettingsConfigurationFieldType.STRING) - .setValue("") - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java index cab2a82fc86c6..5659c46050ad8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java @@ -7,27 +7,19 @@ package org.elasticsearch.xpack.inference.services.openai.embeddings; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; -import org.elasticsearch.inference.SettingsConfiguration; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; -import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionVisitor; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.openai.OpenAiModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.USER; - public class OpenAiEmbeddingsModel extends OpenAiModel { public static OpenAiEmbeddingsModel of(OpenAiEmbeddingsModel model, Map taskSettings) { @@ -105,30 +97,4 @@ public DefaultSecretSettings getSecretSettings() { public ExecutableAction accept(OpenAiActionVisitor creator, Map taskSettings) { return creator.create(this, taskSettings); } - - public static class Configuration { - public static Map get() { - return configuration.getOrCompute(); - } - - private static final LazyInitializable, RuntimeException> configuration = - new LazyInitializable<>(() -> { - var configurationMap = new HashMap(); - - configurationMap.put( - USER, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) - .setLabel("User") - .setOrder(1) - .setRequired(false) - .setSensitive(false) - .setTooltip("Specifies the user issuing the request.") - .setType(SettingsConfigurationFieldType.STRING) - .setValue("") - .build() - ); - - return Collections.unmodifiableMap(configurationMap); - }); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java index 771d2308a502f..15e8128969ddb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java @@ -17,7 +17,6 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SecretSettings; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xcontent.XContentBuilder; @@ -52,16 +51,15 @@ public static DefaultSecretSettings fromMap(@Nullable Map map) { return new DefaultSecretSettings(secureApiToken); } - public static Map toSettingsConfigurationWithTooltip(String tooltip) { + public static Map toSettingsConfigurationWithDescription(String description) { var configurationMap = new HashMap(); configurationMap.put( API_KEY, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.TEXTBOX) + new SettingsConfiguration.Builder().setDescription(description) .setLabel("API Key") - .setOrder(1) .setRequired(true) .setSensitive(true) - .setTooltip(tooltip) + .setUpdatable(true) .setType(SettingsConfigurationFieldType.STRING) .build() ); @@ -69,7 +67,7 @@ public static Map toSettingsConfigurationWithTool } public static Map toSettingsConfiguration() { - return DefaultSecretSettings.toSettingsConfigurationWithTooltip("API Key for the provider you're connecting to."); + return DefaultSecretSettings.toSettingsConfigurationWithDescription("API Key for the provider you're connecting to."); } public DefaultSecretSettings { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java index 416d5bff12ce9..30147a6d24a96 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.configuration.SettingsConfigurationDisplayType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -52,16 +51,15 @@ public static RateLimitSettings of( return requestsPerMinute == null ? defaultValue : new RateLimitSettings(requestsPerMinute); } - public static Map toSettingsConfigurationWithTooltip(String tooltip) { + public static Map toSettingsConfigurationWithDescription(String description) { var configurationMap = new HashMap(); configurationMap.put( FIELD_NAME + "." + REQUESTS_PER_MINUTE_FIELD, - new SettingsConfiguration.Builder().setDisplay(SettingsConfigurationDisplayType.NUMERIC) + new SettingsConfiguration.Builder().setDescription(description) .setLabel("Rate Limit") - .setOrder(6) .setRequired(false) .setSensitive(false) - .setTooltip(tooltip) + .setUpdatable(false) .setType(SettingsConfigurationFieldType.INTEGER) .build() ); @@ -69,7 +67,7 @@ public static Map toSettingsConfigurationWithTool } public static Map toSettingsConfiguration() { - return RateLimitSettings.toSettingsConfigurationWithTooltip("Minimize the number of rate limit errors."); + return RateLimitSettings.toSettingsConfigurationWithDescription("Minimize the number of rate limit errors."); } /** diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java index f8a5bd8a54d31..69b26585d8d49 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java @@ -12,13 +12,10 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.inference.EmptySettingsConfiguration; import org.elasticsearch.inference.InferenceServiceConfiguration; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.SettingsConfiguration; -import org.elasticsearch.inference.TaskSettingsConfiguration; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -177,16 +174,10 @@ public TransportVersion getMinimalSupportedVersion() { @Override public InferenceServiceConfiguration getConfiguration() { - return new InferenceServiceConfiguration.Builder().setProvider("test service") - .setTaskTypes(supportedTaskTypes().stream().map(t -> { - Map taskSettingsConfig; - switch (t) { - // no task settings - default -> taskSettingsConfig = EmptySettingsConfiguration.get(); - } - return new TaskSettingsConfiguration.Builder().setTaskType(t).setConfiguration(taskSettingsConfig).build(); - }).toList()) - .setConfiguration(new HashMap<>()) + return new InferenceServiceConfiguration.Builder().setService("test service") + .setName("Test") + .setTaskTypes(supportedTaskTypes()) + .setConfigurations(new HashMap<>()) .build(); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java index 46c3a062f7db0..e2cb93f731162 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java @@ -438,209 +438,57 @@ public void testGetConfiguration() throws Exception { String content = XContentHelper.stripWhitespace( """ { - "provider": "alibabacloud-ai-search", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": { - "input_type": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Input Type", - "options": [ - { - "label": "ingest", - "value": "ingest" - }, - { - "label": "search", - "value": "search" - } - ], - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the type of input passed to the model.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - }, - { - "task_type": "sparse_embedding", - "configuration": { - "return_token": { - "default_value": null, - "depends_on": [], - "display": "toggle", - "label": "Return Token", - "order": 2, - "required": false, - "sensitive": false, - "tooltip": "If `true`, the token name will be returned in the response. Defaults to `false` which means only the token ID will be returned in the response.", - "type": "bool", - "ui_restrictions": [], - "validations": [], - "value": true - }, - "input_type": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Input Type", - "options": [ - { - "label": "ingest", - "value": "ingest" - }, - { - "label": "search", - "value": "search" - } - ], - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the type of input passed to the model.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - }, - { - "task_type": "rerank", - "configuration": {} - }, - { - "task_type": "completion", - "configuration": {} - } - ], - "configuration": { + "service": "alibabacloud-ai-search", + "name": "AlibabaCloud AI Search", + "task_types": ["text_embedding", "sparse_embedding", "rerank", "completion"], + "configurations": { "workspace": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The name of the workspace used for the {infer} task.", "label": "Workspace", - "order": 5, "required": true, "sensitive": false, - "tooltip": "The name of the workspace used for the {infer} task.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "A valid API key for the AlibabaCloud AI Search API.", "label": "API Key", - "order": 1, "required": true, "sensitive": true, - "tooltip": "A valid API key for the AlibabaCloud AI Search API.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "service_id": { - "default_value": null, - "depends_on": [], - "display": "dropdown", + "description": "The name of the model service to use for the {infer} task.", "label": "Project ID", - "options": [ - { - "label": "ops-text-embedding-001", - "value": "ops-text-embedding-001" - }, - { - "label": "ops-text-embedding-zh-001", - "value": "ops-text-embedding-zh-001" - }, - { - "label": "ops-text-embedding-en-001", - "value": "ops-text-embedding-en-001" - }, - { - "label": "ops-text-embedding-002", - "value": "ops-text-embedding-002" - }, - { - "label": "ops-text-sparse-embedding-001", - "value": "ops-text-sparse-embedding-001" - }, - { - "label": "ops-bge-reranker-larger", - "value": "ops-bge-reranker-larger" - } - ], - "order": 2, "required": true, "sensitive": false, - "tooltip": "The name of the model service to use for the {infer} task.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "host": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The name of the host address used for the {infer} task. You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] of the documentation.", "label": "Host", - "order": 3, "required": true, "sensitive": false, - "tooltip": "The name of the host address used for the {infer} task. You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] of the documentation.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Minimize the number of rate limit errors.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "http_schema": { - "default_value": null, - "depends_on": [], - "display": "dropdown", + "description": "", "label": "HTTP Schema", - "options": [ - { - "label": "https", - "value": "https" - }, - { - "label": "http", - "value": "http" - } - ], - "order": 4, "required": true, "sensitive": false, - "tooltip": "", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index 7ca66d54d8ac3..ce4d928458dca 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -154,192 +154,63 @@ public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOExcepti @SuppressWarnings("checkstyle:LineLength") public void testGetConfiguration() throws Exception { try (var service = createAmazonBedrockService()) { - String content = XContentHelper.stripWhitespace( - """ - { - "provider": "amazonbedrock", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": {} - }, - { - "task_type": "completion", - "configuration": { - "top_p": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Top P", - "order": 3, - "required": false, - "sensitive": false, - "tooltip": "Alternative to temperature. A number in the range of 0.0 to 1.0, to eliminate low-probability tokens.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "max_new_tokens": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Max New Tokens", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Sets the maximum number for the output tokens to be generated.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "top_k": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Top K", - "order": 4, - "required": false, - "sensitive": false, - "tooltip": "Only available for anthropic, cohere, and mistral providers. Alternative to temperature.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "temperature": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Temperature", - "order": 2, - "required": false, - "sensitive": false, - "tooltip": "A number between 0.0 and 1.0 that controls the apparent creativity of the results.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - } - } - } - ], - "configuration": { - "secret_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "Secret Key", - "order": 2, - "required": true, - "sensitive": true, - "tooltip": "A valid AWS secret key that is paired with the access_key.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "provider": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Provider", - "options": [ - { - "label": "amazontitan", - "value": "amazontitan" - }, - { - "label": "anthropic", - "value": "anthropic" - }, - { - "label": "ai21labs", - "value": "ai21labs" - }, - { - "label": "cohere", - "value": "cohere" - }, - { - "label": "meta", - "value": "meta" - }, - { - "label": "mistral", - "value": "mistral" - } - ], - "order": 3, - "required": true, - "sensitive": false, - "tooltip": "The model provider for your deployment.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "access_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "Access Key", - "order": 1, - "required": true, - "sensitive": true, - "tooltip": "A valid AWS access key that has permissions to use Amazon Bedrock.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "model": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "Model", - "order": 4, - "required": true, - "sensitive": false, - "tooltip": "The base model ID or an ARN to a custom model based on a foundational model.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Rate Limit", - "order": 6, - "required": false, - "sensitive": false, - "tooltip": "By default, the amazonbedrock service sets the number of requests allowed per minute to 240.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "region": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "Region", - "order": 5, - "required": true, - "sensitive": false, - "tooltip": "The region that your model or ARN is deployed in.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - } + String content = XContentHelper.stripWhitespace(""" + { + "service": "amazonbedrock", + "name": "Amazon Bedrock", + "task_types": ["text_embedding", "completion"], + "configurations": { + "secret_key": { + "description": "A valid AWS secret key that is paired with the access_key.", + "label": "Secret Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str" + }, + "provider": { + "description": "The model provider for your deployment.", + "label": "Provider", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str" + }, + "access_key": { + "description": "A valid AWS access key that has permissions to use Amazon Bedrock.", + "label": "Access Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str" + }, + "model": { + "description": "The base model ID or an ARN to a custom model based on a foundational model.", + "label": "Model", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str" + }, + "rate_limit.requests_per_minute": { + "description": "By default, the amazonbedrock service sets the number of requests allowed per minute to 240.", + "label": "Rate Limit", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int" + }, + "region": { + "description": "The region that your model or ARN is deployed in.", + "label": "Region", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str" } } - """ - ); + } + """); InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( new BytesArray(content), XContentType.JSON diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java index 0f802637c6700..7eb7ad1d0a19e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java @@ -604,112 +604,33 @@ public void testGetConfiguration() throws Exception { try (var service = createServiceWithMockSender()) { String content = XContentHelper.stripWhitespace(""" { - "provider": "anthropic", - "task_types": [ - { - "task_type": "completion", - "configuration": { - "top_p": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Top P", - "order": 4, - "required": false, - "sensitive": false, - "tooltip": "Specifies to use Anthropic’s nucleus sampling.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "max_tokens": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Max Tokens", - "order": 1, - "required": true, - "sensitive": false, - "tooltip": "The maximum number of tokens to generate before stopping.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "top_k": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Top K", - "order": 3, - "required": false, - "sensitive": false, - "tooltip": "Specifies to only sample from the top K options for each subsequent token.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "temperature": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "Temperature", - "order": 2, - "required": false, - "sensitive": false, - "tooltip": "The amount of randomness injected into the response.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - } - } - } - ], - "configuration": { + "service": "anthropic", + "name": "Anthropic", + "task_types": ["completion"], + "configurations": { "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "API Key for the provider you're connecting to.", "label": "API Key", - "order": 1, "required": true, "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "By default, the anthropic service sets the number of requests allowed per minute to 50.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "By default, the anthropic service sets the number of requests allowed per minute to 50.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "model_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The name of the model to use for the inference task.", "label": "Model ID", - "order": 2, "required": true, "sensitive": false, - "tooltip": "The name of the model to use for the inference task.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java index fe60eaa760e69..72ebd0b96bdc1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java @@ -1389,203 +1389,55 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception { @SuppressWarnings("checkstyle:LineLength") public void testGetConfiguration() throws Exception { try (var service = createService()) { - String content = XContentHelper.stripWhitespace( - """ - { - "provider": "azureaistudio", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": { - "top_p": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Top P", - "order": 4, - "required": false, - "sensitive": false, - "tooltip": "A number in the range of 0.0 to 2.0 that is an alternative value to temperature. Should not be used if temperature is specified.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "max_new_tokens": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Max New Tokens", - "order": 2, - "required": false, - "sensitive": false, - "tooltip": "Provides a hint for the maximum number of output tokens to be generated.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "temperature": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Temperature", - "order": 3, - "required": false, - "sensitive": false, - "tooltip": "A number in the range of 0.0 to 2.0 that specifies the sampling temperature.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "do_sample": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Do Sample", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Instructs the inference process to perform sampling or not.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - } - } - }, - { - "task_type": "completion", - "configuration": { - "user": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "User", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the user issuing the request.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - } - ], - "configuration": { - "endpoint_type": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Endpoint Type", - "options": [ - { - "label": "token", - "value": "token" - }, - { - "label": "realtime", - "value": "realtime" - } - ], - "order": 3, - "required": true, - "sensitive": false, - "tooltip": "Specifies the type of endpoint that is used in your model deployment.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "provider": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Provider", - "options": [ - { - "label": "cohere", - "value": "cohere" - }, - { - "label": "meta", - "value": "meta" - }, - { - "label": "microsoft_phi", - "value": "microsoft_phi" - }, - { - "label": "mistral", - "value": "mistral" - }, - { - "label": "openai", - "value": "openai" - }, - { - "label": "databricks", - "value": "databricks" - } - ], - "order": 3, - "required": true, - "sensitive": false, - "tooltip": "The model provider for your deployment.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "API Key", - "order": 1, - "required": true, - "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Rate Limit", - "order": 6, - "required": false, - "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "target": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "Target", - "order": 2, - "required": true, - "sensitive": false, - "tooltip": "The target URL of your Azure AI Studio model deployment.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - } + String content = XContentHelper.stripWhitespace(""" + { + "service": "azureaistudio", + "name": "Azure AI Studio", + "task_types": ["text_embedding", "completion"], + "configurations": { + "endpoint_type": { + "description": "Specifies the type of endpoint that is used in your model deployment.", + "label": "Endpoint Type", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str" + }, + "provider": { + "description": "The model provider for your deployment.", + "label": "Provider", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str" + }, + "api_key": { + "description": "API Key for the provider you're connecting to.", + "label": "API Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str" + }, + "rate_limit.requests_per_minute": { + "description": "Minimize the number of rate limit errors.", + "label": "Rate Limit", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int" + }, + "target": { + "description": "The target URL of your Azure AI Studio model deployment.", + "label": "Target", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str" } } - """ - ); + } + """); InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( new BytesArray(content), XContentType.JSON diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java index cf79d70749fda..5c69815cbb0ab 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java @@ -1460,131 +1460,57 @@ public void testGetConfiguration() throws Exception { String content = XContentHelper.stripWhitespace( """ { - "provider": "azureopenai", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": { - "user": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "User", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the user issuing the request.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - }, - { - "task_type": "completion", - "configuration": { - "user": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "User", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the user issuing the request.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - } - ], - "configuration": { + "service": "azureopenai", + "name": "Azure OpenAI", + "task_types": ["text_embedding", "completion"], + "configurations": { "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "You must provide either an API key or an Entra ID.", "label": "API Key", - "order": 1, "required": false, "sensitive": true, - "tooltip": "You must provide either an API key or an Entra ID.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "entra_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "You must provide either an API key or an Entra ID.", "label": "Entra ID", - "order": 2, "required": false, "sensitive": true, - "tooltip": "You must provide either an API key or an Entra ID.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "The azureopenai service sets a default number of requests allowed per minute depending on the task type.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "The azureopenai service sets a default number of requests allowed per minute depending on the task type.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "deployment_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The deployment name of your deployed models.", "label": "Deployment ID", - "order": 5, "required": true, "sensitive": false, - "tooltip": "The deployment name of your deployed models.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "resource_name": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The name of your Azure OpenAI resource.", "label": "Resource Name", - "order": 3, "required": true, "sensitive": false, - "tooltip": "The name of your Azure OpenAI resource.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "api_version": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The Azure API version ID to use.", "label": "API Version", - "order": 4, "required": true, "sensitive": false, - "tooltip": "The Azure API version ID to use.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index bf67db3b2a117..dbad76fd46fc4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -1633,147 +1633,31 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception { @SuppressWarnings("checkstyle:LineLength") public void testGetConfiguration() throws Exception { try (var service = createCohereService()) { - String content = XContentHelper.stripWhitespace( - """ - { - "provider": "cohere", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": { - "truncate": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Truncate", - "options": [ - { - "label": "NONE", - "value": "NONE" - }, - { - "label": "START", - "value": "START" - }, - { - "label": "END", - "value": "END" - } - ], - "order": 2, - "required": false, - "sensitive": false, - "tooltip": "Specifies how the API handles inputs longer than the maximum token length.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - }, - "input_type": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Input Type", - "options": [ - { - "label": "classification", - "value": "classification" - }, - { - "label": "clusterning", - "value": "clusterning" - }, - { - "label": "ingest", - "value": "ingest" - }, - { - "label": "search", - "value": "search" - } - ], - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the type of input passed to the model.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - }, - { - "task_type": "rerank", - "configuration": { - "top_n": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Top N", - "order": 2, - "required": false, - "sensitive": false, - "tooltip": "The number of most relevant documents to return, defaults to the number of the documents.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "return_documents": { - "default_value": null, - "depends_on": [], - "display": "toggle", - "label": "Return Documents", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specify whether to return doc text within the results.", - "type": "bool", - "ui_restrictions": [], - "validations": [], - "value": false - } - } - }, - { - "task_type": "completion", - "configuration": {} - } - ], - "configuration": { - "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "API Key", - "order": 1, - "required": true, - "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Rate Limit", - "order": 6, - "required": false, - "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - } + String content = XContentHelper.stripWhitespace(""" + { + "service": "cohere", + "name": "Cohere", + "task_types": ["text_embedding", "rerank", "completion"], + "configurations": { + "api_key": { + "description": "API Key for the provider you're connecting to.", + "label": "API Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str" + }, + "rate_limit.requests_per_minute": { + "description": "Minimize the number of rate limit errors.", + "label": "Rate Limit", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int" } } - """ - ); + } + """); InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( new BytesArray(content), XContentType.JSON diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index 6cdd58c4cd99f..4b2308b9f1565 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -496,55 +496,33 @@ public void testGetConfiguration() throws Exception { try (var service = createServiceWithMockSender()) { String content = XContentHelper.stripWhitespace(""" { - "provider": "elastic", - "task_types": [ - { - "task_type": "sparse_embedding", - "configuration": {} - } - ], - "configuration": { + "service": "elastic", + "name": "Elastic", + "task_types": ["sparse_embedding"], + "configurations": { "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Minimize the number of rate limit errors.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "model_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The name of the model to use for the inference task.", "label": "Model ID", - "order": 2, "required": true, "sensitive": false, - "tooltip": "The name of the model to use for the inference task.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "max_input_tokens": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Allows you to specify the maximum number of tokens per input.", "label": "Maximum Input Tokens", - "order": 3, "required": false, "sensitive": false, - "tooltip": "Allows you to specify the maximum number of tokens per input.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index 21d7efbc7b03c..803151a5b3476 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -1553,100 +1553,36 @@ public void testGetConfiguration() throws Exception { try (var service = createService(mock(Client.class))) { String content = XContentHelper.stripWhitespace(""" { - "provider": "elasticsearch", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": {} - }, - { - "task_type": "sparse_embedding", - "configuration": {} - }, - { - "task_type": "rerank", - "configuration": { - "return_documents": { - "default_value": null, - "depends_on": [], - "display": "toggle", - "label": "Return Documents", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Returns the document instead of only the index.", - "type": "bool", - "ui_restrictions": [], - "validations": [], - "value": true - } - } - } - ], - "configuration": { + "service": "elasticsearch", + "name": "Elasticsearch", + "task_types": ["text_embedding", "sparse_embedding", "rerank"], + "configurations": { "num_allocations": { "default_value": 1, - "depends_on": [], - "display": "numeric", + "description": "The total number of allocations this model is assigned across machine learning nodes.", "label": "Number Allocations", - "order": 2, "required": true, "sensitive": false, - "tooltip": "The total number of allocations this model is assigned across machine learning nodes.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "int" }, "num_threads": { "default_value": 2, - "depends_on": [], - "display": "numeric", + "description": "Sets the number of threads used by each model allocation during inference.", "label": "Number Threads", - "order": 3, "required": true, "sensitive": false, - "tooltip": "Sets the number of threads used by each model allocation during inference.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "model_id": { "default_value": ".multilingual-e5-small", - "depends_on": [], - "display": "dropdown", + "description": "The name of the model to use for the inference task.", "label": "Model ID", - "options": [ - { - "label": ".elser_model_1", - "value": ".elser_model_1" - }, - { - "label": ".elser_model_2", - "value": ".elser_model_2" - }, - { - "label": ".elser_model_2_linux-x86_64", - "value": ".elser_model_2_linux-x86_64" - }, - { - "label": ".multilingual-e5-small", - "value": ".multilingual-e5-small" - }, - { - "label": ".multilingual-e5-small_linux-x86_64", - "value": ".multilingual-e5-small_linux-x86_64" - } - ], - "order": 1, "required": true, "sensitive": false, - "tooltip": "The name of the model to use for the inference task.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java index 3e43bd7c77684..aa45666eb0fb1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java @@ -1123,59 +1123,33 @@ public void testGetConfiguration() throws Exception { try (var service = createGoogleAiStudioService()) { String content = XContentHelper.stripWhitespace(""" { - "provider": "googleaistudio", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": {} - }, - { - "task_type": "completion", - "configuration": {} - } - ], - "configuration": { + "service": "googleaistudio", + "name": "Google AI Studio", + "task_types": ["text_embedding", "completion"], + "configurations": { "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "API Key for the provider you're connecting to.", "label": "API Key", - "order": 1, "required": true, "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Minimize the number of rate limit errors.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "model_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "ID of the LLM you're using.", "label": "Model ID", - "order": 2, "required": true, "sensitive": false, - "tooltip": "ID of the LLM you're using.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java index 2aeba5fcbe209..555e9f0785fa2 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiServiceTests.java @@ -869,149 +869,49 @@ public void testGetConfiguration() throws Exception { String content = XContentHelper.stripWhitespace( """ { - "provider": "googlevertexai", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": { - "input_type": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Input Type", - "options": [ - { - "label": "classification", - "value": "classification" - }, - { - "label": "clustering", - "value": "clustering" - }, - { - "label": "ingest", - "value": "ingest" - }, - { - "label": "search", - "value": "search" - } - ], - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the type of input passed to the model.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - }, - "auto_truncate": { - "default_value": null, - "depends_on": [], - "display": "toggle", - "label": "Auto Truncate", - "order": 2, - "required": false, - "sensitive": false, - "tooltip": "Specifies if the API truncates inputs longer than the maximum token length automatically.", - "type": "bool", - "ui_restrictions": [], - "validations": [], - "value": false - } - } - }, - { - "task_type": "rerank", - "configuration": { - "top_n": { - "default_value": null, - "depends_on": [], - "display": "toggle", - "label": "Top N", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the number of the top n documents, which should be returned.", - "type": "bool", - "ui_restrictions": [], - "validations": [], - "value": false - } - } - } - ], - "configuration": { + "service": "googlevertexai", + "name": "Google Vertex AI", + "task_types": ["text_embedding", "rerank"], + "configurations": { "service_account_json": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "API Key for the provider you're connecting to.", "label": "Credentials JSON", - "order": 1, "required": true, "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "project_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The GCP Project ID which has Vertex AI API(s) enabled. For more information on the URL, refer to the {geminiVertexAIDocs}.", "label": "GCP Project", - "order": 4, "required": true, "sensitive": false, - "tooltip": "The GCP Project ID which has Vertex AI API(s) enabled. For more information on the URL, refer to the {geminiVertexAIDocs}.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "location": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "Please provide the GCP region where the Vertex AI API(s) is enabled. For more information, refer to the {geminiVertexAIDocs}.", "label": "GCP Region", - "order": 3, "required": true, "sensitive": false, - "tooltip": "Please provide the GCP region where the Vertex AI API(s) is enabled. For more information, refer to the {geminiVertexAIDocs}.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Minimize the number of rate limit errors.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "model_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "ID of the LLM you're using.", "label": "Model ID", - "order": 2, "required": true, "sensitive": false, - "tooltip": "ID of the LLM you're using.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java index 12811b6be87b3..0b774badd56b6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java @@ -140,55 +140,33 @@ public void testGetConfiguration() throws Exception { ) { String content = XContentHelper.stripWhitespace(""" { - "provider": "hugging_face_elser", - "task_types": [ - { - "task_type": "sparse_embedding", - "configuration": {} - } - ], - "configuration": { + "service": "hugging_face_elser", + "name": "Hugging Face ELSER", + "task_types": ["sparse_embedding"], + "configurations": { "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "API Key for the provider you're connecting to.", "label": "API Key", - "order": 1, "required": true, "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Minimize the number of rate limit errors.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "url": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The URL endpoint to use for the requests.", "label": "URL", - "order": 1, "required": true, "sensitive": false, - "tooltip": "The URL endpoint to use for the requests.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java index 2d30fd5aba4c2..1399712450af1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java @@ -858,59 +858,34 @@ public void testGetConfiguration() throws Exception { try (var service = createHuggingFaceService()) { String content = XContentHelper.stripWhitespace(""" { - "provider": "hugging_face", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": {} - }, - { - "task_type": "sparse_embedding", - "configuration": {} - } - ], - "configuration": { + "service": "hugging_face", + "name": "Hugging Face", + "task_types": ["text_embedding", "sparse_embedding"], + "configurations": { "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "API Key for the provider you're connecting to.", "label": "API Key", - "order": 1, "required": true, "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Minimize the number of rate limit errors.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "url": { "default_value": "https://api.openai.com/v1/embeddings", - "depends_on": [], - "display": "textbox", + "description": "The URL endpoint to use for the requests.", "label": "URL", - "order": 1, "required": true, "sensitive": false, - "tooltip": "The URL endpoint to use for the requests.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "https://api.openai.com/v1/embeddings" + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java index 3d298823ea19f..6e744fffbff41 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java @@ -975,83 +975,49 @@ public void testGetConfiguration() throws Exception { try (var service = createIbmWatsonxService()) { String content = XContentHelper.stripWhitespace(""" { - "provider": "watsonxai", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": {} - } - ], - "configuration": { + "service": "watsonxai", + "name": "IBM Watsonx", + "task_types": ["text_embedding"], + "configurations": { "project_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "", "label": "Project ID", - "order": 2, "required": true, "sensitive": false, - "tooltip": "", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "model_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The name of the model to use for the inference task.", "label": "Model ID", - "order": 3, "required": true, "sensitive": false, - "tooltip": "The name of the model to use for the inference task.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "api_version": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The IBM Watsonx API version ID to use.", "label": "API Version", - "order": 1, "required": true, "sensitive": false, - "tooltip": "The IBM Watsonx API version ID to use.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "max_input_tokens": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Allows you to specify the maximum number of tokens per input.", "label": "Maximum Input Tokens", - "order": 5, "required": false, "sensitive": false, - "tooltip": "Allows you to specify the maximum number of tokens per input.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "url": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "", "label": "URL", - "order": 4, "required": true, "sensitive": false, - "tooltip": "", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java index 5a1bf8ec383c1..6e68bde6a1266 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java @@ -1831,115 +1831,31 @@ public void testDefaultSimilarity() { @SuppressWarnings("checkstyle:LineLength") public void testGetConfiguration() throws Exception { try (var service = createJinaAIService()) { - String content = XContentHelper.stripWhitespace( - """ - { - "provider": "jinaai", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": { - "task": { - "default_value": null, - "depends_on": [], - "display": "dropdown", - "label": "Task", - "options": [ - { - "label": "retrieval.query", - "value": "retrieval.query" - }, - { - "label": "retrieval.passage", - "value": "retrieval.passage" - }, - { - "label": "classification", - "value": "classification" - }, - { - "label": "separation", - "value": "separation" - } - ], - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the task type passed to the model.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - }, - { - "task_type": "rerank", - "configuration": { - "top_n": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Top N", - "order": 2, - "required": false, - "sensitive": false, - "tooltip": "The number of most relevant documents to return, defaults to the number of the documents.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "return_documents": { - "default_value": null, - "depends_on": [], - "display": "toggle", - "label": "Return Documents", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specify whether to return doc text within the results.", - "type": "bool", - "ui_restrictions": [], - "validations": [], - "value": false - } - } - } - ], - "configuration": { - "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "API Key", - "order": 1, - "required": true, - "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null - }, - "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", - "label": "Rate Limit", - "order": 6, - "required": false, - "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null - } + String content = XContentHelper.stripWhitespace(""" + { + "service": "jinaai", + "name": "Jina AI", + "task_types": ["text_embedding", "rerank"], + "configurations": { + "api_key": { + "description": "API Key for the provider you're connecting to.", + "label": "API Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str" + }, + "rate_limit.requests_per_minute": { + "description": "Minimize the number of rate limit errors.", + "label": "Rate Limit", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int" } } - """ - ); + } + """); InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( new BytesArray(content), XContentType.JSON diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java index f164afe604b43..395e29240dfd4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java @@ -748,69 +748,41 @@ public void testGetConfiguration() throws Exception { try (var service = createService()) { String content = XContentHelper.stripWhitespace(""" { - "provider": "mistral", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": {} - } - ], - "configuration": { + "service": "mistral", + "name": "Mistral", + "task_types": ["text_embedding"], + "configurations": { "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "API Key for the provider you're connecting to.", "label": "API Key", - "order": 1, "required": true, "sensitive": true, - "tooltip": "API Key for the provider you're connecting to.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "model": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "Refer to the Mistral models documentation for the list of available text embedding models.", "label": "Model", - "order": 2, "required": true, "sensitive": false, - "tooltip": "Refer to the Mistral models documentation for the list of available text embedding models.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Minimize the number of rate limit errors.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "Minimize the number of rate limit errors.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "max_input_tokens": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Allows you to specify the maximum number of tokens per input.", "label": "Maximum Input Tokens", - "order": 3, "required": false, "sensitive": false, - "tooltip": "Allows you to specify the maximum number of tokens per input.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index b595862775053..03eacf17a8250 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -1661,117 +1661,50 @@ public void testGetConfiguration() throws Exception { String content = XContentHelper.stripWhitespace( """ { - "provider": "openai", - "task_types": [ - { - "task_type": "text_embedding", - "configuration": { - "user": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "User", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the user issuing the request.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - }, - { - "task_type": "completion", - "configuration": { - "user": { - "default_value": null, - "depends_on": [], - "display": "textbox", - "label": "User", - "order": 1, - "required": false, - "sensitive": false, - "tooltip": "Specifies the user issuing the request.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": "" - } - } - } - ], - "configuration": { + "service": "openai", + "name": "OpenAI", + "task_types": ["text_embedding", "completion"], + "configurations": { "api_key": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The OpenAI API authentication key. For more details about generating OpenAI API keys, refer to the https://platform.openai.com/account/api-keys.", "label": "API Key", - "order": 1, "required": true, "sensitive": true, - "tooltip": "The OpenAI API authentication key. For more details about generating OpenAI API keys, refer to the https://platform.openai.com/account/api-keys.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": true, + "type": "str" }, "organization_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The unique identifier of your organization.", "label": "Organization ID", - "order": 3, "required": false, "sensitive": false, - "tooltip": "The unique identifier of your organization.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "rate_limit.requests_per_minute": { - "default_value": null, - "depends_on": [], - "display": "numeric", + "description": "Default number of requests allowed per minute. For text_embedding is 3000. For completion is 500.", "label": "Rate Limit", - "order": 6, "required": false, "sensitive": false, - "tooltip": "Default number of requests allowed per minute. For text_embedding is 3000. For completion is 500.", - "type": "int", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "int" }, "model_id": { - "default_value": null, - "depends_on": [], - "display": "textbox", + "description": "The name of the model to use for the inference task.", "label": "Model ID", - "order": 2, "required": true, "sensitive": false, - "tooltip": "The name of the model to use for the inference task.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" }, "url": { "default_value": "https://api.openai.com/v1/chat/completions", - "depends_on": [], - "display": "textbox", + "description": "The OpenAI API endpoint URL. For more information on the URL, refer to the https://platform.openai.com/docs/api-reference.", "label": "URL", - "order": 4, "required": true, "sensitive": false, - "tooltip": "The OpenAI API endpoint URL. For more information on the URL, refer to the https://platform.openai.com/docs/api-reference.", - "type": "str", - "ui_restrictions": [], - "validations": [], - "value": null + "updatable": false, + "type": "str" } } } From c3839e1f7642a21bf8e3f8694d3cf5daf95dce6e Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 8 Jan 2025 11:37:03 -0500 Subject: [PATCH 27/52] Add selector syntax to index expressions (#118614) This PR introduces a new syntactical feature to index expression resolution: The selector. Selectors, denoted with a :: followed by a recognized suffix will allow users to specify which component of an index abstraction they would like to operate on within an API call. In this case, an index abstraction is a concrete index, data stream, or alias; Any abstraction that can be resolved to a set of indices/shards. We define a component of an index abstraction to be some searchable unit of the index abstraction. --- .../anomaly-detection/apis/put-job.asciidoc | 1 - modules/data-streams/build.gradle | 13 + .../datastreams/DataStreamsSnapshotsIT.java | 9 +- .../IngestFailureStoreMetricsIT.java | 9 +- .../datastreams/DataStreamOptionsIT.java | 2 +- .../datastreams/FailureStoreQueryParamIT.java | 221 ------ .../DataStreamsStatsTransportAction.java | 14 +- .../lifecycle/DataStreamLifecycleService.java | 25 +- .../DeleteDataStreamLifecycleAction.java | 2 +- .../action/DeleteDataStreamOptionsAction.java | 4 +- .../action/GetDataStreamOptionsAction.java | 4 +- .../action/PutDataStreamOptionsAction.java | 4 +- .../rest/RestGetDataStreamsAction.java | 4 +- .../datastreams/DataStreamsStatsTests.java | 65 +- .../DataStreamLifecycleServiceTests.java | 17 +- .../data_stream/170_modify_data_stream.yml | 3 +- .../210_rollover_failure_store.yml | 32 +- .../rest-api-spec/api/indices.rollover.json | 6 - .../org/elasticsearch/TransportVersions.java | 1 + .../restore/RestoreSnapshotRequest.java | 5 +- .../indices/alias/IndicesAliasesRequest.java | 2 +- .../indices/alias/get/GetAliasesRequest.java | 2 +- .../indices/delete/DeleteIndexRequest.java | 2 +- .../admin/indices/get/GetIndexRequest.java | 9 +- .../mapping/put/PutMappingRequest.java | 2 +- .../put/TransportPutMappingAction.java | 39 +- .../indices/resolve/ResolveIndexAction.java | 67 +- .../indices/rollover/LazyRolloverAction.java | 41 +- .../indices/rollover/RolloverRequest.java | 27 +- .../rollover/TransportRolloverAction.java | 48 +- .../action/bulk/BulkOperation.java | 10 +- .../action/bulk/TransportBulkAction.java | 13 +- .../datastreams/DataStreamsActionUtil.java | 76 +- .../datastreams/DataStreamsStatsAction.java | 5 +- .../datastreams/DeleteDataStreamAction.java | 2 +- .../datastreams/GetDataStreamAction.java | 3 +- .../GetDataStreamLifecycleAction.java | 2 +- .../PutDataStreamLifecycleAction.java | 2 +- .../action/downsample/DownsampleAction.java | 2 +- .../action/support/IndicesOptions.java | 356 ++++----- .../metadata/IndexAbstractionResolver.java | 82 ++- .../metadata/IndexNameExpressionResolver.java | 696 ++++++++++++++---- .../indices/RestRolloverIndexAction.java | 13 +- .../snapshots/RestoreService.java | 1 + .../snapshots/SnapshotsService.java | 3 + .../transport/RemoteClusterAware.java | 22 +- .../action/OriginalIndicesTests.java | 8 +- .../indices/get/GetIndexRequestTests.java | 1 - .../indices/resolve/ResolveIndexTests.java | 6 +- .../MetadataRolloverServiceTests.java | 7 +- .../rollover/RolloverRequestTests.java | 12 +- .../action/bulk/TransportBulkActionTests.java | 8 +- .../DataStreamsActionUtilTests.java | 45 +- .../action/support/IndicesOptionsTests.java | 13 +- .../IndexAbstractionResolverTests.java | 241 ++++++ .../IndexNameExpressionResolverTests.java | 171 ++--- .../metadata/SelectorResolverTests.java | 183 +++++ .../WildcardExpressionResolverTests.java | 385 +++++++--- .../xpack/core/DataStreamRestIT.java | 2 +- .../xpack/core/ilm/LifecyclePolicyUtils.java | 2 +- .../xpack/core/ilm/RolloverStep.java | 7 +- .../core/ilm/WaitForRolloverReadyStep.java | 7 +- .../accesscontrol/IndicesAccessControl.java | 5 +- .../authz/permission/IndicesPermission.java | 90 ++- .../xpack/core/ilm/RolloverStepTests.java | 23 +- .../ilm/WaitForRolloverReadyStepTests.java | 19 +- .../deprecation/DeprecationInfoAction.java | 2 +- .../AnalyticsCollectionResolver.java | 2 +- .../datafeed/DatafeedNodeSelectorTests.java | 11 +- .../authz/IndicesAndAliasesResolver.java | 83 ++- .../xpack/security/authz/RBACEngine.java | 5 + .../authz/IndicesAndAliasesResolverTests.java | 131 +++- .../test/enrich/20_standard_index.yml | 1 + 73 files changed, 2371 insertions(+), 1067 deletions(-) delete mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc index e3c292cc534bf..30a1039f93db0 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc @@ -557,4 +557,3 @@ The API returns the following results: // TESTRESPONSE[s/"job_version" : "8.4.0"/"job_version" : $body.job_version/] // TESTRESPONSE[s/1656087283340/$body.$_path/] // TESTRESPONSE[s/"superuser"/"_es_test_root"/] -// TESTRESPONSE[s/"ignore_throttled" : true/"ignore_throttled" : true,"failure_store":"exclude"/] diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index 97a5fabd79f4c..60bc8d1dc6a92 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -20,6 +20,7 @@ restResources { dependencies { testImplementation project(path: ':test:test-clusters') + testImplementation project(":modules:mapper-extras") internalClusterTestImplementation project(":modules:mapper-extras") } @@ -70,4 +71,16 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("data_stream/200_rollover_failure_store/Lazily roll over a data stream's failure store after a shard failure", "Configuring the failure store via data stream templates is not supported anymore.") task.skipTest("data_stream/200_rollover_failure_store/Don't roll over a data stream's failure store when conditions aren't met", "Configuring the failure store via data stream templates is not supported anymore.") task.skipTest("data_stream/200_rollover_failure_store/Roll over a data stream's failure store with conditions", "Configuring the failure store via data stream templates is not supported anymore.") + + task.skipTest("data_stream/200_rollover_failure_store/Rolling over a failure store on a data stream without the failure store enabled should work", "Rolling over a data stream using target_failure_store is no longer supported.") + task.skipTest("data_stream/200_rollover_failure_store/Rolling over an uninitialized failure store should initialize it", "Rolling over a data stream using target_failure_store is no longer supported.") + + task.skipTest("data_stream/210_rollover_failure_store/A failure store marked for lazy rollover should only be rolled over when there is a failure", "Rolling over a data stream using target_failure_store is no longer supported.") + task.skipTest("data_stream/210_rollover_failure_store/Don't roll over a data stream's failure store when conditions aren't met", "Rolling over a data stream using target_failure_store is no longer supported.") + task.skipTest("data_stream/210_rollover_failure_store/Rolling over a failure store on a data stream without the failure store enabled should work", "Rolling over a data stream using target_failure_store is no longer supported.") + task.skipTest("data_stream/210_rollover_failure_store/Rolling over an uninitialized failure store should initialize it", "Rolling over a data stream using target_failure_store is no longer supported.") + task.skipTest("data_stream/210_rollover_failure_store/Roll over a data stream's failure store with conditions", "Rolling over a data stream using target_failure_store is no longer supported.") + task.skipTest("data_stream/210_rollover_failure_store/Lazily roll over a data stream's failure store after an ingest failure", "Rolling over a data stream using target_failure_store is no longer supported.") + task.skipTest("data_stream/210_rollover_failure_store/Lazily roll over a data stream's failure store after a shard failure", "Rolling over a data stream using target_failure_store is no longer supported.") + task.skipTest("data_stream/210_rollover_failure_store/Roll over a data stream's failure store without conditions", "Rolling over a data stream using target_failure_store is no longer supported.") }) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index 32d080ccc46b1..ac828630b0463 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -31,11 +31,13 @@ import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAlias; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.index.Index; @@ -136,10 +138,7 @@ public void setup() throws Exception { assertTrue(response.isAcknowledged()); // Initialize the failure store. - RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null); - rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build() - ); + RolloverRequest rolloverRequest = new RolloverRequest("with-fs::failures", null); response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get(); assertTrue(response.isAcknowledged()); @@ -345,7 +344,7 @@ public void testFailureStoreSnapshotAndRestore() throws Exception { .cluster() .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT) .setWaitForCompletion(true) - .setIndices(dataStreamName) + .setIndices(IndexNameExpressionResolver.combineSelector(dataStreamName, IndexComponentSelector.ALL_APPLICABLE)) .setIncludeGlobalState(false) .get(); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java index e9eaf7b5faddb..bee3989d20ff0 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java @@ -20,11 +20,12 @@ import org.elasticsearch.action.bulk.FailureStoreMetrics; import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.core.Strings; @@ -194,9 +195,9 @@ public void testRejectionFromFailureStore() throws IOException { createDataStream(); // Initialize failure store. - var rolloverRequest = new RolloverRequest(dataStream, null); - rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build() + var rolloverRequest = new RolloverRequest( + IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES), + null ); var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet(); var failureStoreIndex = rolloverResponse.getNewIndex(); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java index 482867d072fc2..54e21d5155ed1 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamOptionsIT.java @@ -60,7 +60,7 @@ public void setup() throws IOException { assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME))); // Initialize the failure store. - assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store"))); + assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "::failures/_rollover"))); ensureGreen(DATA_STREAM_NAME); final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java deleted file mode 100644 index 85b914be30b2c..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.datastreams; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.junit.Before; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -/** - * This should be a yaml test, but in order to write one we would need to expose the new parameter in the rest-api-spec. - * We do not want to do that until the feature flag is removed. For this reason, we temporarily, test the affected APIs here. - * Please convert this to a yaml test when the feature flag is removed. - */ -public class FailureStoreQueryParamIT extends DisabledSecurityDataStreamTestCase { - - private static final String DATA_STREAM_NAME = "failure-data-stream"; - private String backingIndex; - private String failureStoreIndex; - - @SuppressWarnings("unchecked") - @Before - public void setup() throws IOException { - Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/ds-template"); - putComposableIndexTemplateRequest.setJsonEntity(""" - { - "index_patterns": ["failure-data-stream"], - "template": { - "settings": { - "number_of_replicas": 0 - }, - "data_stream_options": { - "failure_store": { - "enabled": true - } - } - }, - "data_stream": { - } - } - """); - assertOK(client().performRequest(putComposableIndexTemplateRequest)); - - assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME))); - // Initialize the failure store. - assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store"))); - ensureGreen(DATA_STREAM_NAME); - - final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); - List dataStreams = (List) entityAsMap(dataStreamResponse).get("data_streams"); - assertThat(dataStreams.size(), is(1)); - Map dataStream = (Map) dataStreams.get(0); - assertThat(dataStream.get("name"), equalTo(DATA_STREAM_NAME)); - List backingIndices = getIndices(dataStream); - assertThat(backingIndices.size(), is(1)); - List failureStore = getFailureStore(dataStream); - assertThat(failureStore.size(), is(1)); - backingIndex = backingIndices.get(0); - failureStoreIndex = failureStore.get(0); - } - - public void testGetIndexApi() throws IOException { - { - final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME)); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(2)); - assertThat(indices.containsKey(backingIndex), is(true)); - assertThat(indices.containsKey(failureStoreIndex), is(true)); - } - { - final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "?failure_store=exclude")); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(1)); - assertThat(indices.containsKey(backingIndex), is(true)); - } - { - final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "?failure_store=only")); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(1)); - assertThat(indices.containsKey(failureStoreIndex), is(true)); - } - } - - @SuppressWarnings("unchecked") - public void testGetIndexStatsApi() throws IOException { - { - final Response statsResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "/_stats")); - Map indices = (Map) entityAsMap(statsResponse).get("indices"); - assertThat(indices.size(), is(1)); - assertThat(indices.containsKey(backingIndex), is(true)); - } - { - final Response statsResponse = client().performRequest( - new Request("GET", "/" + DATA_STREAM_NAME + "/_stats?failure_store=include") - ); - Map indices = (Map) entityAsMap(statsResponse).get("indices"); - assertThat(indices.size(), is(2)); - assertThat(indices.containsKey(backingIndex), is(true)); - assertThat(indices.containsKey(failureStoreIndex), is(true)); - } - { - final Response statsResponse = client().performRequest( - new Request("GET", "/" + DATA_STREAM_NAME + "/_stats?failure_store=only") - ); - Map indices = (Map) entityAsMap(statsResponse).get("indices"); - assertThat(indices.size(), is(1)); - assertThat(indices.containsKey(failureStoreIndex), is(true)); - } - } - - public void testGetIndexSettingsApi() throws IOException { - { - final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "/_settings")); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(1)); - assertThat(indices.containsKey(backingIndex), is(true)); - } - { - final Response indicesResponse = client().performRequest( - new Request("GET", "/" + DATA_STREAM_NAME + "/_settings?failure_store=include") - ); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(2)); - assertThat(indices.containsKey(backingIndex), is(true)); - assertThat(indices.containsKey(failureStoreIndex), is(true)); - } - { - final Response indicesResponse = client().performRequest( - new Request("GET", "/" + DATA_STREAM_NAME + "/_settings?failure_store=only") - ); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(1)); - assertThat(indices.containsKey(failureStoreIndex), is(true)); - } - } - - public void testGetIndexMappingApi() throws IOException { - { - final Response indicesResponse = client().performRequest(new Request("GET", "/" + DATA_STREAM_NAME + "/_mapping")); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(1)); - assertThat(indices.containsKey(backingIndex), is(true)); - } - { - final Response indicesResponse = client().performRequest( - new Request("GET", "/" + DATA_STREAM_NAME + "/_mapping?failure_store=include") - ); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(2)); - assertThat(indices.containsKey(backingIndex), is(true)); - assertThat(indices.containsKey(failureStoreIndex), is(true)); - } - { - final Response indicesResponse = client().performRequest( - new Request("GET", "/" + DATA_STREAM_NAME + "/_mapping?failure_store=only") - ); - Map indices = entityAsMap(indicesResponse); - assertThat(indices.size(), is(1)); - assertThat(indices.containsKey(failureStoreIndex), is(true)); - } - } - - @SuppressWarnings("unchecked") - public void testPutIndexMappingApi() throws IOException { - { - final Request mappingRequest = new Request("PUT", "/" + DATA_STREAM_NAME + "/_mapping"); - mappingRequest.setJsonEntity(""" - { - "properties": { - "email": { - "type": "keyword" - } - } - } - """); - assertAcknowledged(client().performRequest(mappingRequest)); - } - { - final Request mappingRequest = new Request("PUT", "/" + DATA_STREAM_NAME + "/_mapping?failure_store=include"); - mappingRequest.setJsonEntity(""" - { - "properties": { - "email": { - "type": "keyword" - } - } - } - """); - ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(mappingRequest)); - Map response = entityAsMap(responseException.getResponse()); - assertThat(((Map) response.get("error")).get("reason"), is("failure index not supported")); - } - } - - @SuppressWarnings("unchecked") - private List getFailureStore(Map response) { - var failureStore = (Map) response.get("failure_store"); - return getIndices(failureStore); - - } - - @SuppressWarnings("unchecked") - private List getIndices(Map response) { - List> indices = (List>) response.get("indices"); - return indices.stream().map(index -> index.get("index_name")).toList(); - } -} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java index 1d3b1b676282a..cc5e00d8283ad 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.datastreams.DataStreamsStatsAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -102,10 +103,11 @@ protected ClusterBlockException checkRequestBlock( @Override protected String[] resolveConcreteIndexNames(ClusterState clusterState, DataStreamsStatsAction.Request request) { - return DataStreamsActionUtil.resolveConcreteIndexNames( + return DataStreamsActionUtil.resolveConcreteIndexNamesWithSelector( indexNameExpressionResolver, clusterState, request.indices(), + IndexComponentSelector.ALL_APPLICABLE, request.indicesOptions() ).toArray(String[]::new); } @@ -163,13 +165,17 @@ protected DataStreamsStatsAction.DataStreamShardStats readShardResult(StreamInpu request.indicesOptions(), request.indices() ); - for (String abstractionName : abstractionNames) { - IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName); + for (String abstraction : abstractionNames) { + IndexAbstraction indexAbstraction = indicesLookup.get(abstraction); assert indexAbstraction != null; if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { DataStream dataStream = (DataStream) indexAbstraction; AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats()); - dataStream.getIndices().stream().map(Index::getName).forEach(index -> { + dataStream.getBackingIndices().getIndices().stream().map(Index::getName).forEach(index -> { + stats.backingIndices.add(index); + allBackingIndices.add(index); + }); + dataStream.getFailureIndices().getIndices().stream().map(Index::getName).forEach(index -> { stats.backingIndices.add(index); allBackingIndices.add(index); }); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 7d2828e30d5ab..7de3f180753f8 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -33,7 +33,7 @@ import org.elasticsearch.action.downsample.DownsampleAction; import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; @@ -49,6 +49,9 @@ import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; @@ -944,11 +947,6 @@ private Set maybeExecuteForceMerge(ClusterState state, List indice if ((configuredFloorSegmentMerge == null || configuredFloorSegmentMerge.equals(targetMergePolicyFloorSegment) == false) || (configuredMergeFactor == null || configuredMergeFactor.equals(targetMergePolicyFactor) == false)) { UpdateSettingsRequest updateMergePolicySettingsRequest = new UpdateSettingsRequest(); - updateMergePolicySettingsRequest.indicesOptions( - IndicesOptions.builder(updateMergePolicySettingsRequest.indicesOptions()) - .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE) - .build() - ); updateMergePolicySettingsRequest.indices(indexName); updateMergePolicySettingsRequest.settings( Settings.builder() @@ -998,8 +996,11 @@ private Set maybeExecuteForceMerge(ClusterState state, List indice private void rolloverDataStream(String writeIndexName, RolloverRequest rolloverRequest, ActionListener listener) { // "saving" the rollover target name here so we don't capture the entire request - String rolloverTarget = rolloverRequest.getRolloverTarget(); - logger.trace("Data stream lifecycle issues rollover request for data stream [{}]", rolloverTarget); + ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression( + rolloverRequest.getRolloverTarget(), + rolloverRequest.indicesOptions() + ); + logger.trace("Data stream lifecycle issues rollover request for data stream [{}]", rolloverRequest.getRolloverTarget()); client.admin().indices().rolloverIndex(rolloverRequest, new ActionListener<>() { @Override public void onResponse(RolloverResponse rolloverResponse) { @@ -1014,7 +1015,7 @@ public void onResponse(RolloverResponse rolloverResponse) { logger.info( "Data stream lifecycle successfully rolled over datastream [{}] due to the following met rollover " + "conditions {}. The new index is [{}]", - rolloverTarget, + rolloverRequest.getRolloverTarget(), metConditions, rolloverResponse.getNewIndex() ); @@ -1024,7 +1025,7 @@ public void onResponse(RolloverResponse rolloverResponse) { @Override public void onFailure(Exception e) { - DataStream dataStream = clusterService.state().metadata().dataStreams().get(rolloverTarget); + DataStream dataStream = clusterService.state().metadata().dataStreams().get(resolvedRolloverTarget.resource()); if (dataStream == null || dataStream.getWriteIndex().getName().equals(writeIndexName) == false) { // the data stream has another write index so no point in recording an error for the previous write index we were // attempting to roll over @@ -1407,9 +1408,7 @@ static RolloverRequest getDefaultRolloverRequest( ) { RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null).masterNodeTimeout(TimeValue.MAX_VALUE); if (rolloverFailureStore) { - rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build() - ); + rolloverRequest.setRolloverTarget(IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES)); } rolloverRequest.setConditions(rolloverConfiguration.resolveRolloverConditions(dataRetention)); return rolloverRequest; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java index 1595348649528..7992362d791b1 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java @@ -50,7 +50,7 @@ public static final class Request extends AcknowledgedRequest implement .allowAliasToMultipleIndices(false) .allowClosedIndices(true) .ignoreThrottled(false) - .allowFailureIndices(false) + .allowSelectors(false) .build() ) .build(); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java index 98a29dd636ddf..860bcb5bf2fbe 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/DeleteDataStreamOptionsAction.java @@ -39,7 +39,9 @@ public static final class Request extends AcknowledgedRequest implement .wildcardOptions( IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) ) - .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true).allowSelectors(false) + ) .build(); public Request(StreamInput in) throws IOException { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java index c1354da1129ca..45bda1abd5c02 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/GetDataStreamOptionsAction.java @@ -50,7 +50,9 @@ public static class Request extends MasterNodeReadRequest implements In .wildcardOptions( IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) ) - .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true).allowSelectors(false) + ) .build(); private boolean includeDefaults = false; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java index d055a6972312a..d66b45665d4e2 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/options/action/PutDataStreamOptionsAction.java @@ -71,7 +71,9 @@ public static Request parseRequest(XContentParser parser, Factory factory) { .wildcardOptions( IndicesOptions.WildcardOptions.builder().matchOpen(true).matchClosed(true).allowEmptyExpressions(true).resolveAliases(false) ) - .gatekeeperOptions(IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true)) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder().allowAliasToMultipleIndices(false).allowClosedIndices(true).allowSelectors(false) + ) .build(); private final DataStreamOptions options; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index b61e38297397d..be157608b1c3f 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.set.Sets; @@ -42,8 +41,7 @@ public class RestGetDataStreamsAction extends BaseRestHandler { IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, "verbose" - ), - DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FAILURE_STORE_QUERY_PARAM) : Set.of() + ) ) ); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java index d5c5193948213..e32636fe40d84 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.datastreams.DataStreamsStatsAction; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; @@ -22,8 +23,12 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamFailureStore; +import org.elasticsearch.cluster.metadata.DataStreamOptions; +import org.elasticsearch.cluster.metadata.ResettableValue; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.json.JsonXContent; @@ -40,12 +45,14 @@ import static java.lang.Math.max; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; public class DataStreamsStatsTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return List.of(DataStreamsPlugin.class); + return List.of(DataStreamsPlugin.class, MapperExtrasPlugin.class); } private final Set createdDataStreams = new HashSet<>(); @@ -107,8 +114,30 @@ public void testStatsExistingDataStream() throws Exception { assertEquals(stats.getTotalStoreSize().getBytes(), stats.getDataStreams()[0].getStoreSize().getBytes()); } + public void testStatsExistingDataStreamWithFailureStores() throws Exception { + String dataStreamName = createDataStream(false, true); + createFailedDocument(dataStreamName); + + DataStreamsStatsAction.Response stats = getDataStreamsStats(); + + assertEquals(2, stats.getSuccessfulShards()); + assertEquals(0, stats.getFailedShards()); + assertEquals(1, stats.getDataStreamCount()); + assertEquals(2, stats.getBackingIndices()); + assertNotEquals(0L, stats.getTotalStoreSize().getBytes()); + assertEquals(1, stats.getDataStreams().length); + assertEquals(dataStreamName, stats.getDataStreams()[0].getDataStream()); + assertEquals(2, stats.getDataStreams()[0].getBackingIndices()); + // The timestamp is going to not be something we can validate because + // it captures the time of failure which is uncontrolled in the test + // Just make sure it exists by ensuring it isn't zero + assertThat(stats.getDataStreams()[0].getMaximumTimestamp(), is(greaterThan(0L))); + assertNotEquals(0L, stats.getDataStreams()[0].getStoreSize().getBytes()); + assertEquals(stats.getTotalStoreSize().getBytes(), stats.getDataStreams()[0].getStoreSize().getBytes()); + } + public void testStatsExistingHiddenDataStream() throws Exception { - String dataStreamName = createDataStream(true); + String dataStreamName = createDataStream(true, false); long timestamp = createDocument(dataStreamName); DataStreamsStatsAction.Response stats = getDataStreamsStats(true); @@ -221,14 +250,19 @@ public void testStatsMultipleDataStreams() throws Exception { } private String createDataStream() throws Exception { - return createDataStream(false); + return createDataStream(false, false); } - private String createDataStream(boolean hidden) throws Exception { + private String createDataStream(boolean hidden, boolean failureStore) throws Exception { String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault()); + ResettableValue failureStoreOptions = failureStore == false + ? ResettableValue.undefined() + : ResettableValue.create( + new DataStreamOptions.Template(ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true)))) + ); Template idxTemplate = new Template(null, new CompressedXContent(""" {"properties":{"@timestamp":{"type":"date"},"data":{"type":"keyword"}}} - """), null); + """), null, null, failureStoreOptions); ComposableIndexTemplate template = ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStreamName + "*")) .template(idxTemplate) @@ -269,6 +303,27 @@ private long createDocument(String dataStreamName) throws Exception { return timestamp; } + private long createFailedDocument(String dataStreamName) throws Exception { + // Get some randomized but reasonable timestamps on the data since not all of it is guaranteed to arrive in order. + long timeSeed = System.currentTimeMillis(); + long timestamp = randomLongBetween(timeSeed - TimeUnit.HOURS.toMillis(5), timeSeed); + client().bulk( + new BulkRequest(dataStreamName).add( + new IndexRequest().opType(DocWriteRequest.OpType.CREATE) + .source( + JsonXContent.contentBuilder() + .startObject() + .field("@timestamp", timestamp) + .object("data", b -> b.field("garbage", randomAlphaOfLength(25))) + .endObject() + ) + ) + ).get(); + indicesAdmin().refresh(new RefreshRequest(".fs-" + dataStreamName + "*").indicesOptions(IndicesOptions.lenientExpandOpenHidden())) + .get(); + return timestamp; + } + private DataStreamsStatsAction.Response getDataStreamsStats() throws Exception { return getDataStreamsStats(false); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 698ab427ab040..ac7dabd868a3f 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.action.downsample.DownsampleAction; import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; @@ -46,6 +46,7 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -225,11 +226,12 @@ public void testOperationsExecutedOnce() { assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0); assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA)); assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class)); RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1); - assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES)); + assertThat( + rolloverFailureIndexRequest.getRolloverTarget(), + is(IndexNameExpressionResolver.combineSelector(dataStreamName, IndexComponentSelector.FAILURES)) + ); List deleteRequests = clientSeenRequests.subList(2, 5) .stream() .map(transportRequest -> (DeleteIndexRequest) transportRequest) @@ -1546,11 +1548,12 @@ public void testFailureStoreIsManagedEvenWhenDisabled() { assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0); assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA)); assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class)); RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1); - assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName)); - assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES)); + assertThat( + rolloverFailureIndexRequest.getRolloverTarget(), + is(IndexNameExpressionResolver.combineSelector(dataStreamName, IndexComponentSelector.FAILURES)) + ); assertThat( ((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0], is(dataStream.getFailureIndices().getIndices().get(0).getName()) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml index 13f79e95d99f4..f439cf59bf2d3 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml @@ -148,8 +148,7 @@ # rollover data stream to create new failure store index - do: indices.rollover: - alias: "data-stream-for-modification" - target_failure_store: true + alias: "data-stream-for-modification::failures" - is_true: acknowledged # save index names for later use diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml index cc3a11ffde5e8..51a1e96b1e937 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/210_rollover_failure_store.yml @@ -9,7 +9,7 @@ setup: capabilities: [ 'failure_store_in_template' ] - method: POST path: /{index}/_rollover - capabilities: [ 'lazy-rollover-failure-store' ] + capabilities: [ 'lazy-rollover-failure-store', 'index-expression-selectors' ] - do: allowed_warnings: @@ -58,8 +58,7 @@ teardown: - do: indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true + alias: "data-stream-for-rollover::failures" - match: { acknowledged: true } - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } @@ -92,8 +91,7 @@ teardown: - do: indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true + alias: "data-stream-for-rollover::failures" body: conditions: max_docs: 1 @@ -130,8 +128,7 @@ teardown: - do: indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true + alias: "data-stream-for-rollover::failures" body: conditions: max_primary_shard_docs: 2 @@ -165,8 +162,7 @@ teardown: # Mark the failure store for lazy rollover - do: indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true + alias: "data-stream-for-rollover::failures" lazy: true - match: { acknowledged: true } @@ -263,8 +259,7 @@ teardown: # Mark the failure store for lazy rollover - do: indices.rollover: - alias: data-stream-for-lazy-rollover - target_failure_store: true + alias: data-stream-for-lazy-rollover::failures lazy: true - match: { acknowledged: true } @@ -332,8 +327,7 @@ teardown: # Mark the failure store for lazy rollover - do: indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true + alias: "data-stream-for-rollover::failures" lazy: true - match: { acknowledged: true } @@ -377,16 +371,14 @@ teardown: - do: catch: /Rolling over\/initializing an empty failure store is only supported without conditions\./ indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true + alias: "data-stream-for-rollover::failures" body: conditions: max_docs: 1 - do: indices.rollover: - alias: "data-stream-for-rollover" - target_failure_store: true + alias: "data-stream-for-rollover::failures" - match: { acknowledged: true } - match: { old_index: "_none_" } @@ -424,8 +416,7 @@ teardown: # Initializing should work - do: indices.rollover: - alias: "other-data-stream-for-rollover" - target_failure_store: true + alias: "other-data-stream-for-rollover::failures" - match: { acknowledged: true } - match: { old_index: "_none_" } @@ -448,8 +439,7 @@ teardown: # And "regular" rollover should work - do: indices.rollover: - alias: "other-data-stream-for-rollover" - target_failure_store: true + alias: "other-data-stream-for-rollover::failures" - match: { acknowledged: true } - match: { old_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json index 299c24f987d8d..47a1bee665506 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json @@ -63,12 +63,6 @@ "type":"boolean", "default":"false", "description":"If set to true, the rollover action will only mark a data stream to signal that it needs to be rolled over at the next write. Only allowed on data streams." - }, - "target_failure_store":{ - "type":"boolean", - "description":"If set to true, the rollover action will be applied on the failure store of the data stream.", - "visibility": "feature_flag", - "feature_flag": "es.failure_store_feature_flag_enabled" } }, "body":{ diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 39f9a1f34af3c..da0c91861596d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -153,6 +153,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_ENABLE_NODE_LEVEL_REDUCTION = def(8_818_00_0); public static final TransportVersion JINA_AI_INTEGRATION_ADDED = def(8_819_00_0); public static final TransportVersion TRACK_INDEX_FAILED_DUE_TO_VERSION_CONFLICT_METRIC = def(8_820_00_0); + public static final TransportVersion REPLACE_FAILURE_STORE_OPTIONS_WITH_SELECTOR_SYNTAX = def(8_821_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 03e05ca0e4247..24c427c32d69a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -44,9 +43,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest .allowAliasToMultipleIndices(false) .allowClosedIndices(true) .ignoreThrottled(false) - .allowFailureIndices(true) + .allowSelectors(false) .build() ) .build(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java index 801dbbdee0858..be7aaeec8f69e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.info.ClusterInfoRequest; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.ArrayUtils; @@ -95,13 +94,7 @@ public static Feature[] fromRequest(RestRequest request) { private transient boolean includeDefaults = false; public GetIndexRequest() { - super( - DataStream.isFailureStoreFeatureFlagEnabled() - ? IndicesOptions.builder(IndicesOptions.strictExpandOpen()) - .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE) - .build() - : IndicesOptions.strictExpandOpen() - ); + super(IndicesOptions.strictExpandOpen()); } public GetIndexRequest(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java index 7b782c6da5a84..05cc0d2cf05d8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -82,7 +82,7 @@ public class PutMappingRequest extends AcknowledgedRequest im .allowClosedIndices(true) .allowAliasToMultipleIndices(true) .ignoreThrottled(false) - .allowFailureIndices(false) + .allowSelectors(false) ) .build(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 749470e181deb..24f8735b6bd7f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -20,6 +20,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataMappingService; import org.elasticsearch.cluster.service.ClusterService; @@ -40,6 +42,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.SortedMap; /** * Put mapping action. @@ -106,7 +109,14 @@ protected void masterOperation( return; } - final String message = checkForSystemIndexViolations(systemIndices, concreteIndices, request); + String message = checkForFailureStoreViolations(clusterService.state(), concreteIndices, request); + if (message != null) { + logger.warn(message); + listener.onFailure(new IllegalStateException(message)); + return; + } + + message = checkForSystemIndexViolations(systemIndices, concreteIndices, request); if (message != null) { logger.warn(message); listener.onFailure(new IllegalStateException(message)); @@ -172,6 +182,33 @@ static void performMappingUpdate( metadataMappingService.putMapping(updateRequest, wrappedListener); } + static String checkForFailureStoreViolations(ClusterState clusterState, Index[] concreteIndices, PutMappingRequest request) { + // Requests that a cluster generates itself are permitted to make changes to mappings + // so that rolling upgrade scenarios still work. We check this via the request's origin. + if (Strings.isNullOrEmpty(request.origin()) == false) { + return null; + } + + List violations = new ArrayList<>(); + SortedMap indicesLookup = clusterState.metadata().getIndicesLookup(); + for (Index index : concreteIndices) { + IndexAbstraction indexAbstraction = indicesLookup.get(index.getName()); + if (indexAbstraction != null) { + DataStream maybeDataStream = indexAbstraction.getParentDataStream(); + if (maybeDataStream != null && maybeDataStream.isFailureStoreIndex(index.getName())) { + violations.add(index.getName()); + } + } + } + + if (violations.isEmpty() == false) { + return "Cannot update mappings in " + + violations + + ": mappings for indices contained in data stream failure stores cannot be updated"; + } + return null; + } + static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] concreteIndices, PutMappingRequest request) { // Requests that a cluster generates itself are permitted to have a difference in mappings // so that rolling upgrade scenarios still work. We check this via the request's origin. diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index f5c100b7884bb..4aa022aff1c80 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -59,6 +59,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import java.util.stream.Stream; import static org.elasticsearch.action.search.TransportSearchHelper.checkCCSVersionCompatibility; @@ -598,12 +599,13 @@ private static void mergeResults( private static void enrichIndexAbstraction( ClusterState clusterState, - ResolvedExpression indexAbstraction, + ResolvedExpression resolvedExpression, List indices, List aliases, List dataStreams ) { - IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction.resource()); + SortedMap indicesLookup = clusterState.metadata().getIndicesLookup(); + IndexAbstraction ia = indicesLookup.get(resolvedExpression.resource()); if (ia != null) { switch (ia.getType()) { case CONCRETE_INDEX -> { @@ -632,13 +634,24 @@ private static void enrichIndexAbstraction( ); } case ALIAS -> { - String[] indexNames = ia.getIndices().stream().map(Index::getName).toArray(String[]::new); + String[] indexNames = getAliasIndexStream(resolvedExpression, ia, indicesLookup).map(Index::getName) + .toArray(String[]::new); Arrays.sort(indexNames); aliases.add(new ResolvedAlias(ia.getName(), indexNames)); } case DATA_STREAM -> { DataStream dataStream = (DataStream) ia; - String[] backingIndices = dataStream.getIndices().stream().map(Index::getName).toArray(String[]::new); + Stream dataStreamIndices = resolvedExpression.selector() == null + ? dataStream.getIndices().stream() + : switch (resolvedExpression.selector()) { + case DATA -> dataStream.getBackingIndices().getIndices().stream(); + case FAILURES -> dataStream.getFailureIndices().getIndices().stream(); + case ALL_APPLICABLE -> Stream.concat( + dataStream.getBackingIndices().getIndices().stream(), + dataStream.getFailureIndices().getIndices().stream() + ); + }; + String[] backingIndices = dataStreamIndices.map(Index::getName).toArray(String[]::new); dataStreams.add(new ResolvedDataStream(dataStream.getName(), backingIndices, DataStream.TIMESTAMP_FIELD_NAME)); } default -> throw new IllegalStateException("unknown index abstraction type: " + ia.getType()); @@ -646,6 +659,52 @@ private static void enrichIndexAbstraction( } } + private static Stream getAliasIndexStream( + ResolvedExpression resolvedExpression, + IndexAbstraction ia, + SortedMap indicesLookup + ) { + Stream aliasIndices; + if (resolvedExpression.selector() == null) { + aliasIndices = ia.getIndices().stream(); + } else { + aliasIndices = switch (resolvedExpression.selector()) { + case DATA -> ia.getIndices().stream(); + case FAILURES -> { + assert ia.isDataStreamRelated() : "Illegal selector [failures] used on non data stream alias"; + yield ia.getIndices() + .stream() + .map(Index::getName) + .map(indicesLookup::get) + .map(IndexAbstraction::getParentDataStream) + .filter(Objects::nonNull) + .distinct() + .map(DataStream::getFailureIndices) + .flatMap(failureIndices -> failureIndices.getIndices().stream()); + } + case ALL_APPLICABLE -> { + if (ia.isDataStreamRelated()) { + yield Stream.concat( + ia.getIndices().stream(), + ia.getIndices() + .stream() + .map(Index::getName) + .map(indicesLookup::get) + .map(IndexAbstraction::getParentDataStream) + .filter(Objects::nonNull) + .distinct() + .map(DataStream::getFailureIndices) + .flatMap(failureIndices -> failureIndices.getIndices().stream()) + ); + } else { + yield ia.getIndices().stream(); + } + } + }; + } + return aliasIndices; + } + enum Attribute { OPEN, CLOSED, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java index a677897d79633..7b28acdbd8f84 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java @@ -21,6 +21,8 @@ import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; @@ -119,32 +121,38 @@ protected void masterOperation( : "The auto rollover action does not expect any other parameters in the request apart from the data stream name"; Metadata metadata = clusterState.metadata(); - DataStream dataStream = metadata.dataStreams().get(rolloverRequest.getRolloverTarget()); + ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression( + rolloverRequest.getRolloverTarget(), + rolloverRequest.indicesOptions() + ); + boolean isFailureStoreRollover = resolvedRolloverTarget.selector() != null + && resolvedRolloverTarget.selector().shouldIncludeFailures(); + + DataStream dataStream = metadata.dataStreams().get(resolvedRolloverTarget.resource()); // Skip submitting the task if we detect that the lazy rollover has been already executed. - if (isLazyRolloverNeeded(dataStream, rolloverRequest.targetsFailureStore()) == false) { - DataStream.DataStreamIndices targetIndices = dataStream.getDataStreamIndices(rolloverRequest.targetsFailureStore()); + if (isLazyRolloverNeeded(dataStream, isFailureStoreRollover) == false) { + DataStream.DataStreamIndices targetIndices = dataStream.getDataStreamIndices(isFailureStoreRollover); listener.onResponse(noopLazyRolloverResponse(targetIndices)); return; } // We evaluate the names of the source index as well as what our newly created index would be. final MetadataRolloverService.NameResolution trialRolloverNames = MetadataRolloverService.resolveRolloverNames( clusterState, - rolloverRequest.getRolloverTarget(), + resolvedRolloverTarget.resource(), rolloverRequest.getNewIndexName(), rolloverRequest.getCreateIndexRequest(), - rolloverRequest.targetsFailureStore() + isFailureStoreRollover ); final String trialSourceIndexName = trialRolloverNames.sourceName(); final String trialRolloverIndexName = trialRolloverNames.rolloverName(); MetadataCreateIndexService.validateIndexName(trialRolloverIndexName, clusterState.metadata(), clusterState.routingTable()); - assert metadata.dataStreams().containsKey(rolloverRequest.getRolloverTarget()) : "Auto-rollover applies only to data streams"; + assert metadata.dataStreams().containsKey(resolvedRolloverTarget.resource()) : "Auto-rollover applies only to data streams"; String source = "lazy_rollover source [" + trialSourceIndexName + "] to target [" + trialRolloverIndexName + "]"; // We create a new rollover request to ensure that it doesn't contain any other parameters apart from the data stream name // This will provide a more resilient user experience - var newRolloverRequest = new RolloverRequest(rolloverRequest.getRolloverTarget(), null); - newRolloverRequest.setIndicesOptions(rolloverRequest.indicesOptions()); + var newRolloverRequest = new RolloverRequest(resolvedRolloverTarget.combined(), null); LazyRolloverTask rolloverTask = new LazyRolloverTask(newRolloverRequest, listener); lazyRolloverTaskQueue.submitTask(source, rolloverTask, rolloverRequest.masterNodeTimeout()); } @@ -223,12 +231,19 @@ public ClusterState executeTask( AllocationActionMultiListener allocationActionMultiListener ) throws Exception { + ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression( + rolloverRequest.getRolloverTarget(), + rolloverRequest.indicesOptions() + ); + boolean isFailureStoreRollover = resolvedRolloverTarget.selector() != null + && resolvedRolloverTarget.selector().shouldIncludeFailures(); + // If the data stream has been rolled over since it was marked for lazy rollover, this operation is a noop - final DataStream dataStream = currentState.metadata().dataStreams().get(rolloverRequest.getRolloverTarget()); + final DataStream dataStream = currentState.metadata().dataStreams().get(resolvedRolloverTarget.resource()); assert dataStream != null; - if (isLazyRolloverNeeded(dataStream, rolloverRequest.targetsFailureStore()) == false) { - final DataStream.DataStreamIndices targetIndices = dataStream.getDataStreamIndices(rolloverRequest.targetsFailureStore()); + if (isLazyRolloverNeeded(dataStream, isFailureStoreRollover) == false) { + final DataStream.DataStreamIndices targetIndices = dataStream.getDataStreamIndices(isFailureStoreRollover); var noopResponse = noopLazyRolloverResponse(targetIndices); notifyAllListeners(rolloverTaskContexts, context -> context.getTask().listener.onResponse(noopResponse)); return currentState; @@ -237,7 +252,7 @@ public ClusterState executeTask( // Perform the actual rollover final var rolloverResult = rolloverService.rolloverClusterState( currentState, - rolloverRequest.getRolloverTarget(), + resolvedRolloverTarget.resource(), rolloverRequest.getNewIndexName(), rolloverRequest.getCreateIndexRequest(), List.of(), @@ -246,7 +261,7 @@ public ClusterState executeTask( false, null, null, - rolloverRequest.targetsFailureStore() + isFailureStoreRollover ); results.add(rolloverResult); logger.trace("lazy rollover result [{}]", rolloverResult); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 552ce727d4249..608d32d50a856 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -16,7 +16,8 @@ import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.mapper.MapperService; @@ -81,7 +82,7 @@ public class RolloverRequest extends AcknowledgedRequest implem private RolloverConditions conditions = new RolloverConditions(); // the index name "_na_" is never read back, what matters are settings, mappings and aliases private CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); - private IndicesOptions indicesOptions = IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + private IndicesOptions indicesOptions = IndicesOptions.strictSingleIndexNoExpandForbidClosedAllowSelectors(); public RolloverRequest(StreamInput in) throws IOException { super(in); @@ -125,12 +126,15 @@ public ActionRequestValidationException validate() { ); } - var selector = indicesOptions.selectorOptions().defaultSelector(); - if (selector == IndexComponentSelector.ALL_APPLICABLE) { - validationException = addValidationError( - "rollover cannot be applied to both regular and failure indices at the same time", - validationException - ); + if (rolloverTarget != null) { + ResolvedExpression resolvedExpression = SelectorResolver.parseExpression(rolloverTarget, indicesOptions); + IndexComponentSelector selector = resolvedExpression.selector(); + if (IndexComponentSelector.ALL_APPLICABLE.equals(selector)) { + validationException = addValidationError( + "rollover cannot be applied to both regular and failure indices at the same time", + validationException + ); + } } return validationException; @@ -162,13 +166,6 @@ public IndicesOptions indicesOptions() { return indicesOptions; } - /** - * @return true of the rollover request targets the failure store, false otherwise. - */ - public boolean targetsFailureStore() { - return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.includeFailureIndices(); - } - public void setIndicesOptions(IndicesOptions indicesOptions) { this.indicesOptions = indicesOptions; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index c5c874f9bcddf..4f0aa9c5bade4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -36,6 +36,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadataStats; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SelectorResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; @@ -149,8 +151,7 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState .matchOpen(request.indicesOptions().expandWildcardsOpen()) .matchClosed(request.indicesOptions().expandWildcardsClosed()) .build(), - IndicesOptions.GatekeeperOptions.DEFAULT, - request.indicesOptions().selectorOptions() + IndicesOptions.GatekeeperOptions.DEFAULT ); return state.blocks() @@ -170,11 +171,18 @@ protected void masterOperation( assert task instanceof CancellableTask; Metadata metadata = clusterState.metadata(); + + // Parse the rollover request's target since the expression it may contain a selector on it + ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression( + rolloverRequest.getRolloverTarget(), + rolloverRequest.indicesOptions() + ); + boolean targetFailureStore = resolvedRolloverTarget.selector() != null && resolvedRolloverTarget.selector().shouldIncludeFailures(); + // We evaluate the names of the index for which we should evaluate conditions, as well as what our newly created index *would* be. - boolean targetFailureStore = rolloverRequest.targetsFailureStore(); final MetadataRolloverService.NameResolution trialRolloverNames = MetadataRolloverService.resolveRolloverNames( clusterState, - rolloverRequest.getRolloverTarget(), + resolvedRolloverTarget.resource(), rolloverRequest.getNewIndexName(), rolloverRequest.getCreateIndexRequest(), targetFailureStore @@ -183,7 +191,7 @@ protected void masterOperation( final String trialRolloverIndexName = trialRolloverNames.rolloverName(); MetadataCreateIndexService.validateIndexName(trialRolloverIndexName, metadata, clusterState.routingTable()); - boolean isDataStream = metadata.dataStreams().containsKey(rolloverRequest.getRolloverTarget()); + boolean isDataStream = metadata.dataStreams().containsKey(resolvedRolloverTarget.resource()); if (rolloverRequest.isLazy()) { if (isDataStream == false || rolloverRequest.getConditions().hasConditions()) { String message; @@ -201,7 +209,7 @@ protected void masterOperation( } if (rolloverRequest.isDryRun() == false) { metadataDataStreamsService.setRolloverOnWrite( - rolloverRequest.getRolloverTarget(), + resolvedRolloverTarget.resource(), true, targetFailureStore, rolloverRequest.ackTimeout(), @@ -225,7 +233,7 @@ protected void masterOperation( final IndexAbstraction rolloverTargetAbstraction = clusterState.metadata() .getIndicesLookup() - .get(rolloverRequest.getRolloverTarget()); + .get(resolvedRolloverTarget.resource()); if (rolloverTargetAbstraction.getType() == IndexAbstraction.Type.ALIAS && rolloverTargetAbstraction.isDataStreamRelated()) { listener.onFailure( new IllegalStateException("Aliases to data streams cannot be rolled over. Please rollover the data stream itself.") @@ -246,10 +254,10 @@ protected void masterOperation( final var statsIndicesOptions = new IndicesOptions( IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS, IndicesOptions.WildcardOptions.builder().matchClosed(true).allowEmptyExpressions(false).build(), - IndicesOptions.GatekeeperOptions.DEFAULT, - rolloverRequest.indicesOptions().selectorOptions() + IndicesOptions.GatekeeperOptions.DEFAULT ); - IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getRolloverTarget()) + // Make sure to recombine any selectors on the stats request + IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(resolvedRolloverTarget.combined()) .clear() .indicesOptions(statsIndicesOptions) .docs(true) @@ -266,9 +274,7 @@ protected void masterOperation( listener.delegateFailureAndWrap((delegate, statsResponse) -> { AutoShardingResult rolloverAutoSharding = null; - final IndexAbstraction indexAbstraction = clusterState.metadata() - .getIndicesLookup() - .get(rolloverRequest.getRolloverTarget()); + final IndexAbstraction indexAbstraction = clusterState.metadata().getIndicesLookup().get(resolvedRolloverTarget.resource()); if (indexAbstraction.getType().equals(IndexAbstraction.Type.DATA_STREAM)) { DataStream dataStream = (DataStream) indexAbstraction; final Optional indexStats = Optional.ofNullable(statsResponse) @@ -492,14 +498,20 @@ public ClusterState executeTask( ) throws Exception { final var rolloverTask = rolloverTaskContext.getTask(); final var rolloverRequest = rolloverTask.rolloverRequest(); + ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression( + rolloverRequest.getRolloverTarget(), + rolloverRequest.indicesOptions() + ); + boolean targetFailureStore = resolvedRolloverTarget.selector() != null + && resolvedRolloverTarget.selector().shouldIncludeFailures(); // Regenerate the rollover names, as a rollover could have happened in between the pre-check and the cluster state update final var rolloverNames = MetadataRolloverService.resolveRolloverNames( currentState, - rolloverRequest.getRolloverTarget(), + resolvedRolloverTarget.resource(), rolloverRequest.getNewIndexName(), rolloverRequest.getCreateIndexRequest(), - rolloverRequest.targetsFailureStore() + targetFailureStore ); // Re-evaluate the conditions, now with our final source index name @@ -532,7 +544,7 @@ public ClusterState executeTask( final IndexAbstraction rolloverTargetAbstraction = currentState.metadata() .getIndicesLookup() - .get(rolloverRequest.getRolloverTarget()); + .get(resolvedRolloverTarget.resource()); final IndexMetadataStats sourceIndexStats = rolloverTargetAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM ? IndexMetadataStats.fromStatsResponse(rolloverSourceIndex, rolloverTask.statsResponse()) @@ -541,7 +553,7 @@ public ClusterState executeTask( // Perform the actual rollover final var rolloverResult = rolloverService.rolloverClusterState( currentState, - rolloverRequest.getRolloverTarget(), + resolvedRolloverTarget.resource(), rolloverRequest.getNewIndexName(), rolloverRequest.getCreateIndexRequest(), metConditions, @@ -550,7 +562,7 @@ public ClusterState executeTask( false, sourceIndexStats, rolloverTask.autoShardingResult(), - rolloverRequest.targetsFailureStore() + targetFailureStore ); results.add(rolloverResult); logger.trace("rollover result [{}]", rolloverResult); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index b137809047d18..dd473869fb2d9 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -24,7 +24,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.client.internal.node.NodeClient; @@ -216,11 +216,9 @@ private void rollOverFailureStores(Runnable runnable) { } try (RefCountingRunnable refs = new RefCountingRunnable(runnable)) { for (String dataStream : failureStoresToBeRolledOver) { - RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null); - rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()) - .selectorOptions(IndicesOptions.SelectorOptions.FAILURES) - .build() + RolloverRequest rolloverRequest = new RolloverRequest( + IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES), + null ); // We are executing a lazy rollover because it is an action specialised for this situation, when we want an // unconditional and performant rollover. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 65264faf50129..2a6a789d9d312 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -25,7 +25,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.WriteResponse; import org.elasticsearch.action.support.replication.ReplicationResponse; @@ -425,11 +425,7 @@ private void rollOverDataStreams( RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null); rolloverRequest.masterNodeTimeout(bulkRequest.timeout); if (targetFailureStore) { - rolloverRequest.setIndicesOptions( - IndicesOptions.builder(rolloverRequest.indicesOptions()) - .selectorOptions(IndicesOptions.SelectorOptions.FAILURES) - .build() - ); + rolloverRequest.setRolloverTarget(IndexNameExpressionResolver.combineSelector(dataStream, IndexComponentSelector.FAILURES)); } // We are executing a lazy rollover because it is an action specialised for this situation, when we want an // unconditional and performant rollover. @@ -438,9 +434,8 @@ private void rollOverDataStreams( @Override public void onResponse(RolloverResponse result) { logger.debug( - "Data stream{} {} has {} over, the latest index is {}", - rolloverRequest.targetsFailureStore() ? " failure store" : "", - dataStream, + "Data stream [{}] has {} over, the latest index is {}", + rolloverRequest.getRolloverTarget(), result.isRolledOver() ? "been successfully rolled" : "skipped rolling", result.getNewIndex() ); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java index a0a05138406c5..62caba8f7ed96 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsActionUtil.java @@ -9,16 +9,18 @@ package org.elasticsearch.action.datastreams; +import org.elasticsearch.action.support.IndexComponentSelector; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression; import org.elasticsearch.index.Index; +import java.util.ArrayList; import java.util.List; import java.util.SortedMap; -import java.util.stream.Stream; public class DataStreamsActionUtil { @@ -47,25 +49,79 @@ public static IndicesOptions updateIndicesOptions(IndicesOptions indicesOptions) return indicesOptions; } - public static Stream resolveConcreteIndexNames( + public static List resolveConcreteIndexNames( IndexNameExpressionResolver indexNameExpressionResolver, ClusterState clusterState, String[] names, IndicesOptions indicesOptions ) { - List abstractionNames = getDataStreamNames(indexNameExpressionResolver, clusterState, names, indicesOptions); + List abstractionNames = indexNameExpressionResolver.dataStreams( + clusterState, + updateIndicesOptions(indicesOptions), + names + ); SortedMap indicesLookup = clusterState.getMetadata().getIndicesLookup(); - return abstractionNames.stream().flatMap(abstractionName -> { + List results = new ArrayList<>(abstractionNames.size()); + for (ResolvedExpression abstractionName : abstractionNames) { + IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName.resource()); + assert indexAbstraction != null; + if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { + selectDataStreamIndicesNames( + (DataStream) indexAbstraction, + IndexComponentSelector.FAILURES.equals(abstractionName.selector()), + results + ); + } + } + return results; + } + + /** + * Resolves a list of expressions into data stream names and then collects the concrete indices + * that are applicable for those data streams based on the selector provided in the arguments. + * @param indexNameExpressionResolver resolver object + * @param clusterState state to query + * @param names data stream expressions + * @param selector which component indices of the data stream should be returned + * @param indicesOptions options for expression resolution + * @return A stream of concrete index names that belong to the components specified + * on the data streams returned from the expressions given + */ + public static List resolveConcreteIndexNamesWithSelector( + IndexNameExpressionResolver indexNameExpressionResolver, + ClusterState clusterState, + String[] names, + IndexComponentSelector selector, + IndicesOptions indicesOptions + ) { + assert indicesOptions.allowSelectors() == false : "If selectors are enabled, use resolveConcreteIndexNames instead"; + List abstractionNames = indexNameExpressionResolver.dataStreamNames( + clusterState, + updateIndicesOptions(indicesOptions), + names + ); + SortedMap indicesLookup = clusterState.getMetadata().getIndicesLookup(); + + List results = new ArrayList<>(abstractionNames.size()); + for (String abstractionName : abstractionNames) { IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName); assert indexAbstraction != null; if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { - DataStream dataStream = (DataStream) indexAbstraction; - List indices = dataStream.getIndices(); - return indices.stream().map(Index::getName); - } else { - return Stream.empty(); + if (selector.shouldIncludeData()) { + selectDataStreamIndicesNames((DataStream) indexAbstraction, false, results); + } + if (selector.shouldIncludeFailures()) { + selectDataStreamIndicesNames((DataStream) indexAbstraction, true, results); + } } - }); + } + return results; + } + + private static void selectDataStreamIndicesNames(DataStream indexAbstraction, boolean failureStore, List accumulator) { + for (Index index : indexAbstraction.getDataStreamIndices(failureStore).getIndices()) { + accumulator.add(index.getName()); + } } } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java index 9266bae439b73..82afeec752378 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java @@ -38,8 +38,6 @@ public DataStreamsStatsAction() { public static class Request extends BroadcastRequest { public Request() { - // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and - // a data stream's backing indices are retrieved from its metadata super( null, IndicesOptions.builder() @@ -58,10 +56,9 @@ public Request() { .allowAliasToMultipleIndices(true) .allowClosedIndices(true) .ignoreThrottled(false) - .allowFailureIndices(true) + .allowSelectors(false) .build() ) - .selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE) .build() ); } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java index 4f647d4f02884..640c88918ffc0 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java @@ -61,7 +61,7 @@ public static class Request extends MasterNodeRequest implements Indice .allowAliasToMultipleIndices(false) .allowClosedIndices(true) .ignoreThrottled(false) - .allowFailureIndices(true) + .allowSelectors(false) .build() ) .build(); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 883fc543749c2..c55957787aee7 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -72,10 +72,11 @@ public static class Request extends MasterNodeReadRequest implements In .allowAliasToMultipleIndices(false) .allowClosedIndices(true) .ignoreThrottled(false) - .allowFailureIndices(true) + .allowSelectors(false) .build() ) .build(); + private boolean includeDefaults = false; private boolean verbose = false; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java index a43d29501a7ee..401bd7a27c6fa 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java @@ -63,7 +63,7 @@ public static class Request extends MasterNodeReadRequest implements In .allowAliasToMultipleIndices(false) .allowClosedIndices(true) .ignoreThrottled(false) - .allowFailureIndices(true) + .allowSelectors(false) .build() ) .build(); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java index b054d12890366..c2b7de8d5df8b 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java @@ -94,7 +94,7 @@ public static Request parseRequest(XContentParser parser, Factory factory) { .allowAliasToMultipleIndices(false) .allowClosedIndices(true) .ignoreThrottled(false) - .allowFailureIndices(false) + .allowSelectors(false) .build() ) .build(); diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java index 62771230636c1..cce01aca7685a 100644 --- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java +++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleAction.java @@ -82,7 +82,7 @@ public String[] indices() { @Override public IndicesOptions indicesOptions() { - return IndicesOptions.STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED; + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index ebbd47336e3da..4231d598b2d70 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -13,7 +13,6 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.Nullable; @@ -47,37 +46,13 @@ * @param gatekeeperOptions, applies to all the resolved indices and defines if throttled will be included and if certain type of * aliases or indices are allowed, or they will throw an error. It acts as a gatekeeper when an action * does not support certain options. - * @param selectorOptions, applies to all resolved expressions, and it specifies the index component that should be included, if there - * is no index component defined on the expression level. */ public record IndicesOptions( ConcreteTargetOptions concreteTargetOptions, WildcardOptions wildcardOptions, - GatekeeperOptions gatekeeperOptions, - SelectorOptions selectorOptions + GatekeeperOptions gatekeeperOptions ) implements ToXContentFragment { - /** - * @deprecated this query param will be replaced by the selector `::` on the expression level - */ - @Deprecated - public static final String FAILURE_STORE_QUERY_PARAM = "failure_store"; - /** - * @deprecated this value will be replaced by the selector `::*` on the expression level - */ - @Deprecated - public static final String INCLUDE_ALL = "include"; - /** - * @deprecated this value will be replaced by the selector `::data` on the expression level - */ - @Deprecated - public static final String INCLUDE_ONLY_REGULAR_INDICES = "exclude"; - /** - * @deprecated this value will be replaced by the selector `::failures` on the expression level - */ - @Deprecated - public static final String INCLUDE_ONLY_FAILURE_INDICES = "only"; - public static IndicesOptions.Builder builder() { return new Builder(); } @@ -324,14 +299,14 @@ public static Builder builder(WildcardOptions wildcardOptions) { * - The ignoreThrottled flag, which is a deprecated flag that will filter out frozen indices. * @param allowAliasToMultipleIndices, allow aliases to multiple indices, true by default. * @param allowClosedIndices, allow closed indices, true by default. - * @param allowFailureIndices, allow failure indices in the response, true by default + * @param allowSelectors, allow selectors within index expressions, true by default. * @param ignoreThrottled, filters out throttled (aka frozen indices), defaults to true. This is deprecated and the only one * that only filters and never throws an error. */ public record GatekeeperOptions( boolean allowAliasToMultipleIndices, boolean allowClosedIndices, - boolean allowFailureIndices, + boolean allowSelectors, @Deprecated boolean ignoreThrottled ) implements ToXContentFragment { @@ -355,7 +330,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public static class Builder { private boolean allowAliasToMultipleIndices; private boolean allowClosedIndices; - private boolean allowFailureIndices; + private boolean allowSelectors; private boolean ignoreThrottled; public Builder() { @@ -365,7 +340,7 @@ public Builder() { Builder(GatekeeperOptions options) { allowAliasToMultipleIndices = options.allowAliasToMultipleIndices; allowClosedIndices = options.allowClosedIndices; - allowFailureIndices = options.allowFailureIndices; + allowSelectors = options.allowSelectors; ignoreThrottled = options.ignoreThrottled; } @@ -388,11 +363,12 @@ public Builder allowClosedIndices(boolean allowClosedIndices) { } /** - * Failure indices are accepted when true, otherwise the resolution will throw an error. + * Selectors are allowed within index expressions when true, otherwise the resolution will treat their presence as a syntax + * error when resolving index expressions. * Defaults to true. */ - public Builder allowFailureIndices(boolean allowFailureIndices) { - this.allowFailureIndices = allowFailureIndices; + public Builder allowSelectors(boolean allowSelectors) { + this.allowSelectors = allowSelectors; return this; } @@ -405,7 +381,7 @@ public Builder ignoreThrottled(boolean ignoreThrottled) { } public GatekeeperOptions build() { - return new GatekeeperOptions(allowAliasToMultipleIndices, allowClosedIndices, allowFailureIndices, ignoreThrottled); + return new GatekeeperOptions(allowAliasToMultipleIndices, allowClosedIndices, allowSelectors, ignoreThrottled); } } @@ -418,50 +394,6 @@ public static Builder builder(GatekeeperOptions gatekeeperOptions) { } } - /** - * Defines which selectors should be used by default for an index operation in the event that no selectors are provided. - */ - public record SelectorOptions(IndexComponentSelector defaultSelector) implements Writeable { - - public static final SelectorOptions ALL_APPLICABLE = new SelectorOptions(IndexComponentSelector.ALL_APPLICABLE); - public static final SelectorOptions DATA = new SelectorOptions(IndexComponentSelector.DATA); - public static final SelectorOptions FAILURES = new SelectorOptions(IndexComponentSelector.FAILURES); - /** - * Default instance. Uses 
::data
as the default selector if none are present in an index expression. - */ - public static final SelectorOptions DEFAULT = DATA; - - public static SelectorOptions read(StreamInput in) throws IOException { - if (in.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) { - EnumSet set = in.readEnumSet(IndexComponentSelector.class); - if (set.isEmpty() || set.size() == 2) { - assert set.contains(IndexComponentSelector.DATA) && set.contains(IndexComponentSelector.FAILURES) - : "The enum set only supported ::data and ::failures"; - return SelectorOptions.ALL_APPLICABLE; - } else if (set.contains(IndexComponentSelector.DATA)) { - return SelectorOptions.DATA; - } else { - return SelectorOptions.FAILURES; - } - } else { - return new SelectorOptions(IndexComponentSelector.read(in)); - } - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - if (out.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) { - switch (defaultSelector) { - case ALL_APPLICABLE -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES)); - case DATA -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA)); - case FAILURES -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.FAILURES)); - } - } else { - defaultSelector.writeTo(out); - } - } - } - /** * This class is maintained for backwards compatibility and performance purposes. We use it for serialisation along with {@link Option}. */ @@ -497,7 +429,8 @@ private enum Option { ERROR_WHEN_CLOSED_INDICES, IGNORE_THROTTLED, - ALLOW_FAILURE_INDICES // Added in 8.14 + ALLOW_FAILURE_INDICES, // Added in 8.14, Removed in 8.18 + ALLOW_SELECTORS // Added in 8.18 } private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IndicesOptions.class); @@ -510,8 +443,7 @@ private enum Option { public static final IndicesOptions DEFAULT = new IndicesOptions( ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS, WildcardOptions.DEFAULT, - GatekeeperOptions.DEFAULT, - SelectorOptions.DEFAULT + GatekeeperOptions.DEFAULT ); public static final IndicesOptions STRICT_EXPAND_OPEN = IndicesOptions.builder() @@ -528,10 +460,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -547,10 +478,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.ALL_APPLICABLE) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -566,10 +496,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_NO_SELECTORS = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -585,7 +514,7 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(false) + .allowSelectors(false) .ignoreThrottled(false) ) .build(); @@ -603,10 +532,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -622,10 +550,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -636,10 +563,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) @@ -650,7 +576,7 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(false) + .allowSelectors(false) .ignoreThrottled(false) ) .build(); @@ -668,10 +594,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -682,10 +607,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTORS = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -696,7 +620,7 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(false) + .allowSelectors(false) .ignoreThrottled(false) ) .build(); @@ -714,10 +638,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.ALL_APPLICABLE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -728,10 +651,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.ALL_APPLICABLE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -747,10 +669,9 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(true) .allowClosedIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.ALL_APPLICABLE) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -766,10 +687,9 @@ private enum Option { GatekeeperOptions.builder() .allowClosedIndices(false) .allowAliasToMultipleIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -785,10 +705,9 @@ private enum Option { GatekeeperOptions.builder() .allowClosedIndices(false) .allowAliasToMultipleIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -804,10 +723,9 @@ private enum Option { GatekeeperOptions.builder() .ignoreThrottled(true) .allowClosedIndices(false) - .allowFailureIndices(true) + .allowSelectors(true) .allowAliasToMultipleIndices(true) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -823,10 +741,27 @@ private enum Option { GatekeeperOptions.builder() .allowAliasToMultipleIndices(false) .allowClosedIndices(false) - .allowFailureIndices(true) + .allowSelectors(false) + .ignoreThrottled(false) + ) + .build(); + public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED_ALLOW_SELECTORS = IndicesOptions.builder() + .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) + .wildcardOptions( + WildcardOptions.builder() + .matchOpen(false) + .matchClosed(false) + .includeHidden(false) + .allowEmptyExpressions(true) + .resolveAliases(true) + ) + .gatekeeperOptions( + GatekeeperOptions.builder() + .allowAliasToMultipleIndices(false) + .allowClosedIndices(false) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); public static final IndicesOptions STRICT_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder() .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS) @@ -842,10 +777,9 @@ private enum Option { GatekeeperOptions.builder() .allowClosedIndices(false) .allowAliasToMultipleIndices(true) - .allowFailureIndices(true) + .allowSelectors(true) .ignoreThrottled(false) ) - .selectorOptions(SelectorOptions.DATA) .build(); /** @@ -903,10 +837,10 @@ public boolean forbidClosedIndices() { } /** - * @return Whether execution on failure indices is allowed. + * @return Whether selectors (::) are allowed in the index expression. */ - public boolean allowFailureIndices() { - return gatekeeperOptions.allowFailureIndices(); + public boolean allowSelectors() { + return DataStream.isFailureStoreFeatureFlagEnabled() && gatekeeperOptions.allowSelectors(); } /** @@ -930,20 +864,6 @@ public boolean ignoreThrottled() { return gatekeeperOptions().ignoreThrottled(); } - /** - * @return whether regular indices (stand-alone or backing indices) will be included in the response - */ - public boolean includeRegularIndices() { - return selectorOptions().defaultSelector().shouldIncludeData(); - } - - /** - * @return whether failure indices (only supported by certain data streams) will be included in the response - */ - public boolean includeFailureIndices() { - return selectorOptions().defaultSelector().shouldIncludeFailures(); - } - public void writeIndicesOptions(StreamOutput out) throws IOException { EnumSet