From b80b565583c998e43a809a12e43dca9f9a20f6f0 Mon Sep 17 00:00:00 2001 From: Adi Suresh Date: Wed, 8 Nov 2023 12:07:27 -0600 Subject: [PATCH] Move opensearch source into same module as opensearch sink. (#3607) This change is required to share code between the source and sink plugins. Signed-off-by: Adi Suresh --- .../opensearch-source/README.md | 195 ----------------- .../opensearch-source/build.gradle | 29 --- data-prepper-plugins/opensearch/README.md | 196 ++++++++++++++++++ data-prepper-plugins/opensearch/build.gradle | 10 +- .../source/opensearch/ClientRefresher.java | 0 .../OpenSearchIndexProgressState.java | 0 .../source/opensearch/OpenSearchService.java | 0 .../source/opensearch/OpenSearchSource.java | 0 .../OpenSearchSourceConfiguration.java | 0 .../AwsAuthenticationConfiguration.java | 0 .../ConnectionConfiguration.java | 0 .../IndexParametersConfiguration.java | 0 .../configuration/OpenSearchIndex.java | 0 .../SchedulingParameterConfiguration.java | 0 .../configuration/SearchConfiguration.java | 0 .../OpenSearchSourcePluginMetrics.java | 0 .../worker/NoSearchContextWorker.java | 0 ...nSearchIndexPartitionCreationSupplier.java | 0 .../source/opensearch/worker/PitWorker.java | 0 .../opensearch/worker/ScrollWorker.java | 0 .../opensearch/worker/SearchWorker.java | 0 .../opensearch/worker/WorkerCommonUtils.java | 0 .../worker/client/ClusterClientFactory.java | 0 .../worker/client/ElasticsearchAccessor.java | 0 .../worker/client/OpenSearchAccessor.java | 0 .../client/OpenSearchClientFactory.java | 0 .../worker/client/SearchAccessor.java | 0 .../worker/client/SearchAccessorStrategy.java | 0 .../worker/client/X509TrustAllManager.java | 0 .../exceptions/IndexNotFoundException.java | 0 .../SearchContextLimitException.java | 0 .../model/CreatePointInTimeRequest.java | 0 .../model/CreatePointInTimeResponse.java | 0 .../client/model/CreateScrollRequest.java | 0 .../client/model/CreateScrollResponse.java | 0 .../model/DeletePointInTimeRequest.java | 0 .../client/model/DeleteScrollRequest.java | 0 .../client/model/MetadataKeyAttributes.java | 0 .../model/NoSearchContextSearchRequest.java | 0 .../client/model/SearchContextType.java | 0 .../model/SearchPointInTimeRequest.java | 0 .../client/model/SearchScrollRequest.java | 0 .../client/model/SearchScrollResponse.java | 0 .../model/SearchWithSearchAfterResults.java | 0 .../worker/client/model/SortingOptions.java | 0 .../opensearch/ClientRefresherTest.java | 0 .../opensearch/OpenSearchServiceTest.java | 0 .../OpenSearchSourceConfigurationTest.java | 0 .../opensearch/OpenSearchSourceTest.java | 0 .../AwsAuthenticationConfigurationTest.java | 0 .../ConnectionConfigurationTest.java | 0 .../IndexParametersConfigurationTest.java | 0 .../configuration/OpenSearchIndexTest.java | 0 .../SchedulingParameterConfigurationTest.java | 0 .../SearchConfigurationTest.java | 0 .../worker/NoSearchContextWorkerTest.java | 0 .../opensearch/worker/PitWorkerTest.java | 0 .../opensearch/worker/ScrollWorkerTest.java | 0 .../worker/WorkerCommonUtilsTest.java | 0 .../client/ElasticsearchAccessorTest.java | 0 .../worker/client/OpenSearchAccessorTest.java | 0 .../client/OpenSearchClientFactoryTest.java | 0 ...rchIndexPartitionCreationSupplierTest.java | 0 .../client/SearchAccessStrategyTest.java | 0 settings.gradle | 1 - 65 files changed, 205 insertions(+), 226 deletions(-) delete mode 100644 data-prepper-plugins/opensearch-source/README.md delete mode 100644 data-prepper-plugins/opensearch-source/build.gradle rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresher.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchIndexProgressState.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchService.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSource.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfiguration.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfiguration.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfiguration.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfiguration.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndex.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfiguration.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfiguration.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/metrics/OpenSearchSourcePluginMetrics.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/OpenSearchIndexPartitionCreationSupplier.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/SearchWorker.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ClusterClientFactory.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessor.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessor.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessor.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessorStrategy.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/X509TrustAllManager.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/IndexNotFoundException.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/SearchContextLimitException.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeRequest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeResponse.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollRequest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollResponse.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeletePointInTimeRequest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeleteScrollRequest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/MetadataKeyAttributes.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/NoSearchContextSearchRequest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchContextType.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchPointInTimeRequest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollRequest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollResponse.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchWithSearchAfterResults.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SortingOptions.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresherTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchServiceTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfigurationTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfigurationTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfigurationTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfigurationTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndexTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfigurationTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfigurationTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorkerTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorkerTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtilsTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessorTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessorTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchIndexPartitionCreationSupplierTest.java (100%) rename data-prepper-plugins/{opensearch-source => opensearch}/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessStrategyTest.java (100%) diff --git a/data-prepper-plugins/opensearch-source/README.md b/data-prepper-plugins/opensearch-source/README.md deleted file mode 100644 index 9424bf5093..0000000000 --- a/data-prepper-plugins/opensearch-source/README.md +++ /dev/null @@ -1,195 +0,0 @@ -# OpenSearch Source - -This is the Date Prepper OpenSearch source plugin that processes indices for either OpenSearch, Elasticsearch, -or Amazon OpenSearch Service clusters. It is ideal for migrating index data from a cluster. - -Note: Only fully tested versions will be listed below. It is likely many more versions are supported already, but it is untested. - -The OpenSearch source is compatible with the following OpenSearch versions: -* 2.5 - -And is compatible with the following Elasticsearch versions: -* 7.10 - -# Usages - -### Minimum required config with username and password - -```yaml -opensearch-source-pipeline: - source: - opensearch: - connection: - insecure: true - hosts: [ "https://localhost:9200" ] - username: "username" - password: "password" -``` - -### Full config example - -```yaml -opensearch-source-pipeline: - source: - opensearch: - indices: - include: - - index_name_regex: "test-index-.*" - exclude: - - index_name_regex: "test-index-[1-9].*" - scheduling: - rate: "PT1H" - start_time: "2023-06-02T22:01:30.00Z" - job_count: 2 - search_options: - search_context_type: "none" - batch_size: 1000 - connection: - insecure: false - cert: "/path/to/cert.crt" - socket_timeout: "100ms" - connection_timeout: "100ms" - hosts: [ "https://localhost:9200" ] - username: "username" - password: "password" -``` - -### Amazon OpenSearch Service - -The OpenSearch source can also be configured for an Amazon OpenSearch Service domain. - -```yaml -opensearch-source-pipeline: - source: - opensearch: - connection: - insecure: true - hosts: [ "https://search-my-domain-soopywaovobopgs8ywurr3utsu.us-east-1.es.amazonaws.com" ] - aws: - region: "us-east-1" - sts_role_arn: "arn:aws:iam::123456789012:role/my-domain-role" -``` - -### Using Metadata - -When the OpenSearch source constructs Data Prepper Events from documents in the cluster, the -document index is stored in the `EventMetadata` with an `opensearch-index` key, and the document_id is -stored in the `EventMetadata` with a `opensearch-document_id` key. This allows conditional routing based on the index or document_id, -among other things. For example, one could send to an OpenSearch sink and use the same index and document_id from the source cluster in -the destination cluster. A full config example for this use case is below - -```yaml -opensearch-source-pipeline: - source: - opensearch: - connection: - insecure: true - hosts: [ "https://source-cluster:9200" ] - username: "username" - password: "password" - sink: - - opensearch: - hosts: [ "https://sink-cluster:9200" ] - username: "username" - password: "password" - document_id_field: "getMetadata(\"opensearch-document_id\")" - index: "${getMetadata(\"opensearch-index\"}" -``` - -## Configuration - -- `hosts` (Required) : A list of IP addresses of OpenSearch or Elasticsearch nodes. - - -- `username` (Optional) : A String of username used in the internal users of OpenSearch cluster. Default is null. - - -- `password` (Optional) : A String of password used in the internal users of OpenSearch cluster. Default is null. - - -- `disable_authentication` (Optional) : A boolean that can disable authentication if the cluster supports it. Defaults to false. - - -- `aws` (Optional) : AWS configurations. See [AWS Configuration](#aws_configuration) for details. SigV4 is enabled by default when this option is used. - - -- `search_options` (Optional) : See [Search Configuration](#search_configuration) for details - - -- `indices` (Optional): See [Indices Configurations](#indices_configuration) for filtering options. - - -- `scheduling` (Optional): See [Scheduling Configuration](#scheduling_configuration) for details - - -- `connection` (Optional): See [Connection Configuration](#connection_configuration) - -### AWS Configuration - -* `region` (Optional) : The AWS region to use for credentials. Defaults to [standard SDK behavior to determine the region](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/region-selection.html). - - -* `sts_role_arn` (Optional) : The STS role to assume for requests to AWS. Defaults to null, which will use the [standard SDK behavior for credentials](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html). - - -* `sts_header_overrides` (Optional): A map of header overrides to make when assuming the IAM role for the source plugin. - -### Search Configuration - -* `search_context_type` (Optional) : A direct override for which type of search context should be used to search documents. - Options include `point_in_time`, `scroll`, or `none` (just search after). - By default, the OpenSearch source will attempt to use `point_in_time` on a cluster by auto-detecting that the cluster version and distribution -supports Point in Time. If the cluster does not support `point_in_time`, then `scroll` is the default behavior. - - -* `batch_size` (Optional) : The amount of documents to read in at once while searching. -This size is passed to the search requests for all search context types (`none` (search_after), `point_in_time`, or `scroll`). -Defaults to 1,000. - -### Scheduling Configuration - -Schedule the start time and amount of times an index should be processed. For example, -a `rate` of `PT1H` and a `job_count` of 3 would result in each index getting processed 3 times, starting at `start_time` -and then every hour after the first time the index is processed. - -* `rate` (Optional) : A String that indicates the rate to process an index based on the `job_count`. -Supports ISO_8601 notation Strings ("PT20.345S", "PT15M", etc.) as well as simple notation Strings for seconds ("60s") and milliseconds ("1500ms"). -Defaults to 8 hours, and is only applicable when `job_count` is greater than 1. - - - -* `job_count` (Optional) : An Integer that specifies how many times each index should be processed. Defaults to 1. - - - -* `start_time` (Optional) : A String in the format of a timestamp that is compatible with Java Instant (i.e. `2023-06-02T22:01:30.00Z`). -Processing will be delayed until this timestamp is reached. The default start time is to start immediately. - -### Connection Configuration - -* `insecure` (Optional): A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Default to false. - - -* `cert` (Optional) : CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that the OpenSearch cluster is using. Default is null. - - -* `socket_timeout` (Optional) : A String that indicates the timeout duration for waiting for data. Supports ISO_8601 notation Strings ("PT20.345S", "PT15M", etc.) as well as simple notation Strings for seconds ("60s") and milliseconds ("1500ms"). If this timeout value not set, the underlying Apache HttpClient would rely on operating system settings for managing socket timeouts. - - -* `connection_timeout` (Optional) : A String that indicates the timeout duration used when requesting a connection from the connection manager. Supports ISO_8601 notation Strings ("PT20.345S", "PT15M", etc.) as well as simple notation Strings for seconds ("60s") and milliseconds ("1500ms"). If this timeout value is either negative or not set, the underlying Apache HttpClient would rely on operating system settings for managing connection timeouts. - -### Indices Configuration - -Can be used to filter which indices should be processed. -An index will be processed if its name matches one of the `index_name_regex` -patterns in the `include` list, and does not match any of the pattern in the `exclude` list. -The default behavior is to process all indices. - -* `include` (Optional) : A List of [Index Configuration](#index_configuration) that defines which indices should be processed. Defaults to an empty list. - - -* `exclude` (Optional) : A List of [Index Configuration](#index_configuration) that defines which indices should not be processed. - -#### Index Configuration - -* `index_name_regex`: A regex pattern to represent the index names for filtering diff --git a/data-prepper-plugins/opensearch-source/build.gradle b/data-prepper-plugins/opensearch-source/build.gradle deleted file mode 100644 index c080eff561..0000000000 --- a/data-prepper-plugins/opensearch-source/build.gradle +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -dependencies { - implementation project(path: ':data-prepper-api') - implementation project(':data-prepper-plugins:buffer-common') - implementation project(':data-prepper-plugins:aws-plugin-api') - implementation 'software.amazon.awssdk:apache-client' - implementation 'software.amazon.awssdk:netty-nio-client' - implementation 'io.micrometer:micrometer-core' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2' - implementation 'software.amazon.awssdk:s3' - implementation 'software.amazon.awssdk:sts' - testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - implementation 'org.opensearch.client:opensearch-java:2.5.0' - implementation 'org.opensearch.client:opensearch-rest-client:2.7.0' - implementation 'co.elastic.clients:elasticsearch-java:7.17.0' - implementation libs.commons.lang3 - implementation('org.apache.maven:maven-artifact:3.0.3') { - exclude group: 'org.codehaus.plexus' - } - testImplementation testLibs.mockito.inline -} - -test { - useJUnitPlatform() -} \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/README.md b/data-prepper-plugins/opensearch/README.md index 5bff437fa0..6ea7f3972c 100644 --- a/data-prepper-plugins/opensearch/README.md +++ b/data-prepper-plugins/opensearch/README.md @@ -357,3 +357,199 @@ This plugin is compatible with Java 8. See - [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) - [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/monitoring.md) + +# OpenSearch Source + +This is the Date Prepper OpenSearch source plugin that processes indices for either OpenSearch, Elasticsearch, +or Amazon OpenSearch Service clusters. It is ideal for migrating index data from a cluster. + +Note: Only fully tested versions will be listed below. It is likely many more versions are supported already, but it is untested. + +The OpenSearch source is compatible with the following OpenSearch versions: +* 2.5 + +And is compatible with the following Elasticsearch versions: +* 7.10 + +# Usages + +### Minimum required config with username and password + +```yaml +opensearch-source-pipeline: + source: + opensearch: + connection: + insecure: true + hosts: [ "https://localhost:9200" ] + username: "username" + password: "password" +``` + +### Full config example + +```yaml +opensearch-source-pipeline: + source: + opensearch: + indices: + include: + - index_name_regex: "test-index-.*" + exclude: + - index_name_regex: "test-index-[1-9].*" + scheduling: + rate: "PT1H" + start_time: "2023-06-02T22:01:30.00Z" + job_count: 2 + search_options: + search_context_type: "none" + batch_size: 1000 + connection: + insecure: false + cert: "/path/to/cert.crt" + socket_timeout: "100ms" + connection_timeout: "100ms" + hosts: [ "https://localhost:9200" ] + username: "username" + password: "password" +``` + +### Amazon OpenSearch Service + +The OpenSearch source can also be configured for an Amazon OpenSearch Service domain. + +```yaml +opensearch-source-pipeline: + source: + opensearch: + connection: + insecure: true + hosts: [ "https://search-my-domain-soopywaovobopgs8ywurr3utsu.us-east-1.es.amazonaws.com" ] + aws: + region: "us-east-1" + sts_role_arn: "arn:aws:iam::123456789012:role/my-domain-role" +``` + +### Using Metadata + +When the OpenSearch source constructs Data Prepper Events from documents in the cluster, the +document index is stored in the `EventMetadata` with an `opensearch-index` key, and the document_id is +stored in the `EventMetadata` with a `opensearch-document_id` key. This allows conditional routing based on the index or document_id, +among other things. For example, one could send to an OpenSearch sink and use the same index and document_id from the source cluster in +the destination cluster. A full config example for this use case is below + +```yaml +opensearch-source-pipeline: + source: + opensearch: + connection: + insecure: true + hosts: [ "https://source-cluster:9200" ] + username: "username" + password: "password" + sink: + - opensearch: + hosts: [ "https://sink-cluster:9200" ] + username: "username" + password: "password" + document_id_field: "getMetadata(\"opensearch-document_id\")" + index: "${getMetadata(\"opensearch-index\"}" +``` + +## Configuration + +- `hosts` (Required) : A list of IP addresses of OpenSearch or Elasticsearch nodes. + + +- `username` (Optional) : A String of username used in the internal users of OpenSearch cluster. Default is null. + + +- `password` (Optional) : A String of password used in the internal users of OpenSearch cluster. Default is null. + + +- `disable_authentication` (Optional) : A boolean that can disable authentication if the cluster supports it. Defaults to false. + + +- `aws` (Optional) : AWS configurations. See [AWS Configuration](#aws_configuration) for details. SigV4 is enabled by default when this option is used. + + +- `search_options` (Optional) : See [Search Configuration](#search_configuration) for details + + +- `indices` (Optional): See [Indices Configurations](#indices_configuration) for filtering options. + + +- `scheduling` (Optional): See [Scheduling Configuration](#scheduling_configuration) for details + + +- `connection` (Optional): See [Connection Configuration](#connection_configuration) + +### AWS Configuration + +* `region` (Optional) : The AWS region to use for credentials. Defaults to [standard SDK behavior to determine the region](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/region-selection.html). + + +* `sts_role_arn` (Optional) : The STS role to assume for requests to AWS. Defaults to null, which will use the [standard SDK behavior for credentials](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html). + + +* `sts_header_overrides` (Optional): A map of header overrides to make when assuming the IAM role for the source plugin. + +### Search Configuration + +* `search_context_type` (Optional) : A direct override for which type of search context should be used to search documents. + Options include `point_in_time`, `scroll`, or `none` (just search after). + By default, the OpenSearch source will attempt to use `point_in_time` on a cluster by auto-detecting that the cluster version and distribution + supports Point in Time. If the cluster does not support `point_in_time`, then `scroll` is the default behavior. + + +* `batch_size` (Optional) : The amount of documents to read in at once while searching. + This size is passed to the search requests for all search context types (`none` (search_after), `point_in_time`, or `scroll`). + Defaults to 1,000. + +### Scheduling Configuration + +Schedule the start time and amount of times an index should be processed. For example, +a `rate` of `PT1H` and a `job_count` of 3 would result in each index getting processed 3 times, starting at `start_time` +and then every hour after the first time the index is processed. + +* `rate` (Optional) : A String that indicates the rate to process an index based on the `job_count`. + Supports ISO_8601 notation Strings ("PT20.345S", "PT15M", etc.) as well as simple notation Strings for seconds ("60s") and milliseconds ("1500ms"). + Defaults to 8 hours, and is only applicable when `job_count` is greater than 1. + + + +* `job_count` (Optional) : An Integer that specifies how many times each index should be processed. Defaults to 1. + + + +* `start_time` (Optional) : A String in the format of a timestamp that is compatible with Java Instant (i.e. `2023-06-02T22:01:30.00Z`). + Processing will be delayed until this timestamp is reached. The default start time is to start immediately. + +### Connection Configuration + +* `insecure` (Optional): A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Default to false. + + +* `cert` (Optional) : CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that the OpenSearch cluster is using. Default is null. + + +* `socket_timeout` (Optional) : A String that indicates the timeout duration for waiting for data. Supports ISO_8601 notation Strings ("PT20.345S", "PT15M", etc.) as well as simple notation Strings for seconds ("60s") and milliseconds ("1500ms"). If this timeout value not set, the underlying Apache HttpClient would rely on operating system settings for managing socket timeouts. + + +* `connection_timeout` (Optional) : A String that indicates the timeout duration used when requesting a connection from the connection manager. Supports ISO_8601 notation Strings ("PT20.345S", "PT15M", etc.) as well as simple notation Strings for seconds ("60s") and milliseconds ("1500ms"). If this timeout value is either negative or not set, the underlying Apache HttpClient would rely on operating system settings for managing connection timeouts. + +### Indices Configuration + +Can be used to filter which indices should be processed. +An index will be processed if its name matches one of the `index_name_regex` +patterns in the `include` list, and does not match any of the pattern in the `exclude` list. +The default behavior is to process all indices. + +* `include` (Optional) : A List of [Index Configuration](#index_configuration) that defines which indices should be processed. Defaults to an empty list. + + +* `exclude` (Optional) : A List of [Index Configuration](#index_configuration) that defines which indices should not be processed. + +#### Index Configuration + +* `index_name_regex`: A regex pattern to represent the index names for filtering diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index 17c632a359..2609f148b0 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -7,14 +7,16 @@ dependencies { implementation project(':data-prepper-api') implementation libs.armeria.core testImplementation project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-plugins:aws-plugin-api') + implementation project(':data-prepper-plugins:buffer-common') implementation project(':data-prepper-plugins:common') implementation project(':data-prepper-plugins:failures-common') - implementation project(':data-prepper-plugins:aws-plugin-api') implementation libs.opensearch.client implementation libs.opensearch.rhlc implementation libs.opensearch.java implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2' implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'software.amazon.awssdk:auth' implementation 'software.amazon.awssdk:http-client-spi' @@ -31,11 +33,17 @@ dependencies { implementation libs.commons.lang3 implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' implementation 'software.amazon.awssdk:apache-client' + implementation 'software.amazon.awssdk:netty-nio-client' + implementation 'co.elastic.clients:elasticsearch-java:7.17.0' + implementation('org.apache.maven:maven-artifact:3.0.3') { + exclude group: 'org.codehaus.plexus' + } testImplementation testLibs.junit.vintage testImplementation libs.commons.io testImplementation 'net.bytebuddy:byte-buddy:1.14.8' testImplementation 'net.bytebuddy:byte-buddy-agent:1.14.9' testImplementation testLibs.slf4j.simple + testImplementation testLibs.mockito.inline } sourceSets { diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresher.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresher.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresher.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresher.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchIndexProgressState.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchIndexProgressState.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchIndexProgressState.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchIndexProgressState.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchService.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchService.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchService.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchService.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSource.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSource.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSource.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSource.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfiguration.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfiguration.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfiguration.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfiguration.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfiguration.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfiguration.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfiguration.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfiguration.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfiguration.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfiguration.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfiguration.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfiguration.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndex.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndex.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndex.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndex.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfiguration.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfiguration.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfiguration.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfiguration.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfiguration.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfiguration.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/metrics/OpenSearchSourcePluginMetrics.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/metrics/OpenSearchSourcePluginMetrics.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/metrics/OpenSearchSourcePluginMetrics.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/metrics/OpenSearchSourcePluginMetrics.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/OpenSearchIndexPartitionCreationSupplier.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/OpenSearchIndexPartitionCreationSupplier.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/OpenSearchIndexPartitionCreationSupplier.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/OpenSearchIndexPartitionCreationSupplier.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/SearchWorker.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/SearchWorker.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/SearchWorker.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/SearchWorker.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ClusterClientFactory.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ClusterClientFactory.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ClusterClientFactory.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ClusterClientFactory.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessor.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessor.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessor.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessor.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessor.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessor.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessor.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessor.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactory.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessor.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessor.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessor.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessor.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessorStrategy.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessorStrategy.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessorStrategy.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessorStrategy.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/X509TrustAllManager.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/X509TrustAllManager.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/X509TrustAllManager.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/X509TrustAllManager.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/IndexNotFoundException.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/IndexNotFoundException.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/IndexNotFoundException.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/IndexNotFoundException.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/SearchContextLimitException.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/SearchContextLimitException.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/SearchContextLimitException.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/exceptions/SearchContextLimitException.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeRequest.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeRequest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeRequest.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeRequest.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeResponse.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeResponse.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeResponse.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreatePointInTimeResponse.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollRequest.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollRequest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollRequest.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollRequest.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollResponse.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollResponse.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollResponse.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/CreateScrollResponse.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeletePointInTimeRequest.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeletePointInTimeRequest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeletePointInTimeRequest.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeletePointInTimeRequest.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeleteScrollRequest.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeleteScrollRequest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeleteScrollRequest.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/DeleteScrollRequest.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/MetadataKeyAttributes.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/MetadataKeyAttributes.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/MetadataKeyAttributes.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/MetadataKeyAttributes.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/NoSearchContextSearchRequest.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/NoSearchContextSearchRequest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/NoSearchContextSearchRequest.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/NoSearchContextSearchRequest.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchContextType.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchContextType.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchContextType.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchContextType.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchPointInTimeRequest.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchPointInTimeRequest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchPointInTimeRequest.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchPointInTimeRequest.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollRequest.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollRequest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollRequest.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollRequest.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollResponse.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollResponse.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollResponse.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchScrollResponse.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchWithSearchAfterResults.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchWithSearchAfterResults.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchWithSearchAfterResults.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SearchWithSearchAfterResults.java diff --git a/data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SortingOptions.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SortingOptions.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SortingOptions.java rename to data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/model/SortingOptions.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresherTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresherTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresherTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/ClientRefresherTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchServiceTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchServiceTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchServiceTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchServiceTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfigurationTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfigurationTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfigurationTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceConfigurationTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/OpenSearchSourceTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfigurationTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfigurationTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfigurationTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AwsAuthenticationConfigurationTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfigurationTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfigurationTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfigurationTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/ConnectionConfigurationTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfigurationTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfigurationTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfigurationTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/IndexParametersConfigurationTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndexTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndexTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndexTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/OpenSearchIndexTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfigurationTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfigurationTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfigurationTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SchedulingParameterConfigurationTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfigurationTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfigurationTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfigurationTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/SearchConfigurationTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorkerTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorkerTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorkerTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorkerTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorkerTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorkerTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorkerTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorkerTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtilsTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtilsTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtilsTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtilsTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessorTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessorTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessorTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/ElasticsearchAccessorTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessorTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessorTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessorTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchAccessorTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchClientFactoryTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchIndexPartitionCreationSupplierTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchIndexPartitionCreationSupplierTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchIndexPartitionCreationSupplierTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/OpenSearchIndexPartitionCreationSupplierTest.java diff --git a/data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessStrategyTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessStrategyTest.java similarity index 100% rename from data-prepper-plugins/opensearch-source/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessStrategyTest.java rename to data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/client/SearchAccessStrategyTest.java diff --git a/settings.gradle b/settings.gradle index 5ce7405bbc..b674d1689d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -132,7 +132,6 @@ include 'data-prepper-plugins:newline-codecs' include 'data-prepper-plugins:avro-codecs' include 'data-prepper-plugins:kafka-plugins' include 'data-prepper-plugins:kafka-connect-plugins' -include 'data-prepper-plugins:opensearch-source' include 'data-prepper-plugins:user-agent-processor' include 'data-prepper-plugins:in-memory-source-coordination-store' include 'data-prepper-plugins:aws-plugin-api'