diff --git a/data-prepper-plugins/opensearch/README.md b/data-prepper-plugins/opensearch/README.md index 6d9e28f469..5d42a8c618 100644 --- a/data-prepper-plugins/opensearch/README.md +++ b/data-prepper-plugins/opensearch/README.md @@ -130,9 +130,10 @@ Default is null. ``` - `index`: A String used as index name for custom data type. Applicable and required only If `index_type` is explicitly `custom` or defaults to be `custom`. * This index name can be a plain string, such as `application`, `my-index-name`. - * This index name can also be a plain string plus a date-time pattern as a suffix, such as `application-%{yyyy.MM.dd}`, `my-index-name-%{yyyy.MM.dd.HH}`. When OpenSearch Sink is sending data to OpenSearch, the date-time pattern will be replaced by actual UTC time. The pattern supports all the symbols that represent one hour or above and are listed in [Java DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html). For example, with an index pattern like `my-index-name-%{yyyy.MM.dd}`, a new index is created for each day such as `my-index-name-2022.01.25`. For another example, with an index pattern like `my-index-name-%{yyyy.MM.dd.HH}`, a new index is created for each hour such as `my-index-name-2022.01.25.13`. + * This index name can also be a plain string with a date-time pattern, such as `application-%{yyyy.MM.dd}`, `my-index-name-%{yyyy.MM.dd.HH}`, `index-%{yyyy-MM-dd}-dev`. When OpenSearch Sink is sending data to OpenSearch, the date-time pattern will be replaced by actual UTC time. The pattern supports all the symbols that represent one hour or above and are listed in [Java DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html). For example, with an index pattern like `my-index-name-%{yyyy.MM.dd}`, a new index is created for each day such as `my-index-name-2022.01.25`. For another example, with an index pattern like `my-index-name-%{yyyy.MM.dd.HH}`, a new index is created for each hour such as `my-index-name-2022.01.25.13`. * This index name can also be a formatted string (with or without date-time pattern suffix), such as `my-${index}-name`. When OpenSearchSink is sending data to OpenSearch, the format portion "${index}" will be replaced by it's value in the event that is being processed. The format may also be like "${index1/index2/index3}" in which case the field "index1/index2/index3" is searched in the event and replaced by its value. - Additionally, the formatted string can include expressions to evaluate to format the index name. For example, `my-${index}-${getMetadata(\"some_metadata_key\")}-name` will inject both the `index` value from the Event, as well as the value of `some_metadata_key` from the Event metadata to construct the index name. +- `normalize_index` (optional): If true, the plugin will try to make dynamic index names (index names with format options specified in `${}`) valid according to [index naming restrictions](https://opensearch.org/docs/2.11/api-reference/index-apis/create-index/#index-naming-restrictions). Any invalid characters will be removed. Default value is false. - `template_type`(optional): Defines what type of OpenSearch template to use. The available options are `v1` and `index-template`. The default value is `v1`, which uses the original OpenSearch templates available at the `_template` API endpoints. Select `index-template` to use composable index templates which are available at OpenSearch's `_index_template` endpoint. Note: when `distribution_version` is `es6`, `template_type` is enforced into `v1`. - `template_file`(optional): A json file path or AWS S3 URI to be read as index template for custom data ingestion. The json file content should be the json value of diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 403f8be0ec..38bb5d1caa 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -585,6 +585,57 @@ public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates( } + @ParameterizedTest + @ArgumentsSource(CreateWithIndexTemplateArgumentsProvider.class) + public void testIndexNameWithDateNotAsSuffixCreatesIndexTemplate( + final String templateType, + final String templatePath, + final String templateFile, + final BiFunction, String, Integer> extractVersionFunction, + final BiFunction, String, Map> extractMappingsFunction) throws IOException { + final String testIndexAlias = "prefix-%{yyyy-MM}-suffix"; + final String expectedDate = new SimpleDateFormat("yyyy-MM").format(new Date()); + final String expectedIndexName = "prefix-" + expectedDate + "-suffix"; + final String expectedIndexTemplateName = "prefix-suffix-index-template"; + final String testTemplateFileV1 = getClass().getClassLoader().getResource(templateFile).getFile(); + + PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV1); + OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + + final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( + OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; + Request getTemplateRequest = new Request(HttpMethod.GET, + "/" + templatePath + "/" + expectedIndexTemplateName + extraURI); + Response getTemplateResponse = client.performRequest(getTemplateRequest); + assertThat(getTemplateResponse.getStatusLine().getStatusCode(), equalTo(SC_OK)); + + String responseBody = EntityUtils.toString(getTemplateResponse.getEntity()); + @SuppressWarnings("unchecked") final Integer responseVersion = + extractVersionFunction.apply(createContentParser(XContentType.JSON.xContent(), + responseBody).map(), expectedIndexTemplateName); + @SuppressWarnings("unchecked") final Map templateMappings = + extractMappingsFunction.apply(createContentParser(XContentType.JSON.xContent(), + responseBody).map(), expectedIndexTemplateName); + assertThat(responseVersion, equalTo(Integer.valueOf(1))); + assertThat(templateMappings.isEmpty(), equalTo(false)); + + Request getIndexRequest = new Request(HttpMethod.GET, + "/" + expectedIndexName + extraURI); + Response getIndexResponse = client.performRequest(getIndexRequest); + assertThat(getIndexResponse.getStatusLine().getStatusCode(), equalTo(SC_OK)); + + String getIndexResponseBody = EntityUtils.toString(getIndexResponse.getEntity()); + @SuppressWarnings("unchecked") final Map indexBlob = (Map) createContentParser(XContentType.JSON.xContent(), + getIndexResponseBody).map().get(expectedIndexName); + @SuppressWarnings("unchecked") final Map mappingsBlob = (Map) indexBlob.get("mappings"); + assertThat(mappingsBlob.isEmpty(), equalTo(false)); + + // assert the mappings from index template are applied to the index + assertThat(mappingsBlob, equalTo(templateMappings)); + + sink.shutdown(); + } + static class CreateWithTemplatesArgumentsProvider implements ArgumentsProvider { @Override public Stream provideArguments(ExtensionContext context) { @@ -610,6 +661,35 @@ public Stream provideArguments(ExtensionContext context) { } } + static class CreateWithIndexTemplateArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(ExtensionContext context) { + final List arguments = new ArrayList<>(); + arguments.add( + arguments("v1", "_template", + TEST_TEMPLATE_V1_FILE, + (BiFunction, String, Integer>) (map, templateName) -> + (Integer) ((Map) map.get(templateName)).get("version"), + (BiFunction, String, Map>) (map, templateName) -> + (Map) ((Map) map.get(templateName)).get("mappings") + ) + ); + + if (OpenSearchIntegrationHelper.getVersion().compareTo(DeclaredOpenSearchVersion.OPENDISTRO_1_9) >= 0) { + arguments.add( + arguments("index-template", "_index_template", + TEST_INDEX_TEMPLATE_V1_FILE, + (BiFunction, String, Integer>) (map, unused) -> + (Integer) ((List>>) map.get("index_templates")).get(0).get("index_template").get("version"), + (BiFunction, String, Map>) (map, templateName) -> + (Map) ((List>>>) map.get("index_templates")).get(0).get("index_template").get("template").get("mappings") + ) + ); + } + return arguments.stream(); + } + } + static class CreateSingleWithTemplatesArgumentsProvider implements ArgumentsProvider { @Override public Stream provideArguments(ExtensionContext context) { @@ -1268,6 +1348,42 @@ public void testOpenSearchDynamicIndexWithDate(final String testIndex, final Str sink.shutdown(); } + @ParameterizedTest + @CsvSource({ + "id, yyyy-MM-dd, %{yyyy-MM-dd}-test-${id}-index", + "id, yyyy-MM-dd, test-%{yyyy-MM-dd}-${id}-index", + "id, yyyy-MM-dd, test-${id}-%{yyyy-MM-dd}-index", + }) + public void testOpenSearchDynamicIndexWithDateNotAsSuffix( + final String testIndex, final String testDatePattern, final String dynamicTestIndexAlias) throws IOException { + final String testIndexName = "idx1"; + SimpleDateFormat formatter = new SimpleDateFormat(testDatePattern); + Date date = new Date(); + String expectedDate = formatter.format(date); + final String expectedIndexAlias = dynamicTestIndexAlias + .replace("%{yyyy-MM-dd}", expectedDate) + .replace("${id}", testIndexName); + final String data = UUID.randomUUID().toString(); + final Map dataMap = Map.of("data", data); + final Event testEvent = JacksonEvent.builder() + .withData(dataMap) + .withEventType("event") + .build(); + testEvent.put(testIndex, testIndexName); + + Map expectedMap = testEvent.toMap(); + + final List> testRecords = Collections.singletonList(new Record<>(testEvent)); + + final PluginSetting pluginSetting = generatePluginSetting(null, dynamicTestIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + sink.output(testRecords); + final List> retSources = getSearchResponseDocSources(expectedIndexAlias); + assertThat(retSources.size(), equalTo(1)); + assertThat(retSources, hasItem(expectedMap)); + sink.shutdown(); + } + @ParameterizedTest @ValueSource(strings = {"yyyy-MM", "yyyy-MM-dd", "dd-MM-yyyy"}) public void testOpenSearchIndexWithDate(final String testDatePattern) throws IOException, InterruptedException { @@ -1296,6 +1412,34 @@ public void testOpenSearchIndexWithDate(final String testDatePattern) throws IOE sink.shutdown(); } + @ParameterizedTest + @ValueSource(strings = {"test-%{yyyy-MM-dd}-index", "%{yyyy-MM-dd}-test-index"}) + public void testOpenSearchIndexWithDateNotAsSuffix(final String testIndexAlias) throws IOException, InterruptedException { + final String testDatePattern = "yyyy-MM-dd"; + SimpleDateFormat formatter = new SimpleDateFormat(testDatePattern); + Date date = new Date(); + String expectedIndexName = testIndexAlias.replace("%{yyyy-MM-dd}", formatter.format(date)); + + final String data = UUID.randomUUID().toString(); + final Map dataMap = Map.of("data", data); + final Event testEvent = JacksonEvent.builder() + .withData(dataMap) + .withEventType("event") + .build(); + + Map expectedMap = testEvent.toMap(); + + final List> testRecords = Collections.singletonList(new Record<>(testEvent)); + + final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + sink.output(testRecords); + final List> retSources = getSearchResponseDocSources(expectedIndexName); + assertThat(retSources.size(), equalTo(1)); + assertThat(retSources, hasItem(expectedMap)); + sink.shutdown(); + } + @Test public void testOpenSearchIndexWithInvalidDate() throws IOException, InterruptedException { String invalidDatePattern = "yyyy-MM-dd HH:ss:mm"; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/AbstractIndexManager.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/AbstractIndexManager.java index 86974bec13..bb1565dcd1 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/AbstractIndexManager.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/AbstractIndexManager.java @@ -5,7 +5,6 @@ package org.opensearch.dataprepper.plugins.sink.opensearch.index; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; import org.apache.http.HttpStatus; import org.opensearch.client.RestHighLevelClient; @@ -44,13 +43,13 @@ public abstract class AbstractIndexManager implements IndexManager { = "invalid_index_name_exception"; static final Set NO_ISM_HTTP_STATUS = Set.of(HttpStatus.SC_NOT_FOUND, HttpStatus.SC_BAD_REQUEST); private static final String TIME_PATTERN_STARTING_SYMBOLS = "%{"; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); protected RestHighLevelClient restHighLevelClient; protected OpenSearchClient openSearchClient; protected OpenSearchSinkConfiguration openSearchSinkConfiguration; protected ClusterSettingsParser clusterSettingsParser; protected IsmPolicyManagementStrategy ismPolicyManagementStrategy; private final TemplateStrategy templateStrategy; + private final String configuredIndexAlias; protected String indexPrefix; private Boolean isIndexAlias; private boolean isIndexAliasChecked; @@ -60,12 +59,13 @@ public abstract class AbstractIndexManager implements IndexManager { //For matching a string that begins with a "%{" and ends with a "}". //For a string like "data-prepper-%{yyyy-MM-dd}", "%{yyyy-MM-dd}" is matched. private static final String TIME_PATTERN_REGULAR_EXPRESSION = "%\\{.*?\\}"; + static final Pattern TIME_PATTERN = Pattern.compile(TIME_PATTERN_REGULAR_EXPRESSION); //For matching a string enclosed by "%{" and "}". //For a string like "data-prepper-%{yyyy-MM}", "yyyy-MM" is matched. private static final String TIME_PATTERN_INTERNAL_EXTRACTOR_REGULAR_EXPRESSION = "%\\{(.*?)\\}"; - private Optional indexTimeSuffixFormatter; + private Optional indexDateTimeFormatter; private static final ZoneId UTC_ZONE_ID = ZoneId.of(TimeZone.getTimeZone("UTC").getID()); protected AbstractIndexManager(final RestHighLevelClient restHighLevelClient, @@ -87,7 +87,7 @@ protected AbstractIndexManager(final RestHighLevelClient restHighLevelClient, if (indexAlias == null) { indexAlias = openSearchSinkConfiguration.getIndexConfiguration().getIndexAlias(); } - + configuredIndexAlias = indexAlias; initializeIndexPrefixAndSuffix(indexAlias); } @@ -102,7 +102,6 @@ public static DateTimeFormatter getDatePatternFormatter(final String indexAlias) if(timePattern.contains(TIME_PATTERN_STARTING_SYMBOLS)){ //check if it is a nested pattern such as "data-prepper-%{%{yyyy.MM.dd}}" throw new IllegalArgumentException("An index doesn't allow nested date-time patterns."); } - validateTimePatternIsAtTheEnd(indexAlias, timePattern); validateNoSpecialCharsInTimePattern(timePattern); validateTimePatternGranularity(timePattern); return DateTimeFormatter.ofPattern(timePattern); @@ -112,32 +111,22 @@ public static DateTimeFormatter getDatePatternFormatter(final String indexAlias) public static String getIndexAliasWithDate(final String indexAlias) { final DateTimeFormatter dateFormatter = getDatePatternFormatter(indexAlias); - final String suffix = (dateFormatter != null) ? dateFormatter.format(getCurrentUtcTime()) : ""; - return indexAlias.replaceAll(TIME_PATTERN_REGULAR_EXPRESSION, "") + suffix; - } - - private void initalizeIsIndexAlias(final String indexAlias) { - + final String dateTimeString = (dateFormatter != null) ? dateFormatter.format(getCurrentUtcTime()) : ""; + return TIME_PATTERN.matcher(indexAlias).replaceAll(dateTimeString); } - private void initializeIndexPrefixAndSuffix(final String indexAlias){ + private void initializeIndexPrefixAndSuffix(final String indexAlias) { final DateTimeFormatter dateFormatter = getDatePatternFormatter(indexAlias); if (dateFormatter != null) { - indexTimeSuffixFormatter = Optional.of(dateFormatter); + indexDateTimeFormatter = Optional.of(dateFormatter); } else { - indexTimeSuffixFormatter = Optional.empty(); + indexDateTimeFormatter = Optional.empty(); } - indexPrefix = indexAlias.replaceAll(TIME_PATTERN_REGULAR_EXPRESSION, ""); - } - - /* - Data Prepper only allows time pattern as a suffix. - */ - private static void validateTimePatternIsAtTheEnd(final String indexAlias, final String timePattern) { - if (!indexAlias.endsWith(timePattern + "}")) { - throw new IllegalArgumentException("Time pattern can only be a suffix of an index."); - } + // removes date-time pattern along with leading or trailing hyphen + indexPrefix = indexAlias + .replaceAll("-" + TIME_PATTERN_REGULAR_EXPRESSION, "") + .replaceAll(TIME_PATTERN_REGULAR_EXPRESSION + "-", ""); } /* @@ -170,13 +159,14 @@ private static void validateTimePatternGranularity(final String timePattern) { } } + @Override public String getIndexName(final String dynamicIndexAlias) throws IOException { - if (indexTimeSuffixFormatter.isPresent()) { - final String formattedTimeString = indexTimeSuffixFormatter.get() + if (indexDateTimeFormatter.isPresent()) { + final String formattedTimeString = indexDateTimeFormatter.get() .format(getCurrentUtcTime()); - return indexPrefix + formattedTimeString; + return TIME_PATTERN.matcher(configuredIndexAlias).replaceAll(formattedTimeString); } else { - return indexPrefix; + return configuredIndexAlias; } } @@ -246,25 +236,22 @@ private void checkAndCreateIndexTemplate() throws IOException { } final void checkAndCreateIndexTemplate(final boolean isISMEnabled, final String ismPolicyId) throws IOException { - //If index prefix has a ending dash, then remove it to avoid two consecutive dashes. - final String indexPrefixWithoutTrailingDash = indexPrefix.replaceAll("-$", ""); - final String indexTemplateName = indexPrefixWithoutTrailingDash + "-index-template"; + final String indexTemplateName = indexPrefix + "-index-template"; final Map indexTemplateMap = openSearchSinkConfiguration.getIndexConfiguration() .getIndexTemplate(); final IndexTemplate indexTemplate = templateStrategy.createIndexTemplate(indexTemplateMap); - // Check existing index template version - only overwrite if version is less than or does not exist if (!shouldCreateTemplate(indexTemplateName, indexTemplate)) { return; } if (isISMEnabled) { - attachPolicy(indexTemplate, ismPolicyId, indexPrefixWithoutTrailingDash); + attachPolicy(indexTemplate, ismPolicyId, indexPrefix); } - final List indexPatterns = ismPolicyManagementStrategy.getIndexPatterns(indexPrefixWithoutTrailingDash); + final List indexPatterns = ismPolicyManagementStrategy.getIndexPatterns(configuredIndexAlias); indexTemplate.setTemplateName(indexTemplateName); indexTemplate.setIndexPatterns(indexPatterns); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java index bb10beb1f6..4e4868debf 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexManagerFactory.java @@ -105,8 +105,7 @@ public DefaultIndexManager(final RestHighLevelClient restHighLevelClient, } private String getIndexPolicyName() { - //If index prefix has a ending dash, then remove it to avoid two consecutive dashes. - return indexPrefix.replaceAll("-$", "") + POLICY_NAME_SUFFIX; + return indexPrefix + POLICY_NAME_SUFFIX; } } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagement.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagement.java index 31274f454a..988d00b2e5 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagement.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IsmPolicyManagement.java @@ -39,6 +39,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static org.opensearch.dataprepper.plugins.sink.opensearch.index.AbstractIndexManager.TIME_PATTERN; class IsmPolicyManagement implements IsmPolicyManagementStrategy { private static final Logger LOG = LoggerFactory.getLogger(IsmPolicyManagement.class); @@ -142,7 +143,7 @@ public Optional checkAndCreatePolicy() throws IOException { @Override public List getIndexPatterns(final String indexAlias){ checkArgument(StringUtils.isNotEmpty(indexAlias)); - return Collections.singletonList(indexAlias + "-*"); + return Collections.singletonList(TIME_PATTERN.matcher(indexAlias).replaceAll("*") + "-*"); } @Override diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagement.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagement.java index 29ae4d2a9e..2aa3cbc501 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagement.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagement.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static org.opensearch.dataprepper.plugins.sink.opensearch.index.AbstractIndexManager.TIME_PATTERN; class NoIsmPolicyManagement implements IsmPolicyManagementStrategy { private final RestHighLevelClient restHighLevelClient; @@ -40,7 +41,7 @@ public Optional checkAndCreatePolicy() throws IOException { @Override public List getIndexPatterns(final String indexAlias) { checkArgument(StringUtils.isNotEmpty(indexAlias)); - return Collections.singletonList(indexAlias); + return Collections.singletonList(TIME_PATTERN.matcher(indexAlias).replaceAll("*")); } @Override diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DefaultIndexManagerTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DefaultIndexManagerTests.java index d18c7cbb36..82b8f1b757 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DefaultIndexManagerTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DefaultIndexManagerTests.java @@ -136,27 +136,31 @@ void setup() throws IOException { } @Test - void getIndexAlias_IndexWithTimePattern(){ + void getIndexAlias_IndexWithTimePattern() throws IOException { when(indexConfiguration.getIndexAlias()).thenReturn(INDEX_ALIAS_WITH_TIME_PATTERN); defaultIndexManager = indexManagerFactory.getIndexManager( IndexType.CUSTOM, openSearchClient, restHighLevelClient, openSearchSinkConfiguration, templateStrategy); - try { - final String indexAlias = defaultIndexManager.getIndexName(null); - assertThat(indexAlias, matchesPattern(EXPECTED_INDEX_PATTERN)); - } catch (IOException e){} + + final String indexAlias = defaultIndexManager.getIndexName(null); + assertThat(indexAlias, matchesPattern(EXPECTED_INDEX_PATTERN)); + verify(openSearchSinkConfiguration, times(2)).getIndexConfiguration(); verify(indexConfiguration).getIndexAlias(); verify(indexConfiguration).getIsmPolicyFile(); } @Test - void getIndexAlias_IndexWithTimePattern_Exceptional_NotAsSuffix() { - when(indexConfiguration.getIndexAlias()).thenReturn(INDEX_ALIAS_WITH_TIME_PATTERN + "randomtext"); - assertThrows(IllegalArgumentException.class, - () -> indexManagerFactory.getIndexManager( - IndexType.CUSTOM, openSearchClient, restHighLevelClient, openSearchSinkConfiguration, templateStrategy)); - verify(openSearchSinkConfiguration).getIndexConfiguration(); + void getIndexAlias_IndexWithTimePattern_NotAsSuffix() throws IOException { + when(indexConfiguration.getIndexAlias()).thenReturn(INDEX_ALIAS_WITH_TIME_PATTERN + "-randomtext"); + defaultIndexManager = indexManagerFactory.getIndexManager( + IndexType.CUSTOM, openSearchClient, restHighLevelClient, openSearchSinkConfiguration, templateStrategy); + + final String indexAlias = defaultIndexManager.getIndexName(null); + assertThat(indexAlias, matchesPattern(Pattern.compile(INDEX_ALIAS + "-\\d{4}.\\d{2}.\\d{2}.\\d{2}-randomtext"))); + + verify(openSearchSinkConfiguration, times(2)).getIndexConfiguration(); verify(indexConfiguration).getIndexAlias(); + verify(indexConfiguration).getIsmPolicyFile(); } private static final List INVALID_CHARS = Arrays.asList('#', '\\', '/', '*', '?', '"', '<', '>', '|', ',', ':'); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DynamicIndexManagerTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DynamicIndexManagerTests.java index ffa274a4a2..eef38c6075 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DynamicIndexManagerTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DynamicIndexManagerTests.java @@ -238,6 +238,25 @@ public void getIndexName_DoesRetryOnOpenSearchExceptions_UntilFailure() throws I verify(innerIndexManager, times(3)).setupIndex(); } + @Test + public void getIndexName_IndexWithDateTimePattern_NotAsSuffix() throws IOException { + final String dynamicIndexName = "test-index-%{yyyy.MM.dd}-randomtext"; + final String indexWithDateTimePatternResolved = "test-index-2023.11.11-randomtext"; + when(indexConfiguration.isNormalizeIndex()).thenReturn(true); + innerIndexManager = mock(IndexManager.class); + + when(mockIndexManagerFactory.getIndexManager( + IndexType.CUSTOM, openSearchClient, restHighLevelClient, openSearchSinkConfiguration, templateStrategy, indexWithDateTimePatternResolved)).thenReturn(innerIndexManager); + when(innerIndexManager.getIndexName(indexWithDateTimePatternResolved)).thenReturn(indexWithDateTimePatternResolved); + + try (final MockedStatic abstractIndexManagerMockedStatic = mockStatic(AbstractIndexManager.class)) { + abstractIndexManagerMockedStatic.when(() -> AbstractIndexManager.getIndexAliasWithDate(dynamicIndexName)) + .thenReturn(indexWithDateTimePatternResolved); + final String result = dynamicIndexManager.getIndexName(dynamicIndexName); + assertThat(result, equalTo(indexWithDateTimePatternResolved)); + } + } + @ParameterizedTest @CsvSource(value = {"INVALID_INDEX#, invalid_index", "-AAA:\\\"*+/\\\\|?#><, aaa", "_TeST_InDeX<, test_index", "-- indexManagerFactory.getIndexManager( - IndexType.MANAGEMENT_DISABLED, openSearchClient, restHighLevelClient, openSearchSinkConfiguration, templateStrategy)); + void getIndexAlias_IndexWithTimePattern_Exceptional_NotAsSuffix() throws IOException { + when(indexConfiguration.getIndexAlias()).thenReturn(indexAliasWithTimePattern + "-randomtext"); + final IndexManager objectUnderTest = indexManagerFactory.getIndexManager( + IndexType.MANAGEMENT_DISABLED, openSearchClient, restHighLevelClient, openSearchSinkConfiguration, templateStrategy); + final Pattern expectedIndexPattern = Pattern.compile(baseIndexAlias + "-\\d{4}.\\d{2}.\\d{2}.\\d{2}-randomtext"); + + final String actualIndexPattern = objectUnderTest.getIndexName(null); + assertThat(actualIndexPattern, matchesPattern(expectedIndexPattern)); + verify(openSearchSinkConfiguration).getIndexConfiguration(); verify(indexConfiguration).getIndexAlias(); } diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagementTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagementTests.java index 1977dbf38f..eeb1d1c1fc 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagementTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/NoIsmPolicyManagementTests.java @@ -5,8 +5,10 @@ package org.opensearch.dataprepper.plugins.sink.opensearch.index; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.mockito.Mock; import org.opensearch.client.RestHighLevelClient; import org.opensearch.client.opensearch.OpenSearchClient; @@ -39,14 +41,14 @@ public class NoIsmPolicyManagementTests { @Mock OpenSearchIndicesClient openSearchIndicesClient; - @Before + @BeforeEach public void setup() throws IOException { initMocks(this); ismPolicyManagementStrategy = new NoIsmPolicyManagement(openSearchClient, restHighLevelClient); } @Test - public void constructor_NullRestClient() throws IOException { + public void constructor_NullRestClient() { assertThrows(NullPointerException.class, () -> new NoIsmPolicyManagement(openSearchClient, null)); } @@ -56,13 +58,20 @@ public void checkAndCreatePolicy() throws IOException { assertEquals(Optional.empty(), ismPolicyManagementStrategy.checkAndCreatePolicy()); } - @Test - public void getIndexPatterns() { - assertEquals(Collections.singletonList(INDEX_ALIAS), ismPolicyManagementStrategy.getIndexPatterns(INDEX_ALIAS)); + @ParameterizedTest + @CsvSource({ + "test-index, test-index", + "%{yyyy-MM}-test-index, *-test-index", + "test-%{yyyy-MM}-index, test-*-index", + "test-index-%{yyyy-MM}, test-index-*" + }) + public void getIndexPatterns(final String indexAlias, final String expectedIndexPattern) { + assertEquals(Collections.singletonList(expectedIndexPattern), ismPolicyManagementStrategy.getIndexPatterns(indexAlias)); } @Test public void getIndexPatterns_NullInput_Exception() { + ismPolicyManagementStrategy = new NoIsmPolicyManagement(openSearchClient, restHighLevelClient); assertThrows(IllegalArgumentException.class, () -> ismPolicyManagementStrategy.getIndexPatterns(null) );