From 967919b4153fcac755295abcbec891121c109039 Mon Sep 17 00:00:00 2001 From: Hai Yan <8153134+oeyh@users.noreply.github.com> Date: Fri, 30 Sep 2022 10:30:24 -0500 Subject: [PATCH] Update plugin names to use processor (#1838) Signed-off-by: Hai Yan --- .../aggregate-processor/README.md | 2 +- .../NoOpProcessor.java} | 6 +- .../StringProcessor.java} | 12 +- .../StringProcessorTests.java} | 36 ++--- .../state/ProcessorStateTest.java | 2 +- .../README.md | 8 +- .../build.gradle | 0 .../processor/grok/GrokProcessor.java} | 44 +++--- .../processor/grok/GrokProcessorConfig.java} | 30 ++-- .../grok/GrokProcessorConfigTests.java} | 96 ++++++------ .../processor/grok/GrokProcessorIT.java} | 138 +++++++++--------- .../processor/grok/GrokProcessorTests.java} | 136 ++++++++--------- .../resources/patterns/test_patterns_1.txt | 0 .../resources/patterns/test_patterns_2.txt | 0 .../build.gradle | 0 .../processor/state/MapDbProcessorState.java} | 6 +- .../state/MapDbProcessorStateTest.java} | 6 +- .../service-map-stateful/README.md | 6 +- .../service-map-stateful/build.gradle | 2 +- .../{prepper => processor}/OTelHelper.java | 2 +- .../ServiceMapProcessorConfig.java} | 4 +- .../ServiceMapRelationship.java | 2 +- .../ServiceMapStatefulProcessor.java} | 72 ++++----- .../ServiceMapRelationshipTest.java | 2 +- .../ServiceMapStatefulProcessorTest.java} | 56 +++---- .../ServiceMapTestUtils.java | 6 +- docs/log_analytics.md | 10 +- docs/trace_analytics.md | 4 +- .../trace/EndToEndServiceMapTest.java | 4 +- examples/log-ingestion/README.md | 2 +- settings.gradle | 4 +- 31 files changed, 349 insertions(+), 349 deletions(-) rename data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/{prepper/NoOpPrepper.java => processor/NoOpProcessor.java} (82%) rename data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/{prepper/StringPrepper.java => processor/StringProcessor.java} (91%) rename data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/{prepper/StringPrepperTests.java => processor/StringProcessorTests.java} (81%) rename data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/{prepper => processor}/state/ProcessorStateTest.java (98%) rename data-prepper-plugins/{grok-prepper => grok-processor}/README.md (95%) rename data-prepper-plugins/{grok-prepper => grok-processor}/build.gradle (100%) rename data-prepper-plugins/{grok-prepper/src/main/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepper.java => grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java} (88%) rename data-prepper-plugins/{grok-prepper/src/main/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java => grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java} (79%) rename data-prepper-plugins/{grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java => grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfigTests.java} (50%) rename data-prepper-plugins/{grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperIT.java => grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java} (79%) rename data-prepper-plugins/{grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperTests.java => grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java} (83%) rename data-prepper-plugins/{grok-prepper => grok-processor}/src/test/resources/patterns/test_patterns_1.txt (100%) rename data-prepper-plugins/{grok-prepper => grok-processor}/src/test/resources/patterns/test_patterns_2.txt (100%) rename data-prepper-plugins/{mapdb-prepper-state => mapdb-processor-state}/build.gradle (100%) rename data-prepper-plugins/{mapdb-prepper-state/src/main/java/org/opensearch/dataprepper/plugins/prepper/state/MapDbPrepperState.java => mapdb-processor-state/src/main/java/org/opensearch/dataprepper/plugins/processor/state/MapDbProcessorState.java} (95%) rename data-prepper-plugins/{mapdb-prepper-state/src/test/java/org/opensearch/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java => mapdb-processor-state/src/test/java/org/opensearch/dataprepper/plugins/processor/state/MapDbProcessorStateTest.java} (90%) rename data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/{prepper => processor}/OTelHelper.java (95%) rename data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/{prepper/ServiceMapPrepperConfig.java => processor/ServiceMapProcessorConfig.java} (73%) rename data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/{prepper => processor}/ServiceMapRelationship.java (99%) rename data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/{prepper/ServiceMapStatefulPrepper.java => processor/ServiceMapStatefulProcessor.java} (81%) rename data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/{prepper => processor}/ServiceMapRelationshipTest.java (92%) rename data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/{prepper/ServiceMapStatefulPrepperTest.java => processor/ServiceMapStatefulProcessorTest.java} (88%) rename data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/{prepper => processor}/ServiceMapTestUtils.java (95%) diff --git a/data-prepper-plugins/aggregate-processor/README.md b/data-prepper-plugins/aggregate-processor/README.md index a04d83e9b0..72bdaed829 100644 --- a/data-prepper-plugins/aggregate-processor/README.md +++ b/data-prepper-plugins/aggregate-processor/README.md @@ -6,7 +6,7 @@ It is a great way to reduce unnecessary log volume and create aggregated logs ov ## Usage The following pipeline configuration will aggregate Events based on the entries with keys `sourceIp`, `destinationIp`, and `port`. It uses the [remove_duplicates](#remove_duplicates) action. -While not necessary, a great way to set up the Aggregate Processor [identification_keys](#identification_keys) is with the [Grok Processor](../grok-prepper/README.md) as shown below. +While not necessary, a great way to set up the Aggregate Processor [identification_keys](#identification_keys) is with the [Grok Processor](../grok-processor/README.md) as shown below. ```yaml source: ... diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/prepper/NoOpPrepper.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/NoOpProcessor.java similarity index 82% rename from data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/prepper/NoOpPrepper.java rename to data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/NoOpProcessor.java index dbad9b86fd..ce07757e73 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/prepper/NoOpPrepper.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/NoOpProcessor.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.event.Event; @@ -13,9 +13,9 @@ import java.util.Collection; @DataPrepperPlugin(name = "no-op", pluginType = Processor.class) -public class NoOpPrepper implements Processor, Record> { +public class NoOpProcessor implements Processor, Record> { - public NoOpPrepper() { + public NoOpProcessor() { } diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/prepper/StringPrepper.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java similarity index 91% rename from data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/prepper/StringPrepper.java rename to data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java index 3e413553eb..3b16f18d2d 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/prepper/StringPrepper.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -27,9 +27,9 @@ * A simple String implementation of {@link Processor} which generates new Records with uppercase or lowercase content. The current * simpler implementation does not handle errors (if any). */ -@DataPrepperPlugin(name = "string_converter", pluginType = Processor.class, pluginConfigurationType = StringPrepper.Configuration.class) -public class StringPrepper implements Processor, Record> { - private static Logger LOG = LoggerFactory.getLogger(StringPrepper.class); +@DataPrepperPlugin(name = "string_converter", pluginType = Processor.class, pluginConfigurationType = StringProcessor.Configuration.class) +public class StringProcessor implements Processor, Record> { + private static Logger LOG = LoggerFactory.getLogger(StringProcessor.class); private final ObjectMapper objectMapper = new ObjectMapper(); private final TypeReference> mapTypeReference = new TypeReference>() {}; @@ -51,14 +51,14 @@ public void setUpperCase(final boolean upperCase) { /** * Mandatory constructor for Data Prepper Component - This constructor is used by Data Prepper - * runtime engine to construct an instance of {@link StringPrepper} using an instance of {@link PluginSetting} which + * runtime engine to construct an instance of {@link StringProcessor} using an instance of {@link PluginSetting} which * has access to pluginSetting metadata from pipeline * pluginSetting file. * * @param configuration instance with metadata information from pipeline pluginSetting file. */ @DataPrepperPluginConstructor - public StringPrepper(final Configuration configuration) { + public StringProcessor(final Configuration configuration) { this.upperCase = configuration.getUpperCase(); } diff --git a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/prepper/StringPrepperTests.java b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/StringProcessorTests.java similarity index 81% rename from data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/prepper/StringPrepperTests.java rename to data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/StringProcessorTests.java index dd20aed52e..8ddaf3da02 100644 --- a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/prepper/StringPrepperTests.java +++ b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/StringProcessorTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; @@ -20,7 +20,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertTrue; -public class StringPrepperTests { +public class StringProcessorTests { private final String TEST_EVENT_TYPE = "test_event_type"; private final String TEST_KEY = "test_key"; @@ -39,23 +39,23 @@ public class StringPrepperTests { .build() ); private final List> TEST_RECORDS = Arrays.asList(TEST_RECORD_1, TEST_RECORD_2); - private StringPrepper.Configuration configuration; + private StringProcessor.Configuration configuration; @BeforeEach void setUp() { - configuration = new StringPrepper.Configuration(); + configuration = new StringProcessor.Configuration(); } - private StringPrepper createObjectUnderTest() { - return new StringPrepper(configuration); + private StringProcessor createObjectUnderTest() { + return new StringProcessor(configuration); } @Test public void testStringPrepperDefault() { - final StringPrepper stringPrepper = createObjectUnderTest(); - final List> modifiedRecords = (List>) stringPrepper.execute(TEST_RECORDS); - stringPrepper.shutdown(); + final StringProcessor stringProcessor = createObjectUnderTest(); + final List> modifiedRecords = (List>) stringProcessor.execute(TEST_RECORDS); + stringProcessor.shutdown(); final List modifiedRecordEvents = modifiedRecords.stream().map(Record::getData).collect(Collectors.toList()); @@ -73,9 +73,9 @@ public void testStringPrepperDefault() { @Test public void testStringPrepperLowerCase() { configuration.setUpperCase(false); - final StringPrepper stringPrepper = createObjectUnderTest(); - final List> modifiedRecords = (List>) stringPrepper.execute(TEST_RECORDS); - stringPrepper.shutdown(); + final StringProcessor stringProcessor = createObjectUnderTest(); + final List> modifiedRecords = (List>) stringProcessor.execute(TEST_RECORDS); + stringProcessor.shutdown(); final List modifiedRecordEvents = modifiedRecords.stream().map(Record::getData).collect(Collectors.toList()); @@ -93,9 +93,9 @@ public void testStringPrepperLowerCase() { @Test public void testStringPrepperUpperCase() { configuration.setUpperCase(true); - final StringPrepper stringPrepper = createObjectUnderTest(); - final List> modifiedRecords = (List>) stringPrepper.execute(TEST_RECORDS); - stringPrepper.shutdown(); + final StringProcessor stringProcessor = createObjectUnderTest(); + final List> modifiedRecords = (List>) stringProcessor.execute(TEST_RECORDS); + stringProcessor.shutdown(); final List modifiedRecordEvents = modifiedRecords.stream().map(Record::getData).collect(Collectors.toList()); @@ -112,11 +112,11 @@ public void testStringPrepperUpperCase() { @Test public void testPrepareForShutdown() { - final StringPrepper stringPrepper = createObjectUnderTest(); + final StringProcessor stringProcessor = createObjectUnderTest(); - stringPrepper.prepareForShutdown(); + stringProcessor.prepareForShutdown(); - assertThat(stringPrepper.isReadyForShutdown(), equalTo(true)); + assertThat(stringProcessor.isReadyForShutdown(), equalTo(true)); } } diff --git a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/prepper/state/ProcessorStateTest.java b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/state/ProcessorStateTest.java similarity index 98% rename from data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/prepper/state/ProcessorStateTest.java rename to data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/state/ProcessorStateTest.java index 58cc6e1397..c1a1618132 100644 --- a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/prepper/state/ProcessorStateTest.java +++ b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/state/ProcessorStateTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper.state; +package org.opensearch.dataprepper.plugins.processor.state; import org.opensearch.dataprepper.processor.state.ProcessorState; import org.junit.After; diff --git a/data-prepper-plugins/grok-prepper/README.md b/data-prepper-plugins/grok-processor/README.md similarity index 95% rename from data-prepper-plugins/grok-prepper/README.md rename to data-prepper-plugins/grok-processor/README.md index 84f9da4474..d94b8e3612 100644 --- a/data-prepper-plugins/grok-prepper/README.md +++ b/data-prepper-plugins/grok-processor/README.md @@ -1,9 +1,9 @@ -# Grok Prepper +# Grok Processor -This is a prepper that takes unstructured data and utilizes pattern matching +This is a processor that takes unstructured data and utilizes pattern matching to structure and extract important keys and make data more structured and queryable. -The Grok Prepper uses the [java-grok Library](https://github.com/thekrakken/java-grok) internally and supports all java-grok library compatible patterns. The java-grok library is built using the `java.util.regex` regular expression library. +The Grok Processor uses the [java-grok Library](https://github.com/thekrakken/java-grok) internally and supports all java-grok library compatible patterns. The java-grok library is built using the `java.util.regex` regular expression library. The full set of default patterns can be found [here](https://github.com/thekrakken/java-grok/blob/master/src/main/resources/patterns/patterns). Custom patterns can be added through either the `patterns_definitions` or `patterns_directories` configuration settings. When debugging custom patterns, the [Grok Debugger](https://grokdebug.herokuapp.com/) @@ -231,7 +231,7 @@ The resulting grokked log will now look like this. All of the grok captures were wrapped in an outer key named `grokked`.

* `timeout_millis` (Optional): An `int` that specifies the maximum amount of time, in milliseconds, that matching will be performed on an individual Record before it times out and moves on to the next Record. -Setting a `timeout_millis = 0` will make it so that matching a Record never times out. If a Record does time out, it will remain the same as it was when input to the grok prepper. Default value is `30,000` +Setting a `timeout_millis = 0` will make it so that matching a Record never times out. If a Record does time out, it will remain the same as it was when input to the grok processor. Default value is `30,000` ## Metrics diff --git a/data-prepper-plugins/grok-prepper/build.gradle b/data-prepper-plugins/grok-processor/build.gradle similarity index 100% rename from data-prepper-plugins/grok-prepper/build.gradle rename to data-prepper-plugins/grok-processor/build.gradle diff --git a/data-prepper-plugins/grok-prepper/src/main/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepper.java b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java similarity index 88% rename from data-prepper-plugins/grok-prepper/src/main/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepper.java rename to data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java index 8a6f4c914d..765b5f49b3 100644 --- a/data-prepper-plugins/grok-prepper/src/main/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepper.java +++ b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper.grok; +package org.opensearch.dataprepper.plugins.processor.grok; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; @@ -53,10 +53,10 @@ @SingleThread @DataPrepperPlugin(name = "grok", pluginType = Processor.class) -public class GrokPrepper extends AbstractProcessor, Record> { +public class GrokProcessor extends AbstractProcessor, Record> { static final long EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT = 300L; - private static final Logger LOG = LoggerFactory.getLogger(GrokPrepper.class); + private static final Logger LOG = LoggerFactory.getLogger(GrokProcessor.class); static final String GROK_PROCESSING_MATCH_SUCCESS = "grokProcessingMatchSuccess"; static final String GROK_PROCESSING_MATCH_FAILURE = "grokProcessingMatchFailure"; @@ -72,18 +72,18 @@ public class GrokPrepper extends AbstractProcessor, Record> private final GrokCompiler grokCompiler; private final Map> fieldToGrok; - private final GrokPrepperConfig grokPrepperConfig; + private final GrokProcessorConfig grokProcessorConfig; private final Set keysToOverwrite; private final ExecutorService executorService; - public GrokPrepper(final PluginSetting pluginSetting) { + public GrokProcessor(final PluginSetting pluginSetting) { this(pluginSetting, GrokCompiler.newInstance(), Executors.newSingleThreadExecutor()); } - GrokPrepper(final PluginSetting pluginSetting, final GrokCompiler grokCompiler, final ExecutorService executorService) { + GrokProcessor(final PluginSetting pluginSetting, final GrokCompiler grokCompiler, final ExecutorService executorService) { super(pluginSetting); - this.grokPrepperConfig = GrokPrepperConfig.buildConfig(pluginSetting); - this.keysToOverwrite = new HashSet<>(grokPrepperConfig.getkeysToOverwrite()); + this.grokProcessorConfig = GrokProcessorConfig.buildConfig(pluginSetting); + this.keysToOverwrite = new HashSet<>(grokProcessorConfig.getkeysToOverwrite()); this.grokCompiler = grokCompiler; this.fieldToGrok = new LinkedHashMap<>(); this.executorService = executorService; @@ -113,7 +113,7 @@ public Collection> doExecute(final Collection> recor try { final Event event = record.getData(); - if (grokPrepperConfig.getTimeoutMillis() == 0) { + if (grokProcessorConfig.getTimeoutMillis() == 0) { grokProcessingTime.record(() -> matchAndMerge(event)); } else { runWithTimeout(() -> grokProcessingTime.record(() -> matchAndMerge(event))); @@ -122,7 +122,7 @@ public Collection> doExecute(final Collection> recor final Record grokkedRecord = new Record<>(event, record.getMetadata()); recordsOut.add(grokkedRecord); } catch (TimeoutException e) { - LOG.error("Matching on record [{}] took longer than [{}] and timed out", record.getData(), grokPrepperConfig.getTimeoutMillis()); + LOG.error("Matching on record [{}] took longer than [{}] and timed out", record.getData(), grokProcessorConfig.getTimeoutMillis()); recordsOut.add(record); grokProcessingTimeoutsCounter.increment(); } catch (ExecutionException e) { @@ -170,20 +170,20 @@ public void shutdown() { private void registerPatterns() { grokCompiler.registerDefaultPatterns(); - grokCompiler.register(grokPrepperConfig.getPatternDefinitions()); + grokCompiler.register(grokProcessorConfig.getPatternDefinitions()); registerPatternsDirectories(); } private void registerPatternsDirectories() { - for (final String directory : grokPrepperConfig.getPatternsDirectories()) { + for (final String directory : grokProcessorConfig.getPatternsDirectories()) { final Path path = FileSystems.getDefault().getPath(directory); - try (DirectoryStream stream = Files.newDirectoryStream(path, grokPrepperConfig.getPatternsFilesGlob())) { + try (DirectoryStream stream = Files.newDirectoryStream(path, grokProcessorConfig.getPatternsFilesGlob())) { for (final Path patternFile : stream) { registerPatternsForFile(patternFile.toFile()); } } catch (PatternSyntaxException e) { - LOG.error("Glob pattern {} is invalid", grokPrepperConfig.getPatternsFilesGlob()); + LOG.error("Glob pattern {} is invalid", grokProcessorConfig.getPatternsFilesGlob()); } catch (NotDirectoryException e) { LOG.error("{} is not a directory", directory, e); } catch (IOException e) { @@ -203,10 +203,10 @@ private void registerPatternsForFile(final File file) { } private void compileMatchPatterns() { - for (final Map.Entry> entry : grokPrepperConfig.getMatch().entrySet()) { + for (final Map.Entry> entry : grokProcessorConfig.getMatch().entrySet()) { fieldToGrok.put(entry.getKey(), entry.getValue() .stream() - .map(item -> grokCompiler.compile(item, grokPrepperConfig.isNamedCapturesOnly())) + .map(item -> grokCompiler.compile(item, grokProcessorConfig.isNamedCapturesOnly())) .collect(Collectors.toList())); } } @@ -219,7 +219,7 @@ private void matchAndMerge(final Event event) { final String value = event.get(entry.getKey(), String.class); if (value != null && !value.isEmpty()) { final Match match = grok.match(value); - match.setKeepEmptyCaptures(grokPrepperConfig.isKeepEmptyCaptures()); + match.setKeepEmptyCaptures(grokProcessorConfig.isKeepEmptyCaptures()); final Map captures = match.capture(); mergeCaptures(grokkedCaptures, captures); @@ -234,8 +234,8 @@ private void matchAndMerge(final Event event) { } } - if (grokPrepperConfig.getTargetKey() != null) { - event.put(grokPrepperConfig.getTargetKey(), grokkedCaptures); + if (grokProcessorConfig.getTargetKey() != null) { + event.put(grokProcessorConfig.getTargetKey(), grokkedCaptures); } else { mergeCaptures(event, grokkedCaptures); } @@ -294,11 +294,11 @@ private void mergeValueWithValues(final Object value, final List values) } private boolean shouldBreakOnMatch(final Map captures) { - return captures.size() > 0 && grokPrepperConfig.isBreakOnMatch(); + return captures.size() > 0 && grokProcessorConfig.isBreakOnMatch(); } private void runWithTimeout(final Runnable runnable) throws TimeoutException, ExecutionException, InterruptedException { Future task = executorService.submit(runnable); - task.get(grokPrepperConfig.getTimeoutMillis(), TimeUnit.MILLISECONDS); + task.get(grokProcessorConfig.getTimeoutMillis(), TimeUnit.MILLISECONDS); } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/grok-prepper/src/main/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java similarity index 79% rename from data-prepper-plugins/grok-prepper/src/main/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java rename to data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java index 50cc4204f2..4ceb018f6f 100644 --- a/data-prepper-plugins/grok-prepper/src/main/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java +++ b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java @@ -3,14 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper.grok; +package org.opensearch.dataprepper.plugins.processor.grok; import org.opensearch.dataprepper.model.configuration.PluginSetting; import java.util.List; import java.util.Map; -public class GrokPrepperConfig { +public class GrokProcessorConfig { static final String BREAK_ON_MATCH = "break_on_match"; static final String KEEP_EMPTY_CAPTURES = "keep_empty_captures"; static final String MATCH = "match"; @@ -40,16 +40,16 @@ public class GrokPrepperConfig { private final int timeoutMillis; private final String targetKey; - private GrokPrepperConfig(final boolean breakOnMatch, - final boolean keepEmptyCaptures, - final Map> match, - final boolean namedCapturesOnly, - final List keysToOverwrite, - final List patternsDirectories, - final String patternsFilesGlob, - final Map patternDefinitions, - final int timeoutMillis, - final String targetKey) { + private GrokProcessorConfig(final boolean breakOnMatch, + final boolean keepEmptyCaptures, + final Map> match, + final boolean namedCapturesOnly, + final List keysToOverwrite, + final List patternsDirectories, + final String patternsFilesGlob, + final Map patternDefinitions, + final int timeoutMillis, + final String targetKey) { this.breakOnMatch = breakOnMatch; this.keepEmptyCaptures = keepEmptyCaptures; @@ -63,8 +63,8 @@ private GrokPrepperConfig(final boolean breakOnMatch, this.targetKey = targetKey; } - public static GrokPrepperConfig buildConfig(final PluginSetting pluginSetting) { - return new GrokPrepperConfig(pluginSetting.getBooleanOrDefault(BREAK_ON_MATCH, DEFAULT_BREAK_ON_MATCH), + public static GrokProcessorConfig buildConfig(final PluginSetting pluginSetting) { + return new GrokProcessorConfig(pluginSetting.getBooleanOrDefault(BREAK_ON_MATCH, DEFAULT_BREAK_ON_MATCH), pluginSetting.getBooleanOrDefault(KEEP_EMPTY_CAPTURES, DEFAULT_KEEP_EMPTY_CAPTURES), pluginSetting.getTypedListMap(MATCH, String.class, String.class), pluginSetting.getBooleanOrDefault(NAMED_CAPTURES_ONLY, DEFAULT_NAMED_CAPTURES_ONLY), @@ -115,4 +115,4 @@ public int getTimeoutMillis() { public String getTargetKey() { return targetKey; } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfigTests.java similarity index 50% rename from data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java rename to data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfigTests.java index 6a4180b0c6..c8b2970096 100644 --- a/data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java +++ b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfigTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper.grok; +package org.opensearch.dataprepper.plugins.processor.grok; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.junit.jupiter.api.BeforeAll; @@ -18,14 +18,14 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertThrows; -import static org.opensearch.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_BREAK_ON_MATCH; -import static org.opensearch.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_KEEP_EMPTY_CAPTURES; -import static org.opensearch.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_NAMED_CAPTURES_ONLY; -import static org.opensearch.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_PATTERNS_FILES_GLOB; -import static org.opensearch.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_TARGET_KEY; -import static org.opensearch.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_TIMEOUT_MILLIS; - -public class GrokPrepperConfigTests { +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.DEFAULT_BREAK_ON_MATCH; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.DEFAULT_KEEP_EMPTY_CAPTURES; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.DEFAULT_NAMED_CAPTURES_ONLY; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.DEFAULT_PATTERNS_FILES_GLOB; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.DEFAULT_TARGET_KEY; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.DEFAULT_TIMEOUT_MILLIS; + +public class GrokProcessorConfigTests { private static final String PLUGIN_NAME = "grok"; private static final Map> TEST_MATCH = new HashMap<>(); @@ -61,23 +61,23 @@ public static void setUp() { @Test public void testDefault() { - final GrokPrepperConfig grokPrepperConfig = GrokPrepperConfig.buildConfig(new PluginSetting(PLUGIN_NAME, null)); - - assertThat(grokPrepperConfig.isBreakOnMatch(), equalTo(DEFAULT_BREAK_ON_MATCH)); - assertThat(grokPrepperConfig.isKeepEmptyCaptures(), equalTo(DEFAULT_KEEP_EMPTY_CAPTURES)); - assertThat(grokPrepperConfig.getMatch(), equalTo(Collections.emptyMap())); - assertThat(grokPrepperConfig.getkeysToOverwrite(), equalTo(Collections.emptyList())); - assertThat(grokPrepperConfig.getPatternDefinitions(), equalTo(Collections.emptyMap())); - assertThat(grokPrepperConfig.getPatternsDirectories(), equalTo(Collections.emptyList())); - assertThat(grokPrepperConfig.getPatternsFilesGlob(), equalTo(DEFAULT_PATTERNS_FILES_GLOB)); - assertThat(grokPrepperConfig.getTargetKey(), equalTo(DEFAULT_TARGET_KEY)); - assertThat(grokPrepperConfig.isNamedCapturesOnly(), equalTo(DEFAULT_NAMED_CAPTURES_ONLY)); - assertThat(grokPrepperConfig.getTimeoutMillis(), equalTo(DEFAULT_TIMEOUT_MILLIS)); + final GrokProcessorConfig grokProcessorConfig = GrokProcessorConfig.buildConfig(new PluginSetting(PLUGIN_NAME, null)); + + assertThat(grokProcessorConfig.isBreakOnMatch(), equalTo(DEFAULT_BREAK_ON_MATCH)); + assertThat(grokProcessorConfig.isKeepEmptyCaptures(), equalTo(DEFAULT_KEEP_EMPTY_CAPTURES)); + assertThat(grokProcessorConfig.getMatch(), equalTo(Collections.emptyMap())); + assertThat(grokProcessorConfig.getkeysToOverwrite(), equalTo(Collections.emptyList())); + assertThat(grokProcessorConfig.getPatternDefinitions(), equalTo(Collections.emptyMap())); + assertThat(grokProcessorConfig.getPatternsDirectories(), equalTo(Collections.emptyList())); + assertThat(grokProcessorConfig.getPatternsFilesGlob(), equalTo(DEFAULT_PATTERNS_FILES_GLOB)); + assertThat(grokProcessorConfig.getTargetKey(), equalTo(DEFAULT_TARGET_KEY)); + assertThat(grokProcessorConfig.isNamedCapturesOnly(), equalTo(DEFAULT_NAMED_CAPTURES_ONLY)); + assertThat(grokProcessorConfig.getTimeoutMillis(), equalTo(DEFAULT_TIMEOUT_MILLIS)); } @Test public void testValidConfig() { - final PluginSetting validPluginSetting = completePluginSettingForGrokPrepper( + final PluginSetting validPluginSetting = completePluginSettingForGrokProcessor( false, true, TEST_MATCH, @@ -89,23 +89,23 @@ public void testValidConfig() { TEST_TIMEOUT_MILLIS, TEST_TARGET_KEY); - final GrokPrepperConfig grokPrepperConfig = GrokPrepperConfig.buildConfig(validPluginSetting); - - assertThat(grokPrepperConfig.isBreakOnMatch(), equalTo(false)); - assertThat(grokPrepperConfig.isKeepEmptyCaptures(), equalTo(true)); - assertThat(grokPrepperConfig.getMatch(), equalTo(TEST_MATCH)); - assertThat(grokPrepperConfig.getkeysToOverwrite(), equalTo(TEST_KEYS_TO_OVERWRITE)); - assertThat(grokPrepperConfig.getPatternDefinitions(), equalTo(TEST_PATTERN_DEFINITIONS)); - assertThat(grokPrepperConfig.getPatternsDirectories(), equalTo(TEST_PATTERNS_DIRECTORIES)); - assertThat(grokPrepperConfig.getPatternsFilesGlob(), equalTo(TEST_PATTERNS_FILES_GLOB)); - assertThat(grokPrepperConfig.getTargetKey(), equalTo(TEST_TARGET_KEY)); - assertThat(grokPrepperConfig.isNamedCapturesOnly(), equalTo(false)); - assertThat(grokPrepperConfig.getTimeoutMillis(), equalTo(TEST_TIMEOUT_MILLIS)); + final GrokProcessorConfig grokProcessorConfig = GrokProcessorConfig.buildConfig(validPluginSetting); + + assertThat(grokProcessorConfig.isBreakOnMatch(), equalTo(false)); + assertThat(grokProcessorConfig.isKeepEmptyCaptures(), equalTo(true)); + assertThat(grokProcessorConfig.getMatch(), equalTo(TEST_MATCH)); + assertThat(grokProcessorConfig.getkeysToOverwrite(), equalTo(TEST_KEYS_TO_OVERWRITE)); + assertThat(grokProcessorConfig.getPatternDefinitions(), equalTo(TEST_PATTERN_DEFINITIONS)); + assertThat(grokProcessorConfig.getPatternsDirectories(), equalTo(TEST_PATTERNS_DIRECTORIES)); + assertThat(grokProcessorConfig.getPatternsFilesGlob(), equalTo(TEST_PATTERNS_FILES_GLOB)); + assertThat(grokProcessorConfig.getTargetKey(), equalTo(TEST_TARGET_KEY)); + assertThat(grokProcessorConfig.isNamedCapturesOnly(), equalTo(false)); + assertThat(grokProcessorConfig.getTimeoutMillis(), equalTo(TEST_TIMEOUT_MILLIS)); } @Test public void testInvalidConfig() { - final PluginSetting invalidPluginSetting = completePluginSettingForGrokPrepper( + final PluginSetting invalidPluginSetting = completePluginSettingForGrokProcessor( false, true, TEST_MATCH, @@ -117,12 +117,12 @@ public void testInvalidConfig() { TEST_TIMEOUT_MILLIS, TEST_TARGET_KEY); - invalidPluginSetting.getSettings().put(GrokPrepperConfig.MATCH, TEST_INVALID_MATCH); + invalidPluginSetting.getSettings().put(GrokProcessorConfig.MATCH, TEST_INVALID_MATCH); - assertThrows(IllegalArgumentException.class, () -> GrokPrepperConfig.buildConfig(invalidPluginSetting)); + assertThrows(IllegalArgumentException.class, () -> GrokProcessorConfig.buildConfig(invalidPluginSetting)); } - private PluginSetting completePluginSettingForGrokPrepper(final boolean breakOnMatch, + private PluginSetting completePluginSettingForGrokProcessor(final boolean breakOnMatch, final boolean keepEmptyCaptures, final Map> match, final boolean namedCapturesOnly, @@ -133,16 +133,16 @@ private PluginSetting completePluginSettingForGrokPrepper(final boolean breakOnM final int timeoutMillis, final String targetKey) { final Map settings = new HashMap<>(); - settings.put(GrokPrepperConfig.BREAK_ON_MATCH, breakOnMatch); - settings.put(GrokPrepperConfig.NAMED_CAPTURES_ONLY, namedCapturesOnly); - settings.put(GrokPrepperConfig.MATCH, match); - settings.put(GrokPrepperConfig.KEEP_EMPTY_CAPTURES, keepEmptyCaptures); - settings.put(GrokPrepperConfig.KEYS_TO_OVERWRITE, keysToOverwrite); - settings.put(GrokPrepperConfig.PATTERNS_DIRECTORIES, patternsDirectories); - settings.put(GrokPrepperConfig.PATTERN_DEFINITIONS, patternDefinitions); - settings.put(GrokPrepperConfig.PATTERNS_FILES_GLOB, patternsFilesGlob); - settings.put(GrokPrepperConfig.TIMEOUT_MILLIS, timeoutMillis); - settings.put(GrokPrepperConfig.TARGET_KEY, targetKey); + settings.put(GrokProcessorConfig.BREAK_ON_MATCH, breakOnMatch); + settings.put(GrokProcessorConfig.NAMED_CAPTURES_ONLY, namedCapturesOnly); + settings.put(GrokProcessorConfig.MATCH, match); + settings.put(GrokProcessorConfig.KEEP_EMPTY_CAPTURES, keepEmptyCaptures); + settings.put(GrokProcessorConfig.KEYS_TO_OVERWRITE, keysToOverwrite); + settings.put(GrokProcessorConfig.PATTERNS_DIRECTORIES, patternsDirectories); + settings.put(GrokProcessorConfig.PATTERN_DEFINITIONS, patternDefinitions); + settings.put(GrokProcessorConfig.PATTERNS_FILES_GLOB, patternsFilesGlob); + settings.put(GrokProcessorConfig.TIMEOUT_MILLIS, timeoutMillis); + settings.put(GrokProcessorConfig.TARGET_KEY, targetKey); return new PluginSetting(PLUGIN_NAME, settings); } diff --git a/data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperIT.java b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java similarity index 79% rename from data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperIT.java rename to data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java index 62228948a7..d164a1d8d7 100644 --- a/data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperIT.java +++ b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper.grok; +package org.opensearch.dataprepper.plugins.processor.grok; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; @@ -25,11 +25,11 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.opensearch.dataprepper.plugins.prepper.grok.GrokPrepperTests.buildRecordWithEvent; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorTests.buildRecordWithEvent; -public class GrokPrepperIT { +public class GrokProcessorIT { private PluginSetting pluginSetting; - private GrokPrepper grokPrepper; + private GrokProcessor grokProcessor; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() {}; private final String PLUGIN_NAME = "grok"; @@ -38,17 +38,17 @@ public class GrokPrepperIT { @BeforeEach public void setup() { - pluginSetting = completePluginSettingForGrokPrepper( - GrokPrepperConfig.DEFAULT_BREAK_ON_MATCH, - GrokPrepperConfig.DEFAULT_KEEP_EMPTY_CAPTURES, + pluginSetting = completePluginSettingForGrokProcessor( + GrokProcessorConfig.DEFAULT_BREAK_ON_MATCH, + GrokProcessorConfig.DEFAULT_KEEP_EMPTY_CAPTURES, Collections.emptyMap(), - GrokPrepperConfig.DEFAULT_NAMED_CAPTURES_ONLY, + GrokProcessorConfig.DEFAULT_NAMED_CAPTURES_ONLY, Collections.emptyList(), Collections.emptyList(), - GrokPrepperConfig.DEFAULT_PATTERNS_FILES_GLOB, + GrokProcessorConfig.DEFAULT_PATTERNS_FILES_GLOB, Collections.emptyMap(), - GrokPrepperConfig.DEFAULT_TIMEOUT_MILLIS, - GrokPrepperConfig.DEFAULT_TARGET_KEY); + GrokProcessorConfig.DEFAULT_TIMEOUT_MILLIS, + GrokProcessorConfig.DEFAULT_TARGET_KEY); pluginSetting.setPipelineName("grokPipeline"); @@ -60,10 +60,10 @@ public void setup() { @AfterEach public void tearDown() { - grokPrepper.shutdown(); + grokProcessor.shutdown(); } - private PluginSetting completePluginSettingForGrokPrepper(final boolean breakOnMatch, + private PluginSetting completePluginSettingForGrokProcessor(final boolean breakOnMatch, final boolean keepEmptyCaptures, final Map> match, final boolean namedCapturesOnly, @@ -74,16 +74,16 @@ private PluginSetting completePluginSettingForGrokPrepper(final boolean breakOnM final int timeoutMillis, final String targetKey) { final Map settings = new HashMap<>(); - settings.put(GrokPrepperConfig.BREAK_ON_MATCH, breakOnMatch); - settings.put(GrokPrepperConfig.NAMED_CAPTURES_ONLY, namedCapturesOnly); - settings.put(GrokPrepperConfig.MATCH, match); - settings.put(GrokPrepperConfig.KEEP_EMPTY_CAPTURES, keepEmptyCaptures); - settings.put(GrokPrepperConfig.KEYS_TO_OVERWRITE, keysToOverwrite); - settings.put(GrokPrepperConfig.PATTERNS_DIRECTORIES, patternsDirectories); - settings.put(GrokPrepperConfig.PATTERN_DEFINITIONS, patternDefinitions); - settings.put(GrokPrepperConfig.PATTERNS_FILES_GLOB, patternsFilesGlob); - settings.put(GrokPrepperConfig.TIMEOUT_MILLIS, timeoutMillis); - settings.put(GrokPrepperConfig.TARGET_KEY, targetKey); + settings.put(GrokProcessorConfig.BREAK_ON_MATCH, breakOnMatch); + settings.put(GrokProcessorConfig.NAMED_CAPTURES_ONLY, namedCapturesOnly); + settings.put(GrokProcessorConfig.MATCH, match); + settings.put(GrokProcessorConfig.KEEP_EMPTY_CAPTURES, keepEmptyCaptures); + settings.put(GrokProcessorConfig.KEYS_TO_OVERWRITE, keysToOverwrite); + settings.put(GrokProcessorConfig.PATTERNS_DIRECTORIES, patternsDirectories); + settings.put(GrokProcessorConfig.PATTERN_DEFINITIONS, patternDefinitions); + settings.put(GrokProcessorConfig.PATTERNS_FILES_GLOB, patternsFilesGlob); + settings.put(GrokProcessorConfig.TIMEOUT_MILLIS, timeoutMillis); + settings.put(GrokProcessorConfig.TARGET_KEY, targetKey); return new PluginSetting(PLUGIN_NAME, settings); } @@ -95,15 +95,15 @@ public void testMatchNoCapturesWithExistingAndNonExistingKey() throws JsonProces matchConfig.put("message", Collections.singletonList(nonMatchingPattern)); matchConfig.put("bad_key", Collections.singletonList(nonMatchingPattern)); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", messageInput); final Record record = buildRecordWithEvent(testData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -115,8 +115,8 @@ public void testSingleMatchSinglePatternWithDefaults() throws JsonProcessingExce final Map> matchConfig = new HashMap<>(); matchConfig.put("message", Collections.singletonList("%{COMMONAPACHELOG}")); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -136,7 +136,7 @@ public void testSingleMatchSinglePatternWithDefaults() throws JsonProcessingExce final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -152,9 +152,9 @@ public void testSingleMatchMultiplePatternWithBreakOnMatchFalse() throws JsonPro matchConfig.put("message", patternsToMatchMessage); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - pluginSetting.getSettings().put(GrokPrepperConfig.BREAK_ON_MATCH, false); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.BREAK_ON_MATCH, false); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -176,7 +176,7 @@ public void testSingleMatchMultiplePatternWithBreakOnMatchFalse() throws JsonPro final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -188,8 +188,8 @@ public void testSingleMatchTypeConversionWithDefaults() throws JsonProcessingExc final Map> matchConfig = new HashMap<>(); matchConfig.put("message", Collections.singletonList("\"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response:int} (?:%{NUMBER:bytes:float}|-)")); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -206,7 +206,7 @@ public void testSingleMatchTypeConversionWithDefaults() throws JsonProcessingExc final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -219,9 +219,9 @@ public void testMultipleMatchWithBreakOnMatchFalse() throws JsonProcessingExcept matchConfig.put("message", Collections.singletonList("%{COMMONAPACHELOG}")); matchConfig.put("extra_field", Collections.singletonList("%{GREEDYDATA} %{IPORHOST:host}")); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - pluginSetting.getSettings().put(GrokPrepperConfig.BREAK_ON_MATCH, false); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.BREAK_ON_MATCH, false); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -245,7 +245,7 @@ public void testMultipleMatchWithBreakOnMatchFalse() throws JsonProcessingExcept final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -257,9 +257,9 @@ public void testMatchWithKeepEmptyCapturesTrue() throws JsonProcessingException final Map> matchConfig = new HashMap<>(); matchConfig.put("message", Collections.singletonList("%{COMMONAPACHELOG}")); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - pluginSetting.getSettings().put(GrokPrepperConfig.KEEP_EMPTY_CAPTURES, true); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.KEEP_EMPTY_CAPTURES, true); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", messageInput); @@ -281,7 +281,7 @@ public void testMatchWithKeepEmptyCapturesTrue() throws JsonProcessingException final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -293,9 +293,9 @@ public void testMatchWithNamedCapturesOnlyFalse() throws JsonProcessingException final Map> matchConfig = new HashMap<>(); matchConfig.put("message", Collections.singletonList("%{GREEDYDATA} %{IPORHOST:host} %{NUMBER}")); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - pluginSetting.getSettings().put(GrokPrepperConfig.NAMED_CAPTURES_ONLY, false); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.NAMED_CAPTURES_ONLY, false); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", "This is my greedy data before matching 192.0.2.1 123456"); @@ -310,7 +310,7 @@ public void testMatchWithNamedCapturesOnlyFalse() throws JsonProcessingException final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -325,9 +325,9 @@ public void testPatternDefinitions() throws JsonProcessingException { final Map patternDefinitions = new HashMap<>(); patternDefinitions.put("CUSTOMPHONENUMBERPATTERN", "\\d\\d\\d-\\d\\d\\d-\\d\\d\\d"); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - pluginSetting.getSettings().put(GrokPrepperConfig.PATTERN_DEFINITIONS, patternDefinitions); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.PATTERN_DEFINITIONS, patternDefinitions); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", "This is my greedy data before matching with my phone number 123-456-789"); @@ -341,7 +341,7 @@ public void testPatternDefinitions() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -368,13 +368,13 @@ public void testPatternsDirWithDefaultPatternsFilesGlob() throws JsonProcessingE resultData.put("my_birthday", "April 15, 1991"); resultData.put("my_number", "123-456-789"); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - pluginSetting.getSettings().put(GrokPrepperConfig.PATTERNS_DIRECTORIES, patternsDirectories); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.PATTERNS_DIRECTORIES, patternsDirectories); + grokProcessor = new GrokProcessor(pluginSetting); final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -400,14 +400,14 @@ public void testPatternsDirWithCustomPatternsFilesGlob() throws JsonProcessingEx resultData.put("message", "My phone number is 123-456-789"); resultData.put("my_number", "123-456-789"); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - pluginSetting.getSettings().put(GrokPrepperConfig.PATTERNS_DIRECTORIES, patternsDirectories); - pluginSetting.getSettings().put(GrokPrepperConfig.PATTERNS_FILES_GLOB, "*1.txt"); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.PATTERNS_DIRECTORIES, patternsDirectories); + pluginSetting.getSettings().put(GrokProcessorConfig.PATTERNS_FILES_GLOB, "*1.txt"); + grokProcessor = new GrokProcessor(pluginSetting); final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -416,9 +416,9 @@ public void testPatternsDirWithCustomPatternsFilesGlob() throws JsonProcessingEx final Map> matchConfigWithPatterns2Pattern = new HashMap<>(); matchConfigWithPatterns2Pattern.put("message", Collections.singletonList("My birthday is %{CUSTOMBIRTHDAYPATTERN:my_birthday}")); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfigWithPatterns2Pattern); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfigWithPatterns2Pattern); - Throwable throwable = assertThrows(IllegalArgumentException.class, () -> new GrokPrepper((pluginSetting))); + Throwable throwable = assertThrows(IllegalArgumentException.class, () -> new GrokProcessor((pluginSetting))); assertThat("No definition for key 'CUSTOMBIRTHDAYPATTERN' found, aborting", equalTo(throwable.getMessage())); } @@ -427,8 +427,8 @@ public void testMatchWithNamedCapturesSyntax() throws JsonProcessingException { final Map> matchConfig = new HashMap<>(); matchConfig.put("message", Collections.singletonList("%{GREEDYDATA:greedy_data} (?\\d\\d\\d-\\d\\d\\d-\\d\\d\\d)")); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); - grokPrepper = new GrokPrepper(pluginSetting); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); + grokProcessor = new GrokProcessor(pluginSetting); final Map testData = new HashMap(); testData.put("message", "This is my greedy data before matching with my phone number 123-456-789"); @@ -442,7 +442,7 @@ public void testMatchWithNamedCapturesSyntax() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -452,14 +452,14 @@ public void testMatchWithNamedCapturesSyntax() throws JsonProcessingException { @Test public void testCompileNonRegisteredPatternThrowsIllegalArgumentException() { - grokPrepper = new GrokPrepper(pluginSetting); + grokProcessor = new GrokProcessor(pluginSetting); final Map> matchConfig = new HashMap<>(); matchConfig.put("message", Collections.singletonList("%{NONEXISTENTPATTERN}")); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); - assertThrows(IllegalArgumentException.class, () -> new GrokPrepper(pluginSetting)); + assertThrows(IllegalArgumentException.class, () -> new GrokProcessor(pluginSetting)); } private void assertRecordsAreEqual(final Record first, final Record second) throws JsonProcessingException { diff --git a/data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperTests.java b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java similarity index 83% rename from data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperTests.java rename to data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java index 0e60f786d8..dbe3d17175 100644 --- a/data-prepper-plugins/grok-prepper/src/test/java/org/opensearch/dataprepper/plugins/prepper/grok/GrokPrepperTests.java +++ b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper.grok; +package org.opensearch.dataprepper.plugins.processor.grok; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.configuration.PluginSetting; @@ -51,13 +51,13 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.prepper.grok.GrokPrepper.EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessor.EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT; import static org.opensearch.dataprepper.test.matcher.MapEquals.isEqualWithoutTimestamp; @ExtendWith(MockitoExtension.class) -public class GrokPrepperTests { - private GrokPrepper grokPrepper; +public class GrokProcessorTests { + private GrokProcessor grokProcessor; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() {}; private String messageInput; @@ -113,13 +113,13 @@ public void setup() throws TimeoutException, ExecutionException, InterruptedExce matchPatterns.add("%{PATTERN2}"); matchConfig.put("message", matchPatterns); - pluginSetting.getSettings().put(GrokPrepperConfig.MATCH, matchConfig); + pluginSetting.getSettings().put(GrokProcessorConfig.MATCH, matchConfig); - lenient().when(pluginMetrics.counter(GrokPrepper.GROK_PROCESSING_MATCH_SUCCESS)).thenReturn(grokProcessingMatchSuccessCounter); - lenient().when(pluginMetrics.counter(GrokPrepper.GROK_PROCESSING_MATCH_FAILURE)).thenReturn(grokProcessingMatchFailureCounter); - lenient().when(pluginMetrics.counter(GrokPrepper.GROK_PROCESSING_TIMEOUTS)).thenReturn(grokProcessingTimeoutsCounter); - lenient().when(pluginMetrics.counter(GrokPrepper.GROK_PROCESSING_ERRORS)).thenReturn(grokProcessingErrorsCounter); - lenient().when(pluginMetrics.timer(GrokPrepper.GROK_PROCESSING_TIME)).thenReturn(grokProcessingTime); + lenient().when(pluginMetrics.counter(GrokProcessor.GROK_PROCESSING_MATCH_SUCCESS)).thenReturn(grokProcessingMatchSuccessCounter); + lenient().when(pluginMetrics.counter(GrokProcessor.GROK_PROCESSING_MATCH_FAILURE)).thenReturn(grokProcessingMatchFailureCounter); + lenient().when(pluginMetrics.counter(GrokProcessor.GROK_PROCESSING_TIMEOUTS)).thenReturn(grokProcessingTimeoutsCounter); + lenient().when(pluginMetrics.counter(GrokProcessor.GROK_PROCESSING_ERRORS)).thenReturn(grokProcessingErrorsCounter); + lenient().when(pluginMetrics.timer(GrokProcessor.GROK_PROCESSING_TIME)).thenReturn(grokProcessingTime); lenient().doAnswer(a -> { a.getArgument(0).run(); @@ -139,19 +139,19 @@ public void setup() throws TimeoutException, ExecutionException, InterruptedExce a.getArgument(0).run(); return task; }); - lenient().when(task.get(GrokPrepperConfig.DEFAULT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)).thenReturn(null); + lenient().when(task.get(GrokProcessorConfig.DEFAULT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)).thenReturn(null); } - private GrokPrepper createObjectUnderTest() { + private GrokProcessor createObjectUnderTest() { try (MockedStatic pluginMetricsMockedStatic = mockStatic(PluginMetrics.class)) { pluginMetricsMockedStatic.when(() -> PluginMetrics.fromPluginSetting(pluginSetting)).thenReturn(pluginMetrics); - return new GrokPrepper(pluginSetting, grokCompiler, executorService); + return new GrokProcessor(pluginSetting, grokCompiler, executorService); } } @Test public void testMatchMerge() throws JsonProcessingException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); capture.put("key_capture_2", "value_capture_2"); @@ -169,7 +169,7 @@ public void testMatchMerge() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -181,8 +181,8 @@ public void testMatchMerge() throws JsonProcessingException { @Test public void testTarget() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokPrepperConfig.TARGET_KEY, "test_target"); - grokPrepper = createObjectUnderTest(); + pluginSetting.getSettings().put(GrokProcessorConfig.TARGET_KEY, "test_target"); + grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); capture.put("key_capture_2", "value_capture_2"); @@ -203,7 +203,7 @@ public void testTarget() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -215,8 +215,8 @@ public void testTarget() throws JsonProcessingException { @Test public void testOverwrite() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokPrepperConfig.KEYS_TO_OVERWRITE, Collections.singletonList("message")); - grokPrepper = createObjectUnderTest(); + pluginSetting.getSettings().put(GrokProcessorConfig.KEYS_TO_OVERWRITE, Collections.singletonList("message")); + grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); capture.put("key_capture_2", "value_capture_2"); @@ -235,7 +235,7 @@ public void testOverwrite() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -247,7 +247,7 @@ public void testOverwrite() throws JsonProcessingException { @Test public void testMatchMergeCollisionStrings() throws JsonProcessingException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); capture.put("key_capture_2", "value_capture_2"); @@ -266,7 +266,7 @@ public void testMatchMergeCollisionStrings() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -278,7 +278,7 @@ public void testMatchMergeCollisionStrings() throws JsonProcessingException { @Test public void testMatchMergeCollisionInts() throws JsonProcessingException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", 20); capture.put("key_capture_2", "value_capture_2"); @@ -297,7 +297,7 @@ public void testMatchMergeCollisionInts() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -309,7 +309,7 @@ public void testMatchMergeCollisionInts() throws JsonProcessingException { @Test public void testMatchMergeCollisionWithListMixedTypes() throws JsonProcessingException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); List captureListValues = new ArrayList<>(); captureListValues.add("30"); @@ -333,7 +333,7 @@ public void testMatchMergeCollisionWithListMixedTypes() throws JsonProcessingExc final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -345,7 +345,7 @@ public void testMatchMergeCollisionWithListMixedTypes() throws JsonProcessingExc @Test public void testMatchMergeCollisionWithNullValue() throws JsonProcessingException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); capture.put("key_capture_2", "value_capture_2"); @@ -364,7 +364,7 @@ public void testMatchMergeCollisionWithNullValue() throws JsonProcessingExceptio final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -376,9 +376,9 @@ public void testMatchMergeCollisionWithNullValue() throws JsonProcessingExceptio @Test public void testThatTimeoutExceptionIsCaughtAndProcessingContinues() throws JsonProcessingException, TimeoutException, ExecutionException, InterruptedException { - when(task.get(GrokPrepperConfig.DEFAULT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)).thenThrow(TimeoutException.class); + when(task.get(GrokProcessorConfig.DEFAULT_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)).thenThrow(TimeoutException.class); - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); capture.put("key_capture_2", "value_capture_2"); @@ -388,7 +388,7 @@ public void testThatTimeoutExceptionIsCaughtAndProcessingContinues() throws Json testData.put("message", messageInput); final Record record = buildRecordWithEvent(testData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -399,8 +399,8 @@ public void testThatTimeoutExceptionIsCaughtAndProcessingContinues() throws Json @Test public void testThatProcessingWithTimeoutMillisOfZeroDoesNotInteractWithExecutorServiceAndReturnsCorrectResult() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokPrepperConfig.TIMEOUT_MILLIS, 0); - grokPrepper = createObjectUnderTest(); + pluginSetting.getSettings().put(GrokProcessorConfig.TIMEOUT_MILLIS, 0); + grokProcessor = createObjectUnderTest(); capture.put("key_capture_1", "value_capture_1"); capture.put("key_capture_2", "value_capture_2"); @@ -419,7 +419,7 @@ public void testThatProcessingWithTimeoutMillisOfZeroDoesNotInteractWithExecutor final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); verifyNoInteractions(executorService); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -432,26 +432,26 @@ public void testThatProcessingWithTimeoutMillisOfZeroDoesNotInteractWithExecutor @Test public void testPrepareForShutdown() { - grokPrepper = createObjectUnderTest(); - grokPrepper.prepareForShutdown(); + grokProcessor = createObjectUnderTest(); + grokProcessor.prepareForShutdown(); } @Test public void testShutdown_Successful() throws InterruptedException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); lenient().when(executorService.awaitTermination(eq(EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT), eq(TimeUnit.MILLISECONDS))).thenReturn(true); - grokPrepper.shutdown(); + grokProcessor.shutdown(); verify(executorService).shutdown(); verify(executorService).awaitTermination(eq(EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT), eq(TimeUnit.MILLISECONDS)); } @Test public void testShutdown_Timeout() throws InterruptedException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); lenient().when(executorService.awaitTermination(eq(EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT), eq(TimeUnit.MILLISECONDS))).thenReturn(false); - grokPrepper.shutdown(); + grokProcessor.shutdown(); verify(executorService).shutdown(); verify(executorService).awaitTermination(eq(EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT), eq(TimeUnit.MILLISECONDS)); verify(executorService).shutdownNow(); @@ -459,11 +459,11 @@ public void testShutdown_Timeout() throws InterruptedException { @Test public void testShutdown_InterruptedException() throws InterruptedException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); lenient().when(executorService.awaitTermination(eq(EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT), eq(TimeUnit.MILLISECONDS))) .thenThrow(new InterruptedException()); - grokPrepper.shutdown(); + grokProcessor.shutdown(); verify(executorService).shutdown(); verify(executorService).awaitTermination(eq(EXECUTOR_SERVICE_SHUTDOWN_TIMEOUT), eq(TimeUnit.MILLISECONDS)); verify(executorService).shutdownNow(); @@ -483,7 +483,7 @@ public void setup() { @Test public void testNoCaptures() throws JsonProcessingException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); lenient().when(grokSecondMatch.match(messageInput)).thenReturn(secondMatch); lenient().when(secondMatch.capture()).thenReturn(secondCapture); @@ -492,7 +492,7 @@ public void testNoCaptures() throws JsonProcessingException { testData.put("message", messageInput); final Record record = buildRecordWithEvent(testData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -504,7 +504,7 @@ public void testNoCaptures() throws JsonProcessingException { @Test public void testBreakOnMatchTrue() throws JsonProcessingException { - grokPrepper = createObjectUnderTest(); + grokProcessor = createObjectUnderTest(); lenient().when(grokSecondMatch.match(messageInput)).thenReturn(secondMatch); lenient().when(secondMatch.capture()).thenReturn(secondCapture); @@ -527,7 +527,7 @@ public void testBreakOnMatchTrue() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); verifyNoInteractions(grokSecondMatch, secondMatch); assertThat(grokkedRecords.size(), equalTo(1)); @@ -540,8 +540,8 @@ public void testBreakOnMatchTrue() throws JsonProcessingException { @Test public void testBreakOnMatchFalse() throws JsonProcessingException { - pluginSetting.getSettings().put(GrokPrepperConfig.BREAK_ON_MATCH, false); - grokPrepper = createObjectUnderTest(); + pluginSetting.getSettings().put(GrokProcessorConfig.BREAK_ON_MATCH, false); + grokProcessor = createObjectUnderTest(); when(grokSecondMatch.match(messageInput)).thenReturn(secondMatch); when(secondMatch.capture()).thenReturn(secondCapture); @@ -565,7 +565,7 @@ public void testBreakOnMatchFalse() throws JsonProcessingException { final Record resultRecord = buildRecordWithEvent(resultData); - final List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + final List> grokkedRecords = (List>) grokProcessor.doExecute(Collections.singletonList(record)); assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); @@ -578,20 +578,20 @@ public void testBreakOnMatchFalse() throws JsonProcessingException { private PluginSetting getDefaultPluginSetting() { - return completePluginSettingForGrokPrepper( - GrokPrepperConfig.DEFAULT_BREAK_ON_MATCH, - GrokPrepperConfig.DEFAULT_KEEP_EMPTY_CAPTURES, + return completePluginSettingForGrokProcessor( + GrokProcessorConfig.DEFAULT_BREAK_ON_MATCH, + GrokProcessorConfig.DEFAULT_KEEP_EMPTY_CAPTURES, matchConfig, - GrokPrepperConfig.DEFAULT_NAMED_CAPTURES_ONLY, + GrokProcessorConfig.DEFAULT_NAMED_CAPTURES_ONLY, Collections.emptyList(), Collections.emptyList(), - GrokPrepperConfig.DEFAULT_PATTERNS_FILES_GLOB, + GrokProcessorConfig.DEFAULT_PATTERNS_FILES_GLOB, Collections.emptyMap(), - GrokPrepperConfig.DEFAULT_TIMEOUT_MILLIS, - GrokPrepperConfig.DEFAULT_TARGET_KEY); + GrokProcessorConfig.DEFAULT_TIMEOUT_MILLIS, + GrokProcessorConfig.DEFAULT_TARGET_KEY); } - private PluginSetting completePluginSettingForGrokPrepper(final boolean breakOnMatch, + private PluginSetting completePluginSettingForGrokProcessor(final boolean breakOnMatch, final boolean keepEmptyCaptures, final Map> match, final boolean namedCapturesOnly, @@ -602,16 +602,16 @@ private PluginSetting completePluginSettingForGrokPrepper(final boolean breakOnM final int timeoutMillis, final String targetKey) { final Map settings = new HashMap<>(); - settings.put(GrokPrepperConfig.BREAK_ON_MATCH, breakOnMatch); - settings.put(GrokPrepperConfig.NAMED_CAPTURES_ONLY, namedCapturesOnly); - settings.put(GrokPrepperConfig.MATCH, match); - settings.put(GrokPrepperConfig.KEEP_EMPTY_CAPTURES, keepEmptyCaptures); - settings.put(GrokPrepperConfig.KEYS_TO_OVERWRITE, keysToOverwrite); - settings.put(GrokPrepperConfig.PATTERNS_DIRECTORIES, patternsDirectories); - settings.put(GrokPrepperConfig.PATTERN_DEFINITIONS, patternDefinitions); - settings.put(GrokPrepperConfig.PATTERNS_FILES_GLOB, patternsFilesGlob); - settings.put(GrokPrepperConfig.TIMEOUT_MILLIS, timeoutMillis); - settings.put(GrokPrepperConfig.TARGET_KEY, targetKey); + settings.put(GrokProcessorConfig.BREAK_ON_MATCH, breakOnMatch); + settings.put(GrokProcessorConfig.NAMED_CAPTURES_ONLY, namedCapturesOnly); + settings.put(GrokProcessorConfig.MATCH, match); + settings.put(GrokProcessorConfig.KEEP_EMPTY_CAPTURES, keepEmptyCaptures); + settings.put(GrokProcessorConfig.KEYS_TO_OVERWRITE, keysToOverwrite); + settings.put(GrokProcessorConfig.PATTERNS_DIRECTORIES, patternsDirectories); + settings.put(GrokProcessorConfig.PATTERN_DEFINITIONS, patternDefinitions); + settings.put(GrokProcessorConfig.PATTERNS_FILES_GLOB, patternsFilesGlob); + settings.put(GrokProcessorConfig.TIMEOUT_MILLIS, timeoutMillis); + settings.put(GrokProcessorConfig.TARGET_KEY, targetKey); return new PluginSetting(PLUGIN_NAME, settings); } diff --git a/data-prepper-plugins/grok-prepper/src/test/resources/patterns/test_patterns_1.txt b/data-prepper-plugins/grok-processor/src/test/resources/patterns/test_patterns_1.txt similarity index 100% rename from data-prepper-plugins/grok-prepper/src/test/resources/patterns/test_patterns_1.txt rename to data-prepper-plugins/grok-processor/src/test/resources/patterns/test_patterns_1.txt diff --git a/data-prepper-plugins/grok-prepper/src/test/resources/patterns/test_patterns_2.txt b/data-prepper-plugins/grok-processor/src/test/resources/patterns/test_patterns_2.txt similarity index 100% rename from data-prepper-plugins/grok-prepper/src/test/resources/patterns/test_patterns_2.txt rename to data-prepper-plugins/grok-processor/src/test/resources/patterns/test_patterns_2.txt diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-processor-state/build.gradle similarity index 100% rename from data-prepper-plugins/mapdb-prepper-state/build.gradle rename to data-prepper-plugins/mapdb-processor-state/build.gradle diff --git a/data-prepper-plugins/mapdb-prepper-state/src/main/java/org/opensearch/dataprepper/plugins/prepper/state/MapDbPrepperState.java b/data-prepper-plugins/mapdb-processor-state/src/main/java/org/opensearch/dataprepper/plugins/processor/state/MapDbProcessorState.java similarity index 95% rename from data-prepper-plugins/mapdb-prepper-state/src/main/java/org/opensearch/dataprepper/plugins/prepper/state/MapDbPrepperState.java rename to data-prepper-plugins/mapdb-processor-state/src/main/java/org/opensearch/dataprepper/plugins/processor/state/MapDbProcessorState.java index f95b809a02..f219b7da05 100644 --- a/data-prepper-plugins/mapdb-prepper-state/src/main/java/org/opensearch/dataprepper/plugins/prepper/state/MapDbPrepperState.java +++ b/data-prepper-plugins/mapdb-processor-state/src/main/java/org/opensearch/dataprepper/plugins/processor/state/MapDbProcessorState.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper.state; +package org.opensearch.dataprepper.plugins.processor.state; import org.opensearch.dataprepper.processor.state.ProcessorState; import com.google.common.primitives.SignedBytes; @@ -21,7 +21,7 @@ import java.util.Map; import java.util.function.BiFunction; -public class MapDbPrepperState implements ProcessorState { +public class MapDbProcessorState implements ProcessorState { private static class SignedByteArraySerializer extends SerializerByteArray { @@ -36,7 +36,7 @@ public int compare(byte[] o1, byte[] o2) { private final BTreeMap map; private final File dbFile; - public MapDbPrepperState(final File dbPath, final String dbName, final int concurrencyScale) { + public MapDbProcessorState(final File dbPath, final String dbName, final int concurrencyScale) { // TODO: Cleanup references to file-based map this.dbFile = new File(String.join("/", dbPath.getPath(), dbName)); map = diff --git a/data-prepper-plugins/mapdb-prepper-state/src/test/java/org/opensearch/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java b/data-prepper-plugins/mapdb-processor-state/src/test/java/org/opensearch/dataprepper/plugins/processor/state/MapDbProcessorStateTest.java similarity index 90% rename from data-prepper-plugins/mapdb-prepper-state/src/test/java/org/opensearch/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java rename to data-prepper-plugins/mapdb-processor-state/src/test/java/org/opensearch/dataprepper/plugins/processor/state/MapDbProcessorStateTest.java index 82e76e45b6..47704b30b1 100644 --- a/data-prepper-plugins/mapdb-prepper-state/src/test/java/org/opensearch/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java +++ b/data-prepper-plugins/mapdb-processor-state/src/test/java/org/opensearch/dataprepper/plugins/processor/state/MapDbProcessorStateTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper.state; +package org.opensearch.dataprepper.plugins.processor.state; import org.junit.Assert; import org.junit.Rule; @@ -16,14 +16,14 @@ import java.util.UUID; import java.util.function.BiFunction; -public class MapDbPrepperStateTest extends ProcessorStateTest { +public class MapDbProcessorStateTest extends ProcessorStateTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Override public void setProcessorState() throws Exception { - this.processorState = new MapDbPrepperState<>(temporaryFolder.newFolder(), "testDb", 16); + this.processorState = new MapDbProcessorState<>(temporaryFolder.newFolder(), "testDb", 16); } @Test diff --git a/data-prepper-plugins/service-map-stateful/README.md b/data-prepper-plugins/service-map-stateful/README.md index 44e84de079..897192d9ee 100644 --- a/data-prepper-plugins/service-map-stateful/README.md +++ b/data-prepper-plugins/service-map-stateful/README.md @@ -1,6 +1,6 @@ -# Service-Map Stateful Prepper +# Service-Map Stateful Processor -This is a special prepper that consumes Opentelemetry traces, stores them in a MapDB data store and evaluate relationships at fixed ```window_duration```. +This is a special processor that consumes Opentelemetry traces, stores them in a MapDB data store and evaluate relationships at fixed ```window_duration```. # Usages Example `.yaml` configuration: @@ -14,7 +14,7 @@ processor: * window_duration(Optional) => An `int` represents the fixed time window in seconds to evaluate service-map relationships. Default is ```180```. ## Metrics -Besides common metrics in [AbstractProcessor](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/processor/AbstractProcessor.java), service-map-stateful prepper introduces the following custom metrics. +Besides common metrics in [AbstractProcessor](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/processor/AbstractProcessor.java), service-map-stateful processor introduces the following custom metrics. ### Gauge - `spansDbSize`: measures total spans byte sizes in MapDB across the current and previous window durations. diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index 4acbc15824..55d974d157 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -14,7 +14,7 @@ repositories { dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') - implementation project(':data-prepper-plugins:mapdb-prepper-state') + implementation project(':data-prepper-plugins:mapdb-processor-state') testImplementation project(':data-prepper-api').sourceSets.test.output implementation 'commons-codec:commons-codec:1.15' implementation 'io.micrometer:micrometer-core' diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/OTelHelper.java b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/OTelHelper.java similarity index 95% rename from data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/OTelHelper.java rename to data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/OTelHelper.java index 0d679ee4e1..93859adccb 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/OTelHelper.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/OTelHelper.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import io.opentelemetry.proto.resource.v1.Resource; import io.opentelemetry.proto.trace.v1.Span; diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapPrepperConfig.java b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfig.java similarity index 73% rename from data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapPrepperConfig.java rename to data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfig.java index a7c61ce815..8c337b2737 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapPrepperConfig.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapProcessorConfig.java @@ -3,9 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; -public class ServiceMapPrepperConfig { +public class ServiceMapProcessorConfig { static final String WINDOW_DURATION = "window_duration"; static final int DEFAULT_WINDOW_DURATION = 180; static final String DEFAULT_DB_PATH = "data/service-map/"; diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapRelationship.java b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapRelationship.java similarity index 99% rename from data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapRelationship.java rename to data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapRelationship.java index 1c7f30c1af..4a4f81e745 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapRelationship.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapRelationship.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessor.java similarity index 81% rename from data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java rename to data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessor.java index 312eac92be..60a3e779cf 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessor.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.SingleThread; @@ -19,7 +19,7 @@ import com.google.common.collect.Sets; import com.google.common.primitives.SignedBytes; import org.apache.commons.codec.binary.Hex; -import org.opensearch.dataprepper.plugins.prepper.state.MapDbPrepperState; +import org.opensearch.dataprepper.plugins.processor.state.MapDbProcessorState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,12 +38,12 @@ @SingleThread @DataPrepperPlugin(name = "service_map_stateful", pluginType = Processor.class) -public class ServiceMapStatefulPrepper extends AbstractProcessor, Record> implements RequiresPeerForwarding { +public class ServiceMapStatefulProcessor extends AbstractProcessor, Record> implements RequiresPeerForwarding { public static final String SPANS_DB_SIZE = "spansDbSize"; public static final String TRACE_GROUP_DB_SIZE = "traceGroupDbSize"; - private static final Logger LOG = LoggerFactory.getLogger(ServiceMapStatefulPrepper.class); + private static final Logger LOG = LoggerFactory.getLogger(ServiceMapStatefulProcessor.class); private static final String EMPTY_SUFFIX = "-empty"; private static final String EVENT_TYPE = "event"; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -51,49 +51,49 @@ public class ServiceMapStatefulPrepper extends AbstractProcessor, private static final Integer TO_MILLIS = 1_000; // TODO: This should not be tracked in this class, move it up to the creator - private static final AtomicInteger preppersCreated = new AtomicInteger(0); + private static final AtomicInteger processorsCreated = new AtomicInteger(0); private static long previousTimestamp; private static long windowDurationMillis; private static CyclicBarrier allThreadsCyclicBarrier; - private static volatile MapDbPrepperState previousWindow; - private static volatile MapDbPrepperState currentWindow; - private static volatile MapDbPrepperState previousTraceGroupWindow; - private static volatile MapDbPrepperState currentTraceGroupWindow; + private static volatile MapDbProcessorState previousWindow; + private static volatile MapDbProcessorState currentWindow; + private static volatile MapDbProcessorState previousTraceGroupWindow; + private static volatile MapDbProcessorState currentTraceGroupWindow; //TODO: Consider keeping this state in a db private static final Set RELATIONSHIP_STATE = Sets.newConcurrentHashSet(); private static File dbPath; private static Clock clock; - private final int thisPrepperId; + private final int thisProcessorId; - public ServiceMapStatefulPrepper(final PluginSetting pluginSetting) { - this(pluginSetting.getIntegerOrDefault(ServiceMapPrepperConfig.WINDOW_DURATION, ServiceMapPrepperConfig.DEFAULT_WINDOW_DURATION) * TO_MILLIS, - new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH), + public ServiceMapStatefulProcessor(final PluginSetting pluginSetting) { + this(pluginSetting.getIntegerOrDefault(ServiceMapProcessorConfig.WINDOW_DURATION, ServiceMapProcessorConfig.DEFAULT_WINDOW_DURATION) * TO_MILLIS, + new File(ServiceMapProcessorConfig.DEFAULT_DB_PATH), Clock.systemUTC(), pluginSetting.getNumberOfProcessWorkers(), pluginSetting); } - public ServiceMapStatefulPrepper(final long windowDurationMillis, - final File databasePath, - final Clock clock, - final int processWorkers, - final PluginSetting pluginSetting) { + public ServiceMapStatefulProcessor(final long windowDurationMillis, + final File databasePath, + final Clock clock, + final int processWorkers, + final PluginSetting pluginSetting) { super(pluginSetting); - ServiceMapStatefulPrepper.clock = clock; - this.thisPrepperId = preppersCreated.getAndIncrement(); + ServiceMapStatefulProcessor.clock = clock; + this.thisProcessorId = processorsCreated.getAndIncrement(); if (isMasterInstance()) { - previousTimestamp = ServiceMapStatefulPrepper.clock.millis(); - ServiceMapStatefulPrepper.windowDurationMillis = windowDurationMillis; - ServiceMapStatefulPrepper.dbPath = createPath(databasePath); + previousTimestamp = ServiceMapStatefulProcessor.clock.millis(); + ServiceMapStatefulProcessor.windowDurationMillis = windowDurationMillis; + ServiceMapStatefulProcessor.dbPath = createPath(databasePath); - currentWindow = new MapDbPrepperState<>(dbPath, getNewDbName(), processWorkers); - previousWindow = new MapDbPrepperState<>(dbPath, getNewDbName() + EMPTY_SUFFIX, processWorkers); - currentTraceGroupWindow = new MapDbPrepperState<>(dbPath, getNewTraceDbName(), processWorkers); - previousTraceGroupWindow = new MapDbPrepperState<>(dbPath, getNewTraceDbName() + EMPTY_SUFFIX, processWorkers); + currentWindow = new MapDbProcessorState<>(dbPath, getNewDbName(), processWorkers); + previousWindow = new MapDbProcessorState<>(dbPath, getNewDbName() + EMPTY_SUFFIX, processWorkers); + currentTraceGroupWindow = new MapDbProcessorState<>(dbPath, getNewTraceDbName(), processWorkers); + previousTraceGroupWindow = new MapDbProcessorState<>(dbPath, getNewTraceDbName() + EMPTY_SUFFIX, processWorkers); allThreadsCyclicBarrier = new CyclicBarrier(processWorkers); } @@ -176,8 +176,8 @@ private Collection> evaluateEdges() { try { final Collection> serviceDependencyRecords = new HashSet<>(); - serviceDependencyRecords.addAll(iteratePrepperState(previousWindow)); - serviceDependencyRecords.addAll(iteratePrepperState(currentWindow)); + serviceDependencyRecords.addAll(iterateProcessorState(previousWindow)); + serviceDependencyRecords.addAll(iterateProcessorState(currentWindow)); LOG.info("Done evaluating service map edges"); // Wait for all workers before rotating windows @@ -196,11 +196,11 @@ private Collection> evaluateEdges() { } } - private Collection> iteratePrepperState(final MapDbPrepperState prepperState) { + private Collection> iterateProcessorState(final MapDbProcessorState processorState) { final Collection> serviceDependencyRecords = new HashSet<>(); - if (prepperState.getAll() != null && !prepperState.getAll().isEmpty()) { - prepperState.getIterator(preppersCreated.get(), thisPrepperId).forEachRemaining(entry -> { + if (processorState.getAll() != null && !processorState.getAll().isEmpty()) { + processorState.getIterator(processorsCreated.get(), thisProcessorId).forEachRemaining(entry -> { final ServiceMapStateData child = entry.getValue(); if (child.parentSpanId == null) { @@ -287,12 +287,12 @@ public void shutdown() { /** - * Rotate windows for prepper state + * Rotate windows for processor state */ private void rotateWindows() throws InterruptedException { LOG.info("Rotating service map windows at " + clock.instant().toString()); - MapDbPrepperState tempWindow = previousWindow; + MapDbProcessorState tempWindow = previousWindow; previousWindow = currentWindow; currentWindow = tempWindow; currentWindow.clear(); @@ -348,10 +348,10 @@ private boolean windowDurationHasPassed() { /** * Master instance is needed to do things like window rotation that should only be done once * - * @return Boolean indicating whether this object is the master ServiceMapStatefulPrepper instance + * @return Boolean indicating whether this object is the master ServiceMapStatefulProcessor instance */ private boolean isMasterInstance() { - return thisPrepperId == 0; + return thisProcessorId == 0; } @Override diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapRelationshipTest.java b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapRelationshipTest.java similarity index 92% rename from data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapRelationshipTest.java rename to data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapRelationshipTest.java index 2298ff9853..010793fb79 100644 --- a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapRelationshipTest.java +++ b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapRelationshipTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import org.junit.jupiter.api.Test; diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessorTest.java similarity index 88% rename from data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java rename to data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessorTest.java index ea7200b5b1..c21a3327e1 100644 --- a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java +++ b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapStatefulProcessorTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.MetricsTestUtil; @@ -45,7 +45,7 @@ import static org.mockito.Mockito.when; -public class ServiceMapStatefulPrepperTest { +public class ServiceMapStatefulProcessorTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -56,28 +56,28 @@ public class ServiceMapStatefulPrepperTest { private static final String PASSWORD_DATABASE = "PASS"; private static final String PAYMENT_SERVICE = "PAY"; private static final String CART_SERVICE = "CART"; - private static final PluginSetting PLUGIN_SETTING = new PluginSetting("testServiceMapPrepper", Collections.emptyMap()) {{ + private static final PluginSetting PLUGIN_SETTING = new PluginSetting("testServiceMapProcessor", Collections.emptyMap()) {{ setPipelineName("testPipelineName"); }}; @Before public void setup() throws NoSuchFieldException, IllegalAccessException { - resetServiceMapStatefulPrepperStatic(); + resetServiceMapStatefulProcessorStatic(); MetricsTestUtil.initMetrics(); } - public void resetServiceMapStatefulPrepperStatic() throws NoSuchFieldException, IllegalAccessException { - reflectivelySetField(ServiceMapStatefulPrepper.class, "RELATIONSHIP_STATE", Sets.newConcurrentHashSet()); - reflectivelySetField(ServiceMapStatefulPrepper.class, "preppersCreated", new AtomicInteger(0)); - reflectivelySetField(ServiceMapStatefulPrepper.class, "previousTimestamp", 0); - reflectivelySetField(ServiceMapStatefulPrepper.class, "windowDurationMillis", 0); - reflectivelySetField(ServiceMapStatefulPrepper.class, "dbPath", null); - reflectivelySetField(ServiceMapStatefulPrepper.class, "clock", null); - reflectivelySetField(ServiceMapStatefulPrepper.class, "currentWindow", null); - reflectivelySetField(ServiceMapStatefulPrepper.class, "previousWindow", null); - reflectivelySetField(ServiceMapStatefulPrepper.class, "currentTraceGroupWindow", null); - reflectivelySetField(ServiceMapStatefulPrepper.class, "previousTraceGroupWindow", null); - reflectivelySetField(ServiceMapStatefulPrepper.class, "allThreadsCyclicBarrier", null); + public void resetServiceMapStatefulProcessorStatic() throws NoSuchFieldException, IllegalAccessException { + reflectivelySetField(ServiceMapStatefulProcessor.class, "RELATIONSHIP_STATE", Sets.newConcurrentHashSet()); + reflectivelySetField(ServiceMapStatefulProcessor.class, "processorsCreated", new AtomicInteger(0)); + reflectivelySetField(ServiceMapStatefulProcessor.class, "previousTimestamp", 0); + reflectivelySetField(ServiceMapStatefulProcessor.class, "windowDurationMillis", 0); + reflectivelySetField(ServiceMapStatefulProcessor.class, "dbPath", null); + reflectivelySetField(ServiceMapStatefulProcessor.class, "clock", null); + reflectivelySetField(ServiceMapStatefulProcessor.class, "currentWindow", null); + reflectivelySetField(ServiceMapStatefulProcessor.class, "previousWindow", null); + reflectivelySetField(ServiceMapStatefulProcessor.class, "currentTraceGroupWindow", null); + reflectivelySetField(ServiceMapStatefulProcessor.class, "previousTraceGroupWindow", null); + reflectivelySetField(ServiceMapStatefulProcessor.class, "allThreadsCyclicBarrier", null); } private void reflectivelySetField(final Class clazz, final String fieldName, final Object value) throws NoSuchFieldException, IllegalAccessException { @@ -124,7 +124,7 @@ public void testPluginSettingConstructor() { pluginSetting.setProcessWorkers(4); pluginSetting.setPipelineName("TestPipeline"); //Nothing is accessible to validate, so just verify that no exception is thrown. - final ServiceMapStatefulPrepper serviceMapStatefulPrepper = new ServiceMapStatefulPrepper(pluginSetting); + final ServiceMapStatefulProcessor serviceMapStatefulProcessor = new ServiceMapStatefulProcessor(pluginSetting); } @Test @@ -133,9 +133,9 @@ public void testTraceGroupsWithEventRecordData() throws Exception { Mockito.when(clock.millis()).thenReturn(1L); Mockito.when(clock.instant()).thenReturn(Instant.now()); ExecutorService threadpool = Executors.newCachedThreadPool(); - final File path = new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulPrepper serviceMapStateful1 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING); - final ServiceMapStatefulPrepper serviceMapStateful2 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING); + final File path = new File(ServiceMapProcessorConfig.DEFAULT_DB_PATH); + final ServiceMapStatefulProcessor serviceMapStateful1 = new ServiceMapStatefulProcessor(100, path, clock, 2, PLUGIN_SETTING); + final ServiceMapStatefulProcessor serviceMapStateful2 = new ServiceMapStatefulProcessor(100, path, clock, 2, PLUGIN_SETTING); final byte[] rootSpanId1Bytes = ServiceMapTestUtils.getRandomBytes(8); final byte[] rootSpanId2Bytes = ServiceMapTestUtils.getRandomBytes(8); @@ -263,13 +263,13 @@ public void testTraceGroupsWithEventRecordData() throws Exception { // Verify gauges final List spansDbSizeMeasurement = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper") - .add(ServiceMapStatefulPrepper.SPANS_DB_SIZE).toString()); + new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapProcessor") + .add(ServiceMapStatefulProcessor.SPANS_DB_SIZE).toString()); Assert.assertEquals(1, spansDbSizeMeasurement.size()); final List traceGroupDbSizeMeasurement = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper") - .add(ServiceMapStatefulPrepper.TRACE_GROUP_DB_SIZE).toString()); + new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapProcessor") + .add(ServiceMapStatefulProcessor.TRACE_GROUP_DB_SIZE).toString()); Assert.assertEquals(1, traceGroupDbSizeMeasurement.size()); @@ -303,8 +303,8 @@ public void testTraceGroupsWithEventRecordData() throws Exception { @Test public void testPrepareForShutdownWithEventRecordData() { - final File path = new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulPrepper serviceMapStateful = new ServiceMapStatefulPrepper(100, path, Clock.systemUTC(), 1, PLUGIN_SETTING); + final File path = new File(ServiceMapProcessorConfig.DEFAULT_DB_PATH); + final ServiceMapStatefulProcessor serviceMapStateful = new ServiceMapStatefulProcessor(100, path, Clock.systemUTC(), 1, PLUGIN_SETTING); final byte[] rootSpanId1Bytes = ServiceMapTestUtils.getRandomBytes(8); final byte[] traceId1Bytes = ServiceMapTestUtils.getRandomBytes(16); @@ -336,8 +336,8 @@ public void testGetIdentificationKeys() { pluginSetting.setProcessWorkers(4); pluginSetting.setPipelineName("TestPipeline"); - final ServiceMapStatefulPrepper serviceMapStatefulPrepper = new ServiceMapStatefulPrepper(pluginSetting); - final Collection expectedIdentificationKeys = serviceMapStatefulPrepper.getIdentificationKeys(); + final ServiceMapStatefulProcessor serviceMapStatefulProcessor = new ServiceMapStatefulProcessor(pluginSetting); + final Collection expectedIdentificationKeys = serviceMapStatefulProcessor.getIdentificationKeys(); assertThat(expectedIdentificationKeys, equalTo(Collections.singleton("traceId"))); } diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapTestUtils.java b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapTestUtils.java similarity index 95% rename from data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapTestUtils.java rename to data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapTestUtils.java index afa7dec67d..a90ec8803f 100644 --- a/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/prepper/ServiceMapTestUtils.java +++ b/data-prepper-plugins/service-map-stateful/src/test/java/org/opensearch/dataprepper/plugins/processor/ServiceMapTestUtils.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.prepper; +package org.opensearch.dataprepper.plugins.processor; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; @@ -34,10 +34,10 @@ public static byte[] getRandomBytes(int len) { return bytes; } - public static Future> startExecuteAsync(ExecutorService threadpool, ServiceMapStatefulPrepper prepper, + public static Future> startExecuteAsync(ExecutorService threadpool, ServiceMapStatefulProcessor processor, Collection> records) { return threadpool.submit(() -> { - return prepper.execute(records) + return processor.execute(records) .stream() .map(record -> { try { diff --git a/docs/log_analytics.md b/docs/log_analytics.md index 5258d41565..8eb50c6754 100644 --- a/docs/log_analytics.md +++ b/docs/log_analytics.md @@ -4,7 +4,7 @@ Data Prepper is an extendable, configurable, and scalable solution for log ingestion into OpenSearch and Amazon OpenSearch Service. Currently, Data Prepper is focused on receiving logs from [FluentBit](https://fluentbit.io/) via the -[Http Source](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/http-source/README.md), and processing those logs with a [Grok Prepper](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/grok-prepper/README.md) before ingesting them into OpenSearch through the [OpenSearch sink](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/opensearch/README.md). +[Http Source](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/http-source/README.md), and processing those logs with a [Grok Processor](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/grok-processor/README.md) before ingesting them into OpenSearch through the [OpenSearch sink](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/opensearch/README.md). Here is all of the components for log analytics with FluentBit, Data Prepper, and OpenSearch:
@@ -35,10 +35,10 @@ This source supports industry-standard encryption in the form of TLS/HTTPS and H ## Processor -The Data Prepper 1.2 release will come with a [Grok Prepper](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/grok-prepper/README.md). -The Grok Prepper can be an invaluable tool to structure and extract important fields from your logs in order to make them more queryable. +The Data Prepper 1.2 release will come with a [Grok Processor](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/grok-processor/README.md). +The Grok Processor can be an invaluable tool to structure and extract important fields from your logs in order to make them more queryable. -The Grok Prepper comes with a wide variety of [default patterns](https://github.com/thekrakken/java-grok/blob/master/src/main/resources/patterns/patterns) that match against common log formats like apcahe logs or syslogs, +The Grok Processor comes with a wide variety of [default patterns](https://github.com/thekrakken/java-grok/blob/master/src/main/resources/patterns/patterns) that match against common log formats like apcahe logs or syslogs, but can easily accept any custom patterns that cater to your specific log format. There are a lot of complex Grok features that will not be discussed here, so please read the documentation if you are interested. @@ -126,7 +126,7 @@ log-pipeline: index: apache_logs ``` -This pipeline configuration is an example of apache log ingestion. Don't forget that you can easily configure the Grok Prepper for your own custom logs. +This pipeline configuration is an example of apache log ingestion. Don't forget that you can easily configure the Grok Processor for your own custom logs. You will need to modify the configuration above for your OpenSearch cluster. diff --git a/docs/trace_analytics.md b/docs/trace_analytics.md index 7b803e8065..a734de1130 100644 --- a/docs/trace_analytics.md +++ b/docs/trace_analytics.md @@ -43,8 +43,8 @@ We have two processor for the Trace Analytics feature, We have a generic sink that writes the data to OpenSearch as the destination. The [opensearch sink](../data-prepper-plugins/opensearch/README.md) has configuration options related to OpenSearch cluster like endpoint, SSL/Username, index name, index template, index state management, etc. For the trace analytics feature, the sink has specific configurations which enables the sink to use indices and index templates specific to this feature. Trace analytics specific OpenSearch indices are, -* *otel-v1-apm-span* - This index stores the output from [otel-trace-raw-prepper](../data-prepper-plugins/otel-trace-raw-prepper/README.md). -* *otel-v1-apm-service-map* - This index stores the output from the [service-map-prepper](../data-prepper-plugins/service-map-stateful/README.md). +* *otel-v1-apm-span* - This index stores the output from [otel-trace-raw-processor](../data-prepper-plugins/otel-trace-raw-processor/README.md). +* *otel-v1-apm-service-map* - This index stores the output from the [service-map-processor](../data-prepper-plugins/service-map-stateful/README.md). ### Pipeline Configuration diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java index 680cb8b77a..3598515684 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java @@ -82,7 +82,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { builder.withPassword("admin"); final RestHighLevelClient restHighLevelClient = builder.build().createClient(); - // Wait for service map prepper by 2 * window_duration + // Wait for service map processor by 2 * window_duration await().atLeast(8, TimeUnit.SECONDS).atMost(20, TimeUnit.SECONDS).untilAsserted( () -> { final List> foundSources = getSourcesFromIndex(restHighLevelClient, SERVICE_MAP_INDEX_NAME); @@ -109,7 +109,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException { final List testDataSet2 = Stream.of(TEST_TRACE_2_BATCH_1, TEST_TRACE_2_BATCH_2) .flatMap(Collection::stream).collect(Collectors.toList()); possibleEdges.addAll(getPossibleEdges(TEST_TRACEID_2, testDataSet2)); - // Wait for service map prepper by 2 * window_duration + // Wait for service map processor by 2 * window_duration await().atLeast(8, TimeUnit.SECONDS).atMost(20, TimeUnit.SECONDS).untilAsserted( () -> { final List> foundSources = getSourcesFromIndex(restHighLevelClient, SERVICE_MAP_INDEX_NAME); diff --git a/examples/log-ingestion/README.md b/examples/log-ingestion/README.md index 5cdadfe4cd..26ff468f5a 100644 --- a/examples/log-ingestion/README.md +++ b/examples/log-ingestion/README.md @@ -41,7 +41,7 @@ docker pull opensearchproject/data-prepper:latest ``` 2. Take a look at [log_pipeline.yaml](log_pipeline.yaml). This configuration will take logs sent to the [http source](../../data-prepper-plugins/http-source), -process them with the [Grok Prepper](../../data-prepper-plugins/grok-prepper) by matching against the `COMMONAPACHELOG` pattern, +process them with the [Grok Processor](../../data-prepper-plugins/grok-prepper) by matching against the `COMMONAPACHELOG` pattern, and send the processed logs to a local [OpenSearch sink](../../data-prepper-plugins/opensearch) to an index named `apache_logs`. diff --git a/settings.gradle b/settings.gradle index 57b9a550ec..1744e435b6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -20,7 +20,7 @@ include 'data-prepper-plugins:common' include 'data-prepper-plugins:armeria-common' include 'data-prepper-plugins:opensearch' include 'data-prepper-plugins:service-map-stateful' -include 'data-prepper-plugins:mapdb-prepper-state' +include 'data-prepper-plugins:mapdb-processor-state' include 'data-prepper-plugins:otel-proto-common' include 'data-prepper-plugins:otel-trace-raw-processor' include 'data-prepper-plugins:otel-trace-group-processor' @@ -33,7 +33,7 @@ include 'data-prepper-plugins:http-source' include 'data-prepper-plugins:drop-events-processor' include 'data-prepper-plugins:key-value-processor' include 'data-prepper-plugins:mutate-event-processors' -include 'data-prepper-plugins:grok-prepper' +include 'data-prepper-plugins:grok-processor' include 'data-prepper-plugins:aggregate-processor' include 'data-prepper-plugins:log-generator-source' include 'data-prepper-logstash-configuration'