forked from opensearch-project/data-prepper
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add new metrics for tracking bytes received and processed by KDS (ope…
…nsearch-project#5237) * Add new metrics for tracking bytes received and processed by KDS Signed-off-by: Souvik Bose <[email protected]> * Address review comments. Signed-off-by: Souvik Bose <[email protected]> * Modify the code as per comments Signed-off-by: Souvik Bose <[email protected]> --------- Signed-off-by: Souvik Bose <[email protected]> Co-authored-by: Souvik Bose <[email protected]>
- Loading branch information
Showing
7 changed files
with
259 additions
and
55 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
...org/opensearch/dataprepper/plugins/kinesis/source/processor/KinesisInputOutputRecord.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.kinesis.source.processor; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import org.opensearch.dataprepper.model.event.Event; | ||
| import org.opensearch.dataprepper.model.record.Record; | ||
|
|
||
| @Builder(setterPrefix = "with") | ||
| @Getter | ||
| @AllArgsConstructor | ||
| public class KinesisInputOutputRecord { | ||
| private Record<Event> dataPrepperRecord; | ||
| private long incomingRecordSizeBytes; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
57 changes: 57 additions & 0 deletions
57
.../java/org/opensearch/dataprepper/plugins/kinesis/source/KinesisInputOutputRecordTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.kinesis.source; | ||
|
|
||
| import org.junit.jupiter.api.Test; | ||
| import org.opensearch.dataprepper.model.event.Event; | ||
| import org.opensearch.dataprepper.model.event.JacksonEvent; | ||
| import org.opensearch.dataprepper.model.record.Record; | ||
| import org.opensearch.dataprepper.plugins.kinesis.source.converter.MetadataKeyAttributes; | ||
| import org.opensearch.dataprepper.plugins.kinesis.source.processor.KinesisInputOutputRecord; | ||
| import software.amazon.kinesis.retrieval.KinesisClientRecord; | ||
|
|
||
| import java.nio.ByteBuffer; | ||
| import java.util.UUID; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertEquals; | ||
| import static org.junit.jupiter.api.Assertions.assertNotNull; | ||
| import static org.junit.jupiter.api.Assertions.assertNull; | ||
|
|
||
| public class KinesisInputOutputRecordTest { | ||
|
|
||
| @Test | ||
| void builder_defaultCreatesObjectCorrectly() { | ||
|
|
||
| KinesisInputOutputRecord kinesisInputOutputRecord = KinesisInputOutputRecord.builder().build(); | ||
|
|
||
| assertEquals(0L, kinesisInputOutputRecord.getIncomingRecordSizeBytes()); | ||
| assertNull(kinesisInputOutputRecord.getDataPrepperRecord()); | ||
| } | ||
|
|
||
| @Test | ||
| void builder_createsObjectCorrectly() { | ||
| Event event = JacksonEvent.fromMessage(UUID.randomUUID().toString()); | ||
| event.getMetadata().setAttribute(MetadataKeyAttributes.KINESIS_STREAM_NAME_METADATA_ATTRIBUTE, UUID.randomUUID().toString()); | ||
| Record<Event> record = new Record<>(event); | ||
| KinesisClientRecord kinesisClientRecord = KinesisClientRecord.builder() | ||
| .data(ByteBuffer.wrap(event.toJsonString().getBytes())) | ||
| .sequenceNumber(Integer.toString(1000)).subSequenceNumber(1).build(); | ||
|
|
||
| KinesisInputOutputRecord kinesisInputOutputRecord = KinesisInputOutputRecord.builder() | ||
| .withIncomingRecordSizeBytes(100L) | ||
| .withDataPrepperRecord(record) | ||
| .build(); | ||
|
|
||
| assertNotNull(kinesisInputOutputRecord.getDataPrepperRecord()); | ||
| assertEquals(kinesisInputOutputRecord.getDataPrepperRecord(), record); | ||
| assertEquals(100L, kinesisInputOutputRecord.getIncomingRecordSizeBytes()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.