diff --git a/data-prepper-plugins/geoip-processor/build.gradle b/data-prepper-plugins/geoip-processor/build.gradle index 7f6d88a885..916f78e2f1 100644 --- a/data-prepper-plugins/geoip-processor/build.gradle +++ b/data-prepper-plugins/geoip-processor/build.gradle @@ -24,8 +24,10 @@ dependencies { implementation 'software.amazon.awssdk.crt:aws-crt:0.21.17' implementation 'com.maxmind.geoip2:geoip2:4.0.1' implementation 'com.maxmind.db:maxmind-db:3.0.0' + implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final' implementation libs.commons.lang3 + testImplementation project(':data-prepper-core') testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' testImplementation project(':data-prepper-test-common') diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java index 6265533929..664344b9d0 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java @@ -15,6 +15,7 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.processor.configuration.KeysConfig; import org.opensearch.dataprepper.plugins.processor.databaseenrich.EnrichFailedException; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIpConfigSupplier; import org.opensearch.dataprepper.plugins.processor.utils.IPValidationcheck; import org.opensearch.dataprepper.logging.DataPrepperMarkers; import org.slf4j.Logger; @@ -52,7 +53,8 @@ public class GeoIPProcessor extends AbstractProcessor, Record awsStsHeaderOverrides.forEach(configuration::putHeader)); } - awsCredentialsProvider = StsAssumeRoleCredentialsProvider.builder().stsClient(stsClient) - .refreshRequest(assumeRoleRequestBuilder.build()).build(); + awsCredentialsProvider = StsAssumeRoleCredentialsProvider.builder() + .stsClient(stsClient) + .refreshRequest(assumeRoleRequestBuilder.build()) + .build(); } else { awsCredentialsProvider = DefaultCredentialsProvider.create(); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplier.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplier.java new file mode 100644 index 0000000000..d6a1779972 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplier.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; + +public class DefaultGeoIpConfigSupplier implements GeoIpConfigSupplier { + private final GeoIpServiceConfig geoIpServiceConfig; + + public DefaultGeoIpConfigSupplier(final GeoIpServiceConfig geoIpServiceConfig) { + this.geoIpServiceConfig = geoIpServiceConfig; + } + + @Override + public GeoIPProcessorService getGeoIPProcessorService() { + //TODO: use GeoIpServiceConfig and return GeoIPProcessorService + return null; + } +} diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigExtension.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigExtension.java new file mode 100644 index 0000000000..200541339d --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigExtension.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import org.opensearch.dataprepper.model.annotations.DataPrepperExtensionPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.plugin.ExtensionPlugin; +import org.opensearch.dataprepper.model.plugin.ExtensionPoints; + +@DataPrepperExtensionPlugin(modelType = GeoIpServiceConfig.class, rootKeyJsonPath = "/geoip_service") +public class GeoIpConfigExtension implements ExtensionPlugin { + private final DefaultGeoIpConfigSupplier defaultGeoIpConfigSupplier; + + @DataPrepperPluginConstructor + public GeoIpConfigExtension(final GeoIpServiceConfig geoIpServiceConfig) { + this.defaultGeoIpConfigSupplier = new DefaultGeoIpConfigSupplier(geoIpServiceConfig != null ? geoIpServiceConfig : new GeoIpServiceConfig()); + } + + @Override + public void apply(final ExtensionPoints extensionPoints) { + extensionPoints.addExtensionProvider(new GeoIpConfigProvider(this.defaultGeoIpConfigSupplier)); + } +} diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigProvider.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigProvider.java new file mode 100644 index 0000000000..cc85ecdb2d --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigProvider.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import org.opensearch.dataprepper.model.plugin.ExtensionProvider; + +import java.util.Optional; + +public class GeoIpConfigProvider implements ExtensionProvider { + private final GeoIpConfigSupplier geoIpConfigSupplier; + + public GeoIpConfigProvider(final GeoIpConfigSupplier geoIpConfigSupplier) { + this.geoIpConfigSupplier = geoIpConfigSupplier; + } + + @Override + public Optional provideInstance(Context context) { + return Optional.of(this.geoIpConfigSupplier); + } + + @Override + public Class supportedClass() { + return GeoIpConfigSupplier.class; + } +} diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigSupplier.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigSupplier.java new file mode 100644 index 0000000000..c7adb00c2f --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigSupplier.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; + +/** + * Interface for supplying {@link GeoIPProcessorService} to {@link GeoIpConfigExtension} + * + * @since 2.7 + */ +public interface GeoIpConfigSupplier { + /** + * Returns the {@link GeoIPProcessorService} + * + * @since 2.7 + */ + GeoIPProcessorService getGeoIPProcessorService(); +} diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpServiceConfig.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpServiceConfig.java new file mode 100644 index 0000000000..c311f773ca --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpServiceConfig.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; + +public class GeoIpServiceConfig { + private static final MaxMindConfig DEFAULT_MAXMIND_CONFIG = new MaxMindConfig(); + + public GeoIpServiceConfig() { + // This default constructor is used if geoip_service is not configured + } + + @JsonProperty("maxmind") + @Valid + private MaxMindConfig maxMindConfig = DEFAULT_MAXMIND_CONFIG; + + /** + * Gets the configuration for MaxMind. + * + * @return The MaxMind configuration + * @since 2.7 + */ + public MaxMindConfig getMaxMindConfig() { + return maxMindConfig; + } +} diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/MaxMindConfig.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/MaxMindConfig.java new file mode 100644 index 0000000000..3fea586e67 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/MaxMindConfig.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Min; +import org.hibernate.validator.constraints.time.DurationMax; +import org.hibernate.validator.constraints.time.DurationMin; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +public class MaxMindConfig { + private static final String S3_PREFIX = "s3://"; + + //TODO: Add validations to database paths + //TODO: Make default path to be a public CDN endpoint + private static final List DEFAULT_DATABASE_PATHS = new ArrayList<>(); + private static final Duration DEFAULT_DATABASE_REFRESH_INTERVAL = Duration.ofDays(7); + private static final int DEFAULT_CACHE_SIZE = 4096; + + @JsonProperty("database_paths") + private List databasePaths = DEFAULT_DATABASE_PATHS; + + @JsonProperty("database_refresh_interval") + @DurationMin(days = 1) + @DurationMax(days = 30) + private Duration databaseRefreshInterval = DEFAULT_DATABASE_REFRESH_INTERVAL; + + @JsonProperty("cache_size") + @Min(1) + //TODO: Add a Max limit on cache size + private int cacheSize = DEFAULT_CACHE_SIZE; + + //TODO: Add a destination path to store database files + @JsonProperty("aws") + @Valid + private AwsAuthenticationOptionsConfig awsAuthenticationOptionsConfig; + + public MaxMindConfig() { + // This default constructor is used if maxmind is not configured + } + + @AssertTrue(message = "aws should be configured if any path in database_paths is S3 bucket path.") + boolean isAwsAuthenticationOptionsRequired() { + for (final String databasePath : databasePaths) { + if (databasePath.startsWith(S3_PREFIX)) { + return awsAuthenticationOptionsConfig != null; + } + } + return true; + } + + /** + * Gets the MaxMind database paths + * + * @return The MaxMind database paths + * @since 2.7 + */ + public List getDatabasePaths() { + return databasePaths; + } + + /** + * Gets the database refresh duration. This loads the database from the paths into memory again in case if there are any updates. + * + * @return The refresh duration + * @since 2.7 + */ + public Duration getDatabaseRefreshInterval() { + return databaseRefreshInterval; + } + + /** + * Gets the cache size used in CHM cache for MaxMind DatabaseReader. + * + * @return The cache size + * @since 2.7 + */ + public int getCacheSize() { + return cacheSize; + } + + /** + * Gets the AWS authentication config used for reading from S3 bucket + * + * @return The AWS authentication options + * @since 2.7 + */ + public AwsAuthenticationOptionsConfig getAwsAuthenticationOptionsConfig() { + return awsAuthenticationOptionsConfig; + } +} diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfigTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfigTest.java index b1b6d08c1a..6ed86062c8 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfigTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorConfigTest.java @@ -9,7 +9,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.processor.configuration.AwsAuthenticationOptions; import org.opensearch.dataprepper.plugins.processor.configuration.KeysConfig; import org.opensearch.dataprepper.plugins.processor.configuration.ServiceTypeOptions; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; @@ -30,14 +29,6 @@ void setUp() { geoIPProcessorConfig = new GeoIPProcessorConfig(); } - @Test - void getAwsAuthenticationOptionsTestPositive() throws NoSuchFieldException, IllegalAccessException { - AwsAuthenticationOptions awsAuthenticationOptions = new AwsAuthenticationOptions(); - ReflectivelySetField.setField(GeoIPProcessorConfig.class, - geoIPProcessorConfig, "awsAuthenticationOptions", awsAuthenticationOptions); - assertThat(geoIPProcessorConfig.getAwsAuthenticationOptions(), equalTo(awsAuthenticationOptions)); - } - @Test void getAwsAuthenticationOptionsTestNegative() { assertThat(new GeoIPProcessorConfig().getAwsAuthenticationOptions(), equalTo(null)); diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java index 77243d7c12..e8ab02d3b0 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java @@ -22,6 +22,7 @@ import org.opensearch.dataprepper.plugins.processor.configuration.MaxMindServiceConfig; import org.opensearch.dataprepper.plugins.processor.configuration.ServiceTypeOptions; import org.opensearch.dataprepper.plugins.processor.databaseenrich.EnrichFailedException; +import org.opensearch.dataprepper.plugins.processor.extension.GeoIpConfigSupplier; import org.opensearch.dataprepper.plugins.processor.loadtype.LoadTypeOptions; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; @@ -40,7 +41,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -56,6 +56,8 @@ class GeoIPProcessorTest { @Mock private GeoIPProcessorConfig geoCodingProcessorConfig; @Mock + private GeoIpConfigSupplier geoIpConfigSupplier; + @Mock private PluginSetting pluginSetting; @Mock private ServiceTypeOptions serviceTypeOptions; @@ -164,7 +166,7 @@ private Map prepareGeoData() { } private GeoIPProcessor createObjectUnderTest() throws MalformedURLException { - return new GeoIPProcessor(pluginSetting, geoCodingProcessorConfig); + return new GeoIPProcessor(pluginSetting, geoCodingProcessorConfig, geoIpConfigSupplier); } private List setAttributes() { diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBServiceTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBServiceTest.java index 9f65dd45d8..829e9e7fea 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBServiceTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/databasedownload/S3DBServiceTest.java @@ -11,7 +11,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorConfig; -import org.opensearch.dataprepper.plugins.processor.configuration.AwsAuthenticationOptions; +import org.opensearch.dataprepper.plugins.processor.extension.AwsAuthenticationOptionsConfig; import org.opensearch.dataprepper.plugins.processor.configuration.DatabasePathURLConfig; import org.opensearch.dataprepper.plugins.processor.databaseenrich.DownloadFailedException; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; @@ -36,8 +36,8 @@ class S3DBServiceTest { @BeforeEach void setUp() { - AwsAuthenticationOptions awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); - when(geoIPProcessorConfig.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); + AwsAuthenticationOptionsConfig awsAuthenticationOptionsConfig = mock(AwsAuthenticationOptionsConfig.class); + when(geoIPProcessorConfig.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptionsConfig); when(geoIPProcessorConfig.getAwsAuthenticationOptions().getAwsRegion()).thenReturn(Region.of(S3_REGION)); AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(geoIPProcessorConfig.getAwsAuthenticationOptions().authenticateAwsConfiguration()).thenReturn(awsCredentialsProvider); diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/AwsAuthenticationOptionsTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/AwsAuthenticationOptionsConfigTest.java similarity index 85% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/AwsAuthenticationOptionsTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/AwsAuthenticationOptionsConfigTest.java index a8f125003a..9bc83b9df4 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/AwsAuthenticationOptionsTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/AwsAuthenticationOptionsConfigTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.configuration; +package org.opensearch.dataprepper.plugins.processor.extension; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; @@ -40,15 +40,16 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -class AwsAuthenticationOptionsTest { +class AwsAuthenticationOptionsConfigTest { private ObjectMapper objectMapper; - private AwsAuthenticationOptions awsAuthenticationOptions; + private AwsAuthenticationOptionsConfig awsAuthenticationOptionsConfig; private final String TEST_ROLE = "arn:aws:iam::123456789012:role/test-role"; + @BeforeEach void setUp() { objectMapper = new ObjectMapper(); - awsAuthenticationOptions = new AwsAuthenticationOptions(); + awsAuthenticationOptionsConfig = new AwsAuthenticationOptionsConfig(); } @ParameterizedTest @@ -56,28 +57,28 @@ void setUp() { void getAwsRegion_returns_Region_of(final String regionString) { final Region expectedRegionObject = Region.of(regionString); final Map jsonMap = Map.of("region", regionString); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); + final AwsAuthenticationOptionsConfig objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptionsConfig.class); assertThat(objectUnderTest.getAwsRegion(), equalTo(expectedRegionObject)); } @Test void getAwsRegion_returns_null_when_region_is_null() { final Map jsonMap = Collections.emptyMap(); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); + final AwsAuthenticationOptionsConfig objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptionsConfig.class); assertThat(objectUnderTest.getAwsRegion(), nullValue()); } @Test void authenticateAWSConfiguration_should_return_s3Client_without_sts_role_arn() throws NoSuchFieldException, IllegalAccessException { - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsRegion", "us-east-1"); - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsStsRoleArn", null); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsRegion", "us-east-1"); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsStsRoleArn", null); final DefaultCredentialsProvider mockedCredentialsProvider = mock(DefaultCredentialsProvider.class); final AwsCredentialsProvider actualCredentialsProvider; try (final MockedStatic defaultCredentialsProviderMockedStatic = mockStatic(DefaultCredentialsProvider.class)) { defaultCredentialsProviderMockedStatic.when(DefaultCredentialsProvider::create) .thenReturn(mockedCredentialsProvider); - actualCredentialsProvider = awsAuthenticationOptions.authenticateAwsConfiguration(); + actualCredentialsProvider = awsAuthenticationOptionsConfig.authenticateAwsConfiguration(); } assertThat(actualCredentialsProvider, sameInstance(mockedCredentialsProvider)); @@ -98,8 +99,8 @@ void setUp() { @Test void authenticateAWSConfiguration_should_return_s3Client_with_sts_role_arn() throws NoSuchFieldException, IllegalAccessException { - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsRegion", "us-east-1"); - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsStsRoleArn", TEST_ROLE); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsRegion", "us-east-1"); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsStsRoleArn", TEST_ROLE); when(stsClientBuilder.region(Region.US_EAST_1)).thenReturn(stsClientBuilder); final AssumeRoleRequest.Builder assumeRoleRequestBuilder = mock(AssumeRoleRequest.Builder.class); @@ -113,7 +114,7 @@ void authenticateAWSConfiguration_should_return_s3Client_with_sts_role_arn() thr final MockedStatic assumeRoleRequestMockedStatic = mockStatic(AssumeRoleRequest.class)) { stsClientMockedStatic.when(StsClient::builder).thenReturn(stsClientBuilder); assumeRoleRequestMockedStatic.when(AssumeRoleRequest::builder).thenReturn(assumeRoleRequestBuilder); - actualCredentialsProvider = awsAuthenticationOptions.authenticateAwsConfiguration(); + actualCredentialsProvider = awsAuthenticationOptionsConfig.authenticateAwsConfiguration(); } assertThat(actualCredentialsProvider, instanceOf(AwsCredentialsProvider.class)); @@ -126,16 +127,16 @@ void authenticateAWSConfiguration_should_return_s3Client_with_sts_role_arn() thr @Test void authenticateAWSConfiguration_should_return_s3Client_with_sts_role_arn_when_no_region() throws NoSuchFieldException, IllegalAccessException { - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsRegion", null); - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsStsRoleArn", TEST_ROLE); - assertThat(awsAuthenticationOptions.getAwsRegion(), equalTo(null)); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsRegion", null); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsStsRoleArn", TEST_ROLE); + assertThat(awsAuthenticationOptionsConfig.getAwsRegion(), equalTo(null)); when(stsClientBuilder.region(null)).thenReturn(stsClientBuilder); final AwsCredentialsProvider actualCredentialsProvider; try (final MockedStatic stsClientMockedStatic = mockStatic(StsClient.class)) { stsClientMockedStatic.when(StsClient::builder).thenReturn(stsClientBuilder); - actualCredentialsProvider = awsAuthenticationOptions.authenticateAwsConfiguration(); + actualCredentialsProvider = awsAuthenticationOptionsConfig.authenticateAwsConfiguration(); } assertThat(actualCredentialsProvider, instanceOf(AwsCredentialsProvider.class)); @@ -149,9 +150,9 @@ void authenticateAWSConfiguration_should_override_STS_Headers_when_HeaderOverrid final String headerValue2 = UUID.randomUUID().toString(); final Map overrideHeaders = Map.of(headerName1, headerValue1, headerName2, headerValue2); - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsRegion", "us-east-1"); - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsStsRoleArn", TEST_ROLE); - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsStsHeaderOverrides", overrideHeaders); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsRegion", "us-east-1"); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsStsRoleArn", TEST_ROLE); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsStsHeaderOverrides", overrideHeaders); when(stsClientBuilder.region(Region.US_EAST_1)).thenReturn(stsClientBuilder); @@ -168,7 +169,7 @@ void authenticateAWSConfiguration_should_override_STS_Headers_when_HeaderOverrid final MockedStatic assumeRoleRequestMockedStatic = mockStatic(AssumeRoleRequest.class)) { stsClientMockedStatic.when(StsClient::builder).thenReturn(stsClientBuilder); assumeRoleRequestMockedStatic.when(AssumeRoleRequest::builder).thenReturn(assumeRoleRequestBuilder); - actualCredentialsProvider = awsAuthenticationOptions.authenticateAwsConfiguration(); + actualCredentialsProvider = awsAuthenticationOptionsConfig.authenticateAwsConfiguration(); } assertThat(actualCredentialsProvider, instanceOf(AwsCredentialsProvider.class)); @@ -193,9 +194,9 @@ void authenticateAWSConfiguration_should_override_STS_Headers_when_HeaderOverrid @Test void authenticateAWSConfiguration_should_not_override_STS_Headers_when_HeaderOverrides_are_empty() throws NoSuchFieldException, IllegalAccessException { - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsRegion", "us-east-1"); - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsStsRoleArn", TEST_ROLE); - ReflectivelySetField.setField(AwsAuthenticationOptions.class, awsAuthenticationOptions, "awsStsHeaderOverrides", Collections.emptyMap()); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsRegion", "us-east-1"); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsStsRoleArn", TEST_ROLE); + ReflectivelySetField.setField(AwsAuthenticationOptionsConfig.class, awsAuthenticationOptionsConfig, "awsStsHeaderOverrides", Collections.emptyMap()); when(stsClientBuilder.region(Region.US_EAST_1)).thenReturn(stsClientBuilder); final AssumeRoleRequest.Builder assumeRoleRequestBuilder = mock(AssumeRoleRequest.Builder.class); @@ -209,7 +210,7 @@ void authenticateAWSConfiguration_should_not_override_STS_Headers_when_HeaderOve final MockedStatic assumeRoleRequestMockedStatic = mockStatic(AssumeRoleRequest.class)) { stsClientMockedStatic.when(StsClient::builder).thenReturn(stsClientBuilder); assumeRoleRequestMockedStatic.when(AssumeRoleRequest::builder).thenReturn(assumeRoleRequestBuilder); - actualCredentialsProvider = awsAuthenticationOptions.authenticateAwsConfiguration(); + actualCredentialsProvider = awsAuthenticationOptionsConfig.authenticateAwsConfiguration(); } assertThat(actualCredentialsProvider, instanceOf(AwsCredentialsProvider.class)); diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplierTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplierTest.java new file mode 100644 index 0000000000..18730f166d --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/DefaultGeoIpConfigSupplierTest.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import org.junit.jupiter.api.Test; +import org.mockito.Mock; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +class DefaultGeoIpConfigSupplierTest { + @Mock + private GeoIpServiceConfig geoIpServiceConfig; + + private DefaultGeoIpConfigSupplier createObjectUnderTest() { + return new DefaultGeoIpConfigSupplier(geoIpServiceConfig); + } + + @Test + void getGeoIPProcessorService_returnsNull_whenGeoIpServiceConfigIsNull() { + assertThat(createObjectUnderTest().getGeoIPProcessorService(), equalTo(null)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigExtensionTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigExtensionTest.java new file mode 100644 index 0000000000..ca8ebbf604 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigExtensionTest.java @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.ExtensionPoints; +import org.opensearch.dataprepper.model.plugin.ExtensionProvider; +import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +class GeoIpConfigExtensionTest { + @Mock + private ExtensionPoints extensionPoints; + + @Mock + private GeoIpServiceConfig geoIpServiceConfig; + + @Mock + private ExtensionProvider.Context context; + + private GeoIpConfigExtension createObjectUnderTest() { + return new GeoIpConfigExtension(geoIpServiceConfig); + } + + @Test + void apply_should_addExtensionProvider() { + createObjectUnderTest().apply(extensionPoints); + final ArgumentCaptor extensionProviderArgumentCaptor = + ArgumentCaptor.forClass(ExtensionProvider.class); + + verify(extensionPoints).addExtensionProvider(extensionProviderArgumentCaptor.capture()); + + final ExtensionProvider actualExtensionProvider = extensionProviderArgumentCaptor.getValue(); + + assertThat(actualExtensionProvider, instanceOf(GeoIpConfigProvider.class)); + } + + @Test + void apply_should_addExtensionProvider_and_supplier_should_use_default_config_if_not_configured() { + final GeoIpConfigExtension geoIpConfigExtension = new GeoIpConfigExtension(null); + + geoIpConfigExtension.apply(extensionPoints); + final ArgumentCaptor extensionProviderArgumentCaptor = + ArgumentCaptor.forClass(ExtensionProvider.class); + + verify(extensionPoints).addExtensionProvider(extensionProviderArgumentCaptor.capture()); + + final ExtensionProvider actualExtensionProvider = extensionProviderArgumentCaptor.getValue(); + + assertThat(actualExtensionProvider, instanceOf(GeoIpConfigProvider.class)); + + final GeoIpConfigSupplier geoIpConfigSupplier = (GeoIpConfigSupplier) actualExtensionProvider.provideInstance(context).get(); + + final GeoIPProcessorService geoIPProcessorService = geoIpConfigSupplier.getGeoIPProcessorService(); + + //TODO: Update assertions after updating the supplier with GeoIPProcessorService + assertThat(geoIPProcessorService, nullValue()); + + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigProviderTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigProviderTest.java new file mode 100644 index 0000000000..79b29347f7 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpConfigProviderTest.java @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.ExtensionProvider; + +import java.util.Optional; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; + +@ExtendWith(MockitoExtension.class) +class GeoIpConfigProviderTest { + @Mock + private GeoIpConfigSupplier geoIpConfigSupplier; + + @Mock + private ExtensionProvider.Context context; + + private GeoIpConfigProvider createObjectUnderTest() { + return new GeoIpConfigProvider(geoIpConfigSupplier); + } + + @Test + void supportedClass_returns_geoIptConfigSupplier() { + assertThat(createObjectUnderTest().supportedClass(), equalTo(GeoIpConfigSupplier.class)); + } + + @Test + void provideInstance_returns_the_kafkaConnectConfigSupplier_from_the_constructor() { + final GeoIpConfigProvider objectUnderTest = createObjectUnderTest(); + + final Optional optionalKafkaConnectConfigSupplier = objectUnderTest.provideInstance(context); + assertThat(optionalKafkaConnectConfigSupplier, notNullValue()); + assertThat(optionalKafkaConnectConfigSupplier.isPresent(), equalTo(true)); + assertThat(optionalKafkaConnectConfigSupplier.get(), equalTo(geoIpConfigSupplier)); + + final Optional anotherOptionalKafkaConnectConfigSupplier = objectUnderTest.provideInstance(context); + assertThat(anotherOptionalKafkaConnectConfigSupplier, notNullValue()); + assertThat(anotherOptionalKafkaConnectConfigSupplier.isPresent(), equalTo(true)); + assertThat(anotherOptionalKafkaConnectConfigSupplier.get(), sameInstance(optionalKafkaConnectConfigSupplier.get())); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpServiceConfigTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpServiceConfigTest.java new file mode 100644 index 0000000000..6376c0a935 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/GeoIpServiceConfigTest.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.hamcrest.CoreMatchers; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.parser.DataPrepperDurationDeserializer; +import org.opensearch.dataprepper.parser.model.DataPrepperConfiguration; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.time.Duration; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class GeoIpServiceConfigTest { + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + objectMapper = new ObjectMapper(new YAMLFactory()); + + final SimpleModule simpleModule = new SimpleModule() + .addDeserializer(Duration.class, new DataPrepperDurationDeserializer()); + objectMapper.registerModule(simpleModule); + } + + @Test + void testGeoIpServiceConfig() throws IOException { + final GeoIpServiceConfig geoIpServiceConfig = makeConfig("src/test/resources/geoip_service_config.yaml"); + assertNotNull(geoIpServiceConfig); + + final MaxMindConfig maxMindConfig = geoIpServiceConfig.getMaxMindConfig(); + assertNotNull(maxMindConfig); + assertNotNull(maxMindConfig.getAwsAuthenticationOptionsConfig()); + + assertThat(maxMindConfig, notNullValue()); + assertThat(maxMindConfig.getDatabaseRefreshInterval(), equalTo(Duration.ofDays(10))); + assertThat(maxMindConfig.getDatabasePaths().size(), equalTo(2)); + assertThat(maxMindConfig.getCacheSize(), equalTo(2048)); + } + + private GeoIpServiceConfig makeConfig(final String filePath) throws IOException { + final File configurationFile = new File(filePath); + final DataPrepperConfiguration dataPrepperConfiguration = objectMapper.readValue(configurationFile, DataPrepperConfiguration.class); + + assertThat(dataPrepperConfiguration, CoreMatchers.notNullValue()); + assertThat(dataPrepperConfiguration.getPipelineExtensions(), CoreMatchers.notNullValue()); + + final Map geoipServiceConfigMap = (Map) dataPrepperConfiguration.getPipelineExtensions().getExtensionMap().get("geoip_service"); + String json = objectMapper.writeValueAsString(geoipServiceConfigMap); + final Reader reader = new StringReader(json); + return objectMapper.readValue(reader, GeoIpServiceConfig.class); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/MaxMindConfigTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/MaxMindConfigTest.java new file mode 100644 index 0000000000..6b22d14bf4 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/MaxMindConfigTest.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; + +import java.time.Duration; +import java.util.List; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +class MaxMindConfigTest { + private MaxMindConfig maxMindConfig; + + @BeforeEach + void setup() { + maxMindConfig = new MaxMindConfig(); + } + + @Test + void testDefaultConfig() { + assertThat(maxMindConfig.getDatabasePaths().size(), equalTo(0)); + assertThat(maxMindConfig.getDatabaseRefreshInterval(), equalTo(Duration.ofDays(7))); + assertThat(maxMindConfig.getCacheSize(), equalTo(4096)); + assertThat(maxMindConfig.getAwsAuthenticationOptionsConfig(), equalTo(null)); + } + + @Test + void testCustomConfig() throws NoSuchFieldException, IllegalAccessException { + ReflectivelySetField.setField(MaxMindConfig.class, maxMindConfig, "databaseRefreshInterval", Duration.ofDays(10)); + ReflectivelySetField.setField(MaxMindConfig.class, maxMindConfig, "cacheSize", 2048); + ReflectivelySetField.setField(MaxMindConfig.class, maxMindConfig, "databasePaths", List.of("path1", "path2", "path3")); + + final AwsAuthenticationOptionsConfig awsAuthenticationOptionsConfig = new AwsAuthenticationOptionsConfig(); + ReflectivelySetField.setField(MaxMindConfig.class, maxMindConfig, "awsAuthenticationOptionsConfig", awsAuthenticationOptionsConfig); + + assertThat(maxMindConfig.getDatabaseRefreshInterval(), equalTo(Duration.ofDays(10))); + assertThat(maxMindConfig.getCacheSize(), equalTo(2048)); + assertThat(maxMindConfig.getDatabasePaths().size(), equalTo(3)); + assertThat(maxMindConfig.getAwsAuthenticationOptionsConfig(), equalTo(awsAuthenticationOptionsConfig)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/resources/geoip_service_config.yaml b/data-prepper-plugins/geoip-processor/src/test/resources/geoip_service_config.yaml new file mode 100644 index 0000000000..54d3ebb818 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/resources/geoip_service_config.yaml @@ -0,0 +1,9 @@ +extensions: + geoip_service: + maxmind: + database_paths: ["path1", "path2"] + database_refresh_interval: "P10D" + cache_size: 2048 + aws: + region: "us-east-1" + sts_role_arn: "arn:aws:iam::123456789:role/data-prepper-execution-role" \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index e5b091a14d..c5d0d6c916 100644 --- a/settings.gradle +++ b/settings.gradle @@ -102,7 +102,7 @@ include 'data-prepper-plugins:drop-events-processor' include 'data-prepper-plugins:key-value-processor' include 'data-prepper-plugins:type-conversion-processor' include 'data-prepper-plugins:mutate-event-processors' -//include 'data-prepper-plugins:geoip-processor' +include 'data-prepper-plugins:geoip-processor' include 'data-prepper-plugins:grok-processor' include 'data-prepper-plugins:aggregate-processor' include 'data-prepper-plugins:log-generator-source'