opensearch-project · kkondaka · Feb 11, 2025 · Jan 30, 2025 · Jan 31, 2025 · Jan 31, 2025
@@ -15,7 +15,7 @@ dependencies {
     implementation 'io.micrometer:micrometer-core'
     implementation 'org.projectlombok:lombok:1.18.30'
     annotationProcessor 'org.projectlombok:lombok:1.18.30'
-    implementation("org.springframework:spring-web:${libs.versions.spring.get()}")
+    implementation(libs.spring.web)
 
     implementation(libs.spring.context) {
         exclude group: 'commons-logging', module: 'commons-logging'
@@ -24,7 +24,3 @@ dependencies {
     testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4'
     testImplementation project(path: ':data-prepper-test-common')
 }
-
-test {
-    useJUnitPlatform()
-}
@@ -14,7 +14,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig;
 import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException;
-import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException;
+import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException;
 import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
@@ -52,7 +52,7 @@ protected <T> ResponseEntity<T> invokeRestApi(URI uri, Class<T> responseType) th
                 String statusMessage = ex.getMessage();
                 log.error("An exception has occurred while getting response from Jira search API  {}", ex.getMessage());
                 if (statusCode == HttpStatus.FORBIDDEN) {
-                    throw new UnAuthorizedException(statusMessage);
+                    throw new UnauthorizedException(statusMessage);
                 } else if (statusCode == HttpStatus.UNAUTHORIZED) {
                     log.error(NOISY, "Token expired. We will try to renew the tokens now", ex);
                     authConfig.renewCredentials();

@@ -13,7 +13,7 @@
 import lombok.Getter;
 import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config;
-import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException;
+import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpEntity;
@@ -103,7 +103,7 @@ public String getJiraAccountCloudId() {
                     log.error("Error occurred while accessing resources: ", e);
                 }
             }
-            throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", MAX_RETRIES));
+            throw new UnauthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", MAX_RETRIES));
         }
     }
 

@@ -18,7 +18,7 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig;
-import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException;
+import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.HttpClientErrorException;
@@ -44,7 +44,7 @@ public class AtlassianRestClientTest {
 
     private static Stream<Arguments> provideHttpStatusCodesWithExceptionClass() {
         return Stream.of(
-                Arguments.of(HttpStatus.FORBIDDEN, UnAuthorizedException.class),
+                Arguments.of(HttpStatus.FORBIDDEN, UnauthorizedException.class),
                 Arguments.of(HttpStatus.UNAUTHORIZED, RuntimeException.class),
                 Arguments.of(HttpStatus.TOO_MANY_REQUESTS, RuntimeException.class),
                 Arguments.of(HttpStatus.INSUFFICIENT_STORAGE, RuntimeException.class)

@@ -19,7 +19,7 @@
 import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config;
 import org.opensearch.dataprepper.plugins.source.atlassian.utils.ConfigUtilForTests;
-import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException;
+import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException;
 import org.opensearch.dataprepper.test.helper.ReflectivelySetField;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpMethod;
@@ -163,7 +163,7 @@ void testGetJiraAccountCloudIdUnauthorizedCase() {
         jiraOauthConfig.restTemplate = restTemplateMock;
 
 
-        assertThrows(UnAuthorizedException.class, () -> jiraOauthConfig.initCredentials());
+        assertThrows(UnauthorizedException.class, () -> jiraOauthConfig.initCredentials());
         verify(restTemplateMock, times(6))
                 .exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class));
         verify(restTemplateMock, times(1))

@@ -17,7 +17,6 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'javax.inject:javax.inject:1'
     implementation 'org.jsoup:jsoup:1.18.3'
-    implementation("org.springframework:spring-web:${libs.versions.spring.get()}")
 
     implementation 'org.projectlombok:lombok:1.18.30'
     annotationProcessor 'org.projectlombok:lombok:1.18.30'
@@ -28,8 +27,5 @@ dependencies {
     implementation(libs.spring.context) {
         exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation(libs.spring.web)
 }
-
-test {
-    useJUnitPlatform()
-}
@@ -17,6 +17,7 @@
 import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults;
 import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient;
 import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper;
+import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceContentType;
 import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException;
 import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo;
 import org.springframework.util.CollectionUtils;
@@ -136,6 +137,11 @@ private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configu
         if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration))) {
             validateSpaceFilters(configuration);
         }
+
+        if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration))) {
+            validatePageTypeFilters(configuration);
+        }
+
         String formattedTimeStamp = LocalDateTime.ofInstant(ts, ZoneId.systemDefault())
                 .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"));
         StringBuilder cQl = new StringBuilder(LAST_MODIFIED + GREATER_THAN_EQUALS + "\"" + formattedTimeStamp + "\"");
@@ -164,6 +170,48 @@ private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configu
         return cQl;
     }
 
+    /**
+     * Method for Validating Page Type Filters.
+     *
+     * @param configuration Input Parameter
+     */
+    private void validatePageTypeFilters(ConfluenceSourceConfig configuration) {
+        log.trace("Validating Page Type filters");
+        List<String> badFilters = new ArrayList<>();
+        Set<String> includedPageType = new HashSet<>();
+        List<String> includedAndExcludedPageType = new ArrayList<>();
+        ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration).forEach(pageTypeFilter -> {
+            if (ConfluenceContentType.fromString(pageTypeFilter) == null) {
+                badFilters.add(pageTypeFilter);
+            } else {
+                includedPageType.add(pageTypeFilter);
+            }
+        });
+        ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration).forEach(pageTypeFilter -> {
+            if (includedPageType.contains(pageTypeFilter)) {
+                includedAndExcludedPageType.add(pageTypeFilter);
+            }
+            if (ConfluenceContentType.fromString(pageTypeFilter) == null) {
+                badFilters.add(pageTypeFilter);
+            }
+        });
+        if (!badFilters.isEmpty()) {
+            String filters = String.join("\"" + badFilters + "\"", ", ");
+            log.error("One or more invalid Page Types found in filter configuration: {}", badFilters);
+            throw new BadRequestException("Bad request exception occurred " +
+                    "Invalid Page Type key found in filter configuration "
+                    + filters);
+        }
+        if (!includedAndExcludedPageType.isEmpty()) {
+            String filters = String.join("\"" + includedAndExcludedPageType + "\"", ", ");
+            log.error("One or more Page types found in both include and exclude: {}", includedAndExcludedPageType);
+            throw new BadRequestException("Bad request exception occurred " +
+                    "Page Type filters is invalid because the following Page types are listed in both include and exclude"
+                    + filters);
+        }
+
+    }
+
     /**
      * Method for Validating Space Filters.
      *

@@ -23,4 +23,13 @@ public enum ConfluenceContentType {
 
     @Getter
     private final String type;
+
+    public static ConfluenceContentType fromString(String value) {
+        for (ConfluenceContentType contentType : ConfluenceContentType.values()) {
+            if (contentType.type.equalsIgnoreCase(value)) {
+                return contentType;
+            }
+        }
+        return null;
+    }
 }
@@ -52,7 +52,6 @@
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
@@ -102,7 +101,7 @@ public static ConfluenceSourceConfig createConfluenceConfigurationFromYaml(Strin
     }
 
     public static ConfluenceSourceConfig createConfluenceConfiguration(String auth_type,
-                                                                       List<String> issueType,
+                                                                       List<String> pageTypes,
                                                                        List<String> projectKey) throws JsonProcessingException {
         PluginConfigVariable pcvAccessToken = null;
         PluginConfigVariable pcvRefreshToken = null;
@@ -130,7 +129,7 @@ public static ConfluenceSourceConfig createConfluenceConfiguration(String auth_t
         Map<String, Object> spacesMap = new HashMap<>();
         Map<String, Object> contentTypeMap = new HashMap<>();
 
-        contentTypeMap.put("include", issueType);
+        contentTypeMap.put("include", pageTypes);
         filterMap.put("page_type", contentTypeMap);
 
         Map<String, Object> nameMap = new HashMap<>();
@@ -178,17 +177,17 @@ void testJiraServiceInitialization() throws JsonProcessingException {
     public void testGetPages() throws JsonProcessingException {
         List<String> contentType = new ArrayList<>();
         List<String> spaceKey = new ArrayList<>();
-        contentType.add("page");
+        contentType.add("PAGE");
         spaceKey.add("KAN");
         ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, contentType, spaceKey);
         ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics));
         List<ConfluenceItem> mockPages = new ArrayList<>();
-        ConfluenceItem issue1 = createConfluenceItemBean();
-        mockPages.add(issue1);
-        ConfluenceItem issue2 = createConfluenceItemBean();
-        mockPages.add(issue2);
-        ConfluenceItem issue3 = createConfluenceItemBean();
-        mockPages.add(issue3);
+        ConfluenceItem item1 = createConfluenceItemBean();
+        mockPages.add(item1);
+        ConfluenceItem item2 = createConfluenceItemBean();
+        mockPages.add(item2);
+        ConfluenceItem item3 = createConfluenceItemBean();
+        mockPages.add(item3);
 
         ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class);
         when(mockConfluenceSearchResults.getResults()).thenReturn(mockPages);
@@ -203,10 +202,10 @@ public void testGetPages() throws JsonProcessingException {
 
     @Test
     public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingException {
-        List<String> issueType = new ArrayList<>();
+        List<String> pageType = new ArrayList<>();
         List<String> projectKey = new ArrayList<>();
-        issueType.add("Task");
-        ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, issueType, projectKey);
+        pageType.add("PAGE");
+        ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, pageType, projectKey);
         ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics));
         List<ConfluenceItem> mockIssues = new ArrayList<>();
         Random random = new Random();
@@ -228,17 +227,15 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep
 
     @Test
     public void testBadProjectKeys() throws JsonProcessingException {
-        List<String> issueType = new ArrayList<>();
-        List<String> issueStatus = new ArrayList<>();
+        List<String> pageType = new ArrayList<>();
         List<String> projectKey = new ArrayList<>();
-        issueType.add("Task");
-        issueStatus.add("Done");
+        pageType.add("PAGE");
         projectKey.add("Bad Project Key");
         projectKey.add("A");
         projectKey.add("!@#$");
         projectKey.add("AAAAAAAAAAAAAA");
 
-        ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, issueType, projectKey);
+        ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, pageType, projectKey);
         ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics);
 
         Instant timestamp = Instant.ofEpochSecond(0);
@@ -249,15 +246,12 @@ public void testBadProjectKeys() throws JsonProcessingException {
 
     @Test
     public void testGetPagesException() throws JsonProcessingException {
-        List<String> issueType = new ArrayList<>();
-        List<String> issueStatus = new ArrayList<>();
+        List<String> pageType = new ArrayList<>();
         List<String> projectKey = new ArrayList<>();
-        issueType.add("Task");
-        ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, issueType, projectKey);
+        pageType.add("Task");
+        ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, pageType, projectKey);
         ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics));
 
-        doThrow(RuntimeException.class).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt());
-
         Instant timestamp = Instant.ofEpochSecond(0);
         Queue<ItemInfo> itemInfoQueue = new ConcurrentLinkedQueue<>();
 

@@ -25,7 +25,7 @@
 import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig;
 import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults;
 import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException;
-import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException;
+import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.HttpClientErrorException;
@@ -60,7 +60,7 @@ public class ConfluenceRestClientTest {
 
     private static Stream<Arguments> provideHttpStatusCodesWithExceptionClass() {
         return Stream.of(
-                Arguments.of(HttpStatus.FORBIDDEN, UnAuthorizedException.class),
+                Arguments.of(HttpStatus.FORBIDDEN, UnauthorizedException.class),
                 Arguments.of(HttpStatus.UNAUTHORIZED, RuntimeException.class),
                 Arguments.of(HttpStatus.TOO_MANY_REQUESTS, RuntimeException.class),
                 Arguments.of(HttpStatus.INSUFFICIENT_STORAGE, RuntimeException.class)

@@ -14,12 +14,12 @@
  * Exception to indicate unauthorized access.
  * It could either be caused by invalid credentials supplied by the user or failed renew the credentials.
  */
-public final class UnAuthorizedException extends RuntimeException {
-    public UnAuthorizedException(final String message, final Throwable throwable) {
+public final class UnauthorizedException extends RuntimeException {
+    public UnauthorizedException(final String message, final Throwable throwable) {
         super(message, throwable);
     }
 
-    public UnAuthorizedException(final String message) {
+    public UnauthorizedException(final String message) {
         super(message);
     }
 }
@@ -18,7 +18,7 @@
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.mockito.Mockito.mock;
 
-public class UnAuthorizedExceptionTest {
+public class UnauthorizedExceptionTest {
     private String message;
     private Throwable throwable;
 
@@ -30,8 +30,8 @@ void setUp() {
 
     @Nested
     class MessageOnlyConstructor {
-        private UnAuthorizedException createObjectUnderTest() {
-            return new UnAuthorizedException(message);
+        private UnauthorizedException createObjectUnderTest() {
+            return new UnauthorizedException(message);
         }
 
         @Test
@@ -47,8 +47,8 @@ void getCause_returns_null() {
 
     @Nested
     class MessageThrowableConstructor {
-        private UnAuthorizedException createObjectUnderTest() {
-            return new UnAuthorizedException(message, throwable);
+        private UnauthorizedException createObjectUnderTest() {
+            return new UnauthorizedException(message, throwable);
         }
 
         @Test

@@ -49,6 +49,7 @@ dependencyResolutionManagement {
             version('spring', '5.3.39')
             library('spring-core', 'org.springframework', 'spring-core').versionRef('spring')
             library('spring-context', 'org.springframework', 'spring-context').versionRef('spring')
+            library('spring-web', 'org.springframework', 'spring-web').versionRef('spring')
             version('bouncycastle', '1.78.1')
             library('bouncycastle-bcprov', 'org.bouncycastle', 'bcprov-jdk18on').versionRef('bouncycastle')
             library('bouncycastle-bcpkix', 'org.bouncycastle', 'bcpkix-jdk18on').versionRef('bouncycastle')