Confluence as source plugin (#5404)

* confluence source Signed-off-by: Santhosh Gandhe <[email protected]> * matching config structure to be more inline for Confluence Signed-off-by: Santhosh Gandhe <[email protected]> * Functional confluence version Signed-off-by: Santhosh Gandhe <[email protected]> * Saving page content as text Signed-off-by: Santhosh Gandhe <[email protected]> * injectable plugin metrics Signed-off-by: Santhosh Gandhe <[email protected]> * Introduced atlassian common module Signed-off-by: Santhosh Gandhe <[email protected]> * functional with making use of Atlassian Commons package Signed-off-by: Santhosh Gandhe <[email protected]> * test cases in a running state Signed-off-by: Santhosh Gandhe <[email protected]> * atlassian commons gradle file Signed-off-by: Santhosh Gandhe <[email protected]> * moved the test to the atlassian commons Signed-off-by: Santhosh Gandhe <[email protected]> * additional code coverage Signed-off-by: Santhosh Gandhe <[email protected]> * better names Signed-off-by: Santhosh Gandhe <[email protected]> * better naming Signed-off-by: Santhosh Gandhe <[email protected]> * making a default implementation Signed-off-by: Santhosh Gandhe <[email protected]> * addressing review comments Signed-off-by: Santhosh Gandhe <[email protected]> * validating page type filter Signed-off-by: Santhosh Gandhe <[email protected]> * fixed failing tests Signed-off-by: Santhosh Gandhe <[email protected]> * using InvalidPluginConfigurationException instead of BadRequestException Signed-off-by: Santhosh Gandhe <[email protected]> --------- Signed-off-by: Santhosh Gandhe <[email protected]>
opensearch-project · Feb 11, 2025 · 50f7a39 · 50f7a39
1 parent c2a0972
commit 50f7a39
Show file tree

Hide file tree

Showing 68 changed files with 4,884 additions and 0 deletions.
diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle b/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle
@@ -0,0 +1,26 @@
+plugins {
+    id 'java'
+}
+
+
+dependencies {
+
+    implementation project(path: ':data-prepper-api')
+    implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler')
+    implementation project(path: ':data-prepper-plugins:common')
+
+    implementation 'com.fasterxml.jackson.core:jackson-core'
+    implementation 'com.fasterxml.jackson.core:jackson-databind'
+
+    implementation 'io.micrometer:micrometer-core'
+    implementation 'org.projectlombok:lombok:1.18.30'
+    annotationProcessor 'org.projectlombok:lombok:1.18.30'
+    implementation(libs.spring.web)
+
+    implementation(libs.spring.context) {
+        exclude group: 'commons-logging', module: 'commons-logging'
+    }
+
+    testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4'
+    testImplementation project(path: ':data-prepper-test-common')
+}
diff --git a/.../main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java b/.../main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.atlassian;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.Valid;
+import lombok.Getter;
+import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig;
+import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig;
+
+import java.util.List;
+
+@Getter
+public class AtlassianSourceConfig implements CrawlerSourceConfig {
+
+    private static final int DEFAULT_BATCH_SIZE = 50;
+
+    /**
+     * Jira account url
+     */
+    @JsonProperty("hosts")
+    protected List<String> hosts;
+
+    /**
+     * Authentication Config to Access Jira
+     */
+    @JsonProperty("authentication")
+    @Valid
+    protected AuthenticationConfig authenticationConfig;
+
+    /**
+     * Batch size for fetching tickets
+     */
+    @JsonProperty("batch_size")
+    protected int batchSize = DEFAULT_BATCH_SIZE;
+
+
+    /**
+     * Boolean property indicating end to end acknowledgments state
+     */
+    @JsonProperty("acknowledgments")
+    private boolean acknowledgments = false;
+
+    public String getAccountUrl() {
+        return this.getHosts().get(0);
+    }
+
+    public String getAuthType() {
+        return this.getAuthenticationConfig().getAuthType();
+    }
+}
diff --git a/...g/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java b/...g/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.atlassian.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.AssertTrue;
+import lombok.Getter;
+
+import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.BASIC;
+import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2;
+
+
+@Getter
+public class AuthenticationConfig {
+    @JsonProperty("basic")
+    @Valid
+    private BasicConfig basicConfig;
+
+    @JsonProperty("oauth2")
+    @Valid
+    private Oauth2Config oauth2Config;
+
+    @AssertTrue(message = "Authentication config should have either basic or oauth2")
+    private boolean isValidAuthenticationConfig() {
+        boolean hasBasic = basicConfig != null;
+        boolean hasOauth = oauth2Config != null;
+        return hasBasic ^ hasOauth;
+    }
+
+    public String getAuthType() {
+        if (basicConfig != null) {
+            return BASIC;
+        } else {
+            return OAUTH2;
+        }
+    }
+}
diff --git a/...n/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java b/...n/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.atlassian.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.AssertTrue;
+import lombok.Getter;
+
+@Getter
+public class BasicConfig {
+    @JsonProperty("username")
+    private String username;
+
+    @JsonProperty("password")
+    private String password;
+
+    @AssertTrue(message = "Username and Password are both required for Basic Auth")
+    private boolean isBasicConfigValid() {
+        return username != null && password != null;
+    }
+}
diff --git a/.../java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java b/.../java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.atlassian.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.AssertTrue;
+import lombok.Getter;
+import org.opensearch.dataprepper.model.plugin.PluginConfigVariable;
+
+@Getter
+public class Oauth2Config {
+    @JsonProperty("client_id")
+    private String clientId;
+
+    @JsonProperty("client_secret")
+    private String clientSecret;
+
+    @JsonProperty("access_token")
+    private PluginConfigVariable accessToken;
+
+    @JsonProperty("refresh_token")
+    private PluginConfigVariable refreshToken;
+
+    @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2")
+    private boolean isOauth2ConfigValid() {
+        return clientId != null && clientSecret != null && accessToken != null && refreshToken != null;
+    }
+}
diff --git a/...in/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java b/...in/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.atlassian.rest;
+
+import com.google.common.annotations.VisibleForTesting;
+import lombok.extern.slf4j.Slf4j;
+import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig;
+import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException;
+import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException;
+import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.HttpClientErrorException;
+import org.springframework.web.client.RestTemplate;
+
+import java.net.URI;
+import java.util.List;
+
+import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY;
+import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.MAX_RETRIES;
+
+@Slf4j
+public class AtlassianRestClient {
+
+    public static final List<Integer> RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40);
+    private int sleepTimeMultiplier = 1000;
+    private final RestTemplate restTemplate;
+    private final AtlassianAuthConfig authConfig;
+
+    public AtlassianRestClient(RestTemplate restTemplate, AtlassianAuthConfig authConfig) {
+        this.restTemplate = restTemplate;
+        this.authConfig = authConfig;
+    }
+
+
+    protected <T> ResponseEntity<T> invokeRestApi(URI uri, Class<T> responseType) throws BadRequestException {
+        AddressValidation.validateInetAddress(AddressValidation.getInetAddress(uri.toString()));
+        int retryCount = 0;
+        while (retryCount < MAX_RETRIES) {
+            try {
+                return restTemplate.getForEntity(uri, responseType);
+            } catch (HttpClientErrorException ex) {
+                HttpStatus statusCode = ex.getStatusCode();
+                String statusMessage = ex.getMessage();
+                log.error("An exception has occurred while getting response from Jira search API  {}", ex.getMessage());
+                if (statusCode == HttpStatus.FORBIDDEN) {
+                    throw new UnauthorizedException(statusMessage);
+                } else if (statusCode == HttpStatus.UNAUTHORIZED) {
+                    log.error(NOISY, "Token expired. We will try to renew the tokens now", ex);
+                    authConfig.renewCredentials();
+                } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) {
+                    log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex);
+                }
+                try {
+                    Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException("Sleep in the retry attempt got interrupted", e);
+                }
+            }
+            retryCount++;
+        }
+        String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri);
+        log.error(errorMessage);
+        throw new RuntimeException(errorMessage);
+    }
+
+    @VisibleForTesting
+    public void setSleepTimeMultiplier(int multiplier) {
+        sleepTimeMultiplier = multiplier;
+    }
+}
diff --git a/...n/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java b/...n/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.atlassian.rest;
+
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpRequest;
+import org.springframework.http.client.ClientHttpRequestExecution;
+import org.springframework.http.client.ClientHttpRequestInterceptor;
+import org.springframework.http.client.ClientHttpResponse;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+
+public class BasicAuthInterceptor implements ClientHttpRequestInterceptor {
+    private final String username;
+    private final String password;
+
+    public BasicAuthInterceptor(AtlassianSourceConfig config) {
+        this.username = config.getAuthenticationConfig().getBasicConfig().getUsername();
+        this.password = config.getAuthenticationConfig().getBasicConfig().getPassword();
+    }
+
+    @Override
+    public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException {
+        String auth = username + ":" + password;
+        byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.US_ASCII));
+        String authHeader = "Basic " + new String(encodedAuth);
+        request.getHeaders().set(HttpHeaders.AUTHORIZATION, authHeader);
+        return execution.execute(request, body);
+    }
+}
diff --git a/...va/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java b/...va/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.dataprepper.plugins.source.atlassian.rest;
+
+
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.ClientHttpRequestInterceptor;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.web.client.RestTemplate;
+
+import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2;
+
+@Configuration
+public class CustomRestTemplateConfig {
+
+    @Bean
+    public RestTemplate basicAuthRestTemplate(AtlassianSourceConfig config, AtlassianAuthConfig authConfig) {
+        RestTemplate restTemplate = new RestTemplate();
+        restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory());
+        ClientHttpRequestInterceptor httpInterceptor;
+        if (OAUTH2.equals(config.getAuthType())) {
+            httpInterceptor = new OAuth2RequestInterceptor(authConfig);
+        } else {
+            httpInterceptor = new BasicAuthInterceptor(config);
+        }
+        restTemplate.getInterceptors().add(httpInterceptor);
+        return restTemplate;
+    }
+
+
+}