Jira reuse atlassian commons module (opensearch-project#5441)

* jira making use of atlassian commons * Making OAuth2 flow work for both Confluence and Jira Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com>
divbok · Feb 24, 2025 · c54c0d3 · c54c0d3
1 parent 6397c5d
commit c54c0d3
Show file tree

Hide file tree

Showing 40 changed files with 125 additions and 1,444 deletions.
diff --git a/.../main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java b/.../main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java
@@ -19,7 +19,7 @@
 import java.util.List;
 
 @Getter
-public class AtlassianSourceConfig implements CrawlerSourceConfig {
+public abstract class AtlassianSourceConfig implements CrawlerSourceConfig {
 
     private static final int DEFAULT_BATCH_SIZE = 50;
 
@@ -56,4 +56,6 @@ public String getAccountUrl() {
     public String getAuthType() {
         return this.getAuthenticationConfig().getAuthType();
     }
+
+    public abstract String getOauth2UrlContext();
 }
diff --git a/...a/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java b/...a/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java
@@ -34,12 +34,12 @@
 import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.SLASH;
 
 /**
- * The type Jira service.
+ * The type Atlassian OAuth2 Service Config.
  */
 
 public class AtlassianOauthConfig implements AtlassianAuthConfig {
 
-    public static final String OAuth2_URL = "https://api.atlassian.com/ex/jira/";
+    public static final String OAuth2_URL = "https://api.atlassian.com/ex/";
     public static final String ACCESSIBLE_RESOURCES = "https://api.atlassian.com/oauth/token/accessible-resources";
     public static final String TOKEN_LOCATION = "https://auth.atlassian.com/oauth/token";
 
@@ -60,22 +60,22 @@ public class AtlassianOauthConfig implements AtlassianAuthConfig {
     private String cloudId = null;
     private final String clientId;
     private final String clientSecret;
-    private final AtlassianSourceConfig confluenceSourceConfig;
+    private final AtlassianSourceConfig atlassianSourceConfig;
     private final Object cloudIdFetchLock = new Object();
     private final Object tokenRenewLock = new Object();
 
-    public AtlassianOauthConfig(AtlassianSourceConfig confluenceSourceConfig) {
-        this.confluenceSourceConfig = confluenceSourceConfig;
-        this.accessToken = (String) confluenceSourceConfig.getAuthenticationConfig().getOauth2Config()
+    public AtlassianOauthConfig(AtlassianSourceConfig atlassianSourceConfig) {
+        this.atlassianSourceConfig = atlassianSourceConfig;
+        this.accessToken = (String) atlassianSourceConfig.getAuthenticationConfig().getOauth2Config()
                 .getAccessToken().getValue();
-        this.refreshToken = (String) confluenceSourceConfig.getAuthenticationConfig()
+        this.refreshToken = (String) atlassianSourceConfig.getAuthenticationConfig()
                 .getOauth2Config().getRefreshToken().getValue();
-        this.clientId = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId();
-        this.clientSecret = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret();
+        this.clientId = atlassianSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId();
+        this.clientSecret = atlassianSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret();
     }
 
-    public String getJiraAccountCloudId() {
-        log.info("Getting Jira Account Cloud ID");
+    public String getAtlassianAccountCloudId() {
+        log.info("Getting Atlassian Account Cloud ID");
         synchronized (cloudIdFetchLock) {
             if (this.cloudId != null) {
                 //Someone else must have initialized it
@@ -120,14 +120,14 @@ public void renewCredentials() {
                 return;
             }
 
-            log.info("Renewing access token and refresh token pair for Jira Connector.");
+            log.info("Renewing access token and refresh token pair for Atlassian Connector.");
             HttpHeaders headers = new HttpHeaders();
             headers.setContentType(MediaType.APPLICATION_JSON);
             String payloadTemplate = "{\"grant_type\": \"%s\", \"client_id\": \"%s\", \"client_secret\": \"%s\", \"refresh_token\": \"%s\"}";
             String payload = String.format(payloadTemplate, "refresh_token", clientId, clientSecret, refreshToken);
             HttpEntity<String> entity = new HttpEntity<>(payload, headers);
 
-            Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config();
+            Oauth2Config oauth2Config = atlassianSourceConfig.getAuthenticationConfig().getOauth2Config();
             try {
                 ResponseEntity<Map> responseEntity = restTemplate.postForEntity(TOKEN_LOCATION, entity, Map.class);
                 Map<String, Object> oauthClientResponse = responseEntity.getBody();
@@ -172,12 +172,12 @@ public String getUrl() {
     }
 
     /**
-     * Method for getting Jira url based on auth type.
+     * Method for getting source url based on auth type.
      */
     @Override
     public void initCredentials() {
-        //For OAuth based flow, we use a different Jira url
-        this.cloudId = getJiraAccountCloudId();
-        this.url = OAuth2_URL + this.cloudId + SLASH;
+        //For OAuth based flow, we use a different source url
+        this.cloudId = getAtlassianAccountCloudId();
+        this.url = OAuth2_URL + atlassianSourceConfig.getOauth2UrlContext() + SLASH + this.cloudId + SLASH;
     }
 }
diff --git a/...g/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java b/...g/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java
@@ -124,7 +124,7 @@ void testFailedToRenewAccessToken_with_unauthorized_and_trigger_secrets_refresh(
 
 
     @Test
-    void testGetJiraAccountCloudId() throws InterruptedException {
+    void testGetTestAccountCloudId() throws InterruptedException {
         Map<String, Object> mockGetCallResponse = new HashMap<>();
         mockGetCallResponse.put("id", "test_cloud_id");
         when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class)))
@@ -141,16 +141,16 @@ void testGetJiraAccountCloudId() throws InterruptedException {
         }
         executor.shutdown();
 
-        assertEquals("test_cloud_id", jiraOauthConfig.getJiraAccountCloudId());
-        assertEquals("https://api.atlassian.com/ex/jira/test_cloud_id/", jiraOauthConfig.getUrl());
+        assertEquals("test_cloud_id", jiraOauthConfig.getAtlassianAccountCloudId());
+        assertEquals("https://api.atlassian.com/ex/test/test_cloud_id/", jiraOauthConfig.getUrl());
         //calling second time shouldn't trigger rest call
         jiraOauthConfig.getUrl();
         verify(restTemplateMock, times(1))
                 .exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class));
     }
 
     @Test
-    void testGetJiraAccountCloudIdUnauthorizedCase() {
+    void testGetAtlassianAccountCloudIdUnauthorizedCase() {
 
         when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class)))
                 .thenThrow(new HttpClientErrorException(HttpStatus.UNAUTHORIZED));

diff --git a/.../org/opensearch/dataprepper/plugins/source/atlassian/utils/AtlassianSourceConfigTest.java b/.../org/opensearch/dataprepper/plugins/source/atlassian/utils/AtlassianSourceConfigTest.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.dataprepper.plugins.source.atlassian.utils;
+
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
+
+public class AtlassianSourceConfigTest extends AtlassianSourceConfig {
+    @Override
+    public String getOauth2UrlContext() {
+        return "test";
+    }
+}
diff --git a/...st/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java b/...st/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java
@@ -1,3 +1,12 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
 package org.opensearch.dataprepper.plugins.source.atlassian.utils;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -26,7 +35,7 @@ private static InputStream getResourceAsStream(String resourceName) {
     public static AtlassianSourceConfig createJiraConfigurationFromYaml(String fileName) {
         ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
         try (InputStream inputStream = getResourceAsStream(fileName)) {
-            AtlassianSourceConfig confluenceSourceConfig = objectMapper.readValue(inputStream, AtlassianSourceConfig.class);
+            AtlassianSourceConfig confluenceSourceConfig = objectMapper.readValue(inputStream, AtlassianSourceConfigTest.class);
             Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config();
             if (oauth2Config != null) {
                 ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken",

diff --git a/...src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/...src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java
@@ -111,8 +111,8 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati
             addItemsToQueue(contentList, itemInfoQueue);
             log.debug("Content items fetched so far: {}", total);
             paginationLinks = searchContentItems.getLinks();
+            searchResultsFoundCounter.increment(searchContentItems.getSize());
         } while (paginationLinks != null && paginationLinks.getNext() != null);
-        searchResultsFoundCounter.increment(total);
         log.info("Number of content items found in search api call: {}", total);
     }
 

diff --git a/...ain/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java b/...ain/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java
@@ -33,4 +33,8 @@ public class ConfluenceSourceConfig extends AtlassianSourceConfig implements Cra
     @JsonProperty("acknowledgments")
     private boolean acknowledgments = false;
 
+    @Override
+    public String getOauth2UrlContext() {
+        return "confluence";
+    }
 }
diff --git a/...java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java b/...java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java
@@ -78,7 +78,6 @@ private ConfluenceSourceConfig createConfluenceSourceConfig(String authtype, boo
         Map<String, Object> filterMap = new HashMap<>();
         Map<String, Object> projectMap = new HashMap<>();
         Map<String, Object> issueTypeMap = new HashMap<>();
-        Map<String, Object> statusMap = new HashMap<>();
 
         issueTypeMap.put("include", contentTypeList);
         filterMap.put("page_type", issueTypeMap);
@@ -127,4 +126,10 @@ void testFetchGivenOauthAtrribute() throws Exception {
         assertEquals(clientId, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId());
         assertEquals(clientSecret, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret());
     }
+
+    @Test
+    void testGetOauth2UrlContext() throws Exception {
+        confluenceSourceConfig = createConfluenceSourceConfig(OAUTH2, false);
+        assertEquals("confluence", confluenceSourceConfig.getOauth2UrlContext());
+    }
 }
diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle b/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle
@@ -5,6 +5,7 @@ plugins {
 dependencies {
 
     implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler')
+    implementation project(path: ':data-prepper-plugins:saas-source-plugins:atlassian-commons')
     implementation project(path: ':data-prepper-api')
     implementation project(path: ':data-prepper-plugins:aws-plugin-api')
     implementation project(path: ':data-prepper-plugins:buffer-common')
@@ -15,7 +16,6 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-core'
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'javax.inject:javax.inject:1'
-    implementation("org.springframework:spring-web:${libs.versions.spring.get()}")
 
     implementation 'org.projectlombok:lombok:1.18.30'
     annotationProcessor 'org.projectlombok:lombok:1.18.30'
@@ -26,6 +26,7 @@ dependencies {
     implementation(libs.spring.context) {
         exclude group: 'commons-logging', module: 'commons-logging'
     }
+    implementation(libs.spring.web)
 }
 
 test {

diff --git a/...jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/...jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java
@@ -13,7 +13,7 @@
 import io.micrometer.core.instrument.Counter;
 import lombok.extern.slf4j.Slf4j;
 import org.opensearch.dataprepper.metrics.PluginMetrics;
-import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException;
+import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
 import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean;
 import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults;
 import org.opensearch.dataprepper.plugins.source.jira.rest.JiraRestClient;
@@ -201,14 +201,14 @@ private void validateProjectFilters(JiraSourceConfig configuration) {
         if (!badFilters.isEmpty()) {
             String filters = String.join("\"" + badFilters + "\"", ", ");
             log.error("One or more invalid project keys found in filter configuration: {}", badFilters);
-            throw new BadRequestException("Bad request exception occurred " +
+            throw new InvalidPluginConfigurationException("Bad request exception occurred " +
                     "Invalid project key found in filter configuration for "
                     + filters);
         }
         if (!includedAndExcludedProjects.isEmpty()) {
             String filters = String.join("\"" + includedAndExcludedProjects + "\"", ", ");
             log.error("One or more project keys found in both include and exclude: {}", includedAndExcludedProjects);
-            throw new BadRequestException("Bad request exception occurred " +
+            throw new InvalidPluginConfigurationException("Bad request exception occurred " +
                     "Project filters is invalid because the following projects are listed in both include and exclude"
                     + filters);
         }

diff --git a/.../jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java b/.../jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java
@@ -20,7 +20,8 @@
 import org.opensearch.dataprepper.model.plugin.PluginFactory;
 import org.opensearch.dataprepper.model.record.Record;
 import org.opensearch.dataprepper.model.source.Source;
-import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig;
 import org.opensearch.dataprepper.plugins.source.jira.utils.JiraConfigHelper;
 import org.opensearch.dataprepper.plugins.source.source_crawler.CrawlerApplicationContextMarker;
 import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler;
@@ -39,18 +40,18 @@
 @DataPrepperPlugin(name = PLUGIN_NAME,
         pluginType = Source.class,
         pluginConfigurationType = JiraSourceConfig.class,
-        packagesToScan = {CrawlerApplicationContextMarker.class, JiraSource.class}
+        packagesToScan = {CrawlerApplicationContextMarker.class, AtlassianSourceConfig.class, JiraSource.class}
 )
 public class JiraSource extends CrawlerSourcePlugin {
 
     private static final Logger log = LoggerFactory.getLogger(JiraSource.class);
     private final JiraSourceConfig jiraSourceConfig;
-    private final JiraAuthConfig jiraOauthConfig;
+    private final AtlassianAuthConfig jiraOauthConfig;
 
     @DataPrepperPluginConstructor
     public JiraSource(final PluginMetrics pluginMetrics,
                       final JiraSourceConfig jiraSourceConfig,
-                      final JiraAuthConfig jiraOauthConfig,
+                      final AtlassianAuthConfig jiraOauthConfig,
                       final PluginFactory pluginFactory,
                       final AcknowledgementSetManager acknowledgementSetManager,
                       Crawler crawler,

diff --git a/...source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java b/...source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java
@@ -11,63 +11,36 @@
 package org.opensearch.dataprepper.plugins.source.jira;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import jakarta.validation.Valid;
 import jakarta.validation.constraints.AssertTrue;
 import lombok.Getter;
-import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 import org.opensearch.dataprepper.plugins.source.jira.configuration.FilterConfig;
 import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig;
 
-import java.util.List;
-
 @Getter
-public class JiraSourceConfig implements CrawlerSourceConfig {
+public class JiraSourceConfig extends AtlassianSourceConfig implements CrawlerSourceConfig {
 
     private static final int DEFAULT_BATCH_SIZE = 50;
 
-    /**
-     * Jira account url
-     */
-    @JsonProperty("hosts")
-    private List<String> hosts;
-
-    @AssertTrue(message = "Jira hosts must be a list of length 1")
-    boolean isValidHosts() {
-        return hosts != null && hosts.size() == 1;
-    }
-
-    /**
-     * Authentication Config to Access Jira
-     */
-    @JsonProperty("authentication")
-    @Valid
-    private AuthenticationConfig authenticationConfig;
-
-    /**
-     * Batch size for fetching tickets
-     */
-    @JsonProperty("batch_size")
-    private int batchSize = DEFAULT_BATCH_SIZE;
-
-
     /**
      * Filter Config to filter what tickets get ingested
      */
     @JsonProperty("filter")
     private FilterConfig filterConfig;
-
-
     /**
      * Boolean property indicating end to end acknowledgments state
      */
     @JsonProperty("acknowledgments")
     private boolean acknowledgments = false;
 
-    public String getAccountUrl() {
-        return this.getHosts().get(0);
+    @AssertTrue(message = "Jira hosts must be a list of length 1")
+    boolean isValidHosts() {
+        return hosts != null && hosts.size() == 1;
     }
 
-    public String getAuthType() {
-        return this.getAuthenticationConfig().getAuthType();
+    @Override
+    public String getOauth2UrlContext() {
+        return "jira";
     }
+
 }