diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index 1cd2f54335..39c6477a12 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -4,6 +4,8 @@ */ dependencies { + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' implementation project(':data-prepper-api') implementation libs.armeria.core testImplementation project(':data-prepper-api').sourceSets.test.output diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index d97feb30ba..9df9095fd4 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -38,15 +38,16 @@ import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventType; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.model.plugin.PluginConfigObservable; -import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.SinkContext; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.index.AbstractIndexManager; import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConstants; @@ -89,6 +90,7 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.closeTo; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; @@ -101,12 +103,12 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.isOSBundle; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.waitForClusterStateUpdatesToFinish; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.wipeAllTemplates; -import static org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig.AUTHENTICATION; -import static org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig.PASSWORD; -import static org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig.USERNAME; public class OpenSearchSinkIT { private static final int LUCENE_CHAR_LENGTH_LIMIT = 32_766; + private static final String AUTHENTICATION = "authentication"; + private static final String USERNAME = "username"; + private static final String PASSWORD = "password"; private static final String TEST_STRING_WITH_SPECIAL_CHARS = "Hello! Data-Prepper? #Example123"; private static final String TEST_STRING_WITH_NON_LATIN_CHARS = "Привет,Γειά σας,こんにちは,你好"; private static final String PLUGIN_NAME = "opensearch"; @@ -129,8 +131,7 @@ public class OpenSearchSinkIT { private SinkContext sinkContext; private String testTagsTargetKey; - @Mock - private PluginFactory pluginFactory; + ObjectMapper objectMapper; @Mock private AwsCredentialsSupplier awsCredentialsSupplier; @@ -138,24 +139,36 @@ public class OpenSearchSinkIT { @Mock private ExpressionEvaluator expressionEvaluator; + @Mock + private PipelineDescription pipelineDescription; + + @Mock + private PluginSetting pluginSetting; + @Mock private PluginConfigObservable pluginConfigObservable; - public OpenSearchSink createObjectUnderTest(PluginSetting pluginSetting, boolean doInitialize) { + public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { + when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); + when(pluginSetting.getPipelineName()).thenReturn(PIPELINE_NAME); + when(pluginSetting.getName()).thenReturn(PLUGIN_NAME); OpenSearchSink sink = new OpenSearchSink( - pluginSetting, pluginFactory, null, expressionEvaluator, awsCredentialsSupplier, pluginConfigObservable); + pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { sink.doInitialize(); } return sink; } - public OpenSearchSink createObjectUnderTestWithSinkContext(PluginSetting pluginSetting, boolean doInitialize) { + public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { sinkContext = mock(SinkContext.class); testTagsTargetKey = RandomStringUtils.randomAlphabetic(5); when(sinkContext.getTagsTargetKey()).thenReturn(testTagsTargetKey); + when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); + when(pluginSetting.getPipelineName()).thenReturn(PIPELINE_NAME); + when(pluginSetting.getName()).thenReturn(PLUGIN_NAME); OpenSearchSink sink = new OpenSearchSink( - pluginSetting, pluginFactory, sinkContext, expressionEvaluator, awsCredentialsSupplier, pluginConfigObservable); + pluginSetting, sinkContext, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { sink.doInitialize(); } @@ -166,6 +179,8 @@ public OpenSearchSink createObjectUnderTestWithSinkContext(PluginSetting pluginS public void setup() { pluginConfigObservable = mock(PluginConfigObservable.class); expressionEvaluator = mock(ExpressionEvaluator.class); + pipelineDescription = mock(PipelineDescription.class); + pluginSetting = mock(PluginSetting.class); when(expressionEvaluator.isValidExpressionStatement(any(String.class))).thenReturn(false); } @@ -187,8 +202,8 @@ public void cleanOpenSearch() throws Exception { @Test @DisabledIf(value = "isES6", disabledReason = TRACE_INGESTION_TEST_DISABLED_REASON) public void testInstantiateSinkRawSpanDefault() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); assertThat(indexAlias, equalTo("otel-v1-apm-span")); Request request = new Request(HttpMethod.HEAD, indexAlias); @@ -215,7 +230,7 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK)); // Instantiate sink again - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); // Make sure no new write index *-000001 is created under alias final String rolloverIndexName = String.format("%s-000002", indexAlias); request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias"); @@ -232,8 +247,8 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { @Test @DisabledIf(value = "isES6", disabledReason = LOG_INGESTION_TEST_DISABLED_REASON) public void testInstantiateSinkLogsDefaultLogSink() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting(IndexType.LOG_ANALYTICS.getValue(), null, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.LOG_ANALYTICS.getValue(), null, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.LOG_ANALYTICS); assertThat(indexAlias, equalTo("logs-otel-v1")); Request request = new Request(HttpMethod.HEAD, indexAlias); @@ -260,7 +275,7 @@ public void testInstantiateSinkLogsDefaultLogSink() throws IOException { assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK)); // Instantiate sink again - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); // Make sure no new write index *-000001 is created under alias final String rolloverIndexName = String.format("%s-000002", indexAlias); request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias"); @@ -277,8 +292,8 @@ public void testInstantiateSinkLogsDefaultLogSink() throws IOException { @Test @DisabledIf(value = "isES6", disabledReason = METRIC_INGESTION_TEST_DISABLED_REASON) public void testInstantiateSinkMetricsDefaultMetricSink() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting(IndexType.METRIC_ANALYTICS.getValue(), null, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.METRIC_ANALYTICS.getValue(), null, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.METRIC_ANALYTICS); assertThat(indexAlias, equalTo("metrics-otel-v1")); Request request = new Request(HttpMethod.HEAD, indexAlias); @@ -305,7 +320,7 @@ public void testInstantiateSinkMetricsDefaultMetricSink() throws IOException { assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK)); // Instantiate sink again - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); // Make sure no new write index *-000001 is created under alias final String rolloverIndexName = String.format("%s-000002", indexAlias); request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias"); @@ -326,8 +341,8 @@ public void testInstantiateSinkRawSpanReservedAliasAlreadyUsedAsIndex() throws I final String reservedIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); final Request request = new Request(HttpMethod.PUT, reservedIndexAlias); client.performRequest(request); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, false); Assert.assertThrows(String.format(AbstractIndexManager.INDEX_ALIAS_USED_AS_INDEX_ERROR, reservedIndexAlias), RuntimeException.class, () -> sink.doInitialize()); } @@ -344,9 +359,9 @@ public void testOutputRawSpanDefault(final boolean estimateBulkSizeUsingCompress @SuppressWarnings("unchecked") final Map expData2 = mapper.readValue(testDoc2, Map.class); final List> testRecords = Arrays.asList(jsonStringToRecord(testDoc1), jsonStringToRecord(testDoc2)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, estimateBulkSizeUsingCompression, isRequestCompressionEnabled); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); @@ -413,15 +428,16 @@ public void testOutputRawSpanWithDLQ(final boolean estimateBulkSizeUsingCompress @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc2, Map.class); final List> testRecords = Arrays.asList(jsonStringToRecord(testDoc1), jsonStringToRecord(testDoc2)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, + Map metadata = initializeConfigurationMetadata(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, estimateBulkSizeUsingCompression, isRequestCompressionEnabled); // generate temporary directory for dlq file final File tempDirectory = Files.createTempDirectory("").toFile(); // add dlq file path into setting final String expDLQFile = tempDirectory.getAbsolutePath() + "/test-dlq.txt"; - pluginSetting.getSettings().put(RetryConfiguration.DLQ_FILE, expDLQFile); + metadata.put(RetryConfiguration.DLQ_FILE, expDLQFile); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); sink.shutdown(); @@ -466,8 +482,8 @@ public void testOutputRawSpanWithDLQ(final boolean estimateBulkSizeUsingCompress @Test @DisabledIf(value = "isES6", disabledReason = TRACE_INGESTION_TEST_DISABLED_REASON) public void testInstantiateSinkServiceMapDefault() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_SERVICE_MAP); final Request request = new Request(HttpMethod.HEAD, indexAlias); final Response response = client.performRequest(request); @@ -492,9 +508,9 @@ public void testOutputServiceMapDefault(final boolean estimateBulkSizeUsingCompr @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc, Map.class); final List> testRecords = Collections.singletonList(jsonStringToRecord(testDoc)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null, + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null, estimateBulkSizeUsingCompression, isRequestCompressionEnabled); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_SERVICE_MAP); final List> retSources = getSearchResponseDocSources(expIndexAlias); @@ -524,7 +540,7 @@ public void testOutputServiceMapDefault(final boolean estimateBulkSizeUsingCompr assertThat(bulkRequestSizeBytesMetrics.get(2).getValue(), closeTo(expectedBulkRequestSizeBytes, 0)); // Check restart for index already exists - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.shutdown(); } @@ -533,8 +549,8 @@ public void testInstantiateSinkCustomIndex_NoRollOver() throws IOException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; final Request request = new Request(HttpMethod.HEAD, testIndexAlias + extraURI); @@ -543,7 +559,7 @@ public void testInstantiateSinkCustomIndex_NoRollOver() throws IOException { sink.shutdown(); // Check restart for index already exists - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.shutdown(); } @@ -559,9 +575,9 @@ public void testInstantiateSinkCustomIndex_WithIsmPolicy( final Map metadata = initializeConfigurationMetadata(null, indexAlias, testTemplateFile); metadata.put(IndexConfiguration.ISM_POLICY_FILE, TEST_CUSTOM_INDEX_POLICY_FILE); metadata.put(IndexConfiguration.TEMPLATE_TYPE, templateType); - final PluginSetting pluginSetting = generatePluginSettingByMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; Request request = new Request(HttpMethod.HEAD, indexAlias + extraURI); @@ -605,7 +621,7 @@ public void testInstantiateSinkCustomIndex_WithIsmPolicy( assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK)); // Instantiate sink again - sink = createObjectUnderTest(pluginSetting, true); + sink = createObjectUnderTest(openSearchSinkConfig, true); // Make sure no new write index *-000001 is created under alias final String rolloverIndexName = String.format("%s-000002", indexAlias); request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias"); @@ -633,8 +649,8 @@ public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates( final String testTemplateFileV2 = getClass().getClassLoader().getResource(v2File).getFile(); // Create sink with template version 1 - PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV1); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, templateType, testTemplateFileV1); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; @@ -652,8 +668,8 @@ public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates( sink.shutdown(); // Create sink with template version 2 - pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV2); - sink = createObjectUnderTest(pluginSetting, true); + openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, templateType, testTemplateFileV2); + sink = createObjectUnderTest(openSearchSinkConfig, true); getTemplateRequest = new Request(HttpMethod.GET, "/" + templatePath + "/" + expectedIndexTemplateName + extraURI); @@ -669,8 +685,8 @@ public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates( sink.shutdown(); // Create sink with template version 1 again - pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV1); - sink = createObjectUnderTest(pluginSetting, true); + openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, templateType, testTemplateFileV1); + sink = createObjectUnderTest(openSearchSinkConfig, true); getTemplateRequest = new Request(HttpMethod.GET, "/" + templatePath + "/" + expectedIndexTemplateName + extraURI); @@ -702,8 +718,8 @@ public void testIndexNameWithDateNotAsSuffixCreatesIndexTemplate( final String expectedIndexTemplateName = "prefix-suffix-index-template"; final String testTemplateFileV1 = getClass().getClassLoader().getResource(templateFile).getFile(); - PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, templateType, testTemplateFileV1); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, templateType, testTemplateFileV1); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String extraURI = DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo( OpenSearchIntegrationHelper.getVersion()) >= 0 ? INCLUDE_TYPE_NAME_FALSE_URI : ""; @@ -814,9 +830,10 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -840,10 +857,11 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); - pluginSetting.getSettings().put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -867,16 +885,13 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "create"); - final String actionFormatExpression = "${getMetadata(\"action\")}"; - when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); - when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); - when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); - pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTION, "create"); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -893,28 +908,18 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, } @Test - public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOException, InterruptedException { + public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); - Event event = (Event) testRecords.get(0).getData(); - event.getMetadata().setAttribute("action", "unknown"); - final String actionFormatExpression = "${getMetadata(\"action\")}"; - when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); - when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); - when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); - pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); - sink.output(testRecords); - final List> retSources = getSearchResponseDocSources(testIndexAlias); - assertThat(retSources.size(), equalTo(0)); - assertThat(sink.getInvalidActionErrorsCount(), equalTo(1.0)); - sink.shutdown(); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + metadata.put(IndexConfiguration.ACTION, "unknown"); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + assertThrows(NullPointerException.class, () -> createObjectUnderTest(openSearchSinkConfig, true)); } @Test @@ -926,14 +931,15 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -959,14 +965,15 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -985,8 +992,9 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPDATE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1015,18 +1023,19 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr List> testRecords = Collections.singletonList(jsonStringToRecord(createJsonEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(actionMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1050,8 +1059,9 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.UPDATE.toString()); aList.add(actionMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1078,10 +1088,11 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int actionMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(actionMap); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + metadata.put(IndexConfiguration.ACTIONS, aList); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1104,14 +1115,16 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1130,8 +1143,9 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); retSources = getSearchResponseDocSources(testIndexAlias); @@ -1152,14 +1166,15 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE final String testIdField = "someId"; final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue"))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1188,14 +1203,15 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.DELETE.toString()); aList.add(aMap); - pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + metadata.put(IndexConfiguration.ACTIONS, aList); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(0)); @@ -1214,8 +1230,8 @@ public void testEventOutputWithTags() throws IOException, InterruptedException { final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - final OpenSearchSink sink = createObjectUnderTestWithSinkContext(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); + final OpenSearchSink sink = createObjectUnderTestWithSinkContext(openSearchSinkConfig, true); sink.output(testRecords); final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); @@ -1241,8 +1257,8 @@ public void testEventOutput() throws IOException, InterruptedException { final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); verify(pluginConfigObservable).addPluginConfigObserver(any()); sink.output(testRecords); @@ -1272,8 +1288,8 @@ public void testEventOutputWithSpecialAndExtremeValues(final Object testValue) t final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -1298,9 +1314,10 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testDocumentIdField); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testDocumentIdField); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List docIds = getSearchResponseDocIds(testIndexAlias); @@ -1323,9 +1340,10 @@ public void testOpenSearchRoutingField(final String testRoutingField) throws IOE final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.ROUTING_FIELD, testRoutingField); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.ROUTING_FIELD, testRoutingField); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List routingFields = getSearchResponseRoutingFields(testIndexAlias); @@ -1350,11 +1368,12 @@ public void testOpenSearchRouting(final String testRouting) throws IOException, final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); if (!testRouting.isEmpty()) { - pluginSetting.getSettings().put(IndexConfiguration.ROUTING, "${"+testRouting+"}"); + metadata.put(IndexConfiguration.ROUTING, "${"+testRouting+"}"); } - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List routingFields = getSearchResponseRoutingFields(testIndexAlias); @@ -1381,10 +1400,11 @@ public void testOpenSearchRoutingWithExpressions(final String testRouting) throw final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.ROUTING, "${/"+testRouting+"}"); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.ROUTING, "${/"+testRouting+"}"); when(expressionEvaluator.isValidFormatExpression(any(String.class))).thenReturn(true); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List routingFields = getSearchResponseRoutingFields(testIndexAlias); @@ -1409,10 +1429,11 @@ public void testOpenSearchRoutingWithMixedExpressions(final String testRouting) final String prefix = RandomStringUtils.randomAlphabetic(5); final String suffix = RandomStringUtils.randomAlphabetic(6); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.ROUTING, prefix+"-${/"+testRouting+"}-"+suffix); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); + metadata.put(IndexConfiguration.ROUTING, prefix+"-${/"+testRouting+"}-"+suffix); when(expressionEvaluator.isValidFormatExpression(any(String.class))).thenReturn(true); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final String expectedRouting = prefix+"-"+routing+"-"+suffix; @@ -1441,8 +1462,8 @@ public void testOpenSearchDynamicIndex(final String testIndex) throws IOExceptio final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, dynamicTestIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, dynamicTestIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1474,8 +1495,8 @@ public void testOpenSearchDynamicIndexWithDate(final String testIndex, final Str final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, dynamicTestIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, dynamicTestIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(expectedIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1510,8 +1531,8 @@ public void testOpenSearchDynamicIndexWithDateNotAsSuffix( final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, dynamicTestIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, dynamicTestIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(expectedIndexAlias); assertThat(retSources.size(), equalTo(1)); @@ -1538,8 +1559,8 @@ public void testOpenSearchIndexWithDate(final String testDatePattern) throws IOE final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(expectedIndexName); assertThat(retSources.size(), equalTo(1)); @@ -1566,8 +1587,8 @@ public void testOpenSearchIndexWithDateNotAsSuffix(final String testIndexAlias) final List> testRecords = Collections.singletonList(new Record<>(testEvent)); - final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(expectedIndexName); assertThat(retSources.size(), equalTo(1)); @@ -1579,16 +1600,16 @@ public void testOpenSearchIndexWithDateNotAsSuffix(final String testIndexAlias) public void testOpenSearchIndexWithInvalidDate() throws IOException, InterruptedException { String invalidDatePattern = "yyyy-MM-dd HH:ss:mm"; final String invalidTestIndexAlias = "test-index-%{" + invalidDatePattern + "}"; - final PluginSetting pluginSetting = generatePluginSetting(null, invalidTestIndexAlias, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, invalidTestIndexAlias, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, false); Assert.assertThrows(IllegalArgumentException.class, () -> sink.doInitialize()); } @Test public void testOpenSearchIndexWithInvalidChars() throws IOException, InterruptedException { final String invalidTestIndexAlias = "test#-index"; - final PluginSetting pluginSetting = generatePluginSetting(null, invalidTestIndexAlias, null); - OpenSearchSink sink = createObjectUnderTest(pluginSetting, false); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, invalidTestIndexAlias, null); + OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, false); Assert.assertThrows(RuntimeException.class, () -> sink.doInitialize()); } @@ -1612,10 +1633,11 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept final Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); - metadata.put(AUTHENTICATION, Map.of(USERNAME, username, PASSWORD, password)); + metadata.put(USERNAME, username); + metadata.put(PASSWORD, password); metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); - final PluginSetting pluginSetting = generatePluginSettingByMetadata(metadata); - final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource("management-disabled-index-template.json")).getFile(); @@ -1657,33 +1679,44 @@ private Map initializeConfigurationMetadata(final String indexTy return metadata; } - private PluginSetting generatePluginSetting(final String indexType, final String indexAlias, - final String templateFilePath) { + private Map initializeConfigurationMetadata(final String indexType, final String indexAlias, + final String templateFilePath, final boolean estimateBulkSizeUsingCompression, + final boolean requestCompressionEnabled) throws JsonProcessingException { final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); - return generatePluginSettingByMetadata(metadata); + metadata.put(IndexConfiguration.ESTIMATE_BULK_SIZE_USING_COMPRESSION, estimateBulkSizeUsingCompression); + metadata.put(ConnectionConfiguration.REQUEST_COMPRESSION_ENABLED, requestCompressionEnabled); + return metadata; } - private PluginSetting generatePluginSetting(final String indexType, final String indexAlias, + private OpenSearchSinkConfig generateOpenSearchSinkConfig(final String indexType, final String indexAlias, + final String templateFilePath) throws JsonProcessingException { + final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); + return generateOpenSearchSinkConfigByMetadata(metadata); + } + + private OpenSearchSinkConfig generateOpenSearchSinkConfig(final String indexType, final String indexAlias, final String templateFilePath, final boolean estimateBulkSizeUsingCompression, - final boolean requestCompressionEnabled) { + final boolean requestCompressionEnabled) throws JsonProcessingException { final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); metadata.put(IndexConfiguration.ESTIMATE_BULK_SIZE_USING_COMPRESSION, estimateBulkSizeUsingCompression); metadata.put(ConnectionConfiguration.REQUEST_COMPRESSION_ENABLED, requestCompressionEnabled); - return generatePluginSettingByMetadata(metadata); + return generateOpenSearchSinkConfigByMetadata(metadata); } - private PluginSetting generatePluginSetting(final String indexType, final String indexAlias, + private OpenSearchSinkConfig generateOpenSearchSinkConfig(final String indexType, final String indexAlias, final String templateType, - final String templateFilePath) { + final String templateFilePath) throws JsonProcessingException { final Map metadata = initializeConfigurationMetadata(indexType, indexAlias, templateFilePath); metadata.put(IndexConfiguration.TEMPLATE_TYPE, templateType); - return generatePluginSettingByMetadata(metadata); + return generateOpenSearchSinkConfigByMetadata(metadata); } - private PluginSetting generatePluginSettingByMetadata(final Map configurationMetadata) { - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, configurationMetadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - return pluginSetting; + private OpenSearchSinkConfig generateOpenSearchSinkConfigByMetadata(final Map configurationMetadata) throws JsonProcessingException { + objectMapper = new ObjectMapper(); + String json = new ObjectMapper().writeValueAsString(configurationMetadata); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); + + return openSearchSinkConfig; } private String generateCustomRecordJson(final String idField, final String documentId) throws IOException { diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java index aca748b3ef..f4ff8acafd 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java @@ -15,7 +15,6 @@ import org.opensearch.client.opensearch.core.BulkResponse; import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.rest.RestStatus; @@ -105,9 +104,6 @@ public final class BulkRetryStrategy { private final PluginMetrics pluginMetrics; private final Supplier bulkRequestSupplier; private final int maxRetries; - private final String pluginId; - private final String pluginName; - private final String pipelineName; private final ObjectMapper objectMapper; private final Counter sentDocumentsCounter; @@ -150,15 +146,13 @@ public BulkRetryStrategy(final RequestFunction bulkRequestSupplier, - final PluginSetting pluginSetting) { + final String pipelineName, + final String pluginName) { this.requestFunction = requestFunction; this.logFailure = logFailure; this.pluginMetrics = pluginMetrics; this.bulkRequestSupplier = bulkRequestSupplier; this.maxRetries = maxRetries; - this.pipelineName = pluginSetting.getPipelineName(); - this.pluginId = pluginSetting.getName(); - this.pluginName = pluginSetting.getName(); this.objectMapper = new ObjectMapper(); sentDocumentsCounter = pluginMetrics.counter(DOCUMENTS_SUCCESS); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 4af2ef70a8..d16499c45c 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -5,7 +5,6 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; import org.apache.http.auth.AuthScope; @@ -29,9 +28,11 @@ import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; import org.opensearch.dataprepper.aws.api.AwsRequestSigningApache4Interceptor; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AuthConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ServerlessOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.arns.Arn; @@ -61,36 +62,17 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DISTRIBUTION_VERSION; public class ConnectionConfiguration { - static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final Logger LOG = LoggerFactory.getLogger(OpenSearchSink.class); private static final String AWS_IAM_ROLE = "role"; private static final String AWS_IAM = "iam"; private static final String AOS_SERVICE_NAME = "es"; private static final String AOSS_SERVICE_NAME = "aoss"; - private static final String DEFAULT_AWS_REGION = "us-east-1"; public static final String HOSTS = "hosts"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - public static final String SOCKET_TIMEOUT = "socket_timeout"; - public static final String CONNECT_TIMEOUT = "connect_timeout"; - public static final String CERT_PATH = "cert"; - public static final String INSECURE = "insecure"; - public static final String AWS_OPTION = "aws"; public static final String AWS_SIGV4 = "aws_sigv4"; - public static final String AWS_REGION = "aws_region"; - public static final String AWS_STS_ROLE_ARN = "aws_sts_role_arn"; - public static final String AWS_STS_EXTERNAL_ID = "aws_sts_external_id"; - public static final String AWS_STS_HEADER_OVERRIDES = "aws_sts_header_overrides"; - public static final String PROXY = "proxy"; public static final String SERVERLESS = "serverless"; - public static final String SERVERLESS_OPTIONS = "serverless_options"; - public static final String COLLECTION_NAME = "collection_name"; - public static final String NETWORK_POLICY_NAME = "network_policy_name"; - public static final String VPCE_ID = "vpce_id"; public static final String REQUEST_COMPRESSION_ENABLED = "enable_request_compression"; /** @@ -112,7 +94,6 @@ public class ConnectionConfiguration { private final String awsStsExternalId; private final Map awsStsHeaderOverrides; private final Optional proxy; - private final String pipelineName; private final boolean serverless; private final String serverlessNetworkPolicyName; private final String serverlessCollectionName; @@ -203,23 +184,16 @@ private ConnectionConfiguration(final Builder builder) { this.serverlessCollectionName = builder.serverlessCollectionName; this.serverlessVpceId = builder.serverlessVpceId; this.requestCompressionEnabled = builder.requestCompressionEnabled; - this.pipelineName = builder.pipelineName; this.authConfig = builder.authConfig; } - public static ConnectionConfiguration readConnectionConfiguration(final PluginSetting pluginSetting){ - @SuppressWarnings("unchecked") - final List hosts = (List) pluginSetting.getAttributeFromSettings(HOSTS); + public static ConnectionConfiguration readConnectionConfiguration(final OpenSearchSinkConfig openSearchSinkConfig){ + final List hosts = openSearchSinkConfig.getHosts(); ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); - final String username = (String) pluginSetting.getAttributeFromSettings(USERNAME); - final String password = (String) pluginSetting.getAttributeFromSettings(PASSWORD); - builder.withPipelineName(pluginSetting.getPipelineName()); - final AuthConfig authConfig = AuthConfig.readAuthConfig(pluginSetting); + final String username = openSearchSinkConfig.getUsername(); + final String password = openSearchSinkConfig.getPassword(); + final AuthConfig authConfig = openSearchSinkConfig.getAuthConfig(); if (authConfig != null) { - if (username != null || password != null) { - throw new IllegalStateException("Deprecated username and password should not be set " + - "when authentication is configured."); - } builder = builder.withAuthConfig(authConfig); } else { if (username != null) { @@ -229,74 +203,69 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe builder = builder.withPassword(password); } } - final Integer socketTimeout = pluginSetting.getIntegerOrDefault(SOCKET_TIMEOUT, null); + final Integer socketTimeout = openSearchSinkConfig.getSocketTimeout(); if (socketTimeout != null) { builder = builder.withSocketTimeout(socketTimeout); } - final Integer connectTimeout = pluginSetting.getIntegerOrDefault(CONNECT_TIMEOUT, null); + final Integer connectTimeout = openSearchSinkConfig.getConnectTimeout(); if (connectTimeout != null) { builder = builder.withConnectTimeout(connectTimeout); } - Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); - boolean awsOptionUsed = false; builder.withAwsSigv4(false); - if (awsOption != null && !awsOption.isEmpty()) { - awsOptionUsed = true; - builder.withAwsSigv4(true); - builder.withAwsRegion((String)(awsOption.getOrDefault(AWS_REGION.substring(4), DEFAULT_AWS_REGION))); - builder.withAWSStsRoleArn((String)(awsOption.getOrDefault(AWS_STS_ROLE_ARN.substring(4), null))); - builder.withAWSStsExternalId((String)(awsOption.getOrDefault(AWS_STS_EXTERNAL_ID.substring(4), null))); - builder.withAwsStsHeaderOverrides((Map)awsOption.get(AWS_STS_HEADER_OVERRIDES.substring(4))); - builder.withServerless(OBJECT_MAPPER.convertValue( - awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); - - Map serverlessOptions = (Map) awsOption.get(SERVERLESS_OPTIONS); - if (serverlessOptions != null && !serverlessOptions.isEmpty()) { - builder.withServerlessNetworkPolicyName((String)(serverlessOptions.getOrDefault(NETWORK_POLICY_NAME, null))); - builder.withServerlessCollectionName((String)(serverlessOptions.getOrDefault(COLLECTION_NAME, null))); - builder.withServerlessVpceId((String)(serverlessOptions.getOrDefault(VPCE_ID, null))); - } - } else { - builder.withServerless(false); - } - boolean awsSigv4 = pluginSetting.getBooleanOrDefault(AWS_SIGV4, false); - final String awsOptionConflictMessage = String.format("%s option cannot be used along with %s option", AWS_SIGV4, AWS_OPTION); - if (awsSigv4) { - if (awsOptionUsed) { - throw new RuntimeException(awsOptionConflictMessage); + final AwsAuthenticationConfiguration awsAuthenticationConfiguration = openSearchSinkConfig.getAwsAuthenticationOptions(); + boolean awsSigv4 = openSearchSinkConfig.isAwsSigv4(); + if (awsAuthenticationConfiguration != null) { + builder = builder.withAwsSigv4(true) + .withAwsRegion(awsAuthenticationConfiguration.getAwsRegion().toString()) + .withAWSStsRoleArn(awsAuthenticationConfiguration.getAwsStsRoleArn()) + .withAWSStsExternalId(awsAuthenticationConfiguration.getAwsStsExternalId()) + .withAwsStsHeaderOverrides(awsAuthenticationConfiguration.getAwsStsHeaderOverrides()) + .withServerless(awsAuthenticationConfiguration.isServerlessCollection()); + + final ServerlessOptions serverlessOptions = awsAuthenticationConfiguration.getServerlessOptions(); + if (serverlessOptions != null) { + builder = builder.withServerlessNetworkPolicyName(serverlessOptions.getNetworkPolicyName()) + .withServerlessCollectionName(serverlessOptions.getCollectionName()) + .withServerlessVpceId(serverlessOptions.getVpceId()); + } + } else if (awsSigv4) { + builder = builder.withAwsSigv4(awsSigv4) + .withAwsRegion(openSearchSinkConfig.getAwsRegion()) + .withAWSStsRoleArn(openSearchSinkConfig.getAwsStsRoleArn()) + .withAWSStsExternalId(openSearchSinkConfig.getAwsStsExternalId()) + .withAwsStsHeaderOverrides(openSearchSinkConfig.getAwsStsHeaderOverrides()); + + final ServerlessOptions serverlessOptions = openSearchSinkConfig.getServerlessOptions(); + if (serverlessOptions != null) { + builder = builder.withServerlessNetworkPolicyName(serverlessOptions.getNetworkPolicyName()) + .withServerlessCollectionName(serverlessOptions.getCollectionName()) + .withServerlessVpceId(serverlessOptions.getVpceId()); } - builder.withAwsSigv4(true); - builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); - builder.withAWSStsRoleArn(pluginSetting.getStringOrDefault(AWS_STS_ROLE_ARN, null)); - builder.withAWSStsExternalId(pluginSetting.getStringOrDefault(AWS_STS_EXTERNAL_ID, null)); - builder.withAwsStsHeaderOverrides(pluginSetting.getTypedMap(AWS_STS_HEADER_OVERRIDES, String.class, String.class)); + } else { + builder.withServerless(false); } - final String certPath = pluginSetting.getStringOrDefault(CERT_PATH, null); - final boolean insecure = pluginSetting.getBooleanOrDefault(INSECURE, false); + final String certPath = openSearchSinkConfig.getCertPath(); + final boolean insecure = openSearchSinkConfig.isInsecure(); // Insecure == true will override configured certPath if (insecure) { builder.withInsecure(insecure); } else if (certPath != null) { builder.withCert(certPath); } - final String proxy = pluginSetting.getStringOrDefault(PROXY, null); - builder = builder.withProxy(proxy); + + final String proxy = openSearchSinkConfig.getProxy(); + if (proxy != null) { + builder = builder.withProxy(proxy); + } - final String distributionVersionName = pluginSetting.getStringOrDefault(DISTRIBUTION_VERSION, - DistributionVersion.DEFAULT.getVersion()); - final DistributionVersion distributionVersion = DistributionVersion.fromTypeName(distributionVersionName); - final boolean requestCompressionEnabled = pluginSetting.getBooleanOrDefault( - REQUEST_COMPRESSION_ENABLED, !DistributionVersion.ES6.equals(distributionVersion)); + final boolean requestCompressionEnabled = openSearchSinkConfig.getEnableRequestCompression(); builder = builder.withRequestCompressionEnabled(requestCompressionEnabled); return builder.build(); } - public String getPipelineName() { - return pipelineName; - } public RestHighLevelClient createClient(AwsCredentialsSupplier awsCredentialsSupplier) { final HttpHost[] httpHosts = new HttpHost[hosts.size()]; @@ -632,11 +601,6 @@ public Builder withProxy(final String proxy) { return this; } - public Builder withPipelineName(final String pipelineName) { - this.pipelineName = pipelineName; - return this; - } - public Builder withServerless(boolean serverless) { this.serverless = serverless; return this; diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java index b697fb26bf..bd3b268e2d 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresher.java @@ -3,8 +3,8 @@ import io.micrometer.core.instrument.Counter; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginComponentRefresher; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,7 +13,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; -public class OpenSearchClientRefresher implements PluginComponentRefresher { +public class OpenSearchClientRefresher implements PluginComponentRefresher { static final String CREDENTIALS_CHANGED = "credentialsChanged"; static final String CLIENT_REFRESH_ERRORS = "clientRefreshErrors"; private static final Logger LOG = LoggerFactory.getLogger(OpenSearchClientRefresher.class); @@ -54,8 +54,8 @@ public OpenSearchClient get() { } @Override - public void update(PluginSetting pluginSetting) { - final ConnectionConfiguration newConfig = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + public void update(OpenSearchSinkConfig openSearchSinkConfig) { + final ConnectionConfiguration newConfig = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); if (basicAuthChanged(newConfig)) { credentialsChangeCounter.increment(); readWriteLock.writeLock().lock(); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 71b8df54e4..fe7b1383c5 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -28,7 +28,7 @@ import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; -import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.exceptions.EventKeyNotFoundException; @@ -36,7 +36,6 @@ import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.plugin.PluginConfigObservable; -import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.AbstractSink; import org.opensearch.dataprepper.model.sink.Sink; @@ -46,6 +45,7 @@ import org.opensearch.dataprepper.plugins.common.opensearch.ServerlessOptionsFactory; import org.opensearch.dataprepper.plugins.dlq.DlqProvider; import org.opensearch.dataprepper.plugins.dlq.DlqWriter; +import org.opensearch.dataprepper.plugins.dlq.s3.S3DlqProvider; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.AccumulatingBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.BulkApiWrapper; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.BulkApiWrapperFactory; @@ -53,6 +53,9 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingCompressedBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingUncompressedBulkRequest; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.SerializedJson; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperation; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedBulkOperationConverter; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedDlqData; @@ -76,7 +79,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.StringJoiner; @@ -88,7 +90,7 @@ import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; -@DataPrepperPlugin(name = "opensearch", pluginType = Sink.class) +@DataPrepperPlugin(name = "opensearch", pluginType = Sink.class, pluginConfigurationType = OpenSearchSinkConfig.class) public class OpenSearchSink extends AbstractSink> { public static final String BULKREQUEST_LATENCY = "bulkRequestLatency"; public static final String BULKREQUEST_ERRORS = "bulkRequestErrors"; @@ -96,6 +98,7 @@ public class OpenSearchSink extends AbstractSink> { public static final String BULKREQUEST_SIZE_BYTES = "bulkRequestSizeBytes"; public static final String DYNAMIC_INDEX_DROPPED_EVENTS = "dynamicIndexDroppedEvents"; public static final String INVALID_VERSION_EXPRESSION_DROPPED_EVENTS = "dynamicDocumentVersionDroppedEvents"; + private static final String PLUGIN_NAME = "opensearch"; private static final Logger LOG = LoggerFactory.getLogger(OpenSearchSink.class); private static final int INITIALIZE_RETRY_WAIT_TIME_MS = 5000; @@ -119,7 +122,7 @@ public class OpenSearchSink extends AbstractSink> { private final String routing; private final String pipeline; private final String action; - private final List> actions; + private final List actions; private final String documentRootKey; private String configuredIndexAlias; private final ReentrantLock lock; @@ -136,7 +139,6 @@ public class OpenSearchSink extends AbstractSink> { private OpenSearchClientRefresher openSearchClientRefresher; private ObjectMapper objectMapper; private volatile boolean initialized; - private PluginSetting pluginSetting; private final SinkContext sinkContext; private final ExpressionEvaluator expressionEvaluator; @@ -149,15 +151,17 @@ public class OpenSearchSink extends AbstractSink> { @DataPrepperPluginConstructor public OpenSearchSink(final PluginSetting pluginSetting, - final PluginFactory pluginFactory, final SinkContext sinkContext, final ExpressionEvaluator expressionEvaluator, final AwsCredentialsSupplier awsCredentialsSupplier, - final PluginConfigObservable pluginConfigObservable) { + final PipelineDescription pipelineDescription, + final PluginConfigObservable pluginConfigObservable, + final OpenSearchSinkConfig openSearchSinkConfiguration) { super(pluginSetting, Integer.MAX_VALUE, INITIALIZE_RETRY_WAIT_TIME_MS); this.awsCredentialsSupplier = awsCredentialsSupplier; this.sinkContext = sinkContext != null ? sinkContext : new SinkContext(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); this.expressionEvaluator = expressionEvaluator; + this.pipeline = pipelineDescription.getPipelineName(); bulkRequestTimer = pluginMetrics.timer(BULKREQUEST_LATENCY); bulkRequestErrorsCounter = pluginMetrics.counter(BULKREQUEST_ERRORS); invalidActionErrorsCounter = pluginMetrics.counter(INVALID_ACTION_ERRORS); @@ -165,7 +169,7 @@ public OpenSearchSink(final PluginSetting pluginSetting, bulkRequestSizeBytesSummary = pluginMetrics.summary(BULKREQUEST_SIZE_BYTES); dynamicDocumentVersionDroppedEvents = pluginMetrics.counter(INVALID_VERSION_EXPRESSION_DROPPED_EVENTS); - this.openSearchSinkConfig = OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator); + this.openSearchSinkConfig = OpenSearchSinkConfiguration.readOSConfig(openSearchSinkConfiguration, expressionEvaluator); this.bulkSize = ByteSizeUnit.MB.toBytes(openSearchSinkConfig.getIndexConfiguration().getBulkSize()); this.flushTimeout = openSearchSinkConfig.getIndexConfiguration().getFlushTimeout(); this.indexType = openSearchSinkConfig.getIndexConfiguration().getIndexType(); @@ -173,28 +177,23 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.documentId = openSearchSinkConfig.getIndexConfiguration().getDocumentId(); this.routingField = openSearchSinkConfig.getIndexConfiguration().getRoutingField(); this.routing = openSearchSinkConfig.getIndexConfiguration().getRouting(); - this.pipeline = openSearchSinkConfig.getIndexConfiguration().getPipeline(); this.action = openSearchSinkConfig.getIndexConfiguration().getAction(); this.actions = openSearchSinkConfig.getIndexConfiguration().getActions(); this.documentRootKey = openSearchSinkConfig.getIndexConfiguration().getDocumentRootKey(); this.versionType = openSearchSinkConfig.getIndexConfiguration().getVersionType(); this.versionExpression = openSearchSinkConfig.getIndexConfiguration().getVersionExpression(); this.indexManagerFactory = new IndexManagerFactory(new ClusterSettingsParser()); - this.failedBulkOperationConverter = new FailedBulkOperationConverter(pluginSetting.getPipelineName(), pluginSetting.getName(), - pluginSetting.getName()); + this.failedBulkOperationConverter = new FailedBulkOperationConverter(pipeline, PLUGIN_NAME); this.initialized = false; this.lock = new ReentrantLock(true); - this.pluginSetting = pluginSetting; this.bulkRequestMap = new ConcurrentHashMap<>(); this.lastFlushTimeMap = new ConcurrentHashMap<>(); this.pluginConfigObservable = pluginConfigObservable; this.objectMapper = new ObjectMapper(); - final Optional dlqConfig = openSearchSinkConfig.getRetryConfiguration().getDlq(); + final Optional dlqConfig = openSearchSinkConfig.getRetryConfiguration().getDlq(); if (dlqConfig.isPresent()) { - final PluginSetting dlqPluginSetting = new PluginSetting(dlqConfig.get().getPluginName(), dlqConfig.get().getPluginSettings()); - dlqPluginSetting.setPipelineName(pluginSetting.getPipelineName()); - dlqProvider = pluginFactory.loadPlugin(DlqProvider.class, dlqPluginSetting); + dlqProvider = new S3DlqProvider(dlqConfig.get().getS3DlqWriterConfig()); } } @@ -235,7 +234,7 @@ private void doInitializeInternal() throws IOException { openSearchClientRefresher = new OpenSearchClientRefresher( pluginMetrics, connectionConfiguration, clientFunction); pluginConfigObservable.addPluginConfigObserver( - newPluginSetting -> openSearchClientRefresher.update((PluginSetting) newPluginSetting)); + newOpenSearchSinkConfig -> openSearchClientRefresher.update((OpenSearchSinkConfig) newOpenSearchSinkConfig)); configuredIndexAlias = openSearchSinkConfig.getIndexConfiguration().getIndexAlias(); final IndexTemplateAPIWrapper indexTemplateAPIWrapper = IndexTemplateAPIWrapperFactory.getWrapper( openSearchSinkConfig.getIndexConfiguration(), openSearchClient); @@ -248,8 +247,8 @@ private void doInitializeInternal() throws IOException { dlqFileWriter = Files.newBufferedWriter(Paths.get(dlqFile), StandardOpenOption.CREATE, StandardOpenOption.APPEND); } else if (dlqProvider != null) { Optional potentialDlq = dlqProvider.getDlqWriter(new StringJoiner(MetricNames.DELIMITER) - .add(pluginSetting.getPipelineName()) - .add(pluginSetting.getName()).toString()); + .add(pipeline) + .add(PLUGIN_NAME).toString()); dlqWriter = potentialDlq.isPresent() ? potentialDlq.get() : null; } @@ -280,7 +279,8 @@ private void doInitializeInternal() throws IOException { pluginMetrics, maxRetries, bulkRequestSupplier, - pluginSetting); + pipeline, + PLUGIN_NAME); this.initialized = true; LOG.info("Initialized OpenSearch sink"); @@ -431,9 +431,9 @@ public void doOutput(final Collection> records) { String eventAction = action; if (actions != null) { - for (final Map actionEntry: actions) { - final String condition = (String)actionEntry.get("when"); - eventAction = (String)actionEntry.get("type"); + for (final ActionConfiguration actionEntry: actions) { + final String condition = actionEntry.getWhen(); + eventAction = actionEntry.getType(); if (condition != null && expressionEvaluator.evaluateConditional(condition, event)) { break; @@ -501,7 +501,7 @@ SerializedJson getDocument(final Event event) { String routingValue = null; if (routingField != null) { - routingValue = event.get(routingField, String.class); + routingValue = event.get(routingField, String.class); } else if (routing != null) { try { routingValue = event.formatString(routing, expressionEvaluator); @@ -510,21 +510,9 @@ SerializedJson getDocument(final Event event) { } } - String pipelineValue = null; - if (pipeline != null) { - try { - pipelineValue = event.formatString(pipeline, expressionEvaluator); - } catch (final ExpressionEvaluationException | EventKeyNotFoundException e) { - LOG.error("Unable to construct pipeline with format {}", pipeline, e); - } - if (StringUtils.isEmpty(pipelineValue) || StringUtils.isBlank(pipelineValue)) { - pipelineValue = null; - } - } - final String document = DocumentBuilder.build(event, documentRootKey, sinkContext.getTagsTargetKey(), sinkContext.getIncludeKeys(), sinkContext.getExcludeKeys()); - return SerializedJson.fromStringAndOptionals(document, docId, routingValue, pipelineValue); + return SerializedJson.fromStringAndOptionals(document, docId, routingValue, null); } private void flushBatch(AccumulatingBulkRequest accumulatingBulkRequest) { @@ -566,7 +554,7 @@ private void logFailureForDlqObjects(final List dlqObjects, final Thr }); } else if (dlqWriter != null) { try { - dlqWriter.write(dlqObjects, pluginSetting.getPipelineName(), pluginSetting.getName()); + dlqWriter.write(dlqObjects, pipeline, PLUGIN_NAME); dlqObjects.forEach((dlqObject) -> { dlqObject.releaseEventHandle(true); }); @@ -646,9 +634,9 @@ private DlqObject createDlqObjectFromEvent(final Event event, .withIndex(index) .withMessage(message) .build()) - .withPluginName(pluginSetting.getName()) - .withPipelineName(pluginSetting.getPipelineName()) - .withPluginId(pluginSetting.getName()) + .withPluginName(PLUGIN_NAME) + .withPipelineName(pipeline) + .withPluginId(PLUGIN_NAME) .build(); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java index 4ec8f17bc8..84bf3c28de 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfiguration.java @@ -5,9 +5,9 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; import static com.google.common.base.Preconditions.checkNotNull; @@ -40,15 +40,15 @@ private OpenSearchSinkConfiguration( this.retryConfiguration = retryConfiguration; } - public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting) { - return readESConfig(pluginSetting, null); + public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig) { + return readOSConfig(openSearchSinkConfig, null); } - public static OpenSearchSinkConfiguration readESConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { + public static OpenSearchSinkConfiguration readOSConfig(final OpenSearchSinkConfig openSearchSinkConfig, final ExpressionEvaluator expressionEvaluator) { final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator); - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig, expressionEvaluator); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(openSearchSinkConfig); return new OpenSearchSinkConfiguration(connectionConfiguration, indexConfiguration, retryConfiguration); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java index fa397e9271..a3c45d6a2d 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfiguration.java @@ -5,11 +5,9 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import org.opensearch.dataprepper.model.configuration.PluginModel; -import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Optional; import static com.google.common.base.Preconditions.checkNotNull; @@ -21,13 +19,13 @@ public class RetryConfiguration { private final String dlqFile; private final int maxRetries; - private final PluginModel dlq; + private final DlqConfiguration dlq; public String getDlqFile() { return dlqFile; } - public Optional getDlq() { + public Optional getDlq() { return Optional.ofNullable(dlq); } @@ -42,7 +40,7 @@ public static class Builder { private String dlqFile; private int maxRetries = Integer.MAX_VALUE; - private PluginModel dlq; + private DlqConfiguration dlq; public Builder withDlqFile(final String dlqFile) { checkNotNull(dlqFile, "dlqFile cannot be null."); @@ -56,7 +54,7 @@ public Builder withMaxRetries(final Integer maxRetries) { return this; } - public Builder withDlq(final PluginModel dlq) { + public Builder withDlq(final DlqConfiguration dlq) { checkNotNull(dlq, "dlq cannot be null"); this.dlq = dlq; return this; @@ -72,30 +70,20 @@ private RetryConfiguration(final Builder builder) { this.dlq = builder.dlq; } - public static RetryConfiguration readRetryConfig(final PluginSetting pluginSetting) { - RetryConfiguration.Builder builder = new RetryConfiguration.Builder(); - final String dlqFile = (String) pluginSetting.getAttributeFromSettings(DLQ_FILE); - if (dlqFile != null) { - builder = builder.withDlqFile(dlqFile); - } - final Integer maxRetries = pluginSetting.getIntegerOrDefault(MAX_RETRIES, null); - if (maxRetries != null) { - builder = builder.withMaxRetries(maxRetries); - } - final LinkedHashMap> dlq = (LinkedHashMap) pluginSetting.getAttributeFromSettings(DLQ); - if (dlq != null) { - if (dlqFile != null) { - final String dlqOptionConflictMessage = String.format("%s option cannot be used along with %s option", DLQ_FILE, DLQ); - throw new RuntimeException(dlqOptionConflictMessage); - } - if (dlq.size() != 1) { - throw new RuntimeException("dlq option must declare exactly one dlq configuration"); - } - final Map.Entry> entry = dlq.entrySet().stream() - .findFirst() - .get(); - builder = builder.withDlq(new PluginModel(entry.getKey(), entry.getValue())); - } - return builder.build(); + public static RetryConfiguration readRetryConfig(final OpenSearchSinkConfig openSearchSinkConfig) { + RetryConfiguration.Builder builder = new RetryConfiguration.Builder(); + final String dlqFile = openSearchSinkConfig.getDlqFile(); + if (dlqFile != null) { + builder = builder.withDlqFile(dlqFile); + } + final Integer maxRetries = openSearchSinkConfig.getMaxRetries(); + if (maxRetries != null) { + builder = builder.withMaxRetries(maxRetries); + } + final DlqConfiguration dlqConfiguration = openSearchSinkConfig.getDlq(); + if (dlqConfiguration != null) { + builder = builder.withDlq(dlqConfiguration); + } + return builder.build(); } } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java new file mode 100644 index 0000000000..02c92ea609 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ActionConfiguration.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; +import lombok.Getter; +import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; + +public class ActionConfiguration { + @Size(min = 1, message = "type cannot be empty") + @JsonProperty("type") + private OpenSearchBulkActions type; + + @AssertTrue(message = "action must be one of index, create, update, upsert, delete") + boolean isActionValid() { + if (type == null) { //type will be null if the string doesn't match one of the enums + return false; + } + return true; + } + + public String getType() { + return type.toString(); + } + + @Getter + @JsonProperty("when") + private String when; + +} diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AuthConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AuthConfig.java new file mode 100644 index 0000000000..e41f99043c --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AuthConfig.java @@ -0,0 +1,19 @@ +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class AuthConfig { + @JsonProperty("username") + private String username; + + @JsonProperty("password") + private String password; + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } +} diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java new file mode 100644 index 0000000000..5e8bcd8b63 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/AwsAuthenticationConfiguration.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import software.amazon.awssdk.regions.Region; + +import java.util.Map; + +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_AWS_REGION; + +public class AwsAuthenticationConfiguration { + + @JsonProperty("region") + @Size(min = 1, message = "Region cannot be empty string") + private String awsRegion = DEFAULT_AWS_REGION; + + @JsonProperty("sts_role_arn") + @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") + private String awsStsRoleArn; + + @JsonProperty("sts_external_id") + @Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") + private String awsStsExternalId; + + @JsonProperty("sts_header_overrides") + @Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override") + private Map awsStsHeaderOverrides; + + @JsonProperty("serverless") + private boolean serverless = false; + + @JsonProperty("serverless_options") + private ServerlessOptions serverlessOptions; + + public String getAwsStsRoleArn() { + return awsStsRoleArn; + } + + public String getAwsStsExternalId() { + return awsStsExternalId; + } + + public Region getAwsRegion() { + return awsRegion != null ? Region.of(awsRegion) : null; + } + + public Map getAwsStsHeaderOverrides() { + return awsStsHeaderOverrides; + } + + public boolean isServerlessCollection() { + return serverless; + } + + public ServerlessOptions getServerlessOptions() { + return serverlessOptions; + } +} + diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java new file mode 100644 index 0000000000..2ba4f9805d --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/DlqConfiguration.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import lombok.Getter; +import org.opensearch.dataprepper.plugins.dlq.s3.S3DlqWriterConfig; + +public class DlqConfiguration { + @Getter + @Valid + @JsonProperty("s3") + private S3DlqWriterConfig s3DlqWriterConfig; +} diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java new file mode 100644 index 0000000000..2bde73c184 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/OpenSearchSinkConfig.java @@ -0,0 +1,311 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; +import lombok.Getter; +import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; +import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; +import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class OpenSearchSinkConfig { + public static final long DEFAULT_BULK_SIZE = 5L; + public static final boolean DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION = false; + public static final int DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = 2; + public static final long DEFAULT_FLUSH_TIMEOUT = 60_000L; + public static final String DEFAULT_AWS_REGION = "us-east-1"; + @Getter + @JsonProperty("hosts") + private List hosts; + + @Getter + @JsonProperty("username") + private String username = null; + + @Getter + @JsonProperty("authentication") + private AuthConfig authConfig; + + @AssertTrue(message = "username and password should not be set when authentication is configured.") + public boolean isAuthConfigValid() { + return authConfig == null || (username == null && password == null); + } + + @Getter + @JsonProperty("password") + private String password = null; + + @Getter + @JsonProperty("socket_timeout") + private Integer socketTimeout = null; + + @Getter + @JsonProperty("connect_timeout") + private Integer connectTimeout = null; + + @Getter + @JsonProperty("aws") + @Valid + private AwsAuthenticationConfiguration awsAuthenticationOptions; + + @Getter + @JsonProperty("cert") + private String certPath = null; + + @Getter + @JsonProperty("insecure") + private boolean insecure = false; + + @Getter + @JsonProperty("proxy") + private String proxy = null; + + @Getter + @JsonProperty("distribution_version") + private String distributionVersion = DistributionVersion.DEFAULT.getVersion(); + + @JsonProperty("enable_request_compression") + private Boolean enableRequestCompression = null; + + public boolean getEnableRequestCompression() { + final DistributionVersion distributionVersion = DistributionVersion.fromTypeName(getDistributionVersion()); + return Objects.requireNonNullElse(enableRequestCompression, !DistributionVersion.ES6.equals(distributionVersion)); + } + + @Getter + @JsonProperty("index") + private String indexAlias = null; + + @Getter + @JsonProperty("index_type") + private String indexType = null; + + @Getter + @JsonProperty("template_type") + private String templateType = TemplateType.V1.getTypeName(); + + @Getter + @JsonProperty("template_file") + private String templateFile = null; + + @Getter + @JsonProperty("template_content") + private String templateContent = null; + + @Getter + @JsonProperty("number_of_shards") + private Integer numShards = 0; + + @Getter + @JsonProperty("number_of_replicas") + private Integer numReplicas = 0; + + @Getter + @JsonProperty("bulk_size") + private Long bulkSize = DEFAULT_BULK_SIZE; + + @Getter + @JsonProperty("estimate_bulk_size_using_compression") + private boolean estimateBulkSizeUsingCompression = DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION; + + @Getter + @JsonProperty("max_local_compressions_for_estimation") + private Integer maxLocalCompressionsForEstimation = DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION; + + @Getter + @JsonProperty("flush_timeout") + private Long flushTimeout = DEFAULT_FLUSH_TIMEOUT; + + @Getter + @JsonProperty("document_version_type") + private String versionType = null; + + @Getter + @JsonProperty("document_version") + private String versionExpression = null; + + @Getter + @JsonProperty("normalize_index") + private boolean normalizeIndex = false; + + @Getter + @JsonProperty("document_id") + private String documentId = null; + + @Getter + @JsonProperty("routing") + private String routing = null; + + @Getter + @JsonProperty("ism_policy_file") + private String ismPolicyFile = null; + + @JsonProperty("action") + private OpenSearchBulkActions action = OpenSearchBulkActions.INDEX; + + @AssertTrue(message = "action must be one of index, create, update, upsert, delete") + boolean isActionValid() { + if (action == null) { //action will be null if the string doesn't match one of the enums + return false; + } + return true; + } + + public String getAction() { + return action.toString(); + } + + @Getter + @Valid + @JsonProperty("actions") + private List actions = null; + + @Getter + @JsonProperty("document_root_key") + private String documentRootKey = null; + + @Getter + @JsonProperty("dlq_file") + private String dlqFile = null; + + @Getter + @JsonProperty("max_retries") + private Integer maxRetries = null; + + @Getter + @JsonProperty("dlq") + private DlqConfiguration dlq; + + @AssertTrue(message = "dlq_file option cannot be used along with dlq option") + public boolean isDlqValid() { + if (dlq != null) { + if (dlqFile!= null) { + return false; + } + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("aws_sigv4") + private boolean awsSigv4 = false; + + @Deprecated + @AssertTrue(message = "aws_sigv4 option cannot be used along with aws option. It is preferred to only use aws option as aws_sigv4 is deprecated.") + public boolean isNotAwsSigv4AndAwsOption() { + if (awsAuthenticationOptions != null && awsSigv4) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("aws_region") + @Size(min = 1, message = "Region cannot be empty string") + private String awsRegion = DEFAULT_AWS_REGION; + + @Deprecated + @Getter + @JsonProperty("aws_sts_role_arn") + @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") + private String awsStsRoleArn = null; + + @Deprecated + @AssertTrue(message = "aws_sts_role_arn cannot be used along with aws option. It is preferred to only use aws option as aws_sts_role_arn is deprecated.") + public boolean isNotAwsStsRoleArnAndAws() { + if (awsAuthenticationOptions != null && awsStsRoleArn != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("aws_sts_external_id") + @Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") + private String awsStsExternalId = null; + + @Deprecated + @AssertTrue(message = "aws_sts_external_id cannot be used along with aws option. It is preferred to only use aws option as aws_sts_external_id is deprecated.") + public boolean isNotAwsStsExternalIdAndAws() { + if (awsAuthenticationOptions != null && awsStsExternalId != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("aws_sts_header_overrides") + @Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override") + private Map awsStsHeaderOverrides; + + @Deprecated + @AssertTrue(message = "aws_sts_header_overrides cannot be used along with aws option. It is preferred to only use aws option as aws_sts_header_overrides is deprecated.") + public boolean isNotAwsStsHeaderOverridesAndAws() { + if (awsAuthenticationOptions != null && awsStsHeaderOverrides != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("serverless") + private boolean serverless = false; + + @Deprecated + @AssertTrue(message = "serverless cannot be used along with aws option. It is preferred to only use aws option as serverless is deprecated.") + public boolean isNotServerlessAndAws() { + if (awsAuthenticationOptions != null && serverless) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("serverless_options") + private ServerlessOptions serverlessOptions = null; + + @Deprecated + @AssertTrue(message = "serverless_options cannot be used along with aws option. It is preferred to only use aws option as serverless_options is deprecated.") + public boolean isNotServerlessOptionsAndAws() { + if (awsAuthenticationOptions != null && serverlessOptions != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("document_id_field") + private String documentIdField = null; + + @Deprecated + @AssertTrue(message = "Both document_id_field and document_id cannot be used at the same time. It is preferred to only use document_id as document_id_field is deprecated.") + public boolean isNotDocumentIdFieldAndDocumentId() { + if (documentId != null && documentIdField != null) { + return false; + } + return true; + } + + @Deprecated + @Getter + @JsonProperty("routing_field") + private String routingField = null; +} + diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java new file mode 100644 index 0000000000..2899dfd419 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/configuration/ServerlessOptions.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.sink.opensearch.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; + +public class ServerlessOptions { + + @Size(min = 1, message = "network_policy_name cannot be empty") + @JsonProperty("network_policy_name") + private String networkPolicyName; + + @Size(min = 1, message = "collection_name cannot be empty") + @JsonProperty("collection_name") + private String collectionName; + + @Size(min = 1, message = "vpce_id cannot be empty") + @JsonProperty("vpce_id") + private String vpceId; + + public ServerlessOptions() { + + } + + public ServerlessOptions(String networkPolicyName, String collectionName, String vpceId) { + this.networkPolicyName = networkPolicyName; + this.collectionName = collectionName; + this.vpceId = vpceId; + } + + public String getNetworkPolicyName() { + return networkPolicyName; + } + + public String getCollectionName() { + return collectionName; + } + + public String getVpceId() { + return vpceId; + } + +} + diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverter.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverter.java index 31b24fec8f..6c2e91b5b3 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverter.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverter.java @@ -19,15 +19,12 @@ public class FailedBulkOperationConverter { private final String pluginName; private final String pipelineName; - private final String pluginId; - public FailedBulkOperationConverter(final String pipelineName, final String pluginName, final String pluginId) { + public FailedBulkOperationConverter(final String pipelineName, final String pluginName) { Objects.requireNonNull(pipelineName); - Objects.requireNonNull(pluginId); Objects.requireNonNull(pluginName); this.pluginName = pluginName; this.pipelineName = pipelineName; - this.pluginId = pluginId; } public DlqObject convertToDlqObject(final FailedBulkOperation failedBulkOperation) { @@ -54,7 +51,7 @@ public DlqObject convertToDlqObject(final FailedBulkOperation failedBulkOperatio .withFailedData(failedDlqDataBuilder.build()) .withPluginName(pluginName) .withPipelineName(pipelineName) - .withPluginId(pluginId) + .withPluginId(pluginName) .withEventHandle(bulkOperationWithHandle.getEventHandle()) .build(); } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 10dd892296..7aaa779181 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -10,10 +10,12 @@ import org.apache.commons.lang3.EnumUtils; import org.opensearch.client.opensearch._types.VersionType; import org.opensearch.dataprepper.expression.ExpressionEvaluator; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.ActionConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AwsAuthenticationConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.FileReader; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.S3ClientProvider; import org.opensearch.dataprepper.plugins.sink.opensearch.s3.S3FileReader; @@ -30,12 +32,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_BULK_SIZE; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_FLUSH_TIMEOUT; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION; public class IndexConfiguration { private static final Logger LOG = LoggerFactory.getLogger(IndexConfiguration.class); @@ -59,22 +64,13 @@ public class IndexConfiguration { public static final String ROUTING = "routing"; public static final String PIPELINE = "pipeline"; public static final String ISM_POLICY_FILE = "ism_policy_file"; - public static final long DEFAULT_BULK_SIZE = 5L; - public static final boolean DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION = false; - public static final int DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = 2; - public static final long DEFAULT_FLUSH_TIMEOUT = 60_000L; public static final String ACTION = "action"; public static final String ACTIONS = "actions"; - public static final String S3_AWS_REGION = "s3_aws_region"; - public static final String S3_AWS_STS_ROLE_ARN = "s3_aws_sts_role_arn"; - public static final String S3_AWS_STS_EXTERNAL_ID = "s3_aws_sts_external_id"; public static final String SERVERLESS = "serverless"; public static final String DISTRIBUTION_VERSION = "distribution_version"; public static final String AWS_OPTION = "aws"; public static final String DOCUMENT_ROOT_KEY = "document_root_key"; public static final String DOCUMENT_VERSION_EXPRESSION = "document_version"; - public static final String DOCUMENT_VERSION_TYPE = "document_version_type"; - public static final String NORMALIZE_INDEX = "normalize_index"; private IndexType indexType; private TemplateType templateType; @@ -82,7 +78,6 @@ public class IndexConfiguration { private final Map indexTemplate; private final String documentIdField; private final String documentId; - private final String pipeline; private final String routingField; private final String routing; private final long bulkSize; @@ -91,7 +86,7 @@ public class IndexConfiguration { private final long flushTimeout; private final Optional ismPolicyFile; private final String action; - private final List> actions; + private final List actions; private final String s3AwsRegion; private final String s3AwsStsRoleArn; private final String s3AwsExternalId; @@ -104,7 +99,6 @@ public class IndexConfiguration { private final boolean normalizeIndex; private static final String S3_PREFIX = "s3://"; - private static final String DEFAULT_AWS_REGION = "us-east-1"; @SuppressWarnings("unchecked") private IndexConfiguration(final Builder builder) { @@ -149,7 +143,6 @@ private IndexConfiguration(final Builder builder) { this.flushTimeout = builder.flushTimeout; this.routingField = builder.routingField; this.routing = builder.routing; - this.pipeline = builder.pipeline; String documentIdField = builder.documentIdField; String documentId = builder.documentId; @@ -188,81 +181,46 @@ private void determineTemplateType(Builder builder) { } } - public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting) { - return readIndexConfig(pluginSetting, null); + public static IndexConfiguration readIndexConfig(final OpenSearchSinkConfig openSearchSinkConfig) { + return readIndexConfig(openSearchSinkConfig, null); } - public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetting, final ExpressionEvaluator expressionEvaluator) { + public static IndexConfiguration readIndexConfig(final OpenSearchSinkConfig openSearchSinkConfig, final ExpressionEvaluator expressionEvaluator) { IndexConfiguration.Builder builder = new IndexConfiguration.Builder(); - final String indexAlias = pluginSetting.getStringOrDefault(INDEX_ALIAS, null); + final String indexAlias = openSearchSinkConfig.getIndexAlias(); if (indexAlias != null) { builder = builder.withIndexAlias(indexAlias); } - final String indexType = pluginSetting.getStringOrDefault(INDEX_TYPE, null); - if(indexType != null) { + final String indexType = openSearchSinkConfig.getIndexType(); + if (indexType != null) { builder = builder.withIndexType(indexType); } - final String templateType = pluginSetting.getStringOrDefault(TEMPLATE_TYPE, TemplateType.V1.getTypeName()); - if(templateType != null) { + final String templateType = openSearchSinkConfig.getTemplateType(); + if (templateType != null) { builder = builder.withTemplateType(templateType); } - final String templateFile = pluginSetting.getStringOrDefault(TEMPLATE_FILE, null); + final String templateFile = openSearchSinkConfig.getTemplateFile(); if (templateFile != null) { builder = builder.withTemplateFile(templateFile); } - - final String templateContent = pluginSetting.getStringOrDefault(TEMPLATE_CONTENT, null); + final String templateContent = openSearchSinkConfig.getTemplateContent(); if (templateContent != null) { builder = builder.withTemplateContent(templateContent); } - if (templateContent != null && templateFile != null) { LOG.warn("Both template_content and template_file are configured. Only template_content will be used"); } - builder = builder.withNumShards(pluginSetting.getIntegerOrDefault(NUM_SHARDS, 0)); - builder = builder.withNumReplicas(pluginSetting.getIntegerOrDefault(NUM_REPLICAS, 0)); - final Long batchSize = pluginSetting.getLongOrDefault(BULK_SIZE, DEFAULT_BULK_SIZE); - builder = builder.withBulkSize(batchSize); - final boolean estimateBulkSizeUsingCompression = - pluginSetting.getBooleanOrDefault(ESTIMATE_BULK_SIZE_USING_COMPRESSION, DEFAULT_ESTIMATE_BULK_SIZE_USING_COMPRESSION); - builder = builder.withEstimateBulkSizeUsingCompression(estimateBulkSizeUsingCompression); - - final int maxLocalCompressionsForEstimation = - pluginSetting.getIntegerOrDefault(MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION, DEFAULT_MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION); - builder = builder.withMaxLocalCompressionsForEstimation(maxLocalCompressionsForEstimation); - - final long flushTimeout = pluginSetting.getLongOrDefault(FLUSH_TIMEOUT, DEFAULT_FLUSH_TIMEOUT); - builder = builder.withFlushTimeout(flushTimeout); - final String documentIdField = pluginSetting.getStringOrDefault(DOCUMENT_ID_FIELD, null); - final String documentId = pluginSetting.getStringOrDefault(DOCUMENT_ID, null); - - final String versionExpression = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_EXPRESSION, null); - final String versionType = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_TYPE, null); - final boolean normalizeIndex = pluginSetting.getBooleanOrDefault(NORMALIZE_INDEX, false); - builder = builder.withNormalizeIndex(normalizeIndex); - - builder = builder.withVersionExpression(versionExpression); - if (versionExpression != null && (!expressionEvaluator.isValidFormatExpression(versionExpression))) { - throw new InvalidPluginConfigurationException( - String.format("document_version \"%s\" is not a valid format expression.", versionExpression)); - } - - builder = builder.withVersionType(versionType); - - if (Objects.nonNull(documentIdField) && Objects.nonNull(documentId)) { - throw new InvalidPluginConfigurationException("Both document_id_field and document_id cannot be used at the same time. It is preferred to only use document_id as document_id_field is deprecated."); - } - + final String documentIdField = openSearchSinkConfig.getDocumentIdField(); + final String documentId = openSearchSinkConfig.getDocumentId(); if (documentIdField != null) { LOG.warn("document_id_field is deprecated in favor of document_id, and support for document_id_field will be removed in a future major version release."); builder = builder.withDocumentIdField(documentIdField); } else if (documentId != null) { builder = builder.withDocumentId(documentId); } - - final String routingField = pluginSetting.getStringOrDefault(ROUTING_FIELD, null); - final String routing = pluginSetting.getStringOrDefault(ROUTING, null); + final String routingField = openSearchSinkConfig.getRoutingField(); + final String routing = openSearchSinkConfig.getRouting(); if (routingField != null) { LOG.warn("routing_field is deprecated in favor of routing, and support for routing_field will be removed in a future major version release."); builder = builder.withRoutingField(routingField); @@ -270,51 +228,55 @@ public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetti builder = builder.withRouting(routing); } - final String pipeline = pluginSetting.getStringOrDefault(PIPELINE, null); - if (pipeline != null) { - builder = builder.withPipeline(pipeline); - } + builder = builder.withNumShards(openSearchSinkConfig.getNumShards()) + .withNumReplicas(openSearchSinkConfig.getNumReplicas()) + .withBulkSize(openSearchSinkConfig.getBulkSize()) + .withEstimateBulkSizeUsingCompression(openSearchSinkConfig.isEstimateBulkSizeUsingCompression()) + .withMaxLocalCompressionsForEstimation(openSearchSinkConfig.getMaxLocalCompressionsForEstimation()) + .withFlushTimeout(openSearchSinkConfig.getFlushTimeout()) + .withVersionType(openSearchSinkConfig.getVersionType()) + .withNormalizeIndex(openSearchSinkConfig.isNormalizeIndex()) + .withIsmPolicyFile(openSearchSinkConfig.getIsmPolicyFile()) + .withDocumentRootKey(openSearchSinkConfig.getDocumentRootKey()) + .withDistributionVersion(openSearchSinkConfig.getDistributionVersion()); + - final String ismPolicyFile = pluginSetting.getStringOrDefault(ISM_POLICY_FILE, null); - builder = builder.withIsmPolicyFile(ismPolicyFile); + final String versionExpression = openSearchSinkConfig.getVersionExpression(); + builder = builder.withVersionExpression(versionExpression); + if (versionExpression != null && (!expressionEvaluator.isValidFormatExpression(versionExpression))) { + throw new InvalidPluginConfigurationException( + String.format("document_version \"%s\" is not a valid format expression.", versionExpression)); + } - List> actionsList = pluginSetting.getTypedListOfMaps(ACTIONS, String.class, Object.class); + List actionList = openSearchSinkConfig.getActions(); + if (actionList != null) { + builder = builder.withActions(actionList, expressionEvaluator); + } else { + builder = builder.withAction(openSearchSinkConfig.getAction(), expressionEvaluator); + } - if (actionsList != null) { - builder.withActions(actionsList, expressionEvaluator); + AwsAuthenticationConfiguration awsAuthenticationConfiguration = openSearchSinkConfig.getAwsAuthenticationOptions(); + if (awsAuthenticationConfiguration != null) { + builder = builder.withServerless(awsAuthenticationConfiguration.isServerlessCollection()); } else { - builder.withAction(pluginSetting.getStringOrDefault(ACTION, OpenSearchBulkActions.INDEX.toString()), expressionEvaluator); + builder = builder.withServerless(false); } if ((builder.templateFile != null && builder.templateFile.startsWith(S3_PREFIX)) - || (builder.ismPolicyFile.isPresent() && builder.ismPolicyFile.get().startsWith(S3_PREFIX))) { - builder.withS3AwsRegion(pluginSetting.getStringOrDefault(S3_AWS_REGION, DEFAULT_AWS_REGION)); - builder.withS3AWSStsRoleArn(pluginSetting.getStringOrDefault(S3_AWS_STS_ROLE_ARN, null)); - builder.withS3AWSStsExternalId(pluginSetting.getStringOrDefault(S3_AWS_STS_EXTERNAL_ID, null)); + || (builder.ismPolicyFile.isPresent() && builder.ismPolicyFile.get().startsWith(S3_PREFIX))) { + builder.withS3AwsRegion(awsAuthenticationConfiguration.getAwsRegion().toString()); + builder.withS3AWSStsRoleArn(awsAuthenticationConfiguration.getAwsStsRoleArn()); + builder.withS3AWSStsExternalId(awsAuthenticationConfiguration.getAwsStsExternalId()); final S3ClientProvider clientProvider = new S3ClientProvider( - builder.s3AwsRegion, builder.s3AwsStsRoleArn, builder.s3AwsStsExternalId); + builder.s3AwsRegion, builder.s3AwsStsRoleArn, builder.s3AwsStsExternalId); builder.withS3Client(clientProvider.buildS3Client()); } - Map awsOption = pluginSetting.getTypedMap(AWS_OPTION, String.class, Object.class); - if (awsOption != null && !awsOption.isEmpty()) { - builder.withServerless(OBJECT_MAPPER.convertValue( - awsOption.getOrDefault(SERVERLESS, false), Boolean.class)); - } else { - builder.withServerless(false); - } - - final String documentRootKey = pluginSetting.getStringOrDefault(DOCUMENT_ROOT_KEY, null); - builder.withDocumentRootKey(documentRootKey); - - final String distributionVersion = pluginSetting.getStringOrDefault(DISTRIBUTION_VERSION, - DistributionVersion.DEFAULT.getVersion()); - builder.withDistributionVersion(distributionVersion); - return builder.build(); } + public IndexType getIndexType() { return indexType; } @@ -345,10 +307,6 @@ public String getRouting() { return routing; } - public String getPipeline() { - return pipeline; - } - public long getBulkSize() { return bulkSize; } @@ -373,7 +331,7 @@ public String getAction() { return action; } - public List> getActions() { + public List getActions() { return actions; } @@ -485,7 +443,7 @@ public static class Builder { private long flushTimeout = DEFAULT_FLUSH_TIMEOUT; private Optional ismPolicyFile; private String action; - private List> actions; + private List actions; private String s3AwsRegion; private String s3AwsStsRoleArn; private String s3AwsStsExternalId; @@ -552,11 +510,6 @@ public Builder withRouting(final String routing) { return this; } - public Builder withPipeline(final String pipeline) { - this.pipeline = pipeline; - return this; - } - public Builder withBulkSize(final long bulkSize) { this.bulkSize = bulkSize; return this; @@ -599,9 +552,9 @@ public Builder withAction(final String action, final ExpressionEvaluator express return this; } - public Builder withActions(final List> actions, final ExpressionEvaluator expressionEvaluator) { - for (final Map actionMap: actions) { - String action = (String)actionMap.get("type"); + public Builder withActions(final List actions, final ExpressionEvaluator expressionEvaluator) { + for (final ActionConfiguration actionConfig: actions) { + String action = actionConfig.getType(); if (action != null) { checkArgument((EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, action) || (action.contains("${") && expressionEvaluator.isValidFormatExpression(action))), "action \"" + action + "\". action must be one of the following: " + Arrays.stream(OpenSearchBulkActions.values()).collect(Collectors.toList())); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/TemplateType.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/TemplateType.java index 0c48d8ef41..7b8020a19a 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/TemplateType.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/TemplateType.java @@ -46,7 +46,7 @@ public TemplateStrategy createTemplateStrategy(final IndexTemplateAPIWrapper ind return factoryFunction.apply(indexTemplateAPIWrapper); } - String getTypeName() { + public String getTypeName() { return name; } } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AuthConfig.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AuthConfig.java deleted file mode 100644 index 8893535eb9..0000000000 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/configuration/AuthConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.opensearch.dataprepper.plugins.source.opensearch.configuration; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.opensearch.dataprepper.model.configuration.PluginSetting; - -import java.util.Map; - -public class AuthConfig { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final String AUTHENTICATION = "authentication"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - private String username; - - private String password; - - public String getUsername() { - return username; - } - - public String getPassword() { - return password; - } - - public static AuthConfig readAuthConfig(final PluginSetting pluginSetting) { - final Map authConfigMap = - pluginSetting.getTypedMap(AUTHENTICATION, String.class, Object.class); - return authConfigMap.isEmpty() ? null : OBJECT_MAPPER.convertValue(authConfigMap, AuthConfig.class); - } -} diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java index cc05514502..d30238acb6 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java @@ -124,7 +124,8 @@ public BulkRetryStrategy createObjectUnderTest( pluginMetrics, Integer.MAX_VALUE, bulkRequestSupplier, - pluginSetting); + PIPELINE_NAME, + PLUGIN_NAME); } public BulkRetryStrategy createObjectUnderTest( @@ -139,7 +140,8 @@ public BulkRetryStrategy createObjectUnderTest( pluginMetrics, maxRetries, bulkRequestSupplier, - pluginSetting); + PIPELINE_NAME, + PLUGIN_NAME); } @Test diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index eb12ec5fc5..9a4cf44b3e 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -5,6 +5,8 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -17,8 +19,8 @@ import org.opensearch.client.transport.rest_client.RestClientTransport; import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.plugins.sink.opensearch.bulk.PreSerializedJsonpMapper; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; @@ -58,7 +60,6 @@ class ConnectionConfigurationTests { private final List TEST_HOSTS = Collections.singletonList("http://localhost:9200"); private final String TEST_USERNAME = "admin"; private final String TEST_PASSWORD = "admin"; - private final String TEST_PIPELINE_NAME = "Test-Pipeline"; private final Integer TEST_CONNECT_TIMEOUT = 5; private final Integer TEST_SOCKET_TIMEOUT = 10; private final String TEST_CERT_PATH = Objects.requireNonNull(getClass().getClassLoader().getResource("test-ca.pem")).getFile(); @@ -70,13 +71,15 @@ class ConnectionConfigurationTests { private ApacheHttpClient apacheHttpClient; @Mock private AwsCredentialsSupplier awsCredentialsSupplier; + + ObjectMapper objectMapper; @Test - void testReadConnectionConfigurationDefault() { - final PluginSetting pluginSetting = generatePluginSetting( + void testReadConnectionConfigurationDefault() throws JsonProcessingException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); assertNull(connectionConfiguration.getUsername()); assertNull(connectionConfiguration.getPassword()); @@ -84,37 +87,36 @@ void testReadConnectionConfigurationDefault() { assertNull(connectionConfiguration.getCertPath()); assertNull(connectionConfiguration.getConnectTimeout()); assertNull(connectionConfiguration.getSocketTimeout()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); assertTrue(connectionConfiguration.isRequestCompressionEnabled()); } @Test - void testReadConnectionConfigurationES6Default() { + void testReadConnectionConfigurationES6Default() throws JsonProcessingException { final Map configMetadata = generateConfigurationMetadata( TEST_HOSTS, null, null, null, null, true, null, null, null, false); configMetadata.put(DISTRIBUTION_VERSION, "es6"); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertFalse(connectionConfiguration.isRequestCompressionEnabled()); } @Test - void testReadConnectionConfigurationAwsOptionServerlessDefault() { + void testReadConnectionConfigurationAwsOptionServerlessDefault() throws JsonProcessingException { final String testArn = TEST_ROLE; final Map configMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, true, false, null, testArn, null, TEST_CERT_PATH, false, Collections.emptyMap()); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertTrue(connectionConfiguration.isServerless()); } @Test void testCreateClientDefault() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); @@ -122,10 +124,10 @@ void testCreateClientDefault() throws IOException { @Test void testCreateOpenSearchClientDefault() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); @@ -140,9 +142,9 @@ void testCreateOpenSearchClientAwsServerlessDefault() throws IOException { final Map configMetadata = generateConfigurationMetadata( TEST_HOSTS, null, null, null, null, true, null, null, null, false); configMetadata.put(SERVERLESS, true); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); @@ -165,28 +167,27 @@ void testCreateOpenSearchClientAwsServerlessDefault() throws IOException { } @Test - void testReadConnectionConfigurationWithDeprecatedBasicCredentialsAndNoCert() { - final PluginSetting pluginSetting = generatePluginSetting( + void testReadConnectionConfigurationWithDeprecatedBasicCredentialsAndNoCert() throws JsonProcessingException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); assertEquals(TEST_USERNAME, connectionConfiguration.getUsername()); assertEquals(TEST_PASSWORD, connectionConfiguration.getPassword()); assertEquals(TEST_CONNECT_TIMEOUT, connectionConfiguration.getConnectTimeout()); assertEquals(TEST_SOCKET_TIMEOUT, connectionConfiguration.getSocketTimeout()); assertFalse(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test - void testReadConnectionConfigurationWithBasicCredentialsAndNoCert() { + void testReadConnectionConfigurationWithBasicCredentialsAndNoCert() throws JsonProcessingException { final Map configurationMetadata = generateConfigurationMetadata( TEST_HOSTS, null, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); configurationMetadata.put("authentication", Map.of("username", TEST_USERNAME, "password", TEST_PASSWORD)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); assertNull(connectionConfiguration.getUsername()); assertNull(connectionConfiguration.getPassword()); @@ -196,25 +197,23 @@ void testReadConnectionConfigurationWithBasicCredentialsAndNoCert() { assertEquals(TEST_CONNECT_TIMEOUT, connectionConfiguration.getConnectTimeout()); assertEquals(TEST_SOCKET_TIMEOUT, connectionConfiguration.getSocketTimeout()); assertFalse(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test - void testReadConnectionConfigurationWithBothDeprecatedBasicCredentialsAndAuthConfigShouldThrow() { + void testReadConnectionConfigurationWithBothDeprecatedBasicCredentialsAndAuthConfigShouldThrow() throws JsonProcessingException { final Map configurationMetadata = generateConfigurationMetadata( TEST_HOSTS, TEST_USERNAME, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); configurationMetadata.put("authentication", Map.of("username", TEST_USERNAME, "password", TEST_PASSWORD)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); - assertThrows(IllegalStateException.class, - () -> ConnectionConfiguration.readConnectionConfiguration(pluginSetting)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); + assertFalse(openSearchSinkConfig.isAuthConfigValid()); } @Test void testCreateClientWithDeprecatedBasicCredentialsAndNoCert() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); @@ -224,10 +223,11 @@ void testCreateClientWithDeprecatedBasicCredentialsAndNoCert() throws IOExceptio void testCreateClientWithBasicCredentialsAndNoCert() throws IOException { final Map configurationMetadata = generateConfigurationMetadata( TEST_HOSTS, null, null, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); - configurationMetadata.put("authentication", Map.of("username", TEST_USERNAME, "password", TEST_PASSWORD)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + configurationMetadata.put("username", TEST_USERNAME); + configurationMetadata.put("password", TEST_PASSWORD); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); @@ -235,10 +235,10 @@ void testCreateClientWithBasicCredentialsAndNoCert() throws IOException { @Test void testCreateOpenSearchClientNoCert() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); @@ -249,10 +249,10 @@ void testCreateOpenSearchClientNoCert() throws IOException { @Test void testCreateClientInsecure() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, true); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); @@ -260,10 +260,10 @@ void testCreateClientInsecure() throws IOException { @Test void testCreateOpenSearchClientInsecure() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, true); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); @@ -274,10 +274,10 @@ void testCreateOpenSearchClientInsecure() throws IOException { @Test void testCreateClientWithCertPath() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); client.close(); @@ -286,10 +286,10 @@ void testCreateClientWithCertPath() throws IOException { @Test void testCreateClientWithInsecureAndCertPath() throws IOException { // Insecure should take precedence over cert path when both are set - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, true); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertNull(connectionConfiguration.getCertPath()); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); @@ -298,10 +298,10 @@ void testCreateClientWithInsecureAndCertPath() throws IOException { @Test void testCreateOpenSearchClientWithCertPath() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); final OpenSearchClient openSearchClient = connectionConfiguration.createOpenSearchClient(client, awsCredentialsSupplier); assertNotNull(openSearchClient); @@ -312,10 +312,10 @@ void testCreateOpenSearchClientWithCertPath() throws IOException { @Test void testCreateClientWithAWSSigV4AndRegion() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, true, "us-west-2", null, null, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals("us-west-2", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); } @@ -343,8 +343,8 @@ void testServerlessOptions() throws IOException { metadata.put("socket_timeout", 1); metadata.put("aws", awsOptionMetadata); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); - final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); + final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration.getServerlessNetworkPolicyName(), equalTo(serverlessNetworkPolicyName)); assertThat(connectionConfiguration.getServerlessCollectionName(), equalTo(serverlessCollectionName)); assertThat(connectionConfiguration.getServerlessVpceId(), equalTo(serverlessVpceId)); @@ -352,48 +352,44 @@ void testServerlessOptions() throws IOException { @Test void testCreateClientWithAWSSigV4DefaultRegion() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, true, null, null, null, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test void testCreateClientWithAWSSigV4AndInsecure() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, true, null, null, null, true); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test void testCreateClientWithAWSSigV4AndCertPath() throws IOException { - final PluginSetting pluginSetting = generatePluginSetting( + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, true, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); - assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test - void testCreateClientWithAWSSigV4AndSTSRole() { - final PluginSetting pluginSetting = generatePluginSetting( + void testCreateClientWithAWSSigV4AndSTSRole() throws JsonProcessingException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.getAwsRegion(), equalTo("us-east-1")); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); assertThat(connectionConfiguration.getAwsStsRoleArn(), equalTo(TEST_ROLE)); - assertThat(connectionConfiguration.getPipelineName(), equalTo(TEST_PIPELINE_NAME)); final AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); @@ -410,16 +406,15 @@ void testCreateClientWithAWSSigV4AndSTSRole() { } @Test - void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() { - final PluginSetting pluginSetting = generatePluginSetting( + void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() throws JsonProcessingException { + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig( TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.getAwsRegion(), equalTo("us-east-1")); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); assertThat(connectionConfiguration.getAwsStsRoleArn(), equalTo(TEST_ROLE)); - assertThat(connectionConfiguration.getPipelineName(), equalTo(TEST_PIPELINE_NAME)); final AwsCredentialsProvider awsCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(awsCredentialsProvider); @@ -443,7 +438,7 @@ void testCreateOpenSearchClientWithAWSSigV4AndSTSRole() { } @Test - void testCreateClientWithAWSOption() { + void testCreateClientWithAWSOption() throws JsonProcessingException { final String headerName1 = UUID.randomUUID().toString(); final String headerValue1 = UUID.randomUUID().toString(); final String headerName2 = UUID.randomUUID().toString(); @@ -451,9 +446,9 @@ void testCreateClientWithAWSOption() { final String testArn = TEST_ROLE; final String externalId = UUID.randomUUID().toString(); final Map configurationMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, false, true, null, testArn, externalId, null,false, Map.of(headerName1, headerValue1, headerName2, headerValue2)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); @@ -479,7 +474,7 @@ void testCreateClientWithAWSOption() { } @Test - void testCreateOpenSearchClientWithAWSOption() { + void testCreateOpenSearchClientWithAWSOption() throws JsonProcessingException { final String headerName1 = UUID.randomUUID().toString(); final String headerValue1 = UUID.randomUUID().toString(); final String headerName2 = UUID.randomUUID().toString(); @@ -487,9 +482,9 @@ void testCreateOpenSearchClientWithAWSOption() { final String testArn = TEST_ROLE; final String externalId = UUID.randomUUID().toString(); final Map configurationMetadata = generateConfigurationMetadataWithAwsOption(TEST_HOSTS, null, null, null, null, false, true, null, testArn, externalId, TEST_CERT_PATH, false, Map.of(headerName1, headerValue1, headerName2, headerValue2)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); @@ -520,16 +515,16 @@ void testCreateOpenSearchClientWithAWSOption() { } @Test - void testCreateClientWithAWSSigV4AndHeaderOverrides() { + void testCreateClientWithAWSSigV4AndHeaderOverrides() throws JsonProcessingException { final String headerName1 = UUID.randomUUID().toString(); final String headerValue1 = UUID.randomUUID().toString(); final String headerName2 = UUID.randomUUID().toString(); final String headerValue2 = UUID.randomUUID().toString(); final Map configurationMetadata = generateConfigurationMetadata(TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); configurationMetadata.put("aws_sts_header_overrides", Map.of(headerName1, headerValue1, headerName2, headerValue2)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); @@ -555,16 +550,16 @@ void testCreateClientWithAWSSigV4AndHeaderOverrides() { } @Test - void testCreateOpenSearchClientWithAWSSigV4AndHeaderOverrides() { + void testCreateOpenSearchClientWithAWSSigV4AndHeaderOverrides() throws JsonProcessingException { final String headerName1 = UUID.randomUUID().toString(); final String headerValue1 = UUID.randomUUID().toString(); final String headerName2 = UUID.randomUUID().toString(); final String headerValue2 = UUID.randomUUID().toString(); final Map configurationMetadata = generateConfigurationMetadata(TEST_HOSTS, null, null, null, null, true, null, TEST_ROLE, TEST_CERT_PATH, false); configurationMetadata.put("aws_sts_header_overrides", Map.of(headerName1, headerValue1, headerName2, headerValue2)); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(configurationMetadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(configurationMetadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThat(connectionConfiguration, notNullValue()); assertThat(connectionConfiguration.isAwsSigv4(), equalTo(true)); @@ -602,9 +597,9 @@ void testCreateAllClients_WithValidHttpProxy_HostIP() throws IOException { TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "121.121.121.121:80"; metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); @@ -618,9 +613,9 @@ void testCreateAllClients_WithValidHttpProxy_HostName() throws IOException { TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com:80"; metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); @@ -634,9 +629,9 @@ void testCreateAllClients_WithValidHttpProxy_SchemeProvided() throws IOException TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "http://example.com:4350"; metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); assertNotNull(client); @@ -645,51 +640,51 @@ void testCreateAllClients_WithValidHttpProxy_SchemeProvided() throws IOException } @Test - void testCreateClient_WithInvalidHttpProxy_InvalidPort() { + void testCreateClient_WithInvalidHttpProxy_InvalidPort() throws JsonProcessingException { final Map metadata = generateConfigurationMetadata( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com:port"; metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @Test - void testCreateClient_WithInvalidHttpProxy_NoPort() { + void testCreateClient_WithInvalidHttpProxy_NoPort() throws JsonProcessingException { final Map metadata = generateConfigurationMetadata( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com"; metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @Test - void testCreateClient_WithInvalidHttpProxy_PortNotInRange() { + void testCreateClient_WithInvalidHttpProxy_PortNotInRange() throws JsonProcessingException { final Map metadata = generateConfigurationMetadata( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "example.com:888888"; metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @Test - void testCreateClient_WithInvalidHttpProxy_NotHttp() { + void testCreateClient_WithInvalidHttpProxy_NotHttp() throws JsonProcessingException { final Map metadata = generateConfigurationMetadata( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final String testHttpProxy = "socket://example.com:port"; metadata.put(PROXY_PARAMETER, testHttpProxy); - final PluginSetting pluginSetting = getPluginSettingByConfigurationMetadata(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfigByConfigMetadata(metadata); final ConnectionConfiguration connectionConfiguration = - ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + ConnectionConfiguration.readConnectionConfiguration(openSearchSinkConfig); assertEquals(connectionConfiguration.getProxy().get(), testHttpProxy); assertThrows(IllegalArgumentException.class, () -> connectionConfiguration.createClient(awsCredentialsSupplier)); } @@ -703,13 +698,14 @@ void testCreateClient_WithConnectionConfigurationBuilder_ProxyOptionalObjectShou assertNotNull(client); client.close(); } - - private PluginSetting generatePluginSetting( + + private OpenSearchSinkConfig generateOpenSearchSinkConfig( final List hosts, final String username, final String password, final Integer connectTimeout, final Integer socketTimeout, final boolean awsSigv4, final String awsRegion, - final String awsStsRoleArn, final String certPath, final boolean insecure) { + final String awsStsRoleArn, final String certPath, final boolean insecure) throws JsonProcessingException { + final Map metadata = generateConfigurationMetadata(hosts, username, password, connectTimeout, socketTimeout, awsSigv4, awsRegion, awsStsRoleArn, certPath, insecure); - return getPluginSettingByConfigurationMetadata(metadata); + return getOpenSearchSinkConfigByConfigMetadata(metadata); } private Map generateConfigurationMetadata( @@ -757,9 +753,11 @@ private Map generateConfigurationMetadataWithAwsOption( return metadata; } - private PluginSetting getPluginSettingByConfigurationMetadata(final Map metadata) { - final PluginSetting pluginSetting = new PluginSetting("opensearch", metadata); - pluginSetting.setPipelineName(TEST_PIPELINE_NAME); - return pluginSetting; + private OpenSearchSinkConfig getOpenSearchSinkConfigByConfigMetadata(final Map metadata) throws JsonProcessingException { + objectMapper = new ObjectMapper(); + String json = new ObjectMapper().writeValueAsString(metadata); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); + + return openSearchSinkConfig; } -} +} \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java index 584051dff6..0ceb793785 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchClientRefresherTest.java @@ -9,8 +9,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.plugins.source.opensearch.configuration.AuthConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.AuthConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import java.util.function.Function; @@ -75,7 +75,7 @@ void testGetAfterUpdateWithDeprecatedBasicAuthUnchanged() { verify(clientFunction, times(1)).apply(any()); when(connectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(connectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); when(newConnectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(newConnectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD); @@ -97,7 +97,7 @@ void testGetAfterUpdateWithBasicAuthUnchanged() { when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); when(authConfig.getUsername()).thenReturn(TEST_USERNAME); when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); final AuthConfig newAuthConfig = mock(AuthConfig.class); when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); @@ -121,7 +121,7 @@ void testGetAfterUpdateWithDeprecatedUsernameChanged() { verify(clientFunction, times(1)).apply(any()); assertThat(objectUnderTest.get(), equalTo(openSearchClient)); when(connectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); when(newConnectionConfiguration.getUsername()).thenReturn(TEST_USERNAME + "_changed"); final OpenSearchClient newClient = mock(OpenSearchClient.class); @@ -147,7 +147,7 @@ void testGetAfterUpdateWithUsernameChanged() { when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); when(authConfig.getUsername()).thenReturn(TEST_USERNAME); when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); final AuthConfig newAuthConfig = mock(AuthConfig.class); when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); @@ -174,7 +174,7 @@ void testGetAfterUpdateWithDeprecatedPasswordChanged() { assertThat(objectUnderTest.get(), equalTo(openSearchClient)); when(connectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(connectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); when(newConnectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(newConnectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD + "_changed"); @@ -201,7 +201,7 @@ void testGetAfterUpdateWithPasswordChanged() { when(connectionConfiguration.getAuthConfig()).thenReturn(authConfig); when(authConfig.getUsername()).thenReturn(TEST_USERNAME); when(authConfig.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); final AuthConfig newAuthConfig = mock(AuthConfig.class); when(newConnectionConfiguration.getAuthConfig()).thenReturn(newAuthConfig); @@ -230,7 +230,7 @@ void testGetAfterUpdateClientFailure() { assertThat(objectUnderTest.get(), equalTo(openSearchClient)); when(connectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(connectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD); - final PluginSetting newConfig = mock(PluginSetting.class); + final OpenSearchSinkConfig newConfig = mock(OpenSearchSinkConfig.class); final ConnectionConfiguration newConnectionConfiguration = mock(ConnectionConfiguration.class); when(newConnectionConfiguration.getUsername()).thenReturn(TEST_USERNAME); when(newConnectionConfiguration.getPassword()).thenReturn(TEST_PASSWORD + "_changed"); @@ -247,4 +247,4 @@ void testGetAfterUpdateClientFailure() { verify(clientRefreshErrorsCounter).increment(); verify(clientFunction, times(2)).apply(any()); } -} +} \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java index cb9dfbe898..22e1df982e 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java @@ -5,17 +5,15 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.junit.Test; import org.opensearch.dataprepper.expression.ExpressionEvaluator; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.opensearch.OpenSearchBulkActions; -import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; -import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexType; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; +import java.io.File; +import java.io.IOException; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -25,104 +23,57 @@ import static org.mockito.Mockito.when; public class OpenSearchSinkConfigurationTests { - private final List TEST_HOSTS = Collections.singletonList("http://localhost:9200"); - private static final String PLUGIN_NAME = "opensearch"; - private static final String PIPELINE_NAME = "integTestPipeline"; + private static final String OPEN_SEARCH_SINK_CONFIGURATIONS = "open-search-sink-configurations.yaml"; + private static final String VALID_SINK_CONFIG = "valid-sink"; + private static final String INVALID_ACTION_CONFIG = "invalid-action"; + private static final String INVALID_ACTIONS_CONFIG = "invalid-actions"; + private static final String INVALID_ACTION_WITH_EXPRESSION_CONFIG = "invalid-action-with-expression"; + private static final String INVALID_ACTIONS_WITH_EXPRESSION_CONFIG = "invalid-actions-with-expression"; + private static final String CREATE_ACTION_CONFIG = "create-action"; + private static final String CREATE_ACTIONS_WITH_EXPRESSION_CONFIG = "create-actions-with-expression"; private ExpressionEvaluator expressionEvaluator; + ObjectMapper objectMapper; + @Test - public void testReadESConfig() { - final OpenSearchSinkConfiguration openSearchSinkConfiguration = OpenSearchSinkConfiguration.readESConfig( - generatePluginSetting()); + public void testReadESConfig() throws IOException { + final OpenSearchSinkConfiguration openSearchSinkConfiguration = OpenSearchSinkConfiguration.readOSConfig( + generateOpenSearchSinkConfig(VALID_SINK_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); assertNotNull(openSearchSinkConfiguration.getRetryConfiguration()); assertEquals(OpenSearchBulkActions.INDEX.toString(), openSearchSinkConfiguration.getIndexConfiguration().getAction()); } - @Test(expected = IllegalArgumentException.class) - public void testInvalidAction() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(IndexConfiguration.ACTION, "invalid"); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - - OpenSearchSinkConfiguration.readESConfig(pluginSetting); - + @Test(expected = NullPointerException.class) + public void testInvalidAction() throws IOException { + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTION_CONFIG)); } - @Test(expected = IllegalArgumentException.class) - public void testInvalidActions() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - List> invalidActionList = new ArrayList<>(); - Map actionMap = new HashMap<>(); - actionMap.put("type", "invalid"); - invalidActionList.add(actionMap); - metadata.put(IndexConfiguration.ACTIONS, invalidActionList); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - - OpenSearchSinkConfiguration.readESConfig(pluginSetting); + @Test(expected = NullPointerException.class) + public void testInvalidActions() throws IOException { + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTIONS_CONFIG)); } - @Test(expected = IllegalArgumentException.class) - public void testInvalidActionWithExpression() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(IndexConfiguration.ACTION, "${anInvalidFunction()}"); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - + @Test(expected = NullPointerException.class) + public void testInvalidActionWithExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidExpressionStatement(anyString())).thenReturn(false); - OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTION_WITH_EXPRESSION_CONFIG), expressionEvaluator); } - @Test(expected = IllegalArgumentException.class) - public void testInvalidActionsWithExpression() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - List> invalidActionList = new ArrayList<>(); - Map actionMap = new HashMap<>(); - actionMap.put("type", "${anInvalidFunction()}"); - invalidActionList.add(actionMap); - metadata.put(IndexConfiguration.ACTIONS, invalidActionList); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - + @Test(expected = NullPointerException.class) + public void testInvalidActionsWithExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidExpressionStatement(anyString())).thenReturn(false); - OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(INVALID_ACTIONS_WITH_EXPRESSION_CONFIG), expressionEvaluator); } @Test - public void testReadESConfigWithBulkActionCreate() { - - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - + public void testReadOSConfigWithBulkActionCreate() throws IOException { final OpenSearchSinkConfiguration openSearchSinkConfiguration = - OpenSearchSinkConfiguration.readESConfig(pluginSetting); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(CREATE_ACTION_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); @@ -130,34 +81,31 @@ public void testReadESConfigWithBulkActionCreate() { } @Test - public void testReadESConfigWithBulkActionCreateExpression() { - - final String actionFormatExpression = "${getMetadata(\"action\")}"; - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(IndexConfiguration.ACTION, actionFormatExpression); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); + public void testReadESConfigWithBulkActionCreateExpression() throws IOException { expressionEvaluator = mock(ExpressionEvaluator.class); - when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); + when(expressionEvaluator.isValidFormatExpression("${getMetadata(\"action\")}")).thenReturn(true); + final OpenSearchSinkConfiguration openSearchSinkConfiguration = - OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator); + OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(CREATE_ACTIONS_WITH_EXPRESSION_CONFIG)); assertNotNull(openSearchSinkConfiguration.getConnectionConfiguration()); assertNotNull(openSearchSinkConfiguration.getIndexConfiguration()); assertNotNull(openSearchSinkConfiguration.getRetryConfiguration()); } - private PluginSetting generatePluginSetting() { - final Map metadata = new HashMap<>(); - metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.TRACE_ANALYTICS_RAW.getValue()); - metadata.put(ConnectionConfiguration.HOSTS, TEST_HOSTS); - final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, metadata); - pluginSetting.setPipelineName(PIPELINE_NAME); - return pluginSetting; + private OpenSearchSinkConfig generateOpenSearchSinkConfig(String pipelineName) throws IOException { + final File configurationFile = new File(getClass().getClassLoader().getResource(OPEN_SEARCH_SINK_CONFIGURATIONS).getFile()); + objectMapper = new ObjectMapper(new YAMLFactory()); + final Map pipelineConfigs = objectMapper.readValue(configurationFile, Map.class); + final Map pipelineConfig = (Map) pipelineConfigs.get(pipelineName); + final Map sinkMap = (Map) pipelineConfig.get("sink"); + final Map opensearchSinkMap = (Map) sinkMap.get("opensearch"); + String json = objectMapper.writeValueAsString(opensearchSinkMap); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); + + return openSearchSinkConfig; } -} + +} \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index 31b77e0bf3..59019e0b0a 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -22,15 +22,16 @@ import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventHandle; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.failures.DlqObject; import org.opensearch.dataprepper.model.plugin.PluginConfigObservable; -import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.SinkContext; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import org.opensearch.dataprepper.plugins.sink.opensearch.dlq.FailedDlqData; import org.opensearch.dataprepper.plugins.sink.opensearch.index.DocumentBuilder; import org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration; @@ -41,10 +42,10 @@ import org.opensearch.dataprepper.plugins.sink.opensearch.index.TemplateType; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.Collections; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -53,12 +54,12 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.lenient; import static org.opensearch.dataprepper.model.sink.SinkLatencyMetrics.EXTERNAL_LATENCY; import static org.opensearch.dataprepper.model.sink.SinkLatencyMetrics.INTERNAL_LATENCY; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink.BULKREQUEST_ERRORS; @@ -67,8 +68,8 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink.DYNAMIC_INDEX_DROPPED_EVENTS; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink.INVALID_ACTION_ERRORS; import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSink.INVALID_VERSION_EXPRESSION_DROPPED_EVENTS; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_BULK_SIZE; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DEFAULT_FLUSH_TIMEOUT; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_BULK_SIZE; +import static org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig.DEFAULT_FLUSH_TIMEOUT; @ExtendWith(MockitoExtension.class) public class OpenSearchSinkTest { @@ -79,9 +80,6 @@ public class OpenSearchSinkTest { @Mock private OpenSearchClient openSearchClient; - @Mock - private PluginFactory pluginFactory; - @Mock private SinkContext sinkContext; @@ -97,6 +95,9 @@ public class OpenSearchSinkTest { @Mock private OpenSearchSinkConfiguration openSearchSinkConfiguration; + @Mock + private PipelineDescription pipelineDescription; + @Mock private IndexConfiguration indexConfiguration; @@ -109,6 +110,9 @@ public class OpenSearchSinkTest { @Mock private Timer bulkRequestTimer; + @Mock + private OpenSearchSinkConfig openSearchSinkConfig; + @Mock private Counter bulkRequestErrorsCounter; @@ -129,8 +133,7 @@ public class OpenSearchSinkTest { @BeforeEach void setup() { - when(pluginSetting.getPipelineName()).thenReturn(UUID.randomUUID().toString()); - when(pluginSetting.getName()).thenReturn(UUID.randomUUID().toString()); + when(pipelineDescription.getPipelineName()).thenReturn(UUID.randomUUID().toString()); final RetryConfiguration retryConfiguration = mock(RetryConfiguration.class); when(retryConfiguration.getDlq()).thenReturn(Optional.empty()); @@ -144,9 +147,7 @@ void setup() { when(indexConfiguration.getAction()).thenReturn("index"); when(indexConfiguration.getDocumentId()).thenReturn(null); when(indexConfiguration.getDocumentIdField()).thenReturn(null); - when(indexConfiguration.getRoutingField()).thenReturn(null); when(indexConfiguration.getRouting()).thenReturn(null); - when(indexConfiguration.getPipeline()).thenReturn(null); when(indexConfiguration.getActions()).thenReturn(null); when(indexConfiguration.getDocumentRootKey()).thenReturn(null); lenient().when(indexConfiguration.getVersionType()).thenReturn(null); @@ -184,10 +185,10 @@ private OpenSearchSink createObjectUnderTest() throws IOException { indexManagerFactory = mock; })) { pluginMetricsMockedStatic.when(() -> PluginMetrics.fromPluginSetting(pluginSetting)).thenReturn(pluginMetrics); - openSearchSinkConfigurationMockedStatic.when(() -> OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator)) + openSearchSinkConfigurationMockedStatic.when(() -> OpenSearchSinkConfiguration.readOSConfig(openSearchSinkConfig, expressionEvaluator)) .thenReturn(openSearchSinkConfiguration); return new OpenSearchSink( - pluginSetting, pluginFactory, sinkContext, expressionEvaluator, awsCredentialsSupplier, pluginConfigObservable); + pluginSetting, sinkContext, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); } } @@ -204,7 +205,7 @@ void test_initialization() throws IOException { @Test void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_creates_DLQObject() throws IOException { - + when(pluginSetting.getName()).thenReturn("opensearch"); final String versionExpression = UUID.randomUUID().toString(); when(indexConfiguration.getVersionExpression()).thenReturn(versionExpression); @@ -234,7 +235,7 @@ void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_ when(dlqObjectBuilder.withFailedData(failedDlqData.capture())).thenReturn(dlqObjectBuilder); when(dlqObjectBuilder.withPluginName(pluginSetting.getName())).thenReturn(dlqObjectBuilder); when(dlqObjectBuilder.withPluginId(pluginSetting.getName())).thenReturn(dlqObjectBuilder); - when(dlqObjectBuilder.withPipelineName(pluginSetting.getPipelineName())).thenReturn(dlqObjectBuilder); + when(dlqObjectBuilder.withPipelineName(pipelineDescription.getPipelineName())).thenReturn(dlqObjectBuilder); when(dlqObject.getFailedData()).thenReturn(mock(FailedDlqData.class)); doNothing().when(dlqObject).releaseEventHandle(false); @@ -271,7 +272,6 @@ void test_routing_field_in_document() throws IOException { .withData(Collections.singletonMap(routingFieldKey, routingFieldValue)) .build(); assertThat(objectUnderTest.getDocument(event).getRoutingField(), equalTo(Optional.of(routingFieldValue))); - } @Test @@ -290,25 +290,10 @@ void test_routing_in_document() throws IOException { assertThat(objectUnderTest2.getDocument(event).getRoutingField(), equalTo(Optional.of(routingValue))); } - @Test - void test_pipeline_in_document() throws IOException { - String pipelineValue = UUID.randomUUID().toString(); - String pipelineKey = UUID.randomUUID().toString(); - final OpenSearchSink objectUnderTest = createObjectUnderTest(); - final Event event = JacksonEvent.builder() - .withEventType("event") - .withData(Collections.singletonMap(pipelineKey, pipelineValue)) - .build(); - assertThat(objectUnderTest.getDocument(event).getPipelineField(), equalTo(Optional.empty())); - - when(indexConfiguration.getPipeline()).thenReturn("${"+pipelineKey+"}"); - final OpenSearchSink objectUnderTest2 = createObjectUnderTest(); - assertThat(objectUnderTest2.getDocument(event).getPipelineField(), equalTo(Optional.of(pipelineValue))); - } - @Test void doOutput_with_invalid_version_expression_result_catches_RuntimeException_and_creates_DLQObject() throws IOException { + when(pluginSetting.getName()).thenReturn("opensearch"); final String versionExpression = UUID.randomUUID().toString(); when(indexConfiguration.getVersionExpression()).thenReturn(versionExpression); @@ -338,7 +323,7 @@ void doOutput_with_invalid_version_expression_result_catches_RuntimeException_an when(dlqObjectBuilder.withFailedData(failedDlqData.capture())).thenReturn(dlqObjectBuilder); when(dlqObjectBuilder.withPluginName(pluginSetting.getName())).thenReturn(dlqObjectBuilder); when(dlqObjectBuilder.withPluginId(pluginSetting.getName())).thenReturn(dlqObjectBuilder); - when(dlqObjectBuilder.withPipelineName(pluginSetting.getPipelineName())).thenReturn(dlqObjectBuilder); + when(dlqObjectBuilder.withPipelineName(pipelineDescription.getPipelineName())).thenReturn(dlqObjectBuilder); when(dlqObject.getFailedData()).thenReturn(mock(FailedDlqData.class)); doNothing().when(dlqObject).releaseEventHandle(false); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java index e5b066234f..4eb81de071 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/RetryConfigurationTests.java @@ -5,19 +5,34 @@ package org.opensearch.dataprepper.plugins.sink.opensearch; -import org.opensearch.dataprepper.model.configuration.PluginSetting; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.junit.Test; +import org.opensearch.dataprepper.plugins.dlq.s3.S3DlqWriterConfig; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.DlqConfiguration; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; -import java.util.HashMap; -import java.util.LinkedHashMap; +import java.io.File; +import java.io.IOException; import java.util.Map; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; public class RetryConfigurationTests { + private static final String OPEN_SEARCH_SINK_CONFIGURATIONS = "open-search-sink-configurations.yaml"; + private static final String INVALID_MAX_RETRIES_CONFIG = "invalid-max-retries"; + private static final String NO_DLQ_FILE_PATH = "no-dlq-file-path"; + private static final String DLQ_FILE_PATH_10_RETRIES = "dlq-file-path-10-retries"; + private static final String WITH_DLQ_PLUGIN_10_RETRIES = "with-dlq-plugin-10-retries"; + private static final String DLQ_FILE_AND_DLQ_PLUGIN = "dlq-file-and-dlq-plugin"; + + ObjectMapper objectMapper; + @Test public void testDefaultConfigurationIsNotNull() { final RetryConfiguration retryConfiguration = new RetryConfiguration.Builder().build(); @@ -26,62 +41,52 @@ public void testDefaultConfigurationIsNotNull() { } @Test - public void testReadRetryConfigInvalidMaxRetries() { - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(null, -1, null)); + public void testReadRetryConfigInvalidMaxRetries() throws IOException { + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSinkConfig(INVALID_MAX_RETRIES_CONFIG)); assertThrows(IllegalArgumentException.class, () -> retryConfiguration.getMaxRetries()); } @Test - public void testReadRetryConfigNoDLQFilePath() { - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(null, null, null)); + public void testReadRetryConfigNoDLQFilePath() throws IOException { + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSinkConfig(NO_DLQ_FILE_PATH)); assertNull(retryConfiguration.getDlqFile()); assertEquals(retryConfiguration.getMaxRetries(), Integer.MAX_VALUE); assertFalse(retryConfiguration.getDlq().isPresent()); } @Test - public void testReadRetryConfigWithDLQFilePath() { + public void testReadRetryConfigWithDLQFilePath() throws IOException { final String fakeDlqFilePath = "foo.txt"; final int maxRetries = 10; - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(fakeDlqFilePath, maxRetries, null)); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSinkConfig(DLQ_FILE_PATH_10_RETRIES)); assertEquals(fakeDlqFilePath, retryConfiguration.getDlqFile()); assertEquals(maxRetries, retryConfiguration.getMaxRetries()); } @Test - public void testReadRetryConfigWithDLQPlugin() { - final Map fakePlugin = new LinkedHashMap<>(); - final Map lowLevelPluginSettings = new HashMap<>(); - lowLevelPluginSettings.put("field1", "value1"); - lowLevelPluginSettings.put("field2", "value2"); - fakePlugin.put("another_dlq", lowLevelPluginSettings); + public void testReadRetryConfigWithDLQPlugin() throws IOException { final int maxRetries = 10; - final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generatePluginSetting(null, maxRetries, fakePlugin)); - assertEquals("another_dlq", retryConfiguration.getDlq().get().getPluginName()); + final RetryConfiguration retryConfiguration = RetryConfiguration.readRetryConfig(generateOpenSearchSinkConfig(WITH_DLQ_PLUGIN_10_RETRIES)); + Optional dlqConfiguration = retryConfiguration.getDlq(); + assertInstanceOf(S3DlqWriterConfig.class, dlqConfiguration.get().getS3DlqWriterConfig()); assertEquals(maxRetries, retryConfiguration.getMaxRetries()); } @Test public void testReadRetryConfigWithDLQPluginAndDLQFilePath() { - final String fakeDlqFilePath = "foo.txt"; - final Map fakePlugin = new LinkedHashMap<>(); - fakePlugin.put("another_dlq", "value1"); - final int maxRetries = 10; - assertThrows(RuntimeException.class, () -> RetryConfiguration.readRetryConfig(generatePluginSetting(fakeDlqFilePath, maxRetries, fakePlugin))); + assertThrows(RuntimeException.class, () -> OpenSearchSinkConfiguration.readOSConfig(generateOpenSearchSinkConfig(DLQ_FILE_AND_DLQ_PLUGIN))); } - private PluginSetting generatePluginSetting(final String dlqFilePath, final Integer maxRetries, final Map pluginSettings) { - final Map metadata = new HashMap<>(); - if (dlqFilePath != null) { - metadata.put(RetryConfiguration.DLQ_FILE, dlqFilePath); - } - if (maxRetries != null) { - metadata.put(RetryConfiguration.MAX_RETRIES, maxRetries); - } - if (pluginSettings != null) { - metadata.put(RetryConfiguration.DLQ, pluginSettings); - } + private OpenSearchSinkConfig generateOpenSearchSinkConfig(String pipelineName) throws IOException { + final File configurationFile = new File(getClass().getClassLoader().getResource(OPEN_SEARCH_SINK_CONFIGURATIONS).getFile()); + objectMapper = new ObjectMapper(new YAMLFactory()); + final Map pipelineConfigs = objectMapper.readValue(configurationFile, Map.class); + final Map pipelineConfig = (Map) pipelineConfigs.get(pipelineName); + final Map sinkMap = (Map) pipelineConfig.get("sink"); + final Map opensearchSinkMap = (Map) sinkMap.get("opensearch"); + String json = objectMapper.writeValueAsString(opensearchSinkMap); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); - return new PluginSetting("opensearch", metadata); + return openSearchSinkConfig; } } diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverterTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverterTest.java index 17f0e52079..f02e39fd96 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverterTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/dlq/FailedBulkOperationConverterTest.java @@ -31,7 +31,6 @@ public class FailedBulkOperationConverterTest { private String testId; private String pipelineName; private String pluginName; - private String pluginId; private BulkOperation bulkOperation; private BulkResponseItem bulkResponseItem; private String errorReason; @@ -47,7 +46,6 @@ public void setup() { testId = UUID.randomUUID().toString(); pipelineName = UUID.randomUUID().toString(); pluginName = UUID.randomUUID().toString(); - pluginId = UUID.randomUUID().toString(); errorReason = UUID.randomUUID().toString(); failureMessage = UUID.randomUUID().toString(); @@ -75,7 +73,7 @@ public void setup() { when(bulkResponseItem.error()).thenReturn(errorCause); when(errorCause.reason()).thenReturn(errorReason); - converter = new FailedBulkOperationConverter(pipelineName, pluginName, pluginId); + converter = new FailedBulkOperationConverter(pipelineName, pluginName); } @Test @@ -136,7 +134,7 @@ public void testConvertToDlqObjectWithOnlyBulkResponseItem() { private void validateResponse(final DlqObject result, final String expectedErrorMessage) { assertThat(result, is(notNullValue())); assertThat(result.getPipelineName(), is(equalTo(pipelineName))); - assertThat(result.getPluginId(), is(equalTo(pluginId))); + assertThat(result.getPluginId(), is(equalTo(pluginName))); assertThat(result.getPluginName(), is(equalTo(pluginName))); final Object failedData = result.getFailedData(); assertThat(failedData, is(notNullValue())); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java index e14689e25d..6ba7cbaffc 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfigurationTests.java @@ -14,9 +14,9 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; import org.opensearch.dataprepper.expression.ExpressionEvaluator; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.plugins.sink.opensearch.DistributionVersion; +import org.opensearch.dataprepper.plugins.sink.opensearch.configuration.OpenSearchSinkConfig; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.http.AbortableInputStream; import software.amazon.awssdk.services.s3.S3Client; @@ -52,7 +52,6 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DISTRIBUTION_VERSION; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DOCUMENT_ROOT_KEY; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.DOCUMENT_VERSION_EXPRESSION; -import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.PIPELINE; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.ROUTING; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.ROUTING_FIELD; import static org.opensearch.dataprepper.plugins.sink.opensearch.index.IndexConfiguration.SERVERLESS; @@ -68,6 +67,8 @@ public class IndexConfigurationTests { private static final String DEFAULT_TEMPLATE_FILE = "test-template-withshards.json"; private static final String TEST_CUSTOM_INDEX_POLICY_FILE = "test-custom-index-policy-file.json"; + ObjectMapper objectMapper; + @Test public void testRawAPMSpan() { final IndexConfiguration indexConfiguration = new IndexConfiguration.Builder().withIndexType( @@ -294,9 +295,9 @@ public void testValidCustomWithTemplateContent() throws JsonProcessingException @Test public void readIndexConfigWithTemplateFileAndTemplateContentUsesTemplateContent() throws JsonProcessingException { - final PluginSetting pluginSetting = generatePluginSetting("custom", "test", "test-file", createTemplateContent(), null, null, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig("custom", "test", "test-file", createTemplateContent(), null, null, null); - final IndexConfiguration objectUnderTest = IndexConfiguration.readIndexConfig(pluginSetting); + final IndexConfiguration objectUnderTest = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertThat(objectUnderTest, notNullValue()); assertThat(objectUnderTest.getIndexTemplate(), notNullValue()); @@ -304,12 +305,12 @@ public void readIndexConfigWithTemplateFileAndTemplateContentUsesTemplateContent } @Test - public void invalidTemplateContentThrowsInvalidPluginConfigurationException() { + public void invalidTemplateContentThrowsInvalidPluginConfigurationException() throws JsonProcessingException { final String invalidTemplateContent = UUID.randomUUID().toString(); - final PluginSetting pluginSetting = generatePluginSetting("custom", null, null, invalidTemplateContent, null, null, null); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig("custom", null, null, invalidTemplateContent, null, null, null); - assertThrows(InvalidPluginConfigurationException.class, () -> IndexConfiguration.readIndexConfig(pluginSetting)); + assertThrows(InvalidPluginConfigurationException.class, () -> IndexConfiguration.readIndexConfig(openSearchSinkConfig)); } @@ -322,11 +323,11 @@ public void testInvalidCustom() { } @Test - public void testReadIndexConfig_RawIndexType() { + public void testReadIndexConfig_RawIndexType() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null, null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); final URL expTemplateFile = indexConfiguration .getClass().getClassLoader().getResource(RAW_DEFAULT_TEMPLATE_FILE); assertEquals(IndexType.TRACE_ANALYTICS_RAW, indexConfiguration.getIndexType()); @@ -340,19 +341,19 @@ public void testReadIndexConfig_RawIndexType() { } @Test - public void testReadIndexConfig_InvalidIndexTypeValueString() { + public void testReadIndexConfig_InvalidIndexTypeValueString() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( "i-am-an-illegitimate-index-type", null, null, null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - assertThrows(IllegalArgumentException.class, () -> IndexConfiguration.readIndexConfig(pluginSetting)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + assertThrows(IllegalArgumentException.class, () -> IndexConfiguration.readIndexConfig(openSearchSinkConfig)); } @Test - public void testReadIndexConfig_ServiceMapIndexType() { + public void testReadIndexConfig_ServiceMapIndexType() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null, null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); final URL expTemplateFile = indexConfiguration .getClass().getClassLoader().getResource(SERVICE_MAP_DEFAULT_TEMPLATE_FILE); assertEquals(IndexType.TRACE_ANALYTICS_SERVICE_MAP, indexConfiguration.getIndexType()); @@ -366,18 +367,19 @@ public void testReadIndexConfig_ServiceMapIndexType() { } @Test - public void testReadIndexConfigCustom() { + public void testReadIndexConfigCustom() throws JsonProcessingException { final String defaultTemplateFilePath = Objects.requireNonNull( getClass().getClassLoader().getResource(DEFAULT_TEMPLATE_FILE)).getFile(); final String testIndexAlias = "foo"; final long testBulkSize = 10L; final long testFlushTimeout = 30_000L; final String testIdField = "someId"; - final PluginSetting pluginSetting = generatePluginSetting( + final Map metaData = initializeConfigMetaData( null, testIndexAlias, defaultTemplateFilePath, null, testBulkSize, testFlushTimeout, testIdField); - pluginSetting.getSettings().put(IndexConfiguration.ESTIMATE_BULK_SIZE_USING_COMPRESSION, true); - pluginSetting.getSettings().put(IndexConfiguration.MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION, 5); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + metaData.put(IndexConfiguration.ESTIMATE_BULK_SIZE_USING_COMPRESSION, true); + metaData.put(IndexConfiguration.MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION, 5); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metaData); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(IndexType.CUSTOM, indexConfiguration.getIndexType()); assertEquals(testIndexAlias, indexConfiguration.getIndexAlias()); assertFalse(indexConfiguration.getIndexTemplate().isEmpty()); @@ -389,7 +391,7 @@ public void testReadIndexConfigCustom() { } @Test - public void testReadIndexConfig_ExplicitCustomIndexType() { + public void testReadIndexConfig_ExplicitCustomIndexType() throws JsonProcessingException { final String defaultTemplateFilePath = Objects.requireNonNull( getClass().getClassLoader().getResource(DEFAULT_TEMPLATE_FILE)).getFile(); final String testIndexType = IndexType.CUSTOM.getValue(); @@ -399,8 +401,8 @@ public void testReadIndexConfig_ExplicitCustomIndexType() { final String testIdField = "someId"; final Map metadata = initializeConfigMetaData( testIndexType, testIndexAlias, defaultTemplateFilePath, null, testBulkSize, testFlushTimeout, testIdField); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(IndexType.CUSTOM, indexConfiguration.getIndexType()); assertEquals(testIndexAlias, indexConfiguration.getIndexAlias()); assertFalse(indexConfiguration.getIndexTemplate().isEmpty()); @@ -410,27 +412,27 @@ public void testReadIndexConfig_ExplicitCustomIndexType() { } @Test - public void testReadIndexConfig_awsOptionServerlessDefault() { + public void testReadIndexConfig_awsOptionServerlessDefault() throws JsonProcessingException { final String testIndexAlias = "foo"; final Map metadata = initializeConfigMetaData( null, testIndexAlias, null, null, null, null, null); metadata.put(AWS_OPTION, Map.of(SERVERLESS, true)); metadata.put(TEMPLATE_TYPE, TemplateType.V1.getTypeName()); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(IndexType.MANAGEMENT_DISABLED, indexConfiguration.getIndexType()); assertEquals(testIndexAlias, indexConfiguration.getIndexAlias()); } @Test - public void testReadIndexConfig_awsServerlessIndexTypeOverride() { + public void testReadIndexConfig_awsServerlessIndexTypeOverride() throws JsonProcessingException { final String testIndexAlias = "foo"; final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), testIndexAlias, null, null, null, null, null); metadata.put(AWS_OPTION, Map.of(SERVERLESS, true)); metadata.put(TEMPLATE_TYPE, TemplateType.V1.getTypeName()); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(IndexType.CUSTOM, indexConfiguration.getIndexType()); assertEquals(testIndexAlias, indexConfiguration.getIndexAlias()); assertEquals(TemplateType.INDEX_TEMPLATE, indexConfiguration.getTemplateType()); @@ -438,83 +440,72 @@ public void testReadIndexConfig_awsServerlessIndexTypeOverride() { } @Test - public void testReadIndexConfig_distributionVersionDefault() { + public void testReadIndexConfig_distributionVersionDefault() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( null, "foo", null,null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(indexConfiguration.getDistributionVersion(), DistributionVersion.DEFAULT); } @Test - public void testReadIndexConfig_es6Override() { + public void testReadIndexConfig_es6Override() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( null, "foo", null, null, null, null, null); metadata.put(DISTRIBUTION_VERSION, "es6"); metadata.put(TEMPLATE_TYPE, TemplateType.INDEX_TEMPLATE.getTypeName()); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(indexConfiguration.getDistributionVersion(), DistributionVersion.ES6); assertEquals(TemplateType.V1, indexConfiguration.getTemplateType()); assertEquals(IndexType.CUSTOM, indexConfiguration.getIndexType()); } @Test - public void testReadIndexConfig_documentRootKey() { + public void testReadIndexConfig_documentRootKey() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); final String expectedRootKey = UUID.randomUUID().toString(); metadata.put(DOCUMENT_ROOT_KEY, expectedRootKey); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(expectedRootKey, indexConfiguration.getDocumentRootKey()); } @Test - public void testReadIndexConfig_emptyDocumentRootKey() { + public void testReadIndexConfig_emptyDocumentRootKey() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); metadata.put(DOCUMENT_ROOT_KEY, ""); - final PluginSetting pluginSetting = getPluginSetting(metadata); - assertThrows(IllegalArgumentException.class, () -> IndexConfiguration.readIndexConfig(pluginSetting)); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + assertThrows(IllegalArgumentException.class, () -> IndexConfiguration.readIndexConfig(openSearchSinkConfig)); } @Test - public void testReadIndexConfig_pipeline() { - final Map metadata = initializeConfigMetaData( - IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); - final String expectedPipelineValue = UUID.randomUUID().toString(); - metadata.put(PIPELINE, expectedPipelineValue); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); - assertEquals(expectedPipelineValue, indexConfiguration.getPipeline()); - } - - @Test - public void testReadIndexConfig_routing() { + public void testReadIndexConfig_routing() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); final String expectedRoutingValue = UUID.randomUUID().toString(); metadata.put(ROUTING, expectedRoutingValue); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(expectedRoutingValue, indexConfiguration.getRouting()); } @Test - public void testReadIndexConfig_routingField() { + public void testReadIndexConfig_routingField() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); final String expectedRoutingFieldValue = UUID.randomUUID().toString(); metadata.put(ROUTING_FIELD, expectedRoutingFieldValue); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertEquals(expectedRoutingFieldValue, indexConfiguration.getRoutingField()); } @ParameterizedTest @ValueSource(strings = {"${key}", "${getMetadata(\"key\")}"}) - public void testReadIndexConfig_withValidDocumentVersionExpression(final String versionExpression) { + public void testReadIndexConfig_withValidDocumentVersionExpression(final String versionExpression) throws JsonProcessingException { final ExpressionEvaluator expressionEvaluator = mock(ExpressionEvaluator.class); when(expressionEvaluator.isValidFormatExpression(versionExpression)).thenReturn(true); @@ -523,16 +514,16 @@ public void testReadIndexConfig_withValidDocumentVersionExpression(final String IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); metadata.put(DOCUMENT_VERSION_EXPRESSION, versionExpression); - final PluginSetting pluginSetting = getPluginSetting(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig, expressionEvaluator); assertThat(indexConfiguration, notNullValue()); assertThat(indexConfiguration.getVersionExpression(), equalTo(versionExpression)); } @Test - public void testReadIndexConfig_withInvalidDocumentVersionExpression_throws_InvalidPluginConfigurationException() { + public void testReadIndexConfig_withInvalidDocumentVersionExpression_throws_InvalidPluginConfigurationException() throws JsonProcessingException { final String versionExpression = UUID.randomUUID().toString(); final ExpressionEvaluator expressionEvaluator = mock(ExpressionEvaluator.class); @@ -542,40 +533,44 @@ public void testReadIndexConfig_withInvalidDocumentVersionExpression_throws_Inva IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); metadata.put(DOCUMENT_VERSION_EXPRESSION, versionExpression); - final PluginSetting pluginSetting = getPluginSetting(metadata); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); - assertThrows(InvalidPluginConfigurationException.class, () -> IndexConfiguration.readIndexConfig(pluginSetting, expressionEvaluator)); + assertThrows(InvalidPluginConfigurationException.class, () -> IndexConfiguration.readIndexConfig(openSearchSinkConfig, expressionEvaluator)); } @Test - void getTemplateType_defaults_to_V1() { + void getTemplateType_defaults_to_V1() throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertThat(indexConfiguration.getTemplateType(), equalTo(TemplateType.V1)); } @ParameterizedTest @EnumSource(TemplateType.class) - void getTemplateType_with_configured_templateType(final TemplateType templateType) { + void getTemplateType_with_configured_templateType(final TemplateType templateType) throws JsonProcessingException { final Map metadata = initializeConfigMetaData( IndexType.CUSTOM.getValue(), "foo", null, null, null, null, null); metadata.put(TEMPLATE_TYPE, templateType.getTypeName()); - final PluginSetting pluginSetting = getPluginSetting(metadata); - final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(pluginSetting); + final OpenSearchSinkConfig openSearchSinkConfig = getOpenSearchSinkConfig(metadata); + final IndexConfiguration indexConfiguration = IndexConfiguration.readIndexConfig(openSearchSinkConfig); assertThat(indexConfiguration.getTemplateType(), equalTo(templateType)); } - private PluginSetting generatePluginSetting( + private OpenSearchSinkConfig generateOpenSearchSinkConfig( final String indexType, final String indexAlias, final String templateFilePath, final String templateContent, - final Long bulkSize, final Long flushTimeout, final String documentIdField) { + final Long bulkSize, final Long flushTimeout, final String documentIdField) throws JsonProcessingException { final Map metadata = initializeConfigMetaData(indexType, indexAlias, templateFilePath, templateContent, bulkSize, flushTimeout, documentIdField); - return getPluginSetting(metadata); + return getOpenSearchSinkConfig(metadata); } - private PluginSetting getPluginSetting(Map metadata) { - return new PluginSetting("opensearch", metadata); + private OpenSearchSinkConfig getOpenSearchSinkConfig(Map metadata) throws JsonProcessingException { + objectMapper = new ObjectMapper(); + String json = new ObjectMapper().writeValueAsString(metadata); + OpenSearchSinkConfig openSearchSinkConfig = objectMapper.readValue(json, OpenSearchSinkConfig.class); + + return openSearchSinkConfig; } private Map initializeConfigMetaData( diff --git a/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml b/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml new file mode 100644 index 0000000000..2ccd596dbc --- /dev/null +++ b/data-prepper-plugins/opensearch/src/test/resources/open-search-sink-configurations.yaml @@ -0,0 +1,121 @@ +valid-sink: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + aws: + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + region: "us-east-2" + serverless: true +invalid-action: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "invalid" + aws: + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + region: "us-east-2" + serverless: true +invalid-actions: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + actions: + - type: "invalid" + aws: + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + region: "us-east-2" + serverless: true +invalid-action-with-expression: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "${anInvalidFunction()}" + aws: + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + region: "us-east-2" + serverless: true +invalid-actions-with-expression: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + actions: + - type: "${anInvalidFunction()}" + aws: + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + region: "us-east-2" + serverless: true +create-action: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + action: "update" + aws: + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + region: "us-east-2" + serverless: true +create-actions-with-expression: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + index: "no-more-plugin-setting" + index_type: "trace-analytics-raw" + actions: + - type: "create" + when: "${getMetadata(\"action\")}" + aws: + sts_role_arn: "arn:aws:iam::123456789012:role/test-role" + region: "us-east-2" + serverless: true +invalid-max-retries: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: null + max_retries: -1 +no-dlq-file-path: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: null + max_retries: null +dlq-file-path-10-retries: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: null + dlq_file: foo.txt + max_retries: 10 +with-dlq-plugin-10-retries: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: + s3: + bucket: "my-dlq-bucket" + key_path_prefix: "dlq-files/" + dlq_file: null + max_retries: 10 +dlq-file-and-dlq-plugin: + sink: + opensearch: + hosts: [ "http://localhost:9200" ] + dlq: + s3: + bucket: "my-dlq-bucket" + key_path_prefix: "dlq-files/" + dlq_file: foo.txt + max_retries: 10 \ No newline at end of file