From a1c3d6c18a2a00492128936afa3e068497bb78ae Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Thu, 13 Feb 2025 09:19:37 +0100 Subject: [PATCH 1/4] Update otel proto buf specification Signed-off-by: Tomas Longo --- .../kafka/buffer/KafkaBufferOTelIT.java | 17 -- .../otellogs/OTelLogsGrpcServiceTest.java | 8 +- .../otelmetrics/OTelMetricsProtoHelper.java | 110 -------- ...MetricsPluginExponentialHistogramTest.java | 6 +- .../otelmetrics/MetricsPluginGaugeTest.java | 56 ---- .../MetricsPluginHistogramTest.java | 6 +- .../otelmetrics/MetricsPluginSumTest.java | 26 +- .../otelmetrics/MetricsPluginSummaryTest.java | 6 +- .../OTelMetricsGrpcServiceTest.java | 4 +- .../otelmetrics/OTelMetricsSourceTest.java | 10 +- .../OTelMetricsSource_RetryInfoTest.java | 14 +- .../plugins/otel/codec/OTelProtoCodec.java | 115 +------- .../otel/codec/OTelProtoCodecTest.java | 58 +---- .../test-request-both-span-types.json | 32 +-- .../test-request-instrumentation-library.json | 246 ------------------ .../test/resources/test-request-log-is.json | 34 --- .../resources/test-request-multiple-logs.json | 81 ++---- .../test-request-multiple-traces.json | 58 ----- .../oteltrace/OTelTraceGrpcService.java | 1 + .../oteltrace/OTelTraceGrpcServiceTest.java | 5 +- .../source/oteltrace/OTelTraceSourceTest.java | 10 +- .../OTelTraceSource_RetryInfoTest.java | 7 +- .../trace/EndToEndRawSpanTest.java | 8 +- .../trace/EndToEndServiceMapTest.java | 6 +- settings.gradle | 2 +- 25 files changed, 97 insertions(+), 829 deletions(-) delete mode 100644 data-prepper-plugins/otel-proto-common/src/test/resources/test-request-instrumentation-library.json delete mode 100644 data-prepper-plugins/otel-proto-common/src/test/resources/test-request-log-is.json diff --git a/data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/buffer/KafkaBufferOTelIT.java b/data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/buffer/KafkaBufferOTelIT.java index c8d0dd9e88..e23b3e3539 100644 --- a/data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/buffer/KafkaBufferOTelIT.java +++ b/data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/buffer/KafkaBufferOTelIT.java @@ -45,8 +45,6 @@ import io.opentelemetry.proto.resource.v1.Resource; import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; import io.opentelemetry.proto.common.v1.InstrumentationScope; import io.opentelemetry.proto.metrics.v1.Gauge; import io.opentelemetry.proto.metrics.v1.Sum; @@ -407,24 +405,9 @@ private ExportTraceServiceRequest createExportTraceRequest() { .build()) .build(); - final InstrumentationLibrarySpans ilSpans = InstrumentationLibrarySpans.newBuilder() - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder() - .setName(ilName) - .setVersion(ilVersion) - .build()) - .addSpans(io.opentelemetry.proto.trace.v1.Span.newBuilder() - .setTraceId(ByteString.copyFrom(TraceId2.getBytes())) - .setSpanId(ByteString.copyFrom(SpanId2.getBytes())) - .setKind(io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_INTERNAL) - .setName(ilSpanName) - .setStartTimeUnixNano(currentUnixTimeNano) - .setEndTimeUnixNano(currentUnixTimeNano+TIME_DELTA*1000_000_000) - .build()) - .build(); ResourceSpans resourceSpans = ResourceSpans.newBuilder() .setResource(resource) .addScopeSpans(scopeSpans) - .addInstrumentationLibrarySpans(ilSpans) .build(); return ExportTraceServiceRequest.newBuilder() diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsGrpcServiceTest.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsGrpcServiceTest.java index 28b8cbeca4..5519f03e38 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsGrpcServiceTest.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OTelLogsGrpcServiceTest.java @@ -12,9 +12,10 @@ import io.micrometer.core.instrument.Timer; import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; -import io.opentelemetry.proto.logs.v1.InstrumentationLibraryLogs; import io.opentelemetry.proto.logs.v1.LogRecord; import io.opentelemetry.proto.logs.v1.ResourceLogs; +import io.opentelemetry.proto.logs.v1.ScopeLogs; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -61,9 +62,8 @@ public class OTelLogsGrpcServiceTest { private static final ExportLogsServiceRequest LOGS_REQUEST = ExportLogsServiceRequest.newBuilder() .addResourceLogs(ResourceLogs.newBuilder() - .addInstrumentationLibraryLogs(InstrumentationLibraryLogs.newBuilder() - .addLogRecords(LogRecord.newBuilder()) - .build())).build(); + .addScopeLogs(ScopeLogs.newBuilder().addLogRecords(LogRecord.newBuilder()) .build())) + .build(); private static PluginSetting pluginSetting; private final int bufferWriteTimeoutInMillis = 100000; diff --git a/data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OTelMetricsProtoHelper.java b/data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OTelMetricsProtoHelper.java index 3a90eaea16..377059e66f 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OTelMetricsProtoHelper.java +++ b/data-prepper-plugins/otel-metrics-raw-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/OTelMetricsProtoHelper.java @@ -8,30 +8,21 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; -import io.opentelemetry.proto.common.v1.InstrumentationScope; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; -import io.opentelemetry.proto.metrics.v1.SummaryDataPoint; -import io.opentelemetry.proto.resource.v1.Resource; import org.apache.commons.codec.binary.Hex; import org.opensearch.dataprepper.model.metric.Bucket; import org.opensearch.dataprepper.model.metric.DefaultBucket; import org.opensearch.dataprepper.model.metric.DefaultExemplar; -import org.opensearch.dataprepper.model.metric.DefaultQuantile; import org.opensearch.dataprepper.model.metric.Exemplar; -import org.opensearch.dataprepper.model.metric.Quantile; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Collectors; @@ -40,14 +31,7 @@ public final class OTelMetricsProtoHelper { private static final Logger LOG = LoggerFactory.getLogger(OTelMetricsProtoHelper.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final String SERVICE_NAME = "service.name"; - private static final String METRIC_ATTRIBUTES = "metric.attributes"; - static final String RESOURCE_ATTRIBUTES = "resource.attributes"; static final String EXEMPLAR_ATTRIBUTES = "exemplar.attributes"; - static final String INSTRUMENTATION_LIBRARY_NAME = "instrumentationLibrary.name"; - static final String INSTRUMENTATION_LIBRARY_VERSION = "instrumentationLibrary.version"; - static final String INSTRUMENTATION_SCOPE_NAME = "instrumentationScope.name"; - static final String INSTRUMENTATION_SCOPE_VERSION = "instrumentationScope.version"; /** * To make it ES friendly we will replace '.' in keys with '@' in all the Keys in {@link io.opentelemetry.proto.common.v1.KeyValue} @@ -60,8 +44,6 @@ public final class OTelMetricsProtoHelper { * Span and Resource attributes are essential for kibana so they should not be nested. SO we will prefix them with "metric.attributes" * and "resource.attributes" and "exemplar.attributes". */ - public static final Function PREFIX_AND_METRIC_ATTRIBUTES_REPLACE_DOT_WITH_AT = i -> METRIC_ATTRIBUTES + DOT + i.replace(DOT, AT); - public static final Function PREFIX_AND_RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT = i -> RESOURCE_ATTRIBUTES + DOT + i.replace(DOT, AT); public static final Function PREFIX_AND_EXEMPLAR_ATTRIBUTES_REPLACE_DOT_WITH_AT = i -> EXEMPLAR_ATTRIBUTES + DOT + i.replace(DOT, AT); private OTelMetricsProtoHelper() { @@ -111,30 +93,6 @@ public static Object convertAnyValue(final AnyValue value) { } } - /** - * Converts the keys of all attributes in the {@link NumberDataPoint}. - * Also, casts the underlying data into its actual type - * - * @param numberDataPoint The point to process - * @return A Map containing all attributes of `numberDataPoint` with keys converted into an OS-friendly format - */ - public static Map convertKeysOfDataPointAttributes(final NumberDataPoint numberDataPoint) { - return numberDataPoint.getAttributesList().stream() - .collect(Collectors.toMap(i -> PREFIX_AND_METRIC_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue()))); - } - - /** - * Unpacks the List of {@link KeyValue} object into a Map. - * Converts the keys into an os friendly format and casts the underlying data into its actual type? - * - * @param attributesList The list of {@link KeyValue} objects to process - * @return A Map containing unpacked {@link KeyValue} data - */ - public static Map unpackKeyValueList(List attributesList) { - return attributesList.stream() - .collect(Collectors.toMap(i -> PREFIX_AND_METRIC_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue()))); - } - /** * Unpacks the List of {@link KeyValue} object into a Map. * Converts the keys into an os friendly format and casts the underlying data into its actual type? @@ -184,79 +142,11 @@ public static Double getExemplarValueAsDouble(final io.opentelemetry.proto.metri } } - public static Map getResourceAttributes(final Resource resource) { - return resource.getAttributesList().stream() - .collect(Collectors.toMap(i -> PREFIX_AND_RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue()))); - } - - /** - * Extracts the name and version of the used instrumentation library used - * - * @param instrumentationLibrary instrumentationLibrary - * @return A map, containing information about the instrumentation library - */ - public static Map getInstrumentationLibraryAttributes(final InstrumentationLibrary instrumentationLibrary) { - final Map instrumentationAttr = new HashMap<>(); - if (!instrumentationLibrary.getName().isEmpty()) { - instrumentationAttr.put(INSTRUMENTATION_LIBRARY_NAME, instrumentationLibrary.getName()); - } - if (!instrumentationLibrary.getVersion().isEmpty()) { - instrumentationAttr.put(INSTRUMENTATION_LIBRARY_VERSION, instrumentationLibrary.getVersion()); - } - return instrumentationAttr; - } - - /** - * Extracts the name and version of the used instrumentation scope used - * - * @param instrumentationScope instrumentationScope - * @return A map, containing information about the instrumentation scope - */ - public static Map getInstrumentationScopeAttributes(final InstrumentationScope instrumentationScope) { - final Map instrumentationScopeAttr = new HashMap<>(); - if (!instrumentationScope.getName().isEmpty()) { - instrumentationScopeAttr.put(INSTRUMENTATION_SCOPE_NAME, instrumentationScope.getName()); - } - if (!instrumentationScope.getVersion().isEmpty()) { - instrumentationScopeAttr.put(INSTRUMENTATION_SCOPE_VERSION, instrumentationScope.getVersion()); - } - return instrumentationScopeAttr; - } - public static String convertUnixNanosToISO8601(final long unixNano) { return Instant.ofEpochSecond(0L, unixNano).toString(); } - public static String getStartTimeISO8601(final NumberDataPoint numberDataPoint) { - return convertUnixNanosToISO8601(numberDataPoint.getStartTimeUnixNano()); - } - - public static String getTimeISO8601(final NumberDataPoint ndp) { - return convertUnixNanosToISO8601(ndp.getTimeUnixNano()); - } - - public static Optional getServiceName(final Resource resource) { - return resource.getAttributesList().stream() - .filter(keyValue -> keyValue.getKey().equals(SERVICE_NAME) && !keyValue.getValue().getStringValue().isEmpty()) - .findFirst() - .map(i -> i.getValue().getStringValue()); - } - - - public static Map mergeAllAttributes(final Collection> attributes) { - return attributes.stream() - .flatMap(map -> map.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - - public static List getQuantileValues(List quantileValues) { - return quantileValues.stream() - .map(q -> new DefaultQuantile(q.getQuantile(), q.getValue())) - .collect(Collectors.toList()); - } - /** * Create the buckets, see * the OTel metrics proto spec diff --git a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginExponentialHistogramTest.java b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginExponentialHistogramTest.java index 0112cb2596..f6c926fe8f 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginExponentialHistogramTest.java +++ b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginExponentialHistogramTest.java @@ -12,8 +12,8 @@ import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.metrics.v1.ExponentialHistogram; import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.resource.v1.Resource; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; @@ -139,7 +139,7 @@ private ExportMetricsServiceRequest fillServiceRequest(ExponentialHistogram hist .setName("name") .setDescription("description") .build(); - InstrumentationLibraryMetrics instLib = InstrumentationLibraryMetrics.newBuilder() + ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder() .addMetrics(metric).build(); Resource resource = Resource.newBuilder() @@ -149,7 +149,7 @@ private ExportMetricsServiceRequest fillServiceRequest(ExponentialHistogram hist ).build(); ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() .setResource(resource) - .addInstrumentationLibraryMetrics(instLib) + .addScopeMetrics(scopeMetrics) .build(); return ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build(); } diff --git a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginGaugeTest.java b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginGaugeTest.java index 65df42befc..c3ff0207e9 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginGaugeTest.java +++ b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginGaugeTest.java @@ -10,12 +10,10 @@ import com.google.protobuf.ByteString; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; import io.opentelemetry.proto.common.v1.InstrumentationScope; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.metrics.v1.Exemplar; import io.opentelemetry.proto.metrics.v1.Gauge; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; import io.opentelemetry.proto.metrics.v1.ScopeMetrics; @@ -71,60 +69,6 @@ void init() { rawProcessor = new OTelMetricsRawProcessor(testsettings, new OtelMetricsRawProcessorConfig()); } - @Test - void testInstrumentationLibrary() throws JsonProcessingException { - NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4); - Gauge gauge = Gauge.newBuilder().addDataPoints(p1).build(); - - io.opentelemetry.proto.metrics.v1.Metric.Builder metric = io.opentelemetry.proto.metrics.v1.Metric.newBuilder() - .setGauge(gauge) - .setUnit("seconds") - .setName("name") - .setDescription("description"); - - InstrumentationLibraryMetrics isntLib = InstrumentationLibraryMetrics.newBuilder() - .addMetrics(metric) - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder() - .setName("ilname") - .setVersion("ilversion") - .build()) - .build(); - - Resource resource = Resource.newBuilder() - .addAttributes(KeyValue.newBuilder() - .setKey("service.name") - .setValue(AnyValue.newBuilder().setStringValue("service").build()) - ).build(); - - ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() - .addInstrumentationLibraryMetrics(isntLib) - .setResource(resource) - .build(); - - ExportMetricsServiceRequest exportMetricRequest = ExportMetricsServiceRequest.newBuilder() - .addResourceMetrics(resourceMetrics).build(); - - Record record = new Record<>(exportMetricRequest); - - Collection> records = rawProcessor.doExecute(Collections.singletonList(record)); - List> list = new ArrayList<>(records); - - Record dataPrepperResult = list.get(0); - ObjectMapper objectMapper = new ObjectMapper(); - Map map = objectMapper.readValue(dataPrepperResult.getData().toJsonString(), Map.class); - assertThat(map).contains(entry("kind", Metric.KIND.GAUGE.toString())); - assertThat(map).contains(entry("unit", "seconds")); - assertThat(map).contains(entry("serviceName", "service")); - assertThat(map).contains(entry("resource.attributes.service@name", "service")); - assertThat(map).contains(entry("description", "description")); - assertThat(map).contains(entry("value", 4.0D)); - assertThat(map).contains(entry("startTime", "1970-01-01T00:00:00Z")); - assertThat(map).contains(entry("time", "1970-01-01T00:00:00Z")); - assertThat(map).contains(entry("instrumentationLibrary.name", "ilname")); - assertThat(map).contains(entry("instrumentationLibrary.version", "ilversion")); - - } - @Test void testScopeMetricsLibrary() throws JsonProcessingException { NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4); diff --git a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginHistogramTest.java b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginHistogramTest.java index 807e05ad50..9559987258 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginHistogramTest.java +++ b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginHistogramTest.java @@ -12,8 +12,8 @@ import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.metrics.v1.Histogram; import io.opentelemetry.proto.metrics.v1.HistogramDataPoint; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.resource.v1.Resource; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -119,7 +119,7 @@ private ExportMetricsServiceRequest fillServiceRequest(Histogram histogram) { .setName("name") .setDescription("description") .build(); - InstrumentationLibraryMetrics instLib = InstrumentationLibraryMetrics.newBuilder() + ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder() .addMetrics(metric).build(); Resource resource = Resource.newBuilder() @@ -129,7 +129,7 @@ private ExportMetricsServiceRequest fillServiceRequest(Histogram histogram) { ).build(); ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() .setResource(resource) - .addInstrumentationLibraryMetrics(instLib) + .addScopeMetrics(scopeMetrics) .build(); return ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build(); } diff --git a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginSumTest.java b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginSumTest.java index e6509116df..f8dd361bde 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginSumTest.java +++ b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginSumTest.java @@ -10,13 +10,13 @@ import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.ArrayValue; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; +import io.opentelemetry.proto.common.v1.InstrumentationScope; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.common.v1.KeyValueList; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.metrics.v1.Sum; import io.opentelemetry.proto.resource.v1.Resource; import org.junit.jupiter.api.BeforeEach; @@ -84,8 +84,8 @@ void test() throws JsonProcessingException { .setDescription("description") .build(); - InstrumentationLibraryMetrics instLib = InstrumentationLibraryMetrics.newBuilder() - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder().setVersion("v1").setName("name").build()) + ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder() + .setScope(InstrumentationScope.newBuilder().setVersion("v1").setName("name").build()) .addMetrics(metric).build(); Resource resource = Resource.newBuilder() @@ -96,7 +96,7 @@ void test() throws JsonProcessingException { ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() .setResource(resource) - .addInstrumentationLibraryMetrics(instLib) + .addScopeMetrics(scopeMetrics) .build(); ExportMetricsServiceRequest exportMetricRequest = ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build(); @@ -153,12 +153,12 @@ void missingNameInvalidMetricTest() throws JsonProcessingException { .setDescription("description") .build(); - InstrumentationLibraryMetrics instLib = InstrumentationLibraryMetrics.newBuilder() - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder().setVersion("v1").setName("name").build()) + ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder() + .setScope(InstrumentationScope.newBuilder().setVersion("v1").setName("name").build()) .addMetrics(metric).build(); - InstrumentationLibraryMetrics instLibWithInvalidMetric = InstrumentationLibraryMetrics.newBuilder() - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder().setVersion("v1").setName("name").build()) + ScopeMetrics scopeMetricsWithInvalidMetric = ScopeMetrics.newBuilder() + .setScope(InstrumentationScope.newBuilder().setVersion("v1").setName("name").build()) .addMetrics(metricWithNameMissing).build(); Resource resource = Resource.newBuilder() @@ -169,12 +169,12 @@ void missingNameInvalidMetricTest() throws JsonProcessingException { ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() .setResource(resource) - .addInstrumentationLibraryMetrics(instLib) + .addScopeMetrics(scopeMetrics) .build(); ResourceMetrics resourceMetricsWithInvalidMetric = ResourceMetrics.newBuilder() .setResource(resource) - .addInstrumentationLibraryMetrics(instLibWithInvalidMetric) + .addScopeMetrics(scopeMetricsWithInvalidMetric) .build(); ExportMetricsServiceRequest exportMetricRequest = ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build(); @@ -204,8 +204,8 @@ private void assertSumProcessing(Map map) { assertThat(map).contains(entry("metric.attributes.db@details", "{\"statement@params\":\"us-east-1\",\"statement\":1000}")); assertThat(map).contains(entry("startTime","1970-01-01T00:00:00Z")); assertThat(map).contains(entry("time","1970-01-01T00:00:00Z")); - assertThat(map).contains(entry("instrumentationLibrary.version", "v1")); - assertThat(map).contains(entry("instrumentationLibrary.name", "name")); + assertThat(map).contains(entry("instrumentationScope.version", "v1")); + assertThat(map).contains(entry("instrumentationScope.name", "name")); assertThat(map).contains(entry("metric.attributes.aws@details", "[\"asdf\",2000.123,\"{\\\"statement@params\\\":\\\"us-east-1\\\",\\\"statement\\\":1000}\"]")); assertThat(map).contains(entry("flags", 0)); diff --git a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginSummaryTest.java b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginSummaryTest.java index a0eae0a847..805485de00 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginSummaryTest.java +++ b/data-prepper-plugins/otel-metrics-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/otelmetrics/MetricsPluginSummaryTest.java @@ -10,9 +10,9 @@ import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.metrics.v1.Summary; import io.opentelemetry.proto.metrics.v1.SummaryDataPoint; import io.opentelemetry.proto.resource.v1.Resource; @@ -66,7 +66,7 @@ void testSummaryProcessing() throws JsonProcessingException { .setDescription("description") .build(); - InstrumentationLibraryMetrics instLib = InstrumentationLibraryMetrics.newBuilder() + ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder() .addMetrics(metric).build(); Resource resource = Resource.newBuilder() @@ -77,7 +77,7 @@ void testSummaryProcessing() throws JsonProcessingException { ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() .setResource(resource) - .addInstrumentationLibraryMetrics(instLib) + .addScopeMetrics(scopeMetrics) .build(); ExportMetricsServiceRequest exportMetricRequest = ExportMetricsServiceRequest.newBuilder().addResourceMetrics(resourceMetrics).build(); diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcServiceTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcServiceTest.java index be5c1c817d..812c75f934 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcServiceTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcServiceTest.java @@ -12,7 +12,6 @@ import io.micrometer.core.instrument.Timer; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; @@ -54,6 +53,7 @@ import static org.mockito.Mockito.when; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import io.opentelemetry.proto.metrics.v1.Gauge; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; @ExtendWith(MockitoExtension.class) public class OTelMetricsGrpcServiceTest { @@ -61,7 +61,7 @@ public class OTelMetricsGrpcServiceTest { private static Gauge gauge = Gauge.newBuilder().addDataPoints(p1).build(); private static final ExportMetricsServiceRequest METRICS_REQUEST = ExportMetricsServiceRequest.newBuilder() .addResourceMetrics(ResourceMetrics.newBuilder() - .addInstrumentationLibraryMetrics(InstrumentationLibraryMetrics.newBuilder() + .addScopeMetrics(ScopeMetrics.newBuilder() .addMetrics(Metric.newBuilder().setGauge(gauge).setUnit("seconds").setName("name").build()) .build())).build(); diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java index 4e67c050f5..ee1f65c191 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java @@ -36,14 +36,14 @@ import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import io.opentelemetry.proto.common.v1.InstrumentationScope; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; import io.opentelemetry.proto.metrics.v1.Gauge; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.resource.v1.Resource; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; @@ -1186,9 +1186,9 @@ private ExportMetricsServiceRequest createExportMetricsRequest() { .setUnit("seconds") .setName("name") .setDescription("description"); - InstrumentationLibraryMetrics isntLib = InstrumentationLibraryMetrics.newBuilder() + ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder() .addMetrics(metric) - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder() + .setScope(InstrumentationScope.newBuilder() .setName("ilname") .setVersion("ilversion") .build()) @@ -1197,7 +1197,7 @@ private ExportMetricsServiceRequest createExportMetricsRequest() { final ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() .setResource(resource) - .addInstrumentationLibraryMetrics(isntLib) + .addScopeMetrics(scopeMetrics) .build(); return ExportMetricsServiceRequest.newBuilder() diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java index 403efee46b..32f4570240 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java @@ -47,12 +47,11 @@ import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.metrics.v1.Gauge; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.resource.v1.Resource; @ExtendWith(MockitoExtension.class) @@ -158,18 +157,11 @@ private ExportMetricsServiceRequest createExportMetricsRequest() { .setUnit("seconds") .setName("name") .setDescription("description"); - InstrumentationLibraryMetrics isntLib = InstrumentationLibraryMetrics.newBuilder() - .addMetrics(metric) - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder() - .setName("ilname") - .setVersion("ilversion") - .build()) - .build(); - + ScopeMetrics scopeMetric = ScopeMetrics.newBuilder().addMetrics(metric).build(); final ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() .setResource(resource) - .addInstrumentationLibraryMetrics(isntLib) + .addScopeMetrics(scopeMetric) .build(); return ExportMetricsServiceRequest.newBuilder() diff --git a/data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodec.java b/data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodec.java index 8875e90e53..ff96e73c89 100644 --- a/data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodec.java +++ b/data-prepper-plugins/otel-proto-common/src/main/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodec.java @@ -12,7 +12,6 @@ import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; import io.opentelemetry.proto.common.v1.InstrumentationScope; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.logs.v1.LogRecord; @@ -20,17 +19,14 @@ import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import io.opentelemetry.proto.metrics.v1.SummaryDataPoint; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.resource.v1.Resource; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Status; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; -import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.log.JacksonOtelLog; import org.opensearch.dataprepper.model.log.OpenTelemetryLog; import org.opensearch.dataprepper.model.metric.Bucket; @@ -38,23 +34,24 @@ import org.opensearch.dataprepper.model.metric.DefaultExemplar; import org.opensearch.dataprepper.model.metric.DefaultQuantile; import org.opensearch.dataprepper.model.metric.Exemplar; -import org.opensearch.dataprepper.model.metric.Quantile; import org.opensearch.dataprepper.model.metric.JacksonExponentialHistogram; import org.opensearch.dataprepper.model.metric.JacksonGauge; import org.opensearch.dataprepper.model.metric.JacksonHistogram; import org.opensearch.dataprepper.model.metric.JacksonSum; import org.opensearch.dataprepper.model.metric.JacksonSummary; import org.opensearch.dataprepper.model.metric.Metric; +import org.opensearch.dataprepper.model.metric.Quantile; +import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.trace.DefaultLink; import org.opensearch.dataprepper.model.trace.DefaultSpanEvent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.opensearch.dataprepper.model.trace.DefaultTraceGroupFields; import org.opensearch.dataprepper.model.trace.JacksonSpan; import org.opensearch.dataprepper.model.trace.Link; import org.opensearch.dataprepper.model.trace.Span; import org.opensearch.dataprepper.model.trace.SpanEvent; import org.opensearch.dataprepper.model.trace.TraceGroupFields; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; import java.time.Instant; @@ -88,8 +85,6 @@ public class OTelProtoCodec { protected static final String SERVICE_NAME = "service.name"; protected static final String SPAN_ATTRIBUTES = "span.attributes"; static final String RESOURCE_ATTRIBUTES = "resource.attributes"; - static final String INSTRUMENTATION_LIBRARY_NAME = "instrumentationLibrary.name"; - static final String INSTRUMENTATION_LIBRARY_VERSION = "instrumentationLibrary.version"; static final String STATUS_CODE = "status.code"; static final String STATUS_MESSAGE = "status.message"; @@ -198,17 +193,6 @@ protected Collection parseResourceLogs(ResourceLogs rs, final final Map resourceAttributes = OTelProtoCodec.getResourceAttributes(rs.getResource()); final String schemaUrl = rs.getSchemaUrl(); - Stream mappedInstrumentationLibraryLogs = rs.getInstrumentationLibraryLogsList() - .stream() - .map(ils -> - processLogsList(ils.getLogRecordsList(), - serviceName, - OTelProtoCodec.getInstrumentationLibraryAttributes(ils.getInstrumentationLibrary()), - resourceAttributes, - schemaUrl, - timeReceived)) - .flatMap(Collection::stream); - Stream mappedScopeListLogs = rs.getScopeLogsList() .stream() .map(sls -> @@ -220,7 +204,7 @@ protected Collection parseResourceLogs(ResourceLogs rs, final timeReceived)) .flatMap(Collection::stream); - return Stream.concat(mappedInstrumentationLibraryLogs, mappedScopeListLogs).collect(Collectors.toList()); + return mappedScopeListLogs.collect(Collectors.toList()); } protected Map splitResourceSpansByTraceId(final ResourceSpans resourceSpans) { @@ -229,7 +213,7 @@ protected Map splitResourceSpansByTraceId(final ResourceS Map result = new HashMap<>(); Map resultBuilderMap = new HashMap<>(); - if (resourceSpans.getScopeSpansList().size() > 0) { + if (!resourceSpans.getScopeSpansList().isEmpty()) { for (Map.Entry> entry: splitScopeSpansByTraceId(resourceSpans.getScopeSpansList()).entrySet()) { ResourceSpans.Builder resourceSpansBuilder = ResourceSpans.newBuilder().addAllScopeSpans(entry.getValue()); if (hasResource) { @@ -239,22 +223,6 @@ protected Map splitResourceSpansByTraceId(final ResourceS } } - if (resourceSpans.getInstrumentationLibrarySpansList().size() > 0) { - for (Map.Entry> entry: splitInstrumentationLibrarySpansByTraceId(resourceSpans.getInstrumentationLibrarySpansList()).entrySet()) { - ResourceSpans.Builder resourceSpansBuilder; - String traceId = entry.getKey(); - if (resultBuilderMap.containsKey(traceId)) { - resourceSpansBuilder = resultBuilderMap.get(traceId); - } else { - resourceSpansBuilder = ResourceSpans.newBuilder(); - if (hasResource) { - resourceSpansBuilder.setResource(resource); - } - resultBuilderMap.put(traceId, resourceSpansBuilder); - } - resourceSpansBuilder.addAllInstrumentationLibrarySpans(entry.getValue()); - } - } for (Map.Entry entry: resultBuilderMap.entrySet()) { result.put(entry.getKey(), entry.getValue().build()); } @@ -266,14 +234,10 @@ protected List parseResourceSpans(final ResourceSpans resourceSpans, final final String serviceName = getServiceName(resourceSpans.getResource()).orElse(null); final Map resourceAttributes = getResourceAttributes(resourceSpans.getResource()); - if (resourceSpans.getScopeSpansList().size() > 0) { + if (!resourceSpans.getScopeSpansList().isEmpty()) { return parseScopeSpans(resourceSpans.getScopeSpansList(), serviceName, resourceAttributes, timeReceived); } - if (resourceSpans.getInstrumentationLibrarySpansList().size() > 0) { - return parseInstrumentationLibrarySpans(resourceSpans.getInstrumentationLibrarySpansList(), serviceName, resourceAttributes, timeReceived); - } - LOG.debug("No spans found to parse from ResourceSpans object: {}", resourceSpans); return Collections.emptyList(); } @@ -306,38 +270,6 @@ private Map> splitScopeSpansByTraceId(final List parseInstrumentationLibrarySpans(final List instrumentationLibrarySpansList, - final String serviceName, final Map resourceAttributes, - final Instant timeReceived) { - return instrumentationLibrarySpansList.stream() - .map(instrumentationLibrarySpans -> parseSpans(instrumentationLibrarySpans.getSpansList(), - instrumentationLibrarySpans.getInstrumentationLibrary(), this::getInstrumentationLibraryAttributes, - serviceName, resourceAttributes, timeReceived)) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - } - - private Map> splitInstrumentationLibrarySpansByTraceId(final List instrumentationLibrarySpansList) { - Map> result = new HashMap<>(); - for (InstrumentationLibrarySpans is: instrumentationLibrarySpansList) { - final boolean hasInstrumentationLibrary = is.hasInstrumentationLibrary(); - final io.opentelemetry.proto.common.v1.InstrumentationLibrary instrumentationLibrary = is.getInstrumentationLibrary(); - for (Map.Entry> entry: splitSpansByTraceId(is.getSpansList()).entrySet()) { - String traceId = entry.getKey(); - InstrumentationLibrarySpans.Builder ilSpansBuilder = InstrumentationLibrarySpans.newBuilder().setSchemaUrl(is.getSchemaUrl()).addAllSpans(entry.getValue()); - if (hasInstrumentationLibrary) { - ilSpansBuilder.setInstrumentationLibrary(instrumentationLibrary); - } - - if (!result.containsKey(traceId)) { - result.put(traceId, new ArrayList<>()); - } - result.get(traceId).add(ilSpansBuilder.build()); - } - } - return result; - } - private Map> splitSpansByTraceId(final List spans) { Map> result = new HashMap<>(); @@ -547,17 +479,6 @@ protected TraceGroupFields getTraceGroupFields(final io.opentelemetry.proto.trac return traceGroupFieldsBuilder.build(); } - protected Map getInstrumentationLibraryAttributes(final InstrumentationLibrary instrumentationLibrary) { - final Map instrumentationAttr = new HashMap<>(); - if (!instrumentationLibrary.getName().isEmpty()) { - instrumentationAttr.put(INSTRUMENTATION_SCOPE_NAME, instrumentationLibrary.getName()); - } - if (!instrumentationLibrary.getVersion().isEmpty()) { - instrumentationAttr.put(INSTRUMENTATION_SCOPE_VERSION, instrumentationLibrary.getVersion()); - } - return instrumentationAttr; - } - protected Map getSpanStatusAttributes(final Status status) { final Map statusAttr = new HashMap<>(); statusAttr.put(STATUS_CODE, status.getCodeValue()); @@ -600,11 +521,6 @@ public Collection> parseExportMetricsServiceRequest( final Map resourceAttributes = OTelProtoCodec.getResourceAttributes(rs.getResource()); final String serviceName = OTelProtoCodec.getServiceName(rs.getResource()).orElse(null); - for (InstrumentationLibraryMetrics is : rs.getInstrumentationLibraryMetricsList()) { - final Map ils = OTelProtoCodec.getInstrumentationLibraryAttributes(is.getInstrumentationLibrary()); - recordsOut.addAll(processMetricsList(is.getMetricsList(), serviceName, ils, resourceAttributes, schemaUrl, droppedCounter, exponentialHistogramMaxAllowedScale, timeReceived, calculateHistogramBuckets, calculateExponentialHistogramBuckets, flattenAttributes)); - } - for (ScopeMetrics sm : rs.getScopeMetricsList()) { final Map ils = OTelProtoCodec.getInstrumentationScopeAttributes(sm.getScope()); recordsOut.addAll(processMetricsList(sm.getMetricsList(), serviceName, ils, resourceAttributes, schemaUrl, droppedCounter, exponentialHistogramMaxAllowedScale, timeReceived, calculateHistogramBuckets, calculateExponentialHistogramBuckets, flattenAttributes)); @@ -1171,23 +1087,6 @@ public static Map getResourceAttributes(final Resource resource) .collect(Collectors.toMap(i -> PREFIX_AND_RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue()))); } - /** - * Extracts the name and version of the used instrumentation library used - * - * @param instrumentationLibrary the instrumentation library - * @return A map, containing information about the instrumentation library - */ - public static Map getInstrumentationLibraryAttributes(final InstrumentationLibrary instrumentationLibrary) { - final Map instrumentationAttr = new HashMap<>(); - if (!instrumentationLibrary.getName().isEmpty()) { - instrumentationAttr.put(INSTRUMENTATION_LIBRARY_NAME, instrumentationLibrary.getName()); - } - if (!instrumentationLibrary.getVersion().isEmpty()) { - instrumentationAttr.put(INSTRUMENTATION_LIBRARY_VERSION, instrumentationLibrary.getVersion()); - } - return instrumentationAttr; - } - /** * Extracts the name and version of the used instrumentation scope used * diff --git a/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodecTest.java b/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodecTest.java index b2e42c6c20..945519493a 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodecTest.java +++ b/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodecTest.java @@ -15,7 +15,6 @@ import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.ArrayValue; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; import io.opentelemetry.proto.common.v1.InstrumentationScope; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.common.v1.KeyValueList; @@ -90,7 +89,6 @@ public class OTelProtoCodecTest { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final Random RANDOM = new Random(); private static final String TEST_REQUEST_TRACE_JSON_FILE = "test-request.json"; - private static final String TEST_REQUEST_INSTRUMENTATION_LIBRARY_TRACE_JSON_FILE = "test-request-instrumentation-library.json"; private static final String TEST_REQUEST_BOTH_SPAN_TYPES_JSON_FILE = "test-request-both-span-types.json"; private static final String TEST_REQUEST_NO_SPANS_JSON_FILE = "test-request-no-spans.json"; private static final String TEST_SPAN_EVENT_JSON_FILE = "test-span-event.json"; @@ -99,7 +97,6 @@ public class OTelProtoCodecTest { private static final String TEST_REQUEST_HISTOGRAM_METRICS_JSON_FILE = "test-histogram-metrics.json"; private static final String TEST_REQUEST_HISTOGRAM_METRICS_NO_EXPLICIT_BOUNDS_JSON_FILE = "test-histogram-metrics-no-explicit-bounds.json"; private static final String TEST_REQUEST_LOGS_JSON_FILE = "test-request-log.json"; - private static final String TEST_REQUEST_LOGS_IS_JSON_FILE = "test-request-log-is.json"; private static final String TEST_REQUEST_MULTIPLE_TRACES_FILE = "test-request-multiple-traces.json"; @@ -168,7 +165,6 @@ public void testSplitExportTraceServiceRequestWithMultipleTraces() throws Except assertThat(request.getResourceSpansList().size(), equalTo(1)); ResourceSpans rs = request.getResourceSpansList().get(0); assertThat(rs.getScopeSpansList().size(), equalTo(1)); - assertThat(rs.getInstrumentationLibrarySpansList().size(), equalTo(0)); ScopeSpans ss = rs.getScopeSpansList().get(0); assertThat(ss.getSpansList().size(), equalTo(1)); io.opentelemetry.proto.trace.v1.Span span = ss.getSpansList().get(0); @@ -178,7 +174,6 @@ public void testSplitExportTraceServiceRequestWithMultipleTraces() throws Except assertThat(request.getResourceSpansList().size(), equalTo(1)); ResourceSpans rs = request.getResourceSpansList().get(0); assertThat(rs.getScopeSpansList().size(), equalTo(2)); - assertThat(rs.getInstrumentationLibrarySpansList().size(), equalTo(2)); ScopeSpans ss = rs.getScopeSpansList().get(0); assertThat(ss.getSpansList().size(), equalTo(1)); @@ -196,7 +191,6 @@ public void testSplitExportTraceServiceRequestWithMultipleTraces() throws Except assertThat(request.getResourceSpansList().size(), equalTo(1)); ResourceSpans rs = request.getResourceSpansList().get(0); assertThat(rs.getScopeSpansList().size(), equalTo(1)); - assertThat(rs.getInstrumentationLibrarySpansList().size(), equalTo(0)); ScopeSpans ss = rs.getScopeSpansList().get(0); assertThat(ss.getSpansList().size(), equalTo(1)); io.opentelemetry.proto.trace.v1.Span span = ss.getSpansList().get(0); @@ -216,13 +210,7 @@ public void testParseExportTraceServiceRequest() throws IOException { } @Test - public void testParseExportTraceServiceRequest_InstrumentationLibrarySpans() throws IOException { - final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_INSTRUMENTATION_LIBRARY_TRACE_JSON_FILE); - final List spans = decoderUnderTest.parseExportTraceServiceRequest(exportTraceServiceRequest, Instant.now()); - validateSpans(spans); - } - - @Test + // TODO tlongo is this test still relevant? public void testParseExportTraceServiceRequest_ScopeSpansTakesPrecedenceOverInstrumentationLibrarySpans() throws IOException { final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_BOTH_SPAN_TYPES_JSON_FILE); final List spans = decoderUnderTest.parseExportTraceServiceRequest(exportTraceServiceRequest, Instant.now()); @@ -394,26 +382,6 @@ public void testArrayOfValueAsResourceAttributes() throws JsonProcessingExceptio } - @Test - public void testInstrumentationLibraryAttributes() { - final InstrumentationLibrary il1 = InstrumentationLibrary.newBuilder().setName("Jaeger").setVersion("0.6.0").build(); - final InstrumentationLibrary il2 = InstrumentationLibrary.newBuilder().setName("Jaeger").build(); - final InstrumentationLibrary il3 = InstrumentationLibrary.newBuilder().setVersion("0.6.0").build(); - final InstrumentationLibrary il4 = InstrumentationLibrary.newBuilder().build(); - - assertThat(decoderUnderTest.getInstrumentationLibraryAttributes(il1).size(), equalTo(2)); - assertThat(decoderUnderTest.getInstrumentationLibraryAttributes(il1).get(OTelProtoCodec.INSTRUMENTATION_SCOPE_NAME), equalTo(il1.getName())); - assertThat(decoderUnderTest.getInstrumentationLibraryAttributes(il1).get(OTelProtoCodec.INSTRUMENTATION_SCOPE_VERSION), equalTo(il1.getVersion())); - - assertThat(decoderUnderTest.getInstrumentationLibraryAttributes(il2).size(), equalTo(1)); - assertThat(decoderUnderTest.getInstrumentationLibraryAttributes(il2).get(OTelProtoCodec.INSTRUMENTATION_SCOPE_NAME), equalTo(il2.getName())); - - assertThat(decoderUnderTest.getInstrumentationLibraryAttributes(il3).size(), equalTo(1)); - assertThat(decoderUnderTest.getInstrumentationLibraryAttributes(il3).get(OTelProtoCodec.INSTRUMENTATION_SCOPE_VERSION), equalTo(il3.getVersion())); - - assertThat(decoderUnderTest.getInstrumentationLibraryAttributes(il4).isEmpty(), is(true)); - } - @Test public void testStatusAttributes() { final Status st1 = Status.newBuilder().setCode(Status.StatusCode.STATUS_CODE_ERROR).setMessage("Some message").build(); @@ -498,15 +466,6 @@ public void testParseExportLogsServiceRequest_ScopedLogs() throws IOException { validateLog(logs.get(0)); } - @Test - public void testParseExportLogsServiceRequest_InstrumentationLibraryLogs() throws IOException { - final ExportLogsServiceRequest exportLogsServiceRequest = buildExportLogsServiceRequestFromJsonFile(TEST_REQUEST_LOGS_IS_JSON_FILE); - List logs = decoderUnderTest.parseExportLogsServiceRequest(exportLogsServiceRequest, Instant.now()); - - assertThat(logs.size() , is(equalTo(1))); - validateLog(logs.get(0)); - } - private void validateLog(OpenTelemetryLog logRecord) { assertThat(logRecord.getServiceName(), is("service")); assertThat(logRecord.getTime(), is("2020-05-24T14:00:00Z")); @@ -524,13 +483,6 @@ private void validateLog(OpenTelemetryLog logRecord) { assertThat(mergedAttributes.get("resource.attributes.service@name"), is("service")); } - @Test - public void testParseExportLogsServiceRequest_InstrumentationLibrarySpans() throws IOException { - final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_INSTRUMENTATION_LIBRARY_TRACE_JSON_FILE); - final List spans = decoderUnderTest.parseExportTraceServiceRequest(exportTraceServiceRequest, Instant.now()); - validateSpans(spans); - } - @Test public void testParseExportMetricsServiceRequest_Guage() throws IOException { final ExportMetricsServiceRequest exportMetricsServiceRequest = buildExportMetricsServiceRequestFromJsonFile(TEST_REQUEST_GAUGE_METRICS_JSON_FILE); @@ -743,7 +695,7 @@ public void testEncodeSpanStatusMissingAll() { } @Test - public void testEncodeInstrumentationLibraryComplete() { + public void testEncodeInstrumentationScopeComplete() { final String testName = "test name"; final String testVersion = "1.1"; final String testKeyIrrelevant = "irrelevantKey"; @@ -757,7 +709,7 @@ public void testEncodeInstrumentationLibraryComplete() { } @Test - public void testEncodeInstrumentationLibraryMissingName() { + public void testEncodeInstrumentationScopeMissingName() { final String testVersion = "1.1"; final String testKeyIrrelevant = "irrelevantKey"; final Map testAllAttributes = Map.of( @@ -768,7 +720,7 @@ public void testEncodeInstrumentationLibraryMissingName() { } @Test - public void testEncodeInstrumentationLibraryMissingVersion() { + public void testEncodeInstrumentationScopeMissingVersion() { final String testName = "test name"; final String testKeyIrrelevant = "irrelevantKey"; final Map testAllAttributes = Map.of( @@ -779,7 +731,7 @@ public void testEncodeInstrumentationLibraryMissingVersion() { } @Test - public void testEncodeInstrumentationLibraryMissingAll() { + public void testEncodeInstrumentationScopeMissingAll() { final String testKeyIrrelevant = "irrelevantKey"; final Map testAllAttributes = Map.of(testKeyIrrelevant, 2); final InstrumentationScope instrumentationScope = encoderUnderTest.constructInstrumentationScope(testAllAttributes); diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-both-span-types.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-both-span-types.json index 176d3425bc..62559e23eb 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-both-span-types.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-both-span-types.json @@ -240,37 +240,7 @@ } ] } - ], - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "CFrAgv/Pv40=", - "traceState": "", - "parentSpanId": "yxwHNNFJQP0=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902043168792500", - "endTimeUnixNano": "1597902043215953100", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } ] } ] -} \ No newline at end of file +} diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-instrumentation-library.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-instrumentation-library.json deleted file mode 100644 index 1088034f6d..0000000000 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-instrumentation-library.json +++ /dev/null @@ -1,246 +0,0 @@ -{ - "resourceSpans": [ - { - "resource": { - "attributes": [ - { - "key": "service.name", - "value": { - "stringValue": "analytics-service" - } - }, - { - "key": "telemetry.sdk.language", - "value": { - "stringValue": "java" - } - }, - { - "key": "telemetry.sdk.name", - "value": { - "stringValue": "opentelemetry" - } - }, - { - "key": "telemetry.sdk.version", - "value": { - "stringValue": "0.8.0-SNAPSHOT" - } - }, - { - "key": "array", - "value": { - "arrayValue": { - "values": [ - { - "stringValue": "test string" - }, - { - "boolValue": false - }, - { - "intValue": 0 - } - ] - } - } - }, - { - "key": "kvList", - "value": { - "kvlistValue": { - "values": [ - { - "key": "key1", - "value": { - "stringValue": "value1" - } - }, - { - "key": "key2", - "value": { - "stringValue": "value2" - } - } - ] - } - } - } - ], - "droppedAttributesCount": 0 - }, - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "CFrAgv/Pv40=", - "traceState": "", - "parentSpanId": "yxwHNNFJQP0=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902043168792500", - "endTimeUnixNano": "1597902043215953100", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.apache-httpasyncclient-4.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "G4PRdsC1+0Y=", - "traceState": "", - "parentSpanId": "CFrAgv/Pv40=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902043175204700", - "endTimeUnixNano": "1597902043205117100", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.servlet-3.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "yxwHNNFJQP0=", - "traceState": "", - "parentSpanId": "", - "name": "/logs", - "kind": "SPAN_KIND_SERVER", - "startTimeUnixNano": "1597902043168010200", - "endTimeUnixNano": "1597902043217170200", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "net.peer.port", - "value": { - "intValue": "41164" - } - }, - { - "key": "servlet.path", - "value": { - "stringValue": "/logs" - } - }, - { - "key": "http.response_content_length", - "value": { - "intValue": "7" - } - }, - { - "key": "http.user_agent", - "value": { - "stringValue": "curl/7.54.0" - } - }, - { - "key": "http.flavor", - "value": { - "stringValue": "HTTP/1.1" - } - }, - { - "key": "servlet.context", - "value": { - "stringValue": "" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "http://0.0.0.0:8087/logs" - } - }, - { - "key": "net.peer.ip", - "value": { - "stringValue": "172.29.0.1" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "POST" - } - }, - { - "key": "http.client_ip", - "value": { - "stringValue": "172.29.0.1" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-log-is.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-log-is.json deleted file mode 100644 index f598d05730..0000000000 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-log-is.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "resourceLogs": [{ - "resource": { - "attributes": [{ - "key": "service.name", - "value": { - "stringValue": "service" - } - }] - }, - "schemaUrl": "schemaurl", - "instrumentationLibraryLogs": [{ - "logRecords": [{ - "timeUnixNano": "1590328800000000000", - "severityNumber": "SEVERITY_NUMBER_DEBUG", - "severityText": "Severity value", - "body": { - "stringValue": "Log value" - }, - "attributes": [{ - "key": "statement.params", - "value": { - "stringValue": "us-east-1" - } - }], - "droppedAttributesCount": 3, - "flags": 1, - "traceId": "uhocI7QJO2M=", - "spanId": "LMg6yQ68Rpw=", - "observedTimeUnixNano": "1590328802000000000" - }] - }] - }] -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-logs.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-logs.json index 2ee4beb4b9..9cf99db8e0 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-logs.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-logs.json @@ -10,59 +10,34 @@ } }] }, - "scopeLogs": [{ - "logRecords": [{ - "timeUnixNano": "1590328800000000000", - "severityNumber": "SEVERITY_NUMBER_DEBUG", - "severityText": "Severity value", - "body": { - "stringValue": "Log value" - }, - "attributes": [{ - "key": "statement.params", - "value": { - "stringValue": "us-east-1" + "scopeLogs": [ + { + "logRecords": [ + { + "timeUnixNano": "1590328800000000000", + "severityNumber": "SEVERITY_NUMBER_DEBUG", + "severityText": "Severity value", + "body": { + "stringValue": "Log value" + }, + "attributes": [ + { + "key": "statement.params", + "value": { + "stringValue": "us-east-1" + } + } + ], + "droppedAttributesCount": 3, + "flags": 1, + "traceId": "uhocI7QJO2M=", + "spanId": "LMg6yQ68Rpw=", + "observedTimeUnixNano": "1590328802000000000" } - }], - "droppedAttributesCount": 3, - "flags": 1, - "traceId": "uhocI7QJO2M=", - "spanId": "LMg6yQ68Rpw=", - "observedTimeUnixNano": "1590328802000000000" - }] - }], + ] + } + ], "schemaUrl": "schemaurl" - }, - { - "resource": { - "attributes": [{ - "key": "service.name", - "value": { - "stringValue": "service" - } - }] - }, - "schemaUrl": "schemaurl", - "instrumentationLibraryLogs": [{ - "logRecords": [{ - "timeUnixNano": "1590328800000000000", - "severityNumber": "SEVERITY_NUMBER_DEBUG", - "severityText": "Severity value", - "body": { - "stringValue": "Log value" - }, - "attributes": [{ - "key": "statement.params", - "value": { - "stringValue": "us-east-1" - } - }], - "droppedAttributesCount": 3, - "flags": 1, - "traceId": "uhocI7QJO2M=", - "spanId": "LMg6yQ68Rpw=", - "observedTimeUnixNano": "1590328802000000000" - }] - }] - }] + } + ] } diff --git a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-traces.json b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-traces.json index 461a5c935a..64cf75c2d3 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-traces.json +++ b/data-prepper-plugins/otel-proto-common/src/test/resources/test-request-multiple-traces.json @@ -185,64 +185,6 @@ } ] } - ], - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "VFJBQ0VJRDI=", - "spanId": "VFJBQ0VJRDItU1BBTjE=", - "traceState": "", - "parentSpanId": "yxwHNNFJQP0=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902043168792500", - "endTimeUnixNano": "1597902043215953100", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "VFJBQ0VJRDI=", - "spanId": "VFJBQ0VJRDItU1BBTjI=", - "traceState": "", - "parentSpanId": "yxwHNNFJQP0=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902043168792500", - "endTimeUnixNano": "1597902043215953100", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } ] } ] diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java index a5799605ff..b3cb219684 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java @@ -98,6 +98,7 @@ private void processRequest(final ExportTraceServiceRequest request, final Strea try { if (buffer.isByteBuffer()) { Map requestsMap = oTelProtoDecoder.splitExportTraceServiceRequestByTraceId(request); + // TODO tlongo remove ExportTraceServiceRequest tmp; for (Map.Entry entry: requestsMap.entrySet()) { buffer.writeBytes(entry.getValue().toByteArray(), entry.getKey(), bufferWriteTimeoutInMillis); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcServiceTest.java index ff145a61a9..f083b69096 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcServiceTest.java @@ -14,8 +14,9 @@ import io.micrometer.core.instrument.Timer; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -69,7 +70,7 @@ public class OTelTraceGrpcServiceTest { .setTraceState("SUCCESS").build(); private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() - .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder().addSpans(TEST_SPAN)).build()) + .addScopeSpans(ScopeSpans.newBuilder().addSpans(TEST_SPAN)).build()) .build(); private static PluginSetting pluginSetting; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 03418a9735..75e0d005b7 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -38,8 +38,8 @@ import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; @@ -144,11 +144,11 @@ class OTelTraceSourceTest { private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() - .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder() .addSpans(io.opentelemetry.proto.trace.v1.Span.newBuilder().setTraceState("SUCCESS").build())).build()).build(); private static final ExportTraceServiceRequest FAILURE_REQUEST = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() - .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder() .addSpans(io.opentelemetry.proto.trace.v1.Span.newBuilder().setTraceState("FAILURE").build())).build()).build(); @Mock @@ -1248,7 +1248,7 @@ private ExportTraceServiceRequest createInvalidExportTraceRequest() { .setTraceState("SUCCESS").build(); final ExportTraceServiceRequest successRequest = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() - .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder().addSpans(testSpan)).build()) + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) .build(); return successRequest; @@ -1266,7 +1266,7 @@ private ExportTraceServiceRequest createExportTraceRequest() { return ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() - .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder().addSpans(testSpan)).build()) + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) .build(); } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java index b3f2cb6de8..7894c45598 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java @@ -47,8 +47,8 @@ import io.grpc.StatusRuntimeException; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; @ExtendWith(MockitoExtension.class) @@ -154,9 +154,8 @@ private ExportTraceServiceRequest createExportTraceRequest() { .setEndTimeUnixNano(101) .setTraceState("SUCCESS").build(); + ScopeSpans scopeSpan = ScopeSpans.newBuilder().addSpans(testSpan).build(); return ExportTraceServiceRequest.newBuilder() - .addResourceSpans(ResourceSpans.newBuilder() - .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder().addSpans(testSpan)).build()) - .build(); + .addResourceSpans(ResourceSpans.newBuilder().addScopeSpans(scopeSpan)).build(); } } diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java index 814c1ef93d..fb513e7141 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -20,8 +20,8 @@ import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.resource.v1.Resource; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.Status; import org.opensearch.action.admin.indices.refresh.RefreshRequest; @@ -199,9 +199,9 @@ public static ResourceSpans getResourceSpans(final String serviceName, final Str .setValue(AnyValue.newBuilder().setStringValue(serviceName).build()).build()) .build() ) - .addInstrumentationLibrarySpans( + .addScopeSpans( 0, - InstrumentationLibrarySpans.newBuilder() + ScopeSpans.newBuilder() .addSpans( Span.newBuilder() .setName(spanName) @@ -263,7 +263,7 @@ private List> getExpectedDocuments(ExportTraceServiceRequest for(int i=0; i { final String resourceName = getServiceName(resourceSpans); - resourceSpans.getInstrumentationLibrarySpansList().forEach( instrumentationLibrarySpans -> { + resourceSpans.getScopeSpansList().forEach( instrumentationLibrarySpans -> { instrumentationLibrarySpans.getSpansList().forEach(span -> { expectedDocuments.add(getExpectedEsDocumentSource(span, resourceName)); }); diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java index 79d171c0ef..aead979e6c 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndServiceMapTest.java @@ -16,8 +16,8 @@ import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.resource.v1.Resource; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; import org.junit.Assert; import org.junit.Test; @@ -162,9 +162,9 @@ public static ResourceSpans getResourceSpans(final String serviceName, final Str .setValue(AnyValue.newBuilder().setStringValue(serviceName).build()).build()) .build() ) - .addInstrumentationLibrarySpans( + .addScopeSpans( 0, - InstrumentationLibrarySpans.newBuilder() + ScopeSpans.newBuilder() .addSpans( Span.newBuilder() .setName(spanName) diff --git a/settings.gradle b/settings.gradle index 55536cc7ab..5f9ee9229e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -39,7 +39,7 @@ dependencyResolutionManagement { version('protobuf', '3.24.3') library('protobuf-core', 'com.google.protobuf', 'protobuf-java').versionRef('protobuf') library('protobuf-util', 'com.google.protobuf', 'protobuf-java-util').versionRef('protobuf') - version('opentelemetry', '0.16.0-alpha') + version('opentelemetry', '1.3.2-alpha') library('opentelemetry-proto', 'io.opentelemetry.proto', 'opentelemetry-proto').versionRef('opentelemetry') version('opensearchJava', '2.8.1') library('opensearch-java', 'org.opensearch.client', 'opensearch-java').versionRef('opensearchJava') From 77feae9398b2a875d0d781f99fc97902b52dea2f Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Thu, 13 Feb 2025 14:27:47 +0100 Subject: [PATCH 2/4] Remove todos Signed-off-by: Tomas Longo --- .../plugins/otel/codec/OTelProtoCodecTest.java | 8 -------- .../plugins/source/oteltrace/OTelTraceGrpcService.java | 2 -- .../integration/trace/EndToEndRawSpanTest.java | 4 ++-- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodecTest.java b/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodecTest.java index 945519493a..81277f1eff 100644 --- a/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodecTest.java +++ b/data-prepper-plugins/otel-proto-common/src/test/java/org/opensearch/dataprepper/plugins/otel/codec/OTelProtoCodecTest.java @@ -209,14 +209,6 @@ public void testParseExportTraceServiceRequest() throws IOException { validateSpans(spans); } - @Test - // TODO tlongo is this test still relevant? - public void testParseExportTraceServiceRequest_ScopeSpansTakesPrecedenceOverInstrumentationLibrarySpans() throws IOException { - final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_BOTH_SPAN_TYPES_JSON_FILE); - final List spans = decoderUnderTest.parseExportTraceServiceRequest(exportTraceServiceRequest, Instant.now()); - validateSpans(spans); - } - @Test public void testParseExportTraceServiceRequest_NoSpans() throws IOException { final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_NO_SPANS_JSON_FILE); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java index b3cb219684..e1712585e8 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java @@ -98,8 +98,6 @@ private void processRequest(final ExportTraceServiceRequest request, final Strea try { if (buffer.isByteBuffer()) { Map requestsMap = oTelProtoDecoder.splitExportTraceServiceRequestByTraceId(request); - // TODO tlongo remove - ExportTraceServiceRequest tmp; for (Map.Entry entry: requestsMap.entrySet()) { buffer.writeBytes(entry.getValue().toByteArray(), entry.getKey(), bufferWriteTimeoutInMillis); } diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java index fb513e7141..6a8d033572 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -263,8 +263,8 @@ private List> getExpectedDocuments(ExportTraceServiceRequest for(int i=0; i { final String resourceName = getServiceName(resourceSpans); - resourceSpans.getScopeSpansList().forEach( instrumentationLibrarySpans -> { - instrumentationLibrarySpans.getSpansList().forEach(span -> { + resourceSpans.getScopeSpansList().forEach( scopeSpans -> { + scopeSpans.getSpansList().forEach(span -> { expectedDocuments.add(getExpectedEsDocumentSource(span, resourceName)); }); }); From 103a7ae2fe0b551322c61355ba5a55cd9ab5245f Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 14 Feb 2025 08:55:22 +0100 Subject: [PATCH 3/4] remove last remnants of instrumentation library Signed-off-by: Tomas Longo --- .../src/test/resources/raw-span-missing-trace-group-1.json | 5 ++--- .../src/test/resources/raw-span-missing-trace-group-2.json | 5 ++--- data-prepper-plugins/otel-trace-source/README.md | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/data-prepper-plugins/otel-trace-group-processor/src/test/resources/raw-span-missing-trace-group-1.json b/data-prepper-plugins/otel-trace-group-processor/src/test/resources/raw-span-missing-trace-group-1.json index 239508132a..a1e7fd0e5b 100644 --- a/data-prepper-plugins/otel-trace-group-processor/src/test/resources/raw-span-missing-trace-group-1.json +++ b/data-prepper-plugins/otel-trace-group-processor/src/test/resources/raw-span-missing-trace-group-1.json @@ -25,6 +25,5 @@ "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", "resource.attributes.service@name": "analytics-service", "span.attributes.http@status_code": 200, - "status.code": 1, - "instrumentationLibrary.name": "io.opentelemetry.auto.apache-httpasyncclient-4.0" -} \ No newline at end of file + "status.code": 1 +} diff --git a/data-prepper-plugins/otel-trace-group-processor/src/test/resources/raw-span-missing-trace-group-2.json b/data-prepper-plugins/otel-trace-group-processor/src/test/resources/raw-span-missing-trace-group-2.json index cb4ae8c110..2eea8dcd56 100644 --- a/data-prepper-plugins/otel-trace-group-processor/src/test/resources/raw-span-missing-trace-group-2.json +++ b/data-prepper-plugins/otel-trace-group-processor/src/test/resources/raw-span-missing-trace-group-2.json @@ -25,6 +25,5 @@ "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", "resource.attributes.service@name": "analytics-service", "span.attributes.http@status_code": 200, - "status.code": 1, - "instrumentationLibrary.name": "io.opentelemetry.auto.apache-httpasyncclient-4.0" -} \ No newline at end of file + "status.code": 1 +} diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index 6e5172b1fc..68f5067bdb 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -119,12 +119,12 @@ Make sure to replace the paths for the `sslKeyCertChainFile` and `sslKeyFile` fo Send a sample span with the following https curl command: ``` -curl -k -H 'Content-Type: application/json; charset=utf-8' -d '{"resourceSpans":[{"instrumentationLibrarySpans":[{"spans":[{"spanId":"AAAAAAAAAAM=","name":"test-span"}]}]}]}' https://localhost:21890/opentelemetry.proto.collector.trace.v1.TraceService/Export +curl -k -H 'Content-Type: application/json; charset=utf-8' -d '{"resourceSpans":[{"scopeSpans":[{"spans":[{"spanId":"AAAAAAAAAAM=","name":"test-span"}]}]}]}' https://localhost:21890/opentelemetry.proto.collector.trace.v1.TraceService/Export ``` If `path` option is configured, you can send a sample span to the custom path with the following https curl command: ``` -curl -k -H 'Content-Type: application/json; charset=utf-8' -d '{"resourceSpans":[{"instrumentationLibrarySpans":[{"spans":[{"spanId":"AAAAAAAAAAM=","name":"test-span"}]}]}]}' https://localhost:21890/ +curl -k -H 'Content-Type: application/json; charset=utf-8' -d '{"resourceSpans":[{"scopeSpans":[{"spans":[{"spanId":"AAAAAAAAAAM=","name":"test-span"}]}]}]}' https://localhost:21890/ ``` ## Metrics From 7c5b99d3f7c3d5d3d43d4ddcc240bd160d8be0e2 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 21 Feb 2025 10:30:02 +0100 Subject: [PATCH 4/4] Just test the new otel proto buf spec in the e2e raw span test Signed-off-by: Tomas Longo --- .../data-prepper-trace-analytics-raw-span-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml index 9538f3eb09..f672797f24 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: java: [11, 17, 21, docker] - otelVersion: ['0.9.0-alpha', '0.16.0-alpha'] + otelVersion: ['1.3.2-alpha'] fail-fast: false runs-on: ubuntu-latest