enable to add traceparent header taking over trace id to other serv…

…ices with HTTP

- Add `M3Tracer.processOutgoingHttpRequest` for tracing with various http clients
  currently only support Spring RestTemplate

jvm/core/src/main/kotlin/com/m3/tracing/M3Tracer.kt

@@ -29,6 +29,13 @@ interface M3Tracer: AutoCloseable, TraceContext {
     @CheckReturnValue
     fun processIncomingHttpRequest(request: HttpRequestInfo): HttpRequestSpan
 
+    /**
+     * Start trace for outgoing HTTP request.
+     * Caller MUST close the [HttpRequestSpan] to prevent leak.
+     */
+    @CheckReturnValue
+    fun processOutgoingHttpRequest(request: HttpRequestInfo): HttpRequestSpan
+
     /**
      * Returns context bound to current thread / call stack.
      *

jvm/core/src/main/kotlin/com/m3/tracing/http/HttpRequestInfo.kt

@@ -22,6 +22,11 @@ interface HttpRequestInfo {
      * @return If given header is not available, return null.
      */
     fun tryGetHeader(name: String): String?
+
+    /**
+     * Set value into header.
+     */
+    fun trySetHeader(name: String, value: String)
 }
 
 /**

jvm/core/src/main/kotlin/com/m3/tracing/tracer/logging/M3LoggingTracer.kt

@@ -38,6 +38,23 @@ class M3LoggingTracer: M3Tracer {
         override fun startChildSpan(name: String): TraceSpan = TraceSpanImpl(name)
     }
 
+    override fun processOutgoingHttpRequest(request: HttpRequestInfo): HttpRequestSpan = object: TraceSpanImpl("HTTP ${request.url}"), HttpRequestSpan {
+        private var e: Throwable? = null
+        override fun setError(e: Throwable?) {
+            this.e = e
+        }
+
+        override fun setResponse(response: HttpResponseInfo) {}
+        override fun close() {
+            if (this.e != null) {
+                output.info("Error captured in client request ${request.url}: $e")
+            }
+            super.close()
+        }
+
+        override fun startChildSpan(name: String): TraceSpan = TraceSpanImpl(name)
+    }
+
     // This implementation holds nothing in thread local.
     override val currentContext = object: TraceContext {
         override fun startSpan(name: String): TraceSpan = TraceSpanImpl(name)

jvm/opencensus/src/main/kotlin/com/m3/tracing/tracer/opencensus/HttpRequestTracer.kt

@@ -4,6 +4,7 @@ import com.google.common.annotations.VisibleForTesting
 import com.m3.tracing.http.*
 import io.grpc.Context
 import io.opencensus.common.Scope
+import io.opencensus.contrib.http.HttpClientHandler
 import io.opencensus.contrib.http.HttpServerHandler
 import io.opencensus.trace.Tracer
 import io.opencensus.trace.propagation.TextFormat
@@ -21,13 +22,25 @@ internal class HttpRequestTracer(
         }
     }
     @VisibleForTesting
+    internal val setter = object: TextFormat.Setter<HttpRequestInfo>() {
+        override fun put(carrier: HttpRequestInfo, key: String, value: String) {
+            return carrier.trySetHeader(key, value)
+        }
+    }
+    @VisibleForTesting
     internal val extractor = ExtractorImpl()
     @VisibleForTesting
     internal val handler = HttpServerHandler(tracer, extractor, textFormat, getter, publicEndpoint)
+    @VisibleForTesting
+    internal val clientHandler = HttpClientHandler(tracer, extractor, textFormat, setter)
 
     fun processRequest(request: HttpRequestInfo) = HttpRequestSpanImpl(handler, tracer, request).also {
         it.init()
     }
+
+    fun processClientRequest(request: HttpRequestInfo) = HttpClientRequestSpanImpl(clientHandler, tracer, request).also {
+        it.init()
+    }
 }
 
 internal class HttpRequestSpanImpl(
@@ -85,3 +98,56 @@ internal class HttpRequestSpanImpl(
         }
     }
 }
+
+internal class HttpClientRequestSpanImpl(
+        private val handler: HttpClientHandler<HttpRequestInfo, HttpResponseInfo, HttpRequestInfo>,
+        override val tracer: Tracer,
+        private val request: HttpRequestInfo
+): TraceSpanImpl(null), HttpRequestSpan {
+    companion object {
+        private val logger = LoggerFactory.getLogger(HttpRequestTracer::class.java)
+    }
+
+    val context = handler.handleStart(tracer.currentSpan, request, request).also { context ->
+        request.tryGetMetadata(HttpRequestMetadataKey.ContentLength)?.let { length ->
+            if (length > 0) handler.handleMessageSent(context, length)
+        }
+    }
+
+    override val scopeParentContext: Context? get() = null
+    override val span = handler.getSpanFromContext(context)
+    override val scope: Scope? = tracer.withSpan(span)
+
+    @VisibleForTesting
+    internal var error: Throwable? = null
+    override fun setError(e: Throwable?) {
+        super<TraceSpanImpl>.setError(e)
+        this.error = e
+    }
+
+    @VisibleForTesting
+    internal var response: HttpResponseInfo? = null
+    override fun setResponse(response: HttpResponseInfo) {
+        this.response = response
+    }
+
+    override fun close() {
+        try {
+            captureInfo()
+            handler.handleEnd(context, request, response, error)
+        } catch (e: Throwable) {
+            logger.error("Failed to capture client response detail", e)
+        }
+
+        // Must close scope, span in ANY case to prevent memory leak.
+        // So that MUST call super.close().
+        super.close()
+    }
+
+    private fun captureInfo() {
+        response?.tryGetMetadata(HttpResponseMetadataKey.ContentLength)?.let { length ->
+            handler.handleMessageReceived(context, length)
+        }
+    }
+}
+

jvm/opencensus/src/main/kotlin/com/m3/tracing/tracer/opencensus/M3OpenCensusTracer.kt

@@ -65,6 +65,8 @@ class M3OpenCensusTracer internal constructor(
 
     override fun processIncomingHttpRequest(request: HttpRequestInfo): HttpRequestSpan = httpRequestTracer.processRequest(request)
 
+    override fun processOutgoingHttpRequest(request: HttpRequestInfo): HttpRequestSpan = httpRequestTracer.processClientRequest(request)
+
     private fun createSampler() = SamplerFactory.createSampler(
             Config[samplingConfigName] ?: "never"
     )
@@ -105,4 +107,4 @@ internal class TraceContextImpl(
             return NoopSpan
         }
     }
-}
+}

jvm/servlet/src/main/kotlin/com/m3/tracing/tracer/servlet/ServletHttpRequestInfo.kt

@@ -10,6 +10,7 @@ open class ServletHttpRequestInfo(protected val req: HttpServletRequest): HttpRe
     // Thus this filter should not do anything breaks setCharacterEncoding() even after FilterCain.
 
     override fun tryGetHeader(name: String): String? = req.getHeader(name)
+    override fun trySetHeader(name: String, value: String) = Unit // Do nothing
 
     @Suppress("UNCHECKED_CAST", "IMPLICIT_ANY")
     override fun <T> tryGetMetadata(key: HttpRequestMetadataKey<T>): T? = when(key) {

jvm/spring-web/src/main/kotlin/com/m3/tracing/spring/http/client/M3TracingHttpRequestInterceptor.kt

@@ -15,18 +15,14 @@ class M3TracingHttpRequestInterceptor(
         private val tracer: M3Tracer
 ): ClientHttpRequestInterceptor {
     override fun intercept(request: HttpRequest, body: ByteArray, execution: ClientHttpRequestExecution): ClientHttpResponse {
-        return tracer.startSpan(
-                // Excluded path & query because it might contain dynamic string
-                // Variable string makes huge span summary table and impact to system performance
-                "HTTP ${request.methodValue} ${request.uri.host}"
-        ).use { span ->
+        return tracer.processOutgoingHttpRequest(wrapRequest(request)).use { span ->
             span["client"] = "RestTemplate"
             span["method"] = request.methodValue
             span["uri"] = request.uri.toString()
-
             execution.execute(request, body).also { response ->
                 span["status"] = response.rawStatusCode
             }
         }
     }
+    fun wrapRequest(request: HttpRequest) = SpringHttpRequestInfo(request)
 }

jvm/spring-web/src/main/kotlin/com/m3/tracing/spring/http/client/SpringHttpRequestInfo.kt

@@ -0,0 +1,21 @@
+package com.m3.tracing.spring.http.client
+
+import com.m3.tracing.http.HttpRequestInfo
+import com.m3.tracing.http.HttpRequestMetadataKey
+import org.springframework.http.HttpRequest
+
+open class SpringHttpRequestInfo(protected val req: HttpRequest): HttpRequestInfo {
+    override fun tryGetHeader(name: String): String? = req.headers.getFirst(name)
+    override fun trySetHeader(name: String, value: String) = req.headers.set(name, value)
+
+    @Suppress("UNCHECKED_CAST", "IMPLICIT_ANY")
+    override fun <T> tryGetMetadata(key: HttpRequestMetadataKey<T>): T? = when(key) {
+        HttpRequestMetadataKey.Method -> req.methodValue as T?
+        HttpRequestMetadataKey.Host -> req.headers.host?.hostName as T?
+        HttpRequestMetadataKey.ContentLength -> req.headers.contentLength.let { if (it <= 0) null else it } as T?
+        HttpRequestMetadataKey.Path -> req.uri.path as T?
+        HttpRequestMetadataKey.Url -> req.uri.toString() as T?
+
+        else -> null
+    }
+}