Add OpenTelemetry and Spandex tracing connectors

CHANGELOG.md

@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - `K8s.Client.Runner.Watch.stream/3` - watches a resource and returns an elixir [Stream](https://hexdocs.pm/elixir/1.12/Stream.html) of events #121
 - `K8s.Client.apply/3` - Create a [server-side apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/) operation
+- `K8s.Sys.OpenTelemetry` - BETA! - Connects telemetry spans to an OpenTelemetry tracer
+- `K8s.Sys.Spandex` - BETA! - Connects telemetry spans to a Spandex tracer
 
 ### Changed
 

guides/observability.md

@@ -18,4 +18,25 @@ them.
 
 ## Tracing
 
-Tracing has not been implemented yet.
+There are two connectors to `:telemetry` spans/events. 
+
+### OpenTelemetry
+
+If you're [OpenTelemetry](https://opentelemetry.io/docs/instrumentation/erlang/), attach 
+`:telemetry` spans/events to the OpenTelemetry handler:
+
+```elixir
+K8s.Sys.OpenTelemetry.attach()
+```
+
+### Spandex
+
+:warning: Requires Elixir ~> 1.10
+
+If you're using a [Spandex](https://github.com/spandex-project/spandex) tracer, Attach 
+`:telemetry` spans/events to the Spandex handler. Pass the tracer you created according
+to the Spandex documentation as argument to the `attach/1` function.
+
+```elixir
+K8s.Sys.Spandex.attach(MyApp.Tracer)
+```

lib/k8s/sys/logger.ex

@@ -6,14 +6,19 @@ defmodule K8s.Sys.Logger do
   @doc """
   Attaches telemetry events to the Elixir Logger
   """
-  @spec attach() :: :ok
+  @spec attach() :: :ok | {:error, :already_exists}
   def attach do
     events = K8s.Sys.Telemetry.events()
     :telemetry.attach_many("k8s-events-logger", events, &__MODULE__.log_handler/4, :debug)
   end
 
   @doc false
-  @spec log_handler(keyword, map | integer, map, atom) :: :ok
+  @spec log_handler(
+          :telemetry.event_name(),
+          :telemetry.event_measurements(),
+          :telemetry.event_metadata(),
+          :telemetry.handler_config()
+        ) :: any()
   def log_handler(event, measurements, metadata, preferred_level) do
     event_name = Enum.join(event, ".")
 

lib/k8s/sys/open_telemetry.ex

@@ -0,0 +1,95 @@
+if Code.ensure_loaded?(OpenTelemetry) and Code.ensure_loaded?(OpentelemetryTelemetry) do
+  defmodule K8s.Sys.OpenTelemetry do
+    @moduledoc """
+    This module is still in beta! It has not been tested well and feedback is welcome!
+
+    Converts telemetry spans to opentelemetry tracing spans
+
+    ### Usage
+
+        K8s.Sys.OpenTelemetry.attach()
+    """
+
+    @doc """
+    Attaches telemetry spans to the opentelemetry processor
+    """
+    @spec attach() :: :ok
+    def attach do
+      for span <- K8s.Sys.Telemetry.spans() do
+        span_name = Enum.join(span, ".")
+
+        :ok =
+          :telemetry.attach(
+            "k8s-otel-tracer-#{span_name}-start",
+            span ++ [:start],
+            &__MODULE__.handle_event/4,
+            %{tracer_id: :k8s, type: :start, span_name: "k8s." <> span_name}
+          )
+
+        :ok =
+          :telemetry.attach(
+            "k8s-otel-tracer-#{span_name}-stop",
+            span ++ [:stop],
+            &__MODULE__.handle_event/4,
+            %{tracer_id: :k8s, type: :stop}
+          )
+
+        :ok =
+          :telemetry.attach(
+            "k8s-otel-tracer-#{span_name}-exception",
+            span ++ [:exception],
+            &__MODULE__.handle_event/4,
+            %{tracer_id: :k8s, type: :exception}
+          )
+      end
+
+      :ok
+    end
+
+    @doc false
+    @spec handle_event(
+            :telemetry.event_name(),
+            :telemetry.event_measurements(),
+            :telemetry.event_metadata(),
+            :telemetry.handler_config()
+          ) :: any()
+    def handle_event(
+          _event,
+          %{system_time: start_time},
+          metadata,
+          %{type: :start, tracer_id: tracer_id, span_name: name}
+        ) do
+      start_opts = %{start_time: start_time}
+      OpentelemetryTelemetry.start_telemetry_span(tracer_id, name, metadata, start_opts)
+      :ok
+    end
+
+    def handle_event(
+          _event,
+          %{duration: duration},
+          metadata,
+          %{type: :stop, tracer_id: tracer_id}
+        ) do
+      OpentelemetryTelemetry.set_current_telemetry_span(tracer_id, metadata)
+      OpenTelemetry.Tracer.set_attribute(:duration, duration)
+      OpentelemetryTelemetry.end_telemetry_span(tracer_id, metadata)
+      :ok
+    end
+
+    def handle_event(
+          _event,
+          %{duration: duration},
+          %{kind: kind, reason: reason, stacktrace: stacktrace} = metadata,
+          %{type: :exception, tracer_id: tracer_id}
+        ) do
+      ctx = OpentelemetryTelemetry.set_current_telemetry_span(tracer_id, metadata)
+      status = OpenTelemetry.status(:error, inspect(reason))
+      OpenTelemetry.Span.record_exception(ctx, kind, stacktrace, duration: duration)
+      OpenTelemetry.Tracer.set_status(status)
+      OpentelemetryTelemetry.end_telemetry_span(tracer_id, metadata)
+      :ok
+    end
+
+    def handle_event(_event, _measurements, _metadata, _config), do: :ok
+  end
+end

lib/k8s/sys/spandex.ex

@@ -0,0 +1,96 @@
+if Code.ensure_loaded?(Spandex) do
+  defmodule K8s.Sys.Spandex do
+    @moduledoc """
+    This module is still in beta! It has not been tested well and feedback is welcome!
+
+    Converts telemetry spans to Spandex tracing spans.
+
+    ### Usage
+
+        K8s.Sys.Spandex.attach(MyApp.Tracer)
+    """
+
+    require Spandex
+
+    @doc """
+    Attaches telemetry spans to the spandex processor
+    """
+    @spec attach(atom()) :: :ok
+    def attach(tracer) do
+      for span <- K8s.Sys.Telemetry.spans() do
+        span_name = Enum.join(span, ".")
+
+        :ok =
+          :telemetry.attach(
+            "k8s-spandex-tracer-#{span_name}-start",
+            span ++ [:start],
+            &__MODULE__.handle_event/4,
+            %{tracer: tracer, type: :start, span_name: "k8s." <> span_name}
+          )
+
+        :ok =
+          :telemetry.attach(
+            "k8s-spandex-tracer-#{span_name}-stop",
+            span ++ [:stop],
+            &__MODULE__.handle_event/4,
+            %{tracer: tracer, type: :stop}
+          )
+
+        :ok =
+          :telemetry.attach(
+            "k8s-spandex-tracer-#{span_name}-exception",
+            span ++ [:exception],
+            &__MODULE__.handle_event/4,
+            %{tracer: tracer, type: :exception}
+          )
+      end
+
+      :ok
+    end
+
+    @doc false
+    @spec handle_event(
+            :telemetry.event_name(),
+            :telemetry.event_measurements(),
+            :telemetry.event_metadata(),
+            :telemetry.handler_config()
+          ) :: any()
+    def handle_event(
+          _event,
+          %{system_time: _start_time},
+          metadata,
+          %{type: :start, tracer: tracer, span_name: name}
+        ) do
+      tracer.start_span(name, service: :k8s, type: :custom, tags: Map.to_list(metadata))
+      :ok
+    end
+
+    def handle_event(
+          _event,
+          %{duration: _duration},
+          metadata,
+          %{type: :stop, tracer: tracer}
+        ) do
+      tracer.update_span(tags: Map.to_list(metadata))
+      tracer.finish_span()
+      :ok
+    end
+
+    def handle_event(
+          _event,
+          %{duration: _duration},
+          %{kind: kind, reason: reason, stacktrace: stacktrace} = metadata,
+          %{type: :exception, tracer: tracer}
+        ) do
+      metadata =
+        metadata
+        |> Map.put(:error, reason)
+        |> Map.delete(:reason)
+
+      tracer.span_error(kind, stacktrace, error: reason, tags: metadata)
+      :ok
+    end
+
+    def handle_event(_event, _measurements, _metadata, _config), do: :ok
+  end
+end

lib/k8s/sys/telemetry.ex

@@ -1,12 +1,22 @@
 defmodule K8s.Sys.Telemetry do
   @moduledoc false
 
-  @events [
-    [:http, :request, :start],
-    [:http, :request, :stop],
-    [:http, :request, :exception]
+  @spans [
+    [:http, :request]
   ]
 
+  @spec spans() :: list()
+  def spans, do: @spans
+
   @spec events() :: list()
-  def events, do: @events
+  def events do
+    @spans
+    |> Enum.flat_map(fn span ->
+      [
+        span ++ [:start],
+        span ++ [:stop],
+        span ++ [:exception]
+      ]
+    end)
+  end
 end

mix.exs

@@ -31,11 +31,13 @@ defmodule K8s.MixProject do
 
   # Run "mix help deps" to learn about dependencies.
   defp deps do
-    [
+    deps = [
       {:yaml_elixir, "~> 2.8"},
       {:httpoison, "~> 1.7"},
       {:jason, "~> 1.0"},
       {:telemetry, "~> 1.0"},
+      {:opentelemetry_telemetry, "~> 1.0.0-beta.4", optional: true},
+      {:opentelemetry, "~> 1.0", optional: true},
 
       # dev/test deps (e.g. code coverage)
       {:inch_ex, github: "rrrene/inch_ex", only: [:dev, :test]},
@@ -45,6 +47,11 @@ defmodule K8s.MixProject do
       {:excoveralls, "~> 0.14", only: [:test]},
       {:mix_test_watch, "~> 1.1", only: :dev, runtime: false}
     ]
+
+    # spandex requires 1.10
+    if Version.match?(System.version(), "~> 1.10"),
+      do: [{:spandex, "~> 3.0.3", optional: true} | deps],
+      else: deps
   end
 
   defp package do
@@ -93,7 +100,14 @@ defmodule K8s.MixProject do
   defp dialyzer do
     [
       ignore_warnings: ".dialyzer_ignore.exs",
-      plt_add_apps: [:mix, :eex],
+      plt_add_apps: [
+        :mix,
+        :eex,
+        :opentelemetry,
+        :opentelemetry_telemetry,
+        :opentelemetry_api,
+        :spandex
+      ],
       plt_core_path: "priv/plts",
       plt_file: {:no_warn, "priv/plts/k8s.plt"}
     ]

mix.lock

@@ -11,19 +11,28 @@
   "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"},
   "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
-  "inch_ex": {:git, "https://github.com/rrrene/inch_ex.git", "c8eeaa65312df3ce150e91d7dddb50e2983b3209", []},
+  "inch_ex": {:git, "https://github.com/rrrene/inch_ex.git", "d37c3cd41ceda869696499569547d6f9a416751c", []},
   "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"},
   "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"},
   "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
+  "mime": {:hex, :mime, "2.0.2", "0b9e1a4c840eafb68d820b0e2158ef5c49385d17fb36855ac6e7e087d4b1dcc5", [:mix], [], "hexpm", "e6a3f76b4c277739e36c2e21a2c640778ba4c3846189d5ab19f97f126df5f9b7"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
   "mix_test_watch": {:hex, :mix_test_watch, "1.1.0", "330bb91c8ed271fe408c42d07e0773340a7938d8a0d281d57a14243eae9dc8c3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "52b6b1c476cbb70fd899ca5394506482f12e5f6b0d6acff9df95c7f1e0812ec3"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.2.2", "b99ca56bbce410e9d5ee4f9155a212e942e224e259c7ebbf8f2c86ac21d4fa3c", [:mix], [], "hexpm", "98d51bd64d5f6a2a9c6bb7586ee8129e27dfaab1140b5a4753f24dac0ba27d2f"},
+  "opentelemetry": {:hex, :opentelemetry, "1.0.1", "8e3a7e219db32cfafabb4392bc01a66e8da19176da4f00a0c8d08d63801f67a5", [:rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "4c93fb7a5ffa1ddc15440714132df29d053b84bd7b4b24491704ae9b4e48caf8"},
+  "opentelemetry_api": {:hex, :opentelemetry_api, "1.0.1", "317908954a3dafdad916d3c9ff2f7cd48e393f57bea9bcfa57c6abcf9c0659e5", [:mix, :rebar3], [], "hexpm", "9be65ab0f74fab7ab9cbe197f39ee056420178d8e8de31805b19159eda7fd621"},
+  "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.0.0-beta.7", "ba1df62515aed63f99a80ddf17e7a3873d1f686f23598edebf1633942772856e", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.3.0", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "480f4fa1e992d597f931e7bc9e68478e8d904ad84489d2c5ca6eb6d48bbd7801"},
+  "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm", "1a06ea6a653120226b35b283a1cd10039550f2c566edcdec22b29316d73640fd"},
   "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
+  "plug": {:hex, :plug, "1.13.3", "93b299039c21a8b82cc904d13812bce4ced45cf69153e8d35ca16ffb3e8c5d98", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "98c8003e4faf7b74a9ac41bee99e328b08f069bf932747d4a7532e97ae837a17"},
+  "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
+  "spandex": {:hex, :spandex, "3.0.3", "91aa318f3de696bb4d931adf65f7ebdbe5df25cccce1fe8fd376a44c46bcf69b", [:mix], [{:decorator, "~> 1.2", [hex: :decorator, repo: "hexpm", optional: true]}, {:optimal, "~> 0.3.3", [hex: :optimal, repo: "hexpm", optional: false]}, {:plug, ">= 1.0.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e3e6c319d0ab478ddc9a39102a727a410c962b4d51c0932c72279b86d3b17044"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
   "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"},
+  "telemetry_registry": {:hex, :telemetry_registry, "0.3.0", "6768f151ea53fc0fbca70dbff5b20a8d663ee4e0c0b2ae589590e08658e76f1e", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "492e2adbc609f3e79ece7f29fec363a97a2c484ac78a83098535d6564781e917"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
-  "yamerl": {:hex, :yamerl, "0.8.1", "07da13ffa1d8e13948943789665c62ccd679dfa7b324a4a2ed3149df17f453a4", [:rebar3], [], "hexpm", "96cb30f9d64344fed0ef8a92e9f16f207de6c04dfff4f366752ca79f5bceb23f"},
+  "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},
   "yaml_elixir": {:hex, :yaml_elixir, "2.8.0", "c7ff0034daf57279c2ce902788ce6fdb2445532eb4317e8df4b044209fae6832", [:mix], [{:yamerl, "~> 0.8", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "4b674bd881e373d1ac6a790c64b2ecb69d1fd612c2af3b22de1619c15473830b"},
 }