diff --git a/CHANGELOG.md b/CHANGELOG.md index bb8cd114..3ad52820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `K8s.Client.Runner.Watch.stream/3` - watches a resource and returns an elixir [Stream](https://hexdocs.pm/elixir/1.12/Stream.html) of events #121 - `K8s.Client.apply/3` - Create a [server-side apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/) operation +- `K8s.Sys.OpenTelemetry` - BETA! - Connects telemetry spans to an OpenTelemetry tracer +- `K8s.Sys.Spandex` - BETA! - Connects telemetry spans to a Spandex tracer ### Changed diff --git a/guides/observability.md b/guides/observability.md index 9c8b56eb..0d659ead 100644 --- a/guides/observability.md +++ b/guides/observability.md @@ -18,4 +18,25 @@ them. ## Tracing -Tracing has not been implemented yet. \ No newline at end of file +There are two connectors to `:telemetry` spans/events. + +### OpenTelemetry + +If you're [OpenTelemetry](https://opentelemetry.io/docs/instrumentation/erlang/), attach +`:telemetry` spans/events to the OpenTelemetry handler: + +```elixir +K8s.Sys.OpenTelemetry.attach() +``` + +### Spandex + +:warning: Requires Elixir ~> 1.10 + +If you're using a [Spandex](https://github.com/spandex-project/spandex) tracer, Attach +`:telemetry` spans/events to the Spandex handler. Pass the tracer you created according +to the Spandex documentation as argument to the `attach/1` function. + +```elixir +K8s.Sys.Spandex.attach(MyApp.Tracer) +``` \ No newline at end of file diff --git a/lib/k8s/sys/logger.ex b/lib/k8s/sys/logger.ex index 5e160141..0204f71b 100644 --- a/lib/k8s/sys/logger.ex +++ b/lib/k8s/sys/logger.ex @@ -6,14 +6,19 @@ defmodule K8s.Sys.Logger do @doc """ Attaches telemetry events to the Elixir Logger """ - @spec attach() :: :ok + @spec attach() :: :ok | {:error, :already_exists} def attach do events = K8s.Sys.Telemetry.events() :telemetry.attach_many("k8s-events-logger", events, &__MODULE__.log_handler/4, :debug) end @doc false - @spec log_handler(keyword, map | integer, map, atom) :: :ok + @spec log_handler( + :telemetry.event_name(), + :telemetry.event_measurements(), + :telemetry.event_metadata(), + :telemetry.handler_config() + ) :: any() def log_handler(event, measurements, metadata, preferred_level) do event_name = Enum.join(event, ".") diff --git a/lib/k8s/sys/open_telemetry.ex b/lib/k8s/sys/open_telemetry.ex new file mode 100644 index 00000000..13a810ec --- /dev/null +++ b/lib/k8s/sys/open_telemetry.ex @@ -0,0 +1,95 @@ +if Code.ensure_loaded?(OpenTelemetry) and Code.ensure_loaded?(OpentelemetryTelemetry) do + defmodule K8s.Sys.OpenTelemetry do + @moduledoc """ + This module is still in beta! It has not been tested well and feedback is welcome! + + Converts telemetry spans to opentelemetry tracing spans + + ### Usage + + K8s.Sys.OpenTelemetry.attach() + """ + + @doc """ + Attaches telemetry spans to the opentelemetry processor + """ + @spec attach() :: :ok + def attach do + for span <- K8s.Sys.Telemetry.spans() do + span_name = Enum.join(span, ".") + + :ok = + :telemetry.attach( + "k8s-otel-tracer-#{span_name}-start", + span ++ [:start], + &__MODULE__.handle_event/4, + %{tracer_id: :k8s, type: :start, span_name: "k8s." <> span_name} + ) + + :ok = + :telemetry.attach( + "k8s-otel-tracer-#{span_name}-stop", + span ++ [:stop], + &__MODULE__.handle_event/4, + %{tracer_id: :k8s, type: :stop} + ) + + :ok = + :telemetry.attach( + "k8s-otel-tracer-#{span_name}-exception", + span ++ [:exception], + &__MODULE__.handle_event/4, + %{tracer_id: :k8s, type: :exception} + ) + end + + :ok + end + + @doc false + @spec handle_event( + :telemetry.event_name(), + :telemetry.event_measurements(), + :telemetry.event_metadata(), + :telemetry.handler_config() + ) :: any() + def handle_event( + _event, + %{system_time: start_time}, + metadata, + %{type: :start, tracer_id: tracer_id, span_name: name} + ) do + start_opts = %{start_time: start_time} + OpentelemetryTelemetry.start_telemetry_span(tracer_id, name, metadata, start_opts) + :ok + end + + def handle_event( + _event, + %{duration: duration}, + metadata, + %{type: :stop, tracer_id: tracer_id} + ) do + OpentelemetryTelemetry.set_current_telemetry_span(tracer_id, metadata) + OpenTelemetry.Tracer.set_attribute(:duration, duration) + OpentelemetryTelemetry.end_telemetry_span(tracer_id, metadata) + :ok + end + + def handle_event( + _event, + %{duration: duration}, + %{kind: kind, reason: reason, stacktrace: stacktrace} = metadata, + %{type: :exception, tracer_id: tracer_id} + ) do + ctx = OpentelemetryTelemetry.set_current_telemetry_span(tracer_id, metadata) + status = OpenTelemetry.status(:error, inspect(reason)) + OpenTelemetry.Span.record_exception(ctx, kind, stacktrace, duration: duration) + OpenTelemetry.Tracer.set_status(status) + OpentelemetryTelemetry.end_telemetry_span(tracer_id, metadata) + :ok + end + + def handle_event(_event, _measurements, _metadata, _config), do: :ok + end +end diff --git a/lib/k8s/sys/spandex.ex b/lib/k8s/sys/spandex.ex new file mode 100644 index 00000000..a13eb239 --- /dev/null +++ b/lib/k8s/sys/spandex.ex @@ -0,0 +1,96 @@ +if Code.ensure_loaded?(Spandex) do + defmodule K8s.Sys.Spandex do + @moduledoc """ + This module is still in beta! It has not been tested well and feedback is welcome! + + Converts telemetry spans to Spandex tracing spans. + + ### Usage + + K8s.Sys.Spandex.attach(MyApp.Tracer) + """ + + require Spandex + + @doc """ + Attaches telemetry spans to the spandex processor + """ + @spec attach(atom()) :: :ok + def attach(tracer) do + for span <- K8s.Sys.Telemetry.spans() do + span_name = Enum.join(span, ".") + + :ok = + :telemetry.attach( + "k8s-spandex-tracer-#{span_name}-start", + span ++ [:start], + &__MODULE__.handle_event/4, + %{tracer: tracer, type: :start, span_name: "k8s." <> span_name} + ) + + :ok = + :telemetry.attach( + "k8s-spandex-tracer-#{span_name}-stop", + span ++ [:stop], + &__MODULE__.handle_event/4, + %{tracer: tracer, type: :stop} + ) + + :ok = + :telemetry.attach( + "k8s-spandex-tracer-#{span_name}-exception", + span ++ [:exception], + &__MODULE__.handle_event/4, + %{tracer: tracer, type: :exception} + ) + end + + :ok + end + + @doc false + @spec handle_event( + :telemetry.event_name(), + :telemetry.event_measurements(), + :telemetry.event_metadata(), + :telemetry.handler_config() + ) :: any() + def handle_event( + _event, + %{system_time: _start_time}, + metadata, + %{type: :start, tracer: tracer, span_name: name} + ) do + tracer.start_span(name, service: :k8s, type: :custom, tags: Map.to_list(metadata)) + :ok + end + + def handle_event( + _event, + %{duration: _duration}, + metadata, + %{type: :stop, tracer: tracer} + ) do + tracer.update_span(tags: Map.to_list(metadata)) + tracer.finish_span() + :ok + end + + def handle_event( + _event, + %{duration: _duration}, + %{kind: kind, reason: reason, stacktrace: stacktrace} = metadata, + %{type: :exception, tracer: tracer} + ) do + metadata = + metadata + |> Map.put(:error, reason) + |> Map.delete(:reason) + + tracer.span_error(kind, stacktrace, error: reason, tags: metadata) + :ok + end + + def handle_event(_event, _measurements, _metadata, _config), do: :ok + end +end diff --git a/lib/k8s/sys/telemetry.ex b/lib/k8s/sys/telemetry.ex index 6e342c7d..ef697be3 100644 --- a/lib/k8s/sys/telemetry.ex +++ b/lib/k8s/sys/telemetry.ex @@ -1,12 +1,22 @@ defmodule K8s.Sys.Telemetry do @moduledoc false - @events [ - [:http, :request, :start], - [:http, :request, :stop], - [:http, :request, :exception] + @spans [ + [:http, :request] ] + @spec spans() :: list() + def spans, do: @spans + @spec events() :: list() - def events, do: @events + def events do + @spans + |> Enum.flat_map(fn span -> + [ + span ++ [:start], + span ++ [:stop], + span ++ [:exception] + ] + end) + end end diff --git a/mix.exs b/mix.exs index 5c6493cf..9a470834 100644 --- a/mix.exs +++ b/mix.exs @@ -31,11 +31,13 @@ defmodule K8s.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do - [ + deps = [ {:yaml_elixir, "~> 2.8"}, {:httpoison, "~> 1.7"}, {:jason, "~> 1.0"}, {:telemetry, "~> 1.0"}, + {:opentelemetry_telemetry, "~> 1.0.0-beta.4", optional: true}, + {:opentelemetry, "~> 1.0", optional: true}, # dev/test deps (e.g. code coverage) {:inch_ex, github: "rrrene/inch_ex", only: [:dev, :test]}, @@ -45,6 +47,11 @@ defmodule K8s.MixProject do {:excoveralls, "~> 0.14", only: [:test]}, {:mix_test_watch, "~> 1.1", only: :dev, runtime: false} ] + + # spandex requires 1.10 + if Version.match?(System.version(), "~> 1.10"), + do: [{:spandex, "~> 3.0.3", optional: true} | deps], + else: deps end defp package do @@ -93,7 +100,14 @@ defmodule K8s.MixProject do defp dialyzer do [ ignore_warnings: ".dialyzer_ignore.exs", - plt_add_apps: [:mix, :eex], + plt_add_apps: [ + :mix, + :eex, + :opentelemetry, + :opentelemetry_telemetry, + :opentelemetry_api, + :spandex + ], plt_core_path: "priv/plts", plt_file: {:no_warn, "priv/plts/k8s.plt"} ] diff --git a/mix.lock b/mix.lock index 460f8d36..15ac7897 100644 --- a/mix.lock +++ b/mix.lock @@ -11,19 +11,28 @@ "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, - "inch_ex": {:git, "https://github.com/rrrene/inch_ex.git", "c8eeaa65312df3ce150e91d7dddb50e2983b3209", []}, + "inch_ex": {:git, "https://github.com/rrrene/inch_ex.git", "d37c3cd41ceda869696499569547d6f9a416751c", []}, "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, + "mime": {:hex, :mime, "2.0.2", "0b9e1a4c840eafb68d820b0e2158ef5c49385d17fb36855ac6e7e087d4b1dcc5", [:mix], [], "hexpm", "e6a3f76b4c277739e36c2e21a2c640778ba4c3846189d5ab19f97f126df5f9b7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mix_test_watch": {:hex, :mix_test_watch, "1.1.0", "330bb91c8ed271fe408c42d07e0773340a7938d8a0d281d57a14243eae9dc8c3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "52b6b1c476cbb70fd899ca5394506482f12e5f6b0d6acff9df95c7f1e0812ec3"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.2", "b99ca56bbce410e9d5ee4f9155a212e942e224e259c7ebbf8f2c86ac21d4fa3c", [:mix], [], "hexpm", "98d51bd64d5f6a2a9c6bb7586ee8129e27dfaab1140b5a4753f24dac0ba27d2f"}, + "opentelemetry": {:hex, :opentelemetry, "1.0.1", "8e3a7e219db32cfafabb4392bc01a66e8da19176da4f00a0c8d08d63801f67a5", [:rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "4c93fb7a5ffa1ddc15440714132df29d053b84bd7b4b24491704ae9b4e48caf8"}, + "opentelemetry_api": {:hex, :opentelemetry_api, "1.0.1", "317908954a3dafdad916d3c9ff2f7cd48e393f57bea9bcfa57c6abcf9c0659e5", [:mix, :rebar3], [], "hexpm", "9be65ab0f74fab7ab9cbe197f39ee056420178d8e8de31805b19159eda7fd621"}, + "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.0.0-beta.7", "ba1df62515aed63f99a80ddf17e7a3873d1f686f23598edebf1633942772856e", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.3.0", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "480f4fa1e992d597f931e7bc9e68478e8d904ad84489d2c5ca6eb6d48bbd7801"}, + "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm", "1a06ea6a653120226b35b283a1cd10039550f2c566edcdec22b29316d73640fd"}, "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, + "plug": {:hex, :plug, "1.13.3", "93b299039c21a8b82cc904d13812bce4ced45cf69153e8d35ca16ffb3e8c5d98", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "98c8003e4faf7b74a9ac41bee99e328b08f069bf932747d4a7532e97ae837a17"}, + "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, + "spandex": {:hex, :spandex, "3.0.3", "91aa318f3de696bb4d931adf65f7ebdbe5df25cccce1fe8fd376a44c46bcf69b", [:mix], [{:decorator, "~> 1.2", [hex: :decorator, repo: "hexpm", optional: true]}, {:optimal, "~> 0.3.3", [hex: :optimal, repo: "hexpm", optional: false]}, {:plug, ">= 1.0.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e3e6c319d0ab478ddc9a39102a727a410c962b4d51c0932c72279b86d3b17044"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, + "telemetry_registry": {:hex, :telemetry_registry, "0.3.0", "6768f151ea53fc0fbca70dbff5b20a8d663ee4e0c0b2ae589590e08658e76f1e", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "492e2adbc609f3e79ece7f29fec363a97a2c484ac78a83098535d6564781e917"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, - "yamerl": {:hex, :yamerl, "0.8.1", "07da13ffa1d8e13948943789665c62ccd679dfa7b324a4a2ed3149df17f453a4", [:rebar3], [], "hexpm", "96cb30f9d64344fed0ef8a92e9f16f207de6c04dfff4f366752ca79f5bceb23f"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, "yaml_elixir": {:hex, :yaml_elixir, "2.8.0", "c7ff0034daf57279c2ce902788ce6fdb2445532eb4317e8df4b044209fae6832", [:mix], [{:yamerl, "~> 0.8", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "4b674bd881e373d1ac6a790c64b2ecb69d1fd612c2af3b22de1619c15473830b"}, }