feat: add tempo and otel-collector

devops/ansible/local.yml

@@ -83,3 +83,15 @@
       loki__ports:
         - 3100:3100
       loki__config_template: "{{ playbook_dir }}/templates/loki/local-config.yaml.j2"
+
+    - role: tempo
+      tempo__networks:
+        - name: "{{ docker_network__name }}"
+      tempo__remove_existing_container: true
+      tempo__config_template: "{{ playbook_dir }}/templates/tempo/tempo.yaml.j2"
+
+    - role: otel_collector
+      otel_collector__networks:
+        - name: "{{ docker_network__name }}"
+      otel_collector__remove_existing_container: true
+      otel_collector__config_template: "{{ playbook_dir }}/templates/otel-collector/config.yaml.j2"

devops/ansible/metrics.yml

@@ -35,3 +35,9 @@
       loki__ports:
         - 3100:3100
       loki__config_template: "{{ playbook_dir }}/templates/loki/local-config.yaml.j2"
+
+    - role: tempo
+      tempo__networks:
+        - name: "{{ docker_network__name }}"
+      tempo__remove_existing_container: true
+      tempo__config_template: "{{ playbook_dir }}/templates/tempo/tempo.yaml.j2"

devops/ansible/roles/grafana/tasks/main.yml

@@ -21,19 +21,19 @@
   ansible.builtin.template:
     src: "{{ grafana__config_template }}"
     dest: /etc/grafana/config.ini
-    mode: '0755'
+    mode: "0755"
 
 - name: Template config to /etc/grafana/datasource.yml
   ansible.builtin.template:
     src: "{{ grafana__datasource_template }}"
     dest: /etc/grafana/datasource.yml
-    mode: '0755'
+    mode: "0755"
 
 - name: Template config to /etc/grafana/dashboards.yaml
   ansible.builtin.template:
     src: "{{ grafana__dashboards_template }}"
     dest: /etc/grafana/dashboards.yaml
-    mode: '0755'
+    mode: "0755"
 
 - name: Deploy dashboards files
   ansible.builtin.copy:

devops/ansible/roles/otel_collector/defaults/main.yml

@@ -0,0 +1,17 @@
+---
+otel_collector__container_name: otel-collector
+otel_collector__image: otel/opentelemetry-collector:latest
+otel_collector__network_mode:
+otel_collector__networks: []
+otel_collector__remove_existing_container: true
+otel_collector__env:
+otel_collector__ports:
+  - 1888:1888 # pprof extension
+  - 8888:8888 # Prometheus metrics exposed by the collector
+  - 8889:8889 # Prometheus exporter metrics
+  - 13133:13133 # health_check extension
+  - 4317:4317 # OTLP gRPC receiver
+  - 4318:4318 # OTLP HTTP receiver
+  - 55679:55679 # zpages extension
+otel_collector__restart_policy: always
+otel_collector__config_template: config.yaml.j2

devops/ansible/roles/otel_collector/tasks/main.yml

@@ -0,0 +1,31 @@
+---
+- name: Remove existing container {{ otel_collector__container_name }}
+  community.docker.docker_container:
+    name: "{{ otel_collector__container_name }}"
+    state: absent
+  when: otel_collector__remove_existing_container
+
+- name: Create a directory /etc/otel-collector if it does not exist
+  ansible.builtin.file:
+    path: /etc/otel-collector
+    state: directory
+    mode: "0755"
+
+- name: Template config to /etc/otel-collector/config.yaml
+  ansible.builtin.template:
+    src: "{{ otel_collector__config_template }}"
+    dest: /etc/otel-collector/config.yaml
+    mode: "0755"
+
+- name: Run container {{ otel_collector__container_name }}
+  community.docker.docker_container:
+    name: "{{ otel_collector__container_name }}"
+    image: "{{ otel_collector__image }}"
+    restart_policy: "{{ otel_collector__restart_policy }}"
+    command: [--config=/etc/config.yaml]
+    volumes:
+      - /etc/otel-collector/config.yaml:/etc/config.yaml
+    network_mode: "{{ otel_collector__network_mode }}"
+    networks: "{{ otel_collector__networks }}"
+    expose: "{{ otel_collector__ports }}"
+    ports: "{{ otel_collector__ports }}"

devops/ansible/roles/otel_collector/templates/config.yaml.j2

@@ -0,0 +1,20 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+        cors:
+          allowed_origins:
+            - http://*
+            - https://*
+exporters:
+  otlp:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlp]

devops/ansible/roles/prometheus/tasks/main.yml

@@ -15,7 +15,7 @@
   ansible.builtin.template:
     src: "{{ prometheus__template }}"
     dest: /etc/prometheus/prometheus.yml
-    mode: '0755'
+    mode: "0755"
 
 - name: Run container {{ prometheus__container_name }}
   community.docker.docker_container:

devops/ansible/roles/tempo/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+tempo__container_name: tempo
+tempo__image: grafana/tempo:latest
+tempo__network_mode:
+tempo__networks: []
+tempo__remove_existing_container: true
+tempo__env:
+tempo__ports:
+  - 3200:3200
+tempo__restart_policy: always
+
+tempo__storage_volume: /etc/tempo/data
+tempo__config_template: config.ini.j2

devops/ansible/roles/tempo/tasks/main.yml

@@ -0,0 +1,38 @@
+---
+- name: Remove existing container {{ tempo__container_name }}
+  community.docker.docker_container:
+    name: "{{ tempo__container_name }}"
+    state: absent
+  when: tempo__remove_existing_container
+
+- name: Create a directory /etc/tempo if it does not exist
+  ansible.builtin.file:
+    path: /etc/tempo
+    state: directory
+    mode: "0755"
+
+- name: Create a directory if it does not exist {{ tempo__storage_volume }}
+  ansible.builtin.file:
+    path: "{{ tempo__storage_volume }}"
+    state: directory
+    mode: "0755"
+
+- name: Template config to /etc/tempo/tempo.yaml
+  ansible.builtin.template:
+    src: "{{ tempo__config_template }}"
+    dest: /etc/tempo/tempo.yaml
+    mode: "0755"
+
+- name: Run container {{ tempo__container_name }}
+  community.docker.docker_container:
+    name: "{{ tempo__container_name }}"
+    image: "{{ tempo__image }}"
+    restart_policy: "{{ tempo__restart_policy }}"
+    command: [-config.file=/etc/tempo.yaml]
+    volumes:
+      - /etc/tempo/tempo.yaml:/etc/tempo.yaml
+      - "{{ tempo__storage_volume }}:/tmp/tempo"
+    network_mode: "{{ tempo__network_mode }}"
+    networks: "{{ tempo__networks }}"
+    expose: "{{ tempo__ports }}"
+    ports: "{{ tempo__ports }}"

devops/ansible/roles/tempo/templates/tempo.yaml.j2

@@ -0,0 +1,55 @@
+server:
+  http_listen_port: 3200
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics] # enables metrics generator

devops/ansible/templates/grafana/datasource.yml.j2

@@ -23,4 +23,19 @@ datasources:
   url: http://loki:3100
   basicAuth: false
   version: 1
-  editable: true
+  editable: true
+
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  version: 1
+  editable: true
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+    httpMethod: GET
+    serviceMap:
+      datasourceUid: prometheus

devops/ansible/templates/otel-collector/config.yaml.j2

@@ -0,0 +1,20 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+        cors:
+          allowed_origins:
+            - http://*
+            - https://*
+exporters:
+  otlp:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlp]

devops/ansible/templates/tempo/tempo.yaml.j2

@@ -0,0 +1,55 @@
+server:
+  http_listen_port: 3200
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics] # enables metrics generator