rock/homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add tempo and otel-collector for distributed tracing

Browse Source
This commit is contained in:
rock
2026-08-31 15:02:15 -07:00
parent be55d68571
commit 1af9232613
9 changed files with 573 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/argocd/apps/10-storage-observability.yaml
+26
View File
@@ -223,3 +223,29 @@ spec:
automated: automated:
prune: true prune: true
selfHeal: true selfHeal: true
---
# Distributed tracing: Tempo + OpenTelemetry Collector.
# Receives traces from instrumented services, stores in local volume (72h retention).
# Grafana datasource auto-configured, service graph + latency dashboards included.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: tracing
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "1"
spec:
project: homelab
source:
repoURL: https://forgejo.riotpiao.com/rock/homelab.git
targetRevision: main
path: k8s/infra/tracing
destination:
server: https://kubernetes.default.svc
namespace: tracing
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
k8s/infra/tracing/api-latency-dashboard.yaml
+128
View File
@@ -0,0 +1,128 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: api-latency-dashboard
namespace: logging
labels:
grafana_dashboard: "1"
annotations:
grafana_folder: "Tracing"
data:
api-latency.json: |
{
"title": "API Latency & Traces",
"uid": "api-latency-traces",
"schemaVersion": 39,
"timezone": "browser",
"time": {"from": "now-1h", "to": "now"},
"refresh": "10s",
"panels": [
{
"id": 1,
"title": "Request Rate by Service",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\"}[5m])) by (service)",
"legendFormat": "{{service}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {"fillOpacity": 10}
}
}
},
{
"id": 2,
"title": "P95 Latency by Service",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\"api-gateway|poimen.*\"}[5m])) by (le, service))",
"legendFormat": "{{service}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {"fillOpacity": 10}
}
}
},
{
"id": 3,
"title": "Error Rate by Service",
"type": "timeseries",
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\", status_code=\"STATUS_CODE_ERROR\"}[5m])) by (service)",
"legendFormat": "{{service}} errors"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {"fillOpacity": 10},
"color": {"mode": "palette-classic"}
}
}
},
{
"id": 4,
"title": "Service Dependency Graph",
"type": "nodeGraph",
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
"datasource": {"type": "tempo", "uid": "tempo"},
"targets": [
{
"queryType": "serviceMap"
}
]
},
{
"id": 5,
"title": "Trace Search",
"type": "traces",
"gridPos": {"h": 12, "w": 24, "x": 0, "y": 16},
"datasource": {"type": "tempo", "uid": "tempo"},
"targets": [
{
"queryType": "traceqlSearch",
"serviceName": "api-gateway",
"limit": 20
}
]
},
{
"id": 6,
"title": "LLM Inference Latency",
"type": "timeseries",
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
"legendFormat": "p50 {{service}} {{span_name}}"
},
{
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
"legendFormat": "p95 {{service}} {{span_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {"fillOpacity": 10}
}
}
}
]
}
k8s/infra/tracing/grafana-datasource.yaml
+43
View File
@@ -0,0 +1,43 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-tempo-datasource
namespace: monitoring
labels:
grafana_datasource: "1"
data:
tempo-datasource.yaml: |
apiVersion: 1
datasources:
- name: Tempo
type: tempo
access: proxy
url: http://tempo.tracing.svc.cluster.local:3200
uid: tempo
jsonData:
httpMethod: GET
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: '-1h'
spanEndTimeShift: '1h'
filterByTraceID: true
filterBySpanID: true
customQuery: true
query: '{namespace="$${__span.tags["service.namespace"]}"} | json | trace_id="$${__trace.traceId}"'
tracesToMetrics:
datasourceUid: prometheus
spanStartTimeShift: '-1h'
spanEndTimeShift: '1h'
tags:
- key: service.name
value: service
- key: http.method
value: method
serviceMap:
datasourceUid: prometheus
nodeGraph:
enabled: true
search:
hide: false
lokiSearch:
datasourceUid: loki
k8s/infra/tracing/kustomization.yaml
+10
View File
@@ -0,0 +1,10 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- tempo-config.yaml
- tempo-deployment.yaml
- otel-collector.yaml
- grafana-datasource.yaml
- servicemonitor.yaml
- api-latency-dashboard.yaml
k8s/infra/tracing/namespace.yaml
+6
View File
@@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
name: tracing
labels:
kubernetes.io/metadata.name: tracing
k8s/infra/tracing/otel-collector.yaml
+134
View File
@@ -0,0 +1,134 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: otel-collector-config
namespace: tracing
data:
config.yaml: |
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
batch:
timeout: 1s
send_batch_size: 1024
# Add service.namespace attribute if missing
attributes:
actions:
- key: deployment.environment
value: homelab
action: upsert
exporters:
otlp:
endpoint: tempo.tracing.svc.cluster.local:4317
tls:
insecure: true
# Debug logging for troubleshooting
debug:
verbosity: basic
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
pipelines:
traces:
receivers: [otlp]
processors: [batch, attributes]
exporters: [otlp, debug]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: otel-collector
namespace: tracing
labels:
app.kubernetes.io/name: otel-collector
spec:
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: otel-collector
template:
metadata:
labels:
app.kubernetes.io/name: otel-collector
annotations:
reloader.stakater.com/auto: "true"
spec:
securityContext:
runAsNonRoot: true
runAsUser: 10001
containers:
- name: collector
image: otel/opentelemetry-collector-contrib:0.111.0
args:
- --config=/etc/otel/config.yaml
ports:
- name: otlp-grpc
containerPort: 4317
protocol: TCP
- name: otlp-http
containerPort: 4318
protocol: TCP
- name: health
containerPort: 13133
protocol: TCP
volumeMounts:
- name: config
mountPath: /etc/otel
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 200m
memory: 256Mi
readinessProbe:
httpGet:
path: /
port: health
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /
port: health
initialDelaySeconds: 10
periodSeconds: 30
volumes:
- name: config
configMap:
name: otel-collector-config
---
apiVersion: v1
kind: Service
metadata:
name: otel-collector
namespace: tracing
labels:
app.kubernetes.io/name: otel-collector
spec:
type: ClusterIP
ports:
- name: otlp-grpc
port: 4317
targetPort: otlp-grpc
- name: otlp-http
port: 4318
targetPort: otlp-http
- name: health
port: 13133
targetPort: health
selector:
app.kubernetes.io/name: otel-collector
k8s/infra/tracing/servicemonitor.yaml
+37
View File
@@ -0,0 +1,37 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: tempo
namespace: tracing
labels:
release: prometheus
spec:
selector:
matchLabels:
app.kubernetes.io/name: tempo
namespaceSelector:
matchNames:
- tracing
endpoints:
- port: http
path: /metrics
interval: 30s
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: otel-collector
namespace: tracing
labels:
release: prometheus
spec:
selector:
matchLabels:
app.kubernetes.io/name: otel-collector
namespaceSelector:
matchNames:
- tracing
endpoints:
- port: health
path: /metrics
interval: 30s
k8s/infra/tracing/tempo-config.yaml
+76
View File
@@ -0,0 +1,76 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: tempo-config
namespace: tracing
data:
tempo.yaml: |
# Tempo monolithic mode configuration
server:
http_listen_port: 3200
grpc_listen_port: 9095
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 72h
metrics_generator:
registry:
external_labels:
source: tempo
cluster: homelab
storage:
path: /var/tempo/generator/wal
remote_write:
- url: http://prometheus-kube-prometheus-prometheus.monitoring:9090/api/v1/write
send_exemplars: true
traces_storage:
path: /var/tempo/generator/traces
processor:
service_graphs:
dimensions:
- service.namespace
- http.method
- http.route
span_metrics:
dimensions:
- service.namespace
- http.method
- http.route
- http.status_code
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
querier:
frontend_worker:
frontend_address: localhost:9095
query_frontend:
search:
duration_slo: 5s
throughput_bytes_slo: 1.073741824e+09
overrides:
defaults:
metrics_generator:
processors:
- service-graphs
- span-metrics
k8s/infra/tracing/tempo-deployment.yaml
+113
View File
@@ -0,0 +1,113 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: tempo
namespace: tracing
labels:
app.kubernetes.io/name: tempo
app.kubernetes.io/component: tracing
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: tempo
strategy:
type: Recreate
template:
metadata:
labels:
app.kubernetes.io/name: tempo
annotations:
reloader.stakater.com/auto: "true"
spec:
securityContext:
fsGroup: 10001
runAsNonRoot: true
runAsUser: 10001
containers:
- name: tempo
image: grafana/tempo:2.6.1
args:
- -config.file=/etc/tempo/tempo.yaml
ports:
- name: http
containerPort: 3200
protocol: TCP
- name: grpc
containerPort: 9095
protocol: TCP
- name: otlp-grpc
containerPort: 4317
protocol: TCP
- name: otlp-http
containerPort: 4318
protocol: TCP
volumeMounts:
- name: config
mountPath: /etc/tempo
- name: tempo-data
mountPath: /var/tempo
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 1Gi
readinessProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 10
periodSeconds: 10
livenessProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 30
periodSeconds: 30
volumes:
- name: config
configMap:
name: tempo-config
- name: tempo-data
persistentVolumeClaim:
claimName: tempo-data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: tempo-data
namespace: tracing
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn
resources:
requests:
storage: 10Gi
---
apiVersion: v1
kind: Service
metadata:
name: tempo
namespace: tracing
labels:
app.kubernetes.io/name: tempo
spec:
type: ClusterIP
ports:
- name: http
port: 3200
targetPort: http
- name: grpc
port: 9095
targetPort: grpc
- name: otlp-grpc
port: 4317
targetPort: otlp-grpc
- name: otlp-http
port: 4318
targetPort: otlp-http
selector:
app.kubernetes.io/name: tempo