feat: add tempo and otel-collector for distributed tracing
This commit is contained in:
@@ -223,3 +223,29 @@ spec:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
---
|
||||
# Distributed tracing: Tempo + OpenTelemetry Collector.
|
||||
# Receives traces from instrumented services, stores in local volume (72h retention).
|
||||
# Grafana datasource auto-configured, service graph + latency dashboards included.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: tracing
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "1"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/rock/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/infra/tracing
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: tracing
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: api-latency-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Tracing"
|
||||
data:
|
||||
api-latency.json: |
|
||||
{
|
||||
"title": "API Latency & Traces",
|
||||
"uid": "api-latency-traces",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"refresh": "10s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Request Rate by Service",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\"}[5m])) by (service)",
|
||||
"legendFormat": "{{service}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {"fillOpacity": 10}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "P95 Latency by Service",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\"api-gateway|poimen.*\"}[5m])) by (le, service))",
|
||||
"legendFormat": "{{service}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"custom": {"fillOpacity": 10}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Error Rate by Service",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\", status_code=\"STATUS_CODE_ERROR\"}[5m])) by (service)",
|
||||
"legendFormat": "{{service}} errors"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {"fillOpacity": 10},
|
||||
"color": {"mode": "palette-classic"}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Service Dependency Graph",
|
||||
"type": "nodeGraph",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
|
||||
"datasource": {"type": "tempo", "uid": "tempo"},
|
||||
"targets": [
|
||||
{
|
||||
"queryType": "serviceMap"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Trace Search",
|
||||
"type": "traces",
|
||||
"gridPos": {"h": 12, "w": 24, "x": 0, "y": 16},
|
||||
"datasource": {"type": "tempo", "uid": "tempo"},
|
||||
"targets": [
|
||||
{
|
||||
"queryType": "traceqlSearch",
|
||||
"serviceName": "api-gateway",
|
||||
"limit": 20
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "LLM Inference Latency",
|
||||
"type": "timeseries",
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
|
||||
"legendFormat": "p50 {{service}} {{span_name}}"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
|
||||
"legendFormat": "p95 {{service}} {{span_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"custom": {"fillOpacity": 10}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-tempo-datasource
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_datasource: "1"
|
||||
data:
|
||||
tempo-datasource.yaml: |
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo.tracing.svc.cluster.local:3200
|
||||
uid: tempo
|
||||
jsonData:
|
||||
httpMethod: GET
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: '-1h'
|
||||
spanEndTimeShift: '1h'
|
||||
filterByTraceID: true
|
||||
filterBySpanID: true
|
||||
customQuery: true
|
||||
query: '{namespace="$${__span.tags["service.namespace"]}"} | json | trace_id="$${__trace.traceId}"'
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: '-1h'
|
||||
spanEndTimeShift: '1h'
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
- key: http.method
|
||||
value: method
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
search:
|
||||
hide: false
|
||||
lokiSearch:
|
||||
datasourceUid: loki
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- tempo-config.yaml
|
||||
- tempo-deployment.yaml
|
||||
- otel-collector.yaml
|
||||
- grafana-datasource.yaml
|
||||
- servicemonitor.yaml
|
||||
- api-latency-dashboard.yaml
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: tracing
|
||||
labels:
|
||||
kubernetes.io/metadata.name: tracing
|
||||
@@ -0,0 +1,134 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: otel-collector-config
|
||||
namespace: tracing
|
||||
data:
|
||||
config.yaml: |
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 1024
|
||||
|
||||
# Add service.namespace attribute if missing
|
||||
attributes:
|
||||
actions:
|
||||
- key: deployment.environment
|
||||
value: homelab
|
||||
action: upsert
|
||||
|
||||
exporters:
|
||||
otlp:
|
||||
endpoint: tempo.tracing.svc.cluster.local:4317
|
||||
tls:
|
||||
insecure: true
|
||||
|
||||
# Debug logging for troubleshooting
|
||||
debug:
|
||||
verbosity: basic
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [batch, attributes]
|
||||
exporters: [otlp, debug]
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: otel-collector
|
||||
namespace: tracing
|
||||
labels:
|
||||
app.kubernetes.io/name: otel-collector
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: otel-collector
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: otel-collector
|
||||
annotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
containers:
|
||||
- name: collector
|
||||
image: otel/opentelemetry-collector-contrib:0.111.0
|
||||
args:
|
||||
- --config=/etc/otel/config.yaml
|
||||
ports:
|
||||
- name: otlp-grpc
|
||||
containerPort: 4317
|
||||
protocol: TCP
|
||||
- name: otlp-http
|
||||
containerPort: 4318
|
||||
protocol: TCP
|
||||
- name: health
|
||||
containerPort: 13133
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/otel
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: health
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: health
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: otel-collector-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: otel-collector
|
||||
namespace: tracing
|
||||
labels:
|
||||
app.kubernetes.io/name: otel-collector
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: otlp-grpc
|
||||
port: 4317
|
||||
targetPort: otlp-grpc
|
||||
- name: otlp-http
|
||||
port: 4318
|
||||
targetPort: otlp-http
|
||||
- name: health
|
||||
port: 13133
|
||||
targetPort: health
|
||||
selector:
|
||||
app.kubernetes.io/name: otel-collector
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: tempo
|
||||
namespace: tracing
|
||||
labels:
|
||||
release: prometheus
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: tempo
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- tracing
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: otel-collector
|
||||
namespace: tracing
|
||||
labels:
|
||||
release: prometheus
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: otel-collector
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- tracing
|
||||
endpoints:
|
||||
- port: health
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
@@ -0,0 +1,76 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: tempo-config
|
||||
namespace: tracing
|
||||
data:
|
||||
tempo.yaml: |
|
||||
# Tempo monolithic mode configuration
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
grpc_listen_port: 9095
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 72h
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
cluster: homelab
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus-kube-prometheus-prometheus.monitoring:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
traces_storage:
|
||||
path: /var/tempo/generator/traces
|
||||
processor:
|
||||
service_graphs:
|
||||
dimensions:
|
||||
- service.namespace
|
||||
- http.method
|
||||
- http.route
|
||||
span_metrics:
|
||||
dimensions:
|
||||
- service.namespace
|
||||
- http.method
|
||||
- http.route
|
||||
- http.status_code
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
|
||||
querier:
|
||||
frontend_worker:
|
||||
frontend_address: localhost:9095
|
||||
|
||||
query_frontend:
|
||||
search:
|
||||
duration_slo: 5s
|
||||
throughput_bytes_slo: 1.073741824e+09
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors:
|
||||
- service-graphs
|
||||
- span-metrics
|
||||
@@ -0,0 +1,113 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: tempo
|
||||
namespace: tracing
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: tracing
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: tempo
|
||||
strategy:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
annotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 10001
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
containers:
|
||||
- name: tempo
|
||||
image: grafana/tempo:2.6.1
|
||||
args:
|
||||
- -config.file=/etc/tempo/tempo.yaml
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3200
|
||||
protocol: TCP
|
||||
- name: grpc
|
||||
containerPort: 9095
|
||||
protocol: TCP
|
||||
- name: otlp-grpc
|
||||
containerPort: 4317
|
||||
protocol: TCP
|
||||
- name: otlp-http
|
||||
containerPort: 4318
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/tempo
|
||||
- name: tempo-data
|
||||
mountPath: /var/tempo
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: tempo-config
|
||||
- name: tempo-data
|
||||
persistentVolumeClaim:
|
||||
claimName: tempo-data
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: tempo-data
|
||||
namespace: tracing
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: tempo
|
||||
namespace: tracing
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: http
|
||||
port: 3200
|
||||
targetPort: http
|
||||
- name: grpc
|
||||
port: 9095
|
||||
targetPort: grpc
|
||||
- name: otlp-grpc
|
||||
port: 4317
|
||||
targetPort: otlp-grpc
|
||||
- name: otlp-http
|
||||
port: 4318
|
||||
targetPort: otlp-http
|
||||
selector:
|
||||
app.kubernetes.io/name: tempo
|
||||
Reference in New Issue
Block a user