diff --git a/k8s/argocd/apps/10-storage-observability.yaml b/k8s/argocd/apps/10-storage-observability.yaml index c604333..624f7ae 100644 --- a/k8s/argocd/apps/10-storage-observability.yaml +++ b/k8s/argocd/apps/10-storage-observability.yaml @@ -223,3 +223,29 @@ spec: automated: prune: true selfHeal: true +--- +# Distributed tracing: Tempo + OpenTelemetry Collector. +# Receives traces from instrumented services, stores in local volume (72h retention). +# Grafana datasource auto-configured, service graph + latency dashboards included. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: tracing + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + project: homelab + source: + repoURL: https://forgejo.riotpiao.com/rock/homelab.git + targetRevision: main + path: k8s/infra/tracing + destination: + server: https://kubernetes.default.svc + namespace: tracing + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/k8s/infra/tracing/api-latency-dashboard.yaml b/k8s/infra/tracing/api-latency-dashboard.yaml new file mode 100644 index 0000000..b876e94 --- /dev/null +++ b/k8s/infra/tracing/api-latency-dashboard.yaml @@ -0,0 +1,128 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: api-latency-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "Tracing" +data: + api-latency.json: | + { + "title": "API Latency & Traces", + "uid": "api-latency-traces", + "schemaVersion": 39, + "timezone": "browser", + "time": {"from": "now-1h", "to": "now"}, + "refresh": "10s", + "panels": [ + { + "id": 1, + "title": "Request Rate by Service", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\"}[5m])) by (service)", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": {"fillOpacity": 10} + } + } + }, + { + "id": 2, + "title": "P95 Latency by Service", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\"api-gateway|poimen.*\"}[5m])) by (le, service))", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {"fillOpacity": 10} + } + } + }, + { + "id": 3, + "title": "Error Rate by Service", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\", status_code=\"STATUS_CODE_ERROR\"}[5m])) by (service)", + "legendFormat": "{{service}} errors" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": {"fillOpacity": 10}, + "color": {"mode": "palette-classic"} + } + } + }, + { + "id": 4, + "title": "Service Dependency Graph", + "type": "nodeGraph", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "datasource": {"type": "tempo", "uid": "tempo"}, + "targets": [ + { + "queryType": "serviceMap" + } + ] + }, + { + "id": 5, + "title": "Trace Search", + "type": "traces", + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 16}, + "datasource": {"type": "tempo", "uid": "tempo"}, + "targets": [ + { + "queryType": "traceqlSearch", + "serviceName": "api-gateway", + "limit": 20 + } + ] + }, + { + "id": 6, + "title": "LLM Inference Latency", + "type": "timeseries", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 28}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))", + "legendFormat": "p50 {{service}} {{span_name}}" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))", + "legendFormat": "p95 {{service}} {{span_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": {"fillOpacity": 10} + } + } + } + ] + } diff --git a/k8s/infra/tracing/grafana-datasource.yaml b/k8s/infra/tracing/grafana-datasource.yaml new file mode 100644 index 0000000..bc12a0a --- /dev/null +++ b/k8s/infra/tracing/grafana-datasource.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-tempo-datasource + namespace: monitoring + labels: + grafana_datasource: "1" +data: + tempo-datasource.yaml: | + apiVersion: 1 + datasources: + - name: Tempo + type: tempo + access: proxy + url: http://tempo.tracing.svc.cluster.local:3200 + uid: tempo + jsonData: + httpMethod: GET + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: '-1h' + spanEndTimeShift: '1h' + filterByTraceID: true + filterBySpanID: true + customQuery: true + query: '{namespace="$${__span.tags["service.namespace"]}"} | json | trace_id="$${__trace.traceId}"' + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: '-1h' + spanEndTimeShift: '1h' + tags: + - key: service.name + value: service + - key: http.method + value: method + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + search: + hide: false + lokiSearch: + datasourceUid: loki diff --git a/k8s/infra/tracing/kustomization.yaml b/k8s/infra/tracing/kustomization.yaml new file mode 100644 index 0000000..8ee4504 --- /dev/null +++ b/k8s/infra/tracing/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - tempo-config.yaml + - tempo-deployment.yaml + - otel-collector.yaml + - grafana-datasource.yaml + - servicemonitor.yaml + - api-latency-dashboard.yaml diff --git a/k8s/infra/tracing/namespace.yaml b/k8s/infra/tracing/namespace.yaml new file mode 100644 index 0000000..8ea991e --- /dev/null +++ b/k8s/infra/tracing/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tracing + labels: + kubernetes.io/metadata.name: tracing diff --git a/k8s/infra/tracing/otel-collector.yaml b/k8s/infra/tracing/otel-collector.yaml new file mode 100644 index 0000000..8a859e6 --- /dev/null +++ b/k8s/infra/tracing/otel-collector.yaml @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config + namespace: tracing +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + processors: + batch: + timeout: 1s + send_batch_size: 1024 + + # Add service.namespace attribute if missing + attributes: + actions: + - key: deployment.environment + value: homelab + action: upsert + + exporters: + otlp: + endpoint: tempo.tracing.svc.cluster.local:4317 + tls: + insecure: true + + # Debug logging for troubleshooting + debug: + verbosity: basic + + extensions: + health_check: + endpoint: 0.0.0.0:13133 + + service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch, attributes] + exporters: [otlp, debug] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-collector + namespace: tracing + labels: + app.kubernetes.io/name: otel-collector +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: otel-collector + template: + metadata: + labels: + app.kubernetes.io/name: otel-collector + annotations: + reloader.stakater.com/auto: "true" + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.111.0 + args: + - --config=/etc/otel/config.yaml + ports: + - name: otlp-grpc + containerPort: 4317 + protocol: TCP + - name: otlp-http + containerPort: 4318 + protocol: TCP + - name: health + containerPort: 13133 + protocol: TCP + volumeMounts: + - name: config + mountPath: /etc/otel + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + readinessProbe: + httpGet: + path: / + port: health + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: health + initialDelaySeconds: 10 + periodSeconds: 30 + volumes: + - name: config + configMap: + name: otel-collector-config +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + namespace: tracing + labels: + app.kubernetes.io/name: otel-collector +spec: + type: ClusterIP + ports: + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + - name: otlp-http + port: 4318 + targetPort: otlp-http + - name: health + port: 13133 + targetPort: health + selector: + app.kubernetes.io/name: otel-collector diff --git a/k8s/infra/tracing/servicemonitor.yaml b/k8s/infra/tracing/servicemonitor.yaml new file mode 100644 index 0000000..97353d5 --- /dev/null +++ b/k8s/infra/tracing/servicemonitor.yaml @@ -0,0 +1,37 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: tempo + namespace: tracing + labels: + release: prometheus +spec: + selector: + matchLabels: + app.kubernetes.io/name: tempo + namespaceSelector: + matchNames: + - tracing + endpoints: + - port: http + path: /metrics + interval: 30s +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: otel-collector + namespace: tracing + labels: + release: prometheus +spec: + selector: + matchLabels: + app.kubernetes.io/name: otel-collector + namespaceSelector: + matchNames: + - tracing + endpoints: + - port: health + path: /metrics + interval: 30s diff --git a/k8s/infra/tracing/tempo-config.yaml b/k8s/infra/tracing/tempo-config.yaml new file mode 100644 index 0000000..bf15757 --- /dev/null +++ b/k8s/infra/tracing/tempo-config.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo-config + namespace: tracing +data: + tempo.yaml: | + # Tempo monolithic mode configuration + server: + http_listen_port: 3200 + grpc_listen_port: 9095 + + distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + ingester: + max_block_duration: 5m + + compactor: + compaction: + block_retention: 72h + + metrics_generator: + registry: + external_labels: + source: tempo + cluster: homelab + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus-kube-prometheus-prometheus.monitoring:9090/api/v1/write + send_exemplars: true + traces_storage: + path: /var/tempo/generator/traces + processor: + service_graphs: + dimensions: + - service.namespace + - http.method + - http.route + span_metrics: + dimensions: + - service.namespace + - http.method + - http.route + - http.status_code + + storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + + querier: + frontend_worker: + frontend_address: localhost:9095 + + query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + + overrides: + defaults: + metrics_generator: + processors: + - service-graphs + - span-metrics diff --git a/k8s/infra/tracing/tempo-deployment.yaml b/k8s/infra/tracing/tempo-deployment.yaml new file mode 100644 index 0000000..ff433db --- /dev/null +++ b/k8s/infra/tracing/tempo-deployment.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tempo + namespace: tracing + labels: + app.kubernetes.io/name: tempo + app.kubernetes.io/component: tracing +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: tempo + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: tempo + annotations: + reloader.stakater.com/auto: "true" + spec: + securityContext: + fsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + containers: + - name: tempo + image: grafana/tempo:2.6.1 + args: + - -config.file=/etc/tempo/tempo.yaml + ports: + - name: http + containerPort: 3200 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: otlp-grpc + containerPort: 4317 + protocol: TCP + - name: otlp-http + containerPort: 4318 + protocol: TCP + volumeMounts: + - name: config + mountPath: /etc/tempo + - name: tempo-data + mountPath: /var/tempo + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + volumes: + - name: config + configMap: + name: tempo-config + - name: tempo-data + persistentVolumeClaim: + claimName: tempo-data +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tempo-data + namespace: tracing +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: tempo + namespace: tracing + labels: + app.kubernetes.io/name: tempo +spec: + type: ClusterIP + ports: + - name: http + port: 3200 + targetPort: http + - name: grpc + port: 9095 + targetPort: grpc + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + - name: otlp-http + port: 4318 + targetPort: otlp-http + selector: + app.kubernetes.io/name: tempo