refactor(k8s): consolidate to infra/+apps/ single-source tree, dedicated per-app CNPG (authentik-db/temporal-db), wire monitoring-config, forgejo→cicd ns, drop orphan/stale (data-schemas, ollama, story-crater, sqs/argocd, key-rotation)

k8s/infra/logging/grafana-secrets.enc.yaml

@@ -0,0 +1,18 @@
+adminPassword: ENC[AES256_GCM,data:qykgCu5Kv5IwzM1JFLXL7wgiAyI=,iv:bAph3Jd4fxuyglS7zrWIwzfUgK/wBwRrcEpKusgFhVk=,tag:ymwdnU/Dxnp54eGur2caoQ==,type:str]
+env:
+    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:FwldsYfbOkRYGcycxn5BHZk/dOZsu23e3UCPo7dJKrcyr0n8dXQ9nH5XVSRHDpLOdRk7wbQwl+4Rgb8bGs21/w==,iv:JkNEfbsEIeE9TGouDA1xgjS4fX+v/sDrimaGpc4iaME=,tag:6Z3X29GKbgd2x6ghWA+CUw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQdVNSMWZaVDZuN2RBKyt0
+            eWJHRm5vRlErZ1UweXA0N2xSNGdESE5leWo4CjQ2TTlONnpJK2VZWmxoamdhdHZN
+            ZGVMSU84SUZpVTUxREZlek9JcUxaTXcKLS0tIHlNaEpzZG1iaVpvU0E1UTQzbkVk
+            WnQxSHZ6S3cyZ2hUTlpTNXQyaVZzRU0KwU/0aRJ0D/UsqtFcrJLmuHpNELhTsfYS
+            4WL2TQ/yaUz261s7LEVQdpKUza8JjQXiKqqNcV2gaqEWurdx20ILzQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
+    lastmodified: "2026-07-15T23:02:08Z"
+    mac: ENC[AES256_GCM,data:LeN0mm7i7fsDpwtGQ6La7/UzhHbKfG1f/K30wW7fZJ1XbqPZGh7cdq07kMqrodlR6oWvrq7r4gbyuhcH9ptlxKgN4NuD3OoUjn6/CiPEF+wzgKMMgg2JdqFLsZNxVvGL/Ig3Y5hP5iag91QdtQxXaJmaV6Q9+FF6maYofsOU2rA=,iv:Eik9jEhdu+7qJv1oN7e1Vydnv7rh0pnRpbcBPDhh8e8=,tag:Mkry9bpTsmsf9Xs1jW7b1g==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/infra/logging/grafana-values.yaml

@@ -0,0 +1,200 @@
+# logging/grafana-values.yaml
+# Grafana — dashboarding and log/metrics exploration UI.
+# Deployed in the logging namespace alongside Loki and Promtail.
+#
+# Secrets provided by ArgoCD SOPS CMP plugin:
+#   adminPassword        — mounted from the grafana-admin K8s Secret (admin.existingSecret)
+#   OAuth client secret  — mounted from the grafana-oidc K8s Secret (envFromSecret below)
+
+replicas: 1
+
+# RWO PVC (Longhorn) — old pod must fully terminate before the new one can
+# mount the volume. Recreate avoids the "two pods fighting over one PVC" failure.
+deploymentStrategy:
+  type: Recreate
+
+podAnnotations:
+  secret.reloader.stakater.com/reload: "grafana-oidc,grafana-admin"
+
+adminUser: admin
+admin:
+  existingSecret: grafana-admin
+  existingSecretKey: admin-password
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 500m
+    memory: 256Mi
+
+# ── Persistence ───────────────────────────────────────────────────────────────
+# Stores dashboards saved in the UI, datasource edits, and user sessions.
+# Longhorn provides the RWO block volume; 5Gi is ample for dashboard JSON.
+persistence:
+  enabled: true
+  storageClassName: longhorn
+  accessModes:
+    - ReadWriteOnce
+  size: 5Gi
+
+# ── Grafana config ────────────────────────────────────────────────────────────
+grafana.ini:
+  server:
+    root_url: https://grafana.riotpiao.com
+
+  # No anonymous read access — every user must log in via Authentik SSO.
+  auth.anonymous:
+    enabled: false
+
+  # Explore tab: required for ad-hoc LogQL/PromQL queries against Loki/Prometheus.
+  explore:
+    enabled: true
+
+  # WAL for the embedded SQLite DB — prevents corruption on ungraceful shutdown.
+  database:
+    wal: true
+
+  # ── Authentik OIDC (generic OAuth2) ───────────────────────────────────────
+  # Grafana v10+ supports OIDC auto-discovery; we wire it manually here because
+  # Authentik's discovery endpoint is internal-only (no external DNS for iam.svc).
+  # All URLs use the external hostname so CoreDNS rewrites them to
+  # authentik-server.iam.svc — this keeps the Host header correct so Authentik
+  # doesn't return localhost redirects in its token responses.
+  #
+  # role_attribute_path: JMESPath expression evaluated against the userinfo
+  # response. Members of the 'grafana-admins' Authentik group get Admin role;
+  # everyone else gets Viewer. The group name must match exactly what Authentik
+  # sends in the 'groups' claim.
+  auth.generic_oauth:
+    enabled: true
+    name: Authentik
+    allow_sign_up: true
+    client_id: grafana
+    scopes: openid email profile
+    auth_url: https://authentik.riotpiao.com/application/o/authorize/
+    token_url: https://authentik.riotpiao.com/application/o/token/
+    api_url: https://authentik.riotpiao.com/application/o/userinfo/
+    role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
+    use_pkce: false
+    use_refresh_token: false
+    skip_org_role_sync: false
+    tls_skip_verify_insecure: true   # Authentik uses self-signed cert; verify in prod
+
+# GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s
+# Secret (created by k8s/talos-iam/setup_talos_iam.sh). envFromSecret mounts
+# every key in that secret as an environment variable — keeps secrets out of
+# values files and out of git.
+envFromSecret: grafana-oidc
+
+# ── Datasources ───────────────────────────────────────────────────────────────
+# Provisioned at install — no manual "Add datasource" step in the UI.
+# Loki is the default (log exploration); Prometheus is secondary (metrics).
+# Both use in-cluster service DNS — Grafana never leaves the cluster for data.
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    datasources:
+      - name: Loki
+        type: loki
+        uid: loki
+        access: proxy
+        url: http://loki.logging.svc.cluster.local:3100
+        isDefault: true
+        version: 1
+        editable: true
+        jsonData:
+          maxLines: 1000
+          timeout: 60
+      - name: Prometheus
+        type: prometheus
+        uid: prometheus
+        access: proxy
+        url: http://prometheus-kube-prometheus-prometheus.monitoring.svc.cluster.local:9090
+        isDefault: false
+        version: 1
+        editable: true
+        jsonData:
+          timeInterval: 30s
+          timeout: 60
+
+# ── Dashboard providers ───────────────────────────────────────────────────────
+# Tells Grafana to watch a directory for dashboard JSON files.
+# The `dashboards` block below populates that directory via an init container
+# that downloads from grafana.com at install time.
+dashboardProviders:
+  dashboardproviders.yaml:
+    apiVersion: 1
+    providers:
+      - name: default
+        orgId: 1
+        folder: ""
+        type: file
+        disableDeletion: false
+        editable: true
+        options:
+          path: /var/lib/grafana/dashboards/default
+
+# ── Pre-loaded dashboards ──────────────────────────────────────────────────────
+# Fetched from grafana.com by an init container at helm-install time and baked
+# into ConfigMaps. gnetId + revision pin the exact dashboard version so an
+# upstream change on grafana.com doesn't silently alter what's deployed.
+dashboards:
+  default:
+    node-exporter-full:
+      gnetId: 1860
+      revision: 37
+      datasource: Prometheus
+    kubernetes-cluster:
+      gnetId: 7249
+      revision: 1
+      datasource: Prometheus
+
+# Allow scheduling on the control-plane node (talos-cp-1 carries NoSchedule taint).
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+
+# Pin to az-a (talos-cp-1) — the only node running Longhorn. Without this the
+# scheduler may place grafana on cp-2/cp-3, where its Longhorn PVC cannot attach
+# ("CSINode does not contain driver driver.longhorn.io").
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: topology.kubernetes.io/zone
+              operator: In
+              values: [az-a]
+
+# ClusterIP only — access via ingress (grafana.riotpiao.com) or port-forward.
+service:
+  type: ClusterIP
+  port: 80
+
+# Ingress managed by k8s/ingress/ingress.yaml — not the chart's built-in ingress.
+ingress:
+  enabled: false
+
+serviceMonitor:
+  enabled: false
+
+# ── Dashboard-as-code (sidecar discovery) ──────────────────────────────────────
+# Additive to the gnetId-download mechanism above. The sidecar watches for
+# ConfigMaps labeled grafana_dashboard=1 in ANY namespace and loads them live —
+# no Grafana restart needed when a new dashboard ConfigMap is applied.
+sidecar:
+  dashboards:
+    enabled: true
+    label: grafana_dashboard
+    labelValue: "1"
+    folder: /var/lib/grafana/dashboards/custom
+    folderAnnotation: grafana_folder
+    provider:
+      name: custom
+      folder: "Homelab"
+      disableDelete: false
+      foldersFromFilesStructure: true
+    searchNamespace: ALL

k8s/infra/logging/loki-secrets.enc.yaml

@@ -0,0 +1,24 @@
+loki:
+    storage:
+        s3:
+            accessKeyId: ENC[AES256_GCM,data:rUe9BzMuNEk7Lg==,iv:a6oz4kQciHc/sqT1lu05CqI8+iJmi82JAQJ9BM5yBjo=,tag:M1CJmBQlbz/H08S2Y7eUCA==,type:str]
+            secretAccessKey: ENC[AES256_GCM,data:WEvUewgbriABBYllwMtyKsQw850Eb0J6CHc/MEY/NlsolWp9+eq6wKChknI=,iv:0rq4mv8cQ1za8ZLlR8voePiwZFdzK8sbRDDqgloO68g=,tag:r4GNYDa2fL1+d0gY5ndG1Q==,type:str]
+            bucketNames:
+                chunks: ENC[AES256_GCM,data:cAwa5qibAOvuhCk=,iv:Jqt3iHhLaHae6g4CsylytZxg+3fnTdw4PUinhaUaOCI=,tag:jVduzj+ggN9vQSD62qZ0jw==,type:str]
+                ruler: ENC[AES256_GCM,data:G1r90YhXWGkUOw==,iv:K1ddZrCVFAo+CZet+BV7KJh6RVPzGbsnVIkoZpjWCU0=,tag:o1xK1XLVtOqehTBIn9cNHg==,type:str]
+                admin: ENC[AES256_GCM,data:LhWklk60wExptQ==,iv:DpuT40Jl9H1nH04u+gm7pHFox7QrZ52WqNQuBqGiz8Y=,tag:ylGjb9J5CGXq/CxHtq732g==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRVW4vV0xvaGw4VUlGL1Fa
+            Tjh4V3RUV2d5L1padVZRVkE1V3BTRjg4bzNzCndjSHowU0s5a3I4NGsxNlBUVXFo
+            M1kwQjh3Rkw0VnlDWnh4Q3NSMmdUcXMKLS0tIGl4RWNvVzVyMHo5TC93SXJyWU1G
+            QzBJK0RrSnFiaTgxNnNiZWthM1J4TnMKPb3uoQLHPSTW0+6Kgsapa5/u6cN7aJa1
+            NcQIhd/PPibvvGb47OgLrh8djg7uGpVDMMn3v0Kv3Pu5bshJjZ2gBQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
+    lastmodified: "2026-07-15T23:02:08Z"
+    mac: ENC[AES256_GCM,data:Fg9WsV1LGCgCc9TQtxJ83rthGZCpi9Is/0ab7Udry90TxUHAnfhktP/V74lh8XQq+mVOjCT5ZAw1mpM93lj92/o3ZTb2LnGoaxD4Mj/3eL1T3oH6GQH9VRb7iq+aD5BkYZQd4jmxiIVET3K8a6a7Tt1hrWeKUYz1fcsC9IFIOJQ=,iv:sF8T/Q7+RBrU3dXBBDGS2j4SSiVIpE5wBlGgK7YKGTY=,tag:wqKm9p2SctnsS0jrk4jF/w==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/infra/logging/loki-values.yaml

@@ -0,0 +1,178 @@
+# logging/loki-values.yaml
+# Grafana Loki — log aggregation backend for the homelab.
+# Deployed in SingleBinary mode: one pod handles ingest, query, and compaction.
+# Chunks are stored in MinIO (S3-compatible) — no local PVC needed for log data.
+#
+# MinIO credentials are injected at deploy time via helmfile --set:
+#   loki.storage.s3.accessKeyId    ← MINIO_ROOT_USER
+#   loki.storage.s3.secretAccessKey ← MINIO_ROOT_PASSWORD
+# The placeholder values below are overridden and never used.
+
+# ── Deployment mode ───────────────────────────────────────────────────────────
+# SingleBinary collapses all Loki components (ingester, querier, compactor, ruler)
+# into one Deployment. Simpler ops for a homelab — no inter-component networking
+# or separate scaling to worry about. The tradeoff is no horizontal scaling.
+deploymentMode: SingleBinary
+
+loki:
+  # auth_enabled: false skips tenant header (X-Scope-OrgID) enforcement.
+  # All Promtail → Loki traffic is internal; multi-tenancy adds no value here.
+  auth_enabled: false
+
+  # ── Replication ─────────────────────────────────────────────────────────────
+  # replication_factor: 1 — single replica, no write quorum needed.
+  # Higher values require multiple ingesters (only valid outside SingleBinary).
+  commonConfig:
+    replication_factor: 1
+
+  # ── Storage backend ──────────────────────────────────────────────────────────
+  # s3 type with s3ForcePathStyle: MinIO exposes buckets as paths
+  # (http://host:9000/bucket) not subdomains (http://bucket.host:9000).
+  # insecure: true — MinIO in this cluster has no TLS; traffic stays in-cluster.
+  # Three buckets: chunks (log data), ruler (recording/alerting rules), admin (index).
+  storage:
+    type: s3
+    s3:
+      endpoint: minio.storage.svc.cluster.local:80   # operator svc maps :80 -> pod :9000
+      region: us-east-1          # MinIO ignores region but Loki's S3 client requires it
+      s3ForcePathStyle: true
+      insecure: true
+      # Interpolated at runtime from the loki-s3-creds Secret (SOPS-CMP emitted),
+      # injected as env vars via extraEnvFrom below. Requires -config.expand-env.
+      access_key_id: "${access_key_id}"
+      secret_access_key: "${secret_access_key}"
+    bucketNames:
+      chunks: loki-chunks
+      ruler: loki-ruler
+      admin: loki-admin
+
+  # ── Schema ───────────────────────────────────────────────────────────────────
+  # v13 + TSDB is the current recommended schema (Loki 3.x).
+  # from: sets the date after which this schema applies — logs before this date
+  # would use a previous schema config (none exists here, so all logs use v13).
+  # period: 24h means one index table per day in the object store.
+  schemaConfig:
+    configs:
+      - from: "2024-01-01"
+        store: boltdb-shipper
+        object_store: s3
+        schema: v13
+        index:
+          prefix: index_
+          period: 24h
+
+  # ── Ingester ─────────────────────────────────────────────────────────────────
+  # Controls how log chunks are buffered before being flushed to MinIO.
+  # chunk_idle_period: flush a chunk if no new logs arrive for 3m (reduces
+  # open chunk count). chunk_retain_period: keep flushed chunks in memory
+  # briefly so late-arriving out-of-order logs can still be appended.
+  # WAL persists in-memory chunks to disk — required for boltdb-shipper.
+  ingester:
+    chunk_idle_period: 3m
+    chunk_block_size: 262144
+    chunk_retain_period: 1m
+    wal:
+      dir: /var/loki/wal
+
+  # ── Compactor ────────────────────────────────────────────────────────────────
+  # Merges small index files written by ingesters into larger ones, and
+  # enforces retention by deleting chunks older than retention_period.
+  # retention_delete_delay: waits 2h after marking chunks for deletion before
+  # actually removing them — safety window if a query is still reading them.
+  compactor:
+    working_directory: /var/loki/compactor
+    compaction_interval: 10m
+    retention_enabled: true
+    retention_delete_delay: 2h
+    retention_delete_worker_count: 150
+    delete_request_store: s3
+
+  # ── Limits ───────────────────────────────────────────────────────────────────
+  # retention_period: 10 days. Homelab — no long-term log storage needed.
+  # ingestion_rate_mb / burst: rate limits per tenant (single tenant here).
+  #   4 MB/s steady, 6 MB/s burst — plenty for a 3-node cluster.
+  # max_query_series: caps how many unique label combinations a single query
+  #   can return — prevents runaway cardinality queries from OOMing the pod.
+  # max_query_lookback: hard cap matching retention_period (no point querying
+  #   further back than what's stored).
+  # allow_structured_metadata: false — required for boltdb-shipper index store.
+  limits_config:
+    retention_period: 240h
+    ingestion_rate_mb: 4
+    ingestion_burst_size_mb: 6
+    max_query_series: 5000
+    max_query_lookback: 240h
+    max_label_names_per_series: 30
+    allow_structured_metadata: false
+    # Query timeout: increased to 120s to tolerate 5+ second network latency spikes
+    # Default: 30s — too aggressive when pod-to-pod latency hits 5-10s
+    query_timeout: 120s
+
+# ── Single binary pod ─────────────────────────────────────────────────────────
+singleBinary:
+  replicas: 1
+
+  # Interpolate ${access_key_id}/${secret_access_key} in the S3 config from the
+  # loki-s3-creds Secret (SOPS-CMP emitted).
+  extraArgs:
+    - "-config.expand-env=true"
+  extraEnvFrom:
+    - secretRef:
+        name: loki-s3-creds
+
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 1000m
+      memory: 512Mi
+
+  # Persistent storage for indices + WAL. Durable log chunks live in MinIO.
+  persistence:
+    enabled: true
+    storageClassName: longhorn
+    size: 5Gi
+
+  # Pin to az-a (talos-cp-1) — sole Longhorn node; otherwise the PVC fails to
+  # attach on cp-2/cp-3.
+  nodeSelector:
+    topology.kubernetes.io/zone: az-a
+
+# ── Disable micro-service replicas ───────────────────────────────────────────
+# In SingleBinary mode the chart still templates read/write/backend Deployments
+# unless explicitly set to 0. Setting replicas: 0 keeps them out of the cluster.
+read:
+  replicas: 0
+write:
+  replicas: 0
+backend:
+  replicas: 0
+
+# Nginx gateway is only needed for multi-replica deployments that split read
+# and write paths. Not used in SingleBinary.
+gateway:
+  enabled: false
+
+# Disable the bundled MinIO subchart — we run our own minio-az-a/az-b releases
+# in the storage namespace with site replication.
+minio:
+  enabled: false
+
+# ── Monitoring ────────────────────────────────────────────────────────────────
+# Self-monitoring ships a Grafana Agent operator to scrape Loki's own metrics.
+# We use kube-prometheus-stack for that instead — avoid running two agents.
+# lokiCanary sends synthetic log lines to verify the write→read pipeline;
+# useful in production, too noisy for a homelab.
+monitoring:
+  selfMonitoring:
+    enabled: false
+    grafanaAgent:
+      installOperator: false
+  lokiCanary:
+    enabled: false
+  serviceMonitor:
+    enabled: false
+
+test:
+  enabled: false

k8s/infra/logging/promtail-values.yaml

@@ -0,0 +1,238 @@
+# logging/promtail-values.yaml
+# Promtail — log shipper DaemonSet. One pod per node; reads container logs
+# from /var/log/pods and Talos kernel/service logs from /var/log/journal,
+# then pushes them to Loki.
+
+daemonset:
+  enabled: true
+
+config:
+  logLevel: warn   # info is noisy at homelab scale; warn keeps Promtail quiet
+  serverPort: 3101
+
+  # Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false).
+  clients:
+    - url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push
+
+  snippets:
+    scrapeConfigs: |
+      # ═══════════════════════════════════════════════════════════════════════
+      # LOG AGGREGATION STRATEGY
+      # ═══════════════════════════════════════════════════════════════════════
+      # Three scrape jobs balance log volume, visibility, and cost:
+      #
+      # 1. kubernetes-pods-critical (no sampling)
+      #    Namespaces: iam, monitoring, temporal, cicd, cert-manager
+      #    Strategy: Keep ALL logs (full fidelity for security/observability)
+      #    Rationale: Critical services justify full log retention
+      #
+      # 2. kubernetes-pods (50% sampling + selective drops)
+      #    Namespaces: all others (sqs, storage, ingress-nginx, llm, etc.)
+      #    Drops: health/heartbeat/ping/keepalive + DEBUG from verbose services
+      #    Sampling: 50% of INFO/DEBUG (ERROR/WARN always kept)
+      #    Rationale: Reduce noise while preserving error visibility
+      #
+      # 3. systemd-journal (no filtering)
+      #    Source: Talos node kernel/kubelet logs
+      #    Strategy: Keep all for system-level debugging
+      #
+      # Result: ~60-70% total log reduction vs. 90% before
+      #         All errors/warnings captured; info logs sampled intelligently
+      # ═══════════════════════════════════════════════════════════════════════
+
+      # ── Pod logs ──────────────────────────────────────────────────────────
+      # Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod).
+      # cri pipeline stage parses the CRI-O/containerd log format so timestamps
+      # and stream (stdout/stderr) are extracted properly before the line is
+      # forwarded to Loki.
+      # relabel_configs build useful labels: namespace, pod, container, node,
+      # and a job label of the form "namespace/pod-name" for easy filtering.
+      - job_name: kubernetes-pods
+        kubernetes_sd_configs:
+          - role: pod
+        relabel_configs:
+          # Exclude critical namespaces (handled separately by kubernetes-pods-critical)
+          - source_labels: [__meta_kubernetes_namespace]
+            regex: '(iam|monitoring|temporal|cicd|cert-manager)'
+            action: drop
+          # Apply standard pod labels
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: __host__
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          - action: replace
+            replacement: $1
+            separator: /
+            source_labels:
+              - __meta_kubernetes_namespace
+              - __meta_kubernetes_pod_name
+            target_label: job
+          - action: replace
+            source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_container_name]
+            target_label: container
+          - replacement: /var/log/pods/*$1/*.log
+            separator: /
+            source_labels:
+              - __meta_kubernetes_pod_uid
+              - __meta_kubernetes_pod_container_name
+            target_label: __path__
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: node
+        pipeline_stages:
+          # ── Drop Rule 1: Noise — all services ──────────────────────────
+          # Discard health checks, keep-alives, pings, trace logs
+          - drop:
+              expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE|timeout.*retrying|level="trace"|level="TRACE")'
+
+          # ── Drop Rule 2: DEBUG from verbose services ────────────────────
+          # storage, ingress-nginx, kube-system have chatty DEBUG logs
+          # Example: {namespace="storage"...level="debug"}
+          - drop:
+              expression: 'namespace="(storage|ingress-nginx|kube-system|sqs)".*level="(debug|DEBUG)"'
+
+          # ── Sampling: 50% keep rate on INFO + DEBUG after drops ────────
+          # Keep 100% of ERROR/WARN; keep 50% of INFO (if not dropped above)
+          # This balances log volume with visibility
+          - sampling:
+              rate: 0.5
+              enabled: true
+
+      # ── Pod logs from critical services (no sampling) ────────────────────
+      # iam, monitoring, temporal, cicd = always keep all logs (no sampling)
+      # These services are critical for debugging and deserve full log retention
+      - job_name: kubernetes-pods-critical
+        kubernetes_sd_configs:
+          - role: pod
+        pipeline_stages:
+          # Only drop obvious noise; keep everything else
+          - drop:
+              expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE)'
+        relabel_configs:
+          # Only select pods from critical namespaces
+          - source_labels: [__meta_kubernetes_namespace]
+            regex: '(iam|monitoring|temporal|cicd|cert-manager)'
+            action: keep
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: __host__
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          - action: replace
+            replacement: $1
+            separator: /
+            source_labels:
+              - __meta_kubernetes_namespace
+              - __meta_kubernetes_pod_name
+            target_label: job
+          - action: replace
+            source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_container_name]
+            target_label: container
+          - replacement: /var/log/pods/*$1/*.log
+            separator: /
+            source_labels:
+              - __meta_kubernetes_pod_uid
+              - __meta_kubernetes_pod_container_name
+            target_label: __path__
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: node
+
+      # ── Talos systemd journal ──────────────────────────────────────────────
+      # Talos runs containerd, kubelet, and kernel messages through systemd-
+      # journald — they never appear in /var/log/pods. This job reads the
+      # binary journal directly and emits unit (systemd unit name) and node
+      # labels so you can filter by service (e.g. unit="kubelet.service").
+      # max_age: 12h — only tail recent journal entries on startup; prevents
+      # Promtail from replaying hours of history after a pod restart.
+      - job_name: systemd-journal
+        journal:
+          path: /var/log/journal
+          max_age: 12h
+          labels:
+            job: systemd-journal
+        relabel_configs:
+          - source_labels: [__journal__systemd_unit]
+            target_label: unit
+          - source_labels: [__journal__hostname]
+            target_label: node
+
+# ── Volume mounts ─────────────────────────────────────────────────────────────
+# hostPath mounts give Promtail access to the node's log directories.
+# /var/log/pods — container stdout/stderr (written by containerd's CRI layer)
+# /var/log/journal — Talos systemd journal (binary format, read via journald API)
+# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes
+# before journald has written anything.
+defaultVolumes:
+  - name: pods-logs
+    hostPath:
+      path: /var/log/pods
+  - name: journal
+    hostPath:
+      path: /var/log/journal
+      type: DirectoryOrCreate
+
+defaultVolumeMounts:
+  - name: pods-logs
+    mountPath: /var/log/pods
+    readOnly: true
+  - name: journal
+    mountPath: /var/log/journal
+    readOnly: true
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 64Mi
+  limits:
+    cpu: 200m
+    memory: 128Mi
+
+# ── Security context ──────────────────────────────────────────────────────────
+# privileged: true — required to open the binary journal on Talos (journald
+# uses file locking that non-privileged processes can't bypass).
+# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods
+# even with a read-only root filesystem. Without this, pod logs from containers
+# running as non-root UIDs would be unreadable.
+# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to
+# write to its own container filesystem.
+# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+
+# rejects privileged containers that explicitly set allowPrivilegeEscalation: false.
+containerSecurityContext:
+  privileged: true
+  allowPrivilegeEscalation: true
+  readOnlyRootFilesystem: true
+  capabilities:
+    drop:
+      - ALL
+    add:
+      - DAC_READ_SEARCH
+
+# runAsUser/Group: 0 (root) — needed to access journal files and pod log
+# directories that are owned by root on Talos nodes.
+podSecurityContext:
+  runAsUser: 0
+  runAsGroup: 0
+
+# Tolerate every taint so Promtail runs on ALL nodes including the CP.
+# Without this, the control-plane node's logs (etcd, kube-apiserver) would
+# be missing from Loki entirely.
+tolerations:
+  - effect: NoSchedule
+    operator: Exists
+  - effect: NoExecute
+    operator: Exists
+
+serviceMonitor:
+  enabled: false