k8s/monitoring: add prometheus grafana loki observability
- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,13 @@
|
||||
# logging/.env.example
|
||||
# Copy to logging/.env and fill in real values.
|
||||
# The real .env is gitignored — never commit it.
|
||||
# Used by: k8s/logging/bootstrap.sh and k8s/monitoring/bootstrap.sh (Grafana upgrade step)
|
||||
|
||||
# Grafana admin UI password
|
||||
# openssl rand -base64 24
|
||||
GRAFANA_ADMIN_PASSWORD=
|
||||
|
||||
# MinIO credentials — must match the values used in k8s/storage/.env
|
||||
# Loki uses these to authenticate to the MinIO S3 backend
|
||||
MINIO_ROOT_USER=
|
||||
MINIO_ROOT_PASSWORD=
|
||||
Executable
+124
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env bash
|
||||
# logging/bootstrap.sh
|
||||
# Deploys MinIO + Loki + Promtail + Grafana into the logging namespace.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
|
||||
# - helm >= 3.x installed
|
||||
# - logging/.env file containing:
|
||||
# GRAFANA_ADMIN_PASSWORD=...
|
||||
# MINIO_ROOT_USER=...
|
||||
# MINIO_ROOT_PASSWORD=...
|
||||
# OR those variables already exported in the calling shell.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../cluster-config/kubeconfig}"
|
||||
export KUBECONFIG
|
||||
|
||||
# ── Load credentials ──────────────────────────────────────────────────────────
|
||||
if [[ -f "${SCRIPT_DIR}/.env" ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/.env"
|
||||
fi
|
||||
|
||||
for var in GRAFANA_ADMIN_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do
|
||||
if [[ -z "${!var:-}" ]]; then
|
||||
echo "ERROR: ${var} is not set. Export it or place it in logging/.env"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Namespace ─────────────────────────────────────────────────────────────────
|
||||
echo "==> Creating logging namespace with privileged pod security..."
|
||||
kubectl create namespace logging --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl label namespace logging \
|
||||
pod-security.kubernetes.io/enforce=privileged \
|
||||
pod-security.kubernetes.io/enforce-version=latest \
|
||||
--overwrite
|
||||
|
||||
# ── Helm repos ────────────────────────────────────────────────────────────────
|
||||
echo "==> Adding Helm repositories..."
|
||||
helm repo add minio https://charts.min.io
|
||||
helm repo add grafana https://grafana.github.io/helm-charts
|
||||
helm repo update minio grafana
|
||||
|
||||
# ── MinIO ─────────────────────────────────────────────────────────────────────
|
||||
echo "==> Installing MinIO..."
|
||||
helm upgrade --install minio minio/minio \
|
||||
--namespace logging \
|
||||
--values "${SCRIPT_DIR}/minio-values.yaml" \
|
||||
--set auth.rootUser="${MINIO_ROOT_USER}" \
|
||||
--set auth.rootPassword="${MINIO_ROOT_PASSWORD}" \
|
||||
--wait \
|
||||
--timeout 5m
|
||||
|
||||
echo "==> Waiting for MinIO Deployment to be ready..."
|
||||
kubectl rollout status deployment/minio -n logging --timeout=120s
|
||||
|
||||
# ── Create Loki buckets in MinIO ──────────────────────────────────────────────
|
||||
echo "==> Creating Loki buckets in MinIO..."
|
||||
MINIO_POD=$(kubectl get pod -n logging -l app=minio,release=minio -o jsonpath='{.items[0].metadata.name}')
|
||||
|
||||
# Read the actual credentials MinIO is running with from the cluster secret
|
||||
ACTUAL_MINIO_USER=$(kubectl get secret minio -n logging -o jsonpath='{.data.rootUser}' | base64 --decode)
|
||||
ACTUAL_MINIO_PASS=$(kubectl get secret minio -n logging -o jsonpath='{.data.rootPassword}' | base64 --decode)
|
||||
|
||||
kubectl exec -n logging "${MINIO_POD}" -- \
|
||||
mc alias set local http://localhost:9000 "${ACTUAL_MINIO_USER}" "${ACTUAL_MINIO_PASS}"
|
||||
for bucket in loki-chunks loki-ruler loki-admin; do
|
||||
kubectl exec -n logging "${MINIO_POD}" -- \
|
||||
mc mb --ignore-existing "local/${bucket}"
|
||||
echo " bucket: ${bucket} ready"
|
||||
done
|
||||
|
||||
# ── Loki ──────────────────────────────────────────────────────────────────────
|
||||
echo "==> Installing Loki (SingleBinary + MinIO backend)..."
|
||||
helm upgrade --install loki grafana/loki \
|
||||
--namespace logging \
|
||||
--values "${SCRIPT_DIR}/loki-values.yaml" \
|
||||
--set loki.storage.s3.access_key_id="${ACTUAL_MINIO_USER}" \
|
||||
--set loki.storage.s3.secret_access_key="${ACTUAL_MINIO_PASS}" \
|
||||
--wait \
|
||||
--timeout 5m
|
||||
|
||||
echo "==> Waiting for Loki StatefulSet to be ready..."
|
||||
kubectl rollout status statefulset/loki -n logging --timeout=120s
|
||||
|
||||
# ── Promtail ──────────────────────────────────────────────────────────────────
|
||||
echo "==> Installing Promtail..."
|
||||
helm upgrade --install promtail grafana/promtail \
|
||||
--namespace logging \
|
||||
--values "${SCRIPT_DIR}/promtail-values.yaml" \
|
||||
--wait \
|
||||
--timeout 3m
|
||||
|
||||
echo "==> Waiting for Promtail DaemonSet to be ready..."
|
||||
kubectl rollout status daemonset/promtail -n logging --timeout=60s
|
||||
|
||||
# ── Grafana ───────────────────────────────────────────────────────────────────
|
||||
echo "==> Installing Grafana..."
|
||||
helm upgrade --install grafana grafana/grafana \
|
||||
--namespace logging \
|
||||
--values "${SCRIPT_DIR}/grafana-values.yaml" \
|
||||
--set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \
|
||||
--wait \
|
||||
--timeout 5m
|
||||
|
||||
echo "==> Waiting for Grafana Deployment to be ready..."
|
||||
kubectl rollout status deployment/grafana -n logging --timeout=120s
|
||||
|
||||
# ── Done ──────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "==> Logging stack is up."
|
||||
echo ""
|
||||
echo "Grafana (log explorer UI):"
|
||||
echo " kubectl port-forward -n logging svc/grafana 3000:80"
|
||||
echo " http://localhost:3000 (admin / <your GRAFANA_ADMIN_PASSWORD>)"
|
||||
echo ""
|
||||
echo "MinIO console (S3 object browser):"
|
||||
echo " kubectl port-forward -n logging svc/minio 9001:9001"
|
||||
echo " http://localhost:9001 (${MINIO_ROOT_USER} / <your MINIO_ROOT_PASSWORD>)"
|
||||
echo ""
|
||||
echo "MinIO S3 endpoint for other apps:"
|
||||
echo " http://minio.logging.svc.cluster.local:9000"
|
||||
@@ -0,0 +1,185 @@
|
||||
# logging/grafana-values.yaml
|
||||
# Grafana — dashboarding and log/metrics exploration UI.
|
||||
# Deployed in the logging namespace alongside Loki and Promtail.
|
||||
#
|
||||
# Secrets never set here:
|
||||
# adminPassword — injected via helmfile --set (GRAFANA_ADMIN_PASSWORD)
|
||||
# OAuth client secret — mounted from the grafana-oidc K8s Secret (envFromSecret below)
|
||||
|
||||
replicas: 1
|
||||
|
||||
# RWO PVC (Longhorn) — old pod must fully terminate before the new one can
|
||||
# mount the volume. Recreate avoids the "two pods fighting over one PVC" failure.
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
|
||||
podAnnotations:
|
||||
secret.reloader.stakater.com/reload: "grafana-oidc"
|
||||
|
||||
adminUser: admin
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
|
||||
# ── Persistence ───────────────────────────────────────────────────────────────
|
||||
# Stores dashboards saved in the UI, datasource edits, and user sessions.
|
||||
# Longhorn provides the RWO block volume; 5Gi is ample for dashboard JSON.
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClassName: longhorn
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
size: 5Gi
|
||||
|
||||
# ── Grafana config ────────────────────────────────────────────────────────────
|
||||
grafana.ini:
|
||||
server:
|
||||
root_url: https://grafana.riotpiao.homelab.com
|
||||
|
||||
# No anonymous read access — every user must log in via Authentik SSO.
|
||||
auth.anonymous:
|
||||
enabled: false
|
||||
|
||||
# Explore tab: required for ad-hoc LogQL/PromQL queries against Loki/Prometheus.
|
||||
explore:
|
||||
enabled: true
|
||||
|
||||
# WAL for the embedded SQLite DB — prevents corruption on ungraceful shutdown.
|
||||
database:
|
||||
wal: true
|
||||
|
||||
# ── Authentik OIDC (generic OAuth2) ───────────────────────────────────────
|
||||
# Grafana v10+ supports OIDC auto-discovery; we wire it manually here because
|
||||
# Authentik's discovery endpoint is internal-only (no external DNS for iam.svc).
|
||||
# All URLs use the external hostname so CoreDNS rewrites them to
|
||||
# authentik-server.iam.svc — this keeps the Host header correct so Authentik
|
||||
# doesn't return localhost redirects in its token responses.
|
||||
#
|
||||
# role_attribute_path: JMESPath expression evaluated against the userinfo
|
||||
# response. Members of the 'grafana-admins' Authentik group get Admin role;
|
||||
# everyone else gets Viewer. The group name must match exactly what Authentik
|
||||
# sends in the 'groups' claim.
|
||||
auth.generic_oauth:
|
||||
enabled: true
|
||||
name: Authentik
|
||||
allow_sign_up: true
|
||||
client_id: grafana
|
||||
scopes: openid email profile
|
||||
auth_url: https://authentik.riotpiao.homelab.com/application/o/authorize/
|
||||
token_url: https://authentik.riotpiao.homelab.com/application/o/token/
|
||||
api_url: https://authentik.riotpiao.homelab.com/application/o/userinfo/
|
||||
role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
|
||||
use_pkce: false
|
||||
use_refresh_token: false
|
||||
skip_org_role_sync: false
|
||||
tls_skip_verify_insecure: true # Authentik uses self-signed cert; verify in prod
|
||||
|
||||
# GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s
|
||||
# Secret (created by k8s/talos-iam/setup_talos_iam.sh). envFromSecret mounts
|
||||
# every key in that secret as an environment variable — keeps secrets out of
|
||||
# values files and out of git.
|
||||
envFromSecret: grafana-oidc
|
||||
|
||||
# ── Datasources ───────────────────────────────────────────────────────────────
|
||||
# Provisioned at install — no manual "Add datasource" step in the UI.
|
||||
# Loki is the default (log exploration); Prometheus is secondary (metrics).
|
||||
# Both use in-cluster service DNS — Grafana never leaves the cluster for data.
|
||||
datasources:
|
||||
datasources.yaml:
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
access: proxy
|
||||
url: http://loki.logging.svc.cluster.local:3100
|
||||
isDefault: true
|
||||
version: 1
|
||||
editable: true
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
timeout: 60
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus-kube-prometheus-prometheus.monitoring.svc.cluster.local:9090
|
||||
isDefault: false
|
||||
version: 1
|
||||
editable: true
|
||||
jsonData:
|
||||
timeInterval: 30s
|
||||
timeout: 60
|
||||
|
||||
# ── Dashboard providers ───────────────────────────────────────────────────────
|
||||
# Tells Grafana to watch a directory for dashboard JSON files.
|
||||
# The `dashboards` block below populates that directory via an init container
|
||||
# that downloads from grafana.com at install time.
|
||||
dashboardProviders:
|
||||
dashboardproviders.yaml:
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: default
|
||||
orgId: 1
|
||||
folder: ""
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards/default
|
||||
|
||||
# ── Pre-loaded dashboards ──────────────────────────────────────────────────────
|
||||
# Fetched from grafana.com by an init container at helm-install time and baked
|
||||
# into ConfigMaps. gnetId + revision pin the exact dashboard version so an
|
||||
# upstream change on grafana.com doesn't silently alter what's deployed.
|
||||
dashboards:
|
||||
default:
|
||||
node-exporter-full:
|
||||
gnetId: 1860
|
||||
revision: 37
|
||||
datasource: Prometheus
|
||||
kubernetes-cluster:
|
||||
gnetId: 7249
|
||||
revision: 1
|
||||
datasource: Prometheus
|
||||
|
||||
# Allow scheduling on the control-plane node (talos-cp-1 carries NoSchedule taint).
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
|
||||
# ClusterIP only — access via ingress (grafana.riotpiao.homelab.com) or port-forward.
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 80
|
||||
|
||||
# Ingress managed by k8s/ingress/ingress.yaml — not the chart's built-in ingress.
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
|
||||
# ── Dashboard-as-code (sidecar discovery) ──────────────────────────────────────
|
||||
# Additive to the gnetId-download mechanism above. The sidecar watches for
|
||||
# ConfigMaps labeled grafana_dashboard=1 in ANY namespace and loads them live —
|
||||
# no Grafana restart needed when a new dashboard ConfigMap is applied.
|
||||
sidecar:
|
||||
dashboards:
|
||||
enabled: true
|
||||
label: grafana_dashboard
|
||||
labelValue: "1"
|
||||
folder: /var/lib/grafana/dashboards/custom
|
||||
folderAnnotation: grafana_folder
|
||||
provider:
|
||||
name: custom
|
||||
folder: "Homelab"
|
||||
disableDelete: false
|
||||
foldersFromFilesStructure: true
|
||||
searchNamespace: ALL
|
||||
@@ -0,0 +1,163 @@
|
||||
# logging/loki-values.yaml
|
||||
# Grafana Loki — log aggregation backend for the homelab.
|
||||
# Deployed in SingleBinary mode: one pod handles ingest, query, and compaction.
|
||||
# Chunks are stored in MinIO (S3-compatible) — no local PVC needed for log data.
|
||||
#
|
||||
# MinIO credentials are injected at deploy time via helmfile --set:
|
||||
# loki.storage.s3.accessKeyId ← MINIO_ROOT_USER
|
||||
# loki.storage.s3.secretAccessKey ← MINIO_ROOT_PASSWORD
|
||||
# The placeholder values below are overridden and never used.
|
||||
|
||||
# ── Deployment mode ───────────────────────────────────────────────────────────
|
||||
# SingleBinary collapses all Loki components (ingester, querier, compactor, ruler)
|
||||
# into one Deployment. Simpler ops for a homelab — no inter-component networking
|
||||
# or separate scaling to worry about. The tradeoff is no horizontal scaling.
|
||||
deploymentMode: SingleBinary
|
||||
|
||||
loki:
|
||||
# auth_enabled: false skips tenant header (X-Scope-OrgID) enforcement.
|
||||
# All Promtail → Loki traffic is internal; multi-tenancy adds no value here.
|
||||
auth_enabled: false
|
||||
|
||||
# ── Replication ─────────────────────────────────────────────────────────────
|
||||
# replication_factor: 1 — single replica, no write quorum needed.
|
||||
# Higher values require multiple ingesters (only valid outside SingleBinary).
|
||||
commonConfig:
|
||||
replication_factor: 1
|
||||
|
||||
# ── Storage backend ──────────────────────────────────────────────────────────
|
||||
# s3 type with s3ForcePathStyle: MinIO exposes buckets as paths
|
||||
# (http://host:9000/bucket) not subdomains (http://bucket.host:9000).
|
||||
# insecure: true — MinIO in this cluster has no TLS; traffic stays in-cluster.
|
||||
# Three buckets: chunks (log data), ruler (recording/alerting rules), admin (index).
|
||||
storage:
|
||||
type: s3
|
||||
s3:
|
||||
endpoint: minio.storage.svc.cluster.local:9000
|
||||
region: us-east-1 # MinIO ignores region but Loki's S3 client requires it
|
||||
s3ForcePathStyle: true
|
||||
insecure: true
|
||||
access_key_id: "" # overridden by helmfile --set (MINIO_ROOT_USER)
|
||||
secret_access_key: "" # overridden by helmfile --set (MINIO_ROOT_PASSWORD)
|
||||
bucketNames:
|
||||
chunks: loki-chunks
|
||||
ruler: loki-ruler
|
||||
admin: loki-admin
|
||||
|
||||
# ── Schema ───────────────────────────────────────────────────────────────────
|
||||
# v13 + TSDB is the current recommended schema (Loki 3.x).
|
||||
# from: sets the date after which this schema applies — logs before this date
|
||||
# would use a previous schema config (none exists here, so all logs use v13).
|
||||
# period: 24h means one index table per day in the object store.
|
||||
schemaConfig:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: boltdb-shipper
|
||||
object_store: s3
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
# ── Ingester ─────────────────────────────────────────────────────────────────
|
||||
# Controls how log chunks are buffered before being flushed to MinIO.
|
||||
# chunk_idle_period: flush a chunk if no new logs arrive for 3m (reduces
|
||||
# open chunk count). chunk_retain_period: keep flushed chunks in memory
|
||||
# briefly so late-arriving out-of-order logs can still be appended.
|
||||
# WAL persists in-memory chunks to disk — required for boltdb-shipper.
|
||||
ingester:
|
||||
chunk_idle_period: 3m
|
||||
chunk_block_size: 262144
|
||||
chunk_retain_period: 1m
|
||||
wal:
|
||||
dir: /var/loki/wal
|
||||
|
||||
# ── Compactor ────────────────────────────────────────────────────────────────
|
||||
# Merges small index files written by ingesters into larger ones, and
|
||||
# enforces retention by deleting chunks older than retention_period.
|
||||
# retention_delete_delay: waits 2h after marking chunks for deletion before
|
||||
# actually removing them — safety window if a query is still reading them.
|
||||
compactor:
|
||||
working_directory: /var/loki/compactor
|
||||
compaction_interval: 10m
|
||||
retention_enabled: true
|
||||
retention_delete_delay: 2h
|
||||
retention_delete_worker_count: 150
|
||||
delete_request_store: s3
|
||||
|
||||
# ── Limits ───────────────────────────────────────────────────────────────────
|
||||
# retention_period: 10 days. Homelab — no long-term log storage needed.
|
||||
# ingestion_rate_mb / burst: rate limits per tenant (single tenant here).
|
||||
# 4 MB/s steady, 6 MB/s burst — plenty for a 3-node cluster.
|
||||
# max_query_series: caps how many unique label combinations a single query
|
||||
# can return — prevents runaway cardinality queries from OOMing the pod.
|
||||
# max_query_lookback: hard cap matching retention_period (no point querying
|
||||
# further back than what's stored).
|
||||
# allow_structured_metadata: false — required for boltdb-shipper index store.
|
||||
limits_config:
|
||||
retention_period: 240h
|
||||
ingestion_rate_mb: 4
|
||||
ingestion_burst_size_mb: 6
|
||||
max_query_series: 5000
|
||||
max_query_lookback: 240h
|
||||
max_label_names_per_series: 30
|
||||
allow_structured_metadata: false
|
||||
# Query timeout: increased to 120s to tolerate 5+ second network latency spikes
|
||||
# Default: 30s — too aggressive when pod-to-pod latency hits 5-10s
|
||||
query_timeout: 120s
|
||||
|
||||
# ── Single binary pod ─────────────────────────────────────────────────────────
|
||||
singleBinary:
|
||||
replicas: 1
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 512Mi
|
||||
|
||||
# Persistent storage for indices + WAL. Durable log chunks live in MinIO.
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClassName: longhorn
|
||||
size: 5Gi
|
||||
|
||||
# ── Disable micro-service replicas ───────────────────────────────────────────
|
||||
# In SingleBinary mode the chart still templates read/write/backend Deployments
|
||||
# unless explicitly set to 0. Setting replicas: 0 keeps them out of the cluster.
|
||||
read:
|
||||
replicas: 0
|
||||
write:
|
||||
replicas: 0
|
||||
backend:
|
||||
replicas: 0
|
||||
|
||||
# Nginx gateway is only needed for multi-replica deployments that split read
|
||||
# and write paths. Not used in SingleBinary.
|
||||
gateway:
|
||||
enabled: false
|
||||
|
||||
# Disable the bundled MinIO subchart — we run our own minio-az-a/az-b releases
|
||||
# in the storage namespace with site replication.
|
||||
minio:
|
||||
enabled: false
|
||||
|
||||
# ── Monitoring ────────────────────────────────────────────────────────────────
|
||||
# Self-monitoring ships a Grafana Agent operator to scrape Loki's own metrics.
|
||||
# We use kube-prometheus-stack for that instead — avoid running two agents.
|
||||
# lokiCanary sends synthetic log lines to verify the write→read pipeline;
|
||||
# useful in production, too noisy for a homelab.
|
||||
monitoring:
|
||||
selfMonitoring:
|
||||
enabled: false
|
||||
grafanaAgent:
|
||||
installOperator: false
|
||||
lokiCanary:
|
||||
enabled: false
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
|
||||
test:
|
||||
enabled: false
|
||||
@@ -0,0 +1,42 @@
|
||||
# logging/minio-values.yaml
|
||||
# Official MinIO chart (minio/minio from https://charts.min.io).
|
||||
# Single pod with console built-in on port 9001.
|
||||
# rootUser and rootPassword are injected via --set at install time from .env.
|
||||
|
||||
mode: standalone
|
||||
|
||||
rootUser: "" # injected via --set
|
||||
rootPassword: "" # injected via --set
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: longhorn
|
||||
accessMode: ReadWriteOnce
|
||||
size: 100Gi
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 9000
|
||||
|
||||
consoleService:
|
||||
type: ClusterIP
|
||||
port: 9001
|
||||
|
||||
# Disable ingress — access via kubectl port-forward
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
consoleIngress:
|
||||
enabled: false
|
||||
|
||||
metrics:
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
@@ -0,0 +1,152 @@
|
||||
# logging/promtail-values.yaml
|
||||
# Promtail — log shipper DaemonSet. One pod per node; reads container logs
|
||||
# from /var/log/pods and Talos kernel/service logs from /var/log/journal,
|
||||
# then pushes them to Loki.
|
||||
|
||||
daemonset:
|
||||
enabled: true
|
||||
|
||||
config:
|
||||
logLevel: warn # info is noisy at homelab scale; warn keeps Promtail quiet
|
||||
serverPort: 3101
|
||||
|
||||
# Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false).
|
||||
clients:
|
||||
- url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push
|
||||
|
||||
snippets:
|
||||
scrapeConfigs: |
|
||||
# ── Pod logs ──────────────────────────────────────────────────────────
|
||||
# Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod).
|
||||
# cri pipeline stage parses the CRI-O/containerd log format so timestamps
|
||||
# and stream (stdout/stderr) are extracted properly before the line is
|
||||
# forwarded to Loki.
|
||||
# relabel_configs build useful labels: namespace, pod, container, node,
|
||||
# and a job label of the form "namespace/pod-name" for easy filtering.
|
||||
- job_name: kubernetes-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
pipeline_stages:
|
||||
- drop:
|
||||
expression: '(health|heartbeat|ping|keepalive|level="debug"|"timeout".*"retrying")'
|
||||
- sampling:
|
||||
rate: 0.1
|
||||
enabled: true
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: __host__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- action: replace
|
||||
replacement: $1
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_namespace
|
||||
- __meta_kubernetes_pod_name
|
||||
target_label: job
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- replacement: /var/log/pods/*$1/*.log
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_pod_uid
|
||||
- __meta_kubernetes_pod_container_name
|
||||
target_label: __path__
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
|
||||
# ── Talos systemd journal ──────────────────────────────────────────────
|
||||
# Talos runs containerd, kubelet, and kernel messages through systemd-
|
||||
# journald — they never appear in /var/log/pods. This job reads the
|
||||
# binary journal directly and emits unit (systemd unit name) and node
|
||||
# labels so you can filter by service (e.g. unit="kubelet.service").
|
||||
# max_age: 12h — only tail recent journal entries on startup; prevents
|
||||
# Promtail from replaying hours of history after a pod restart.
|
||||
- job_name: systemd-journal
|
||||
journal:
|
||||
path: /var/log/journal
|
||||
max_age: 12h
|
||||
labels:
|
||||
job: systemd-journal
|
||||
relabel_configs:
|
||||
- source_labels: [__journal__systemd_unit]
|
||||
target_label: unit
|
||||
- source_labels: [__journal__hostname]
|
||||
target_label: node
|
||||
|
||||
# ── Volume mounts ─────────────────────────────────────────────────────────────
|
||||
# hostPath mounts give Promtail access to the node's log directories.
|
||||
# /var/log/pods — container stdout/stderr (written by containerd's CRI layer)
|
||||
# /var/log/journal — Talos systemd journal (binary format, read via journald API)
|
||||
# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes
|
||||
# before journald has written anything.
|
||||
defaultVolumes:
|
||||
- name: pods-logs
|
||||
hostPath:
|
||||
path: /var/log/pods
|
||||
- name: journal
|
||||
hostPath:
|
||||
path: /var/log/journal
|
||||
type: DirectoryOrCreate
|
||||
|
||||
defaultVolumeMounts:
|
||||
- name: pods-logs
|
||||
mountPath: /var/log/pods
|
||||
readOnly: true
|
||||
- name: journal
|
||||
mountPath: /var/log/journal
|
||||
readOnly: true
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
|
||||
# ── Security context ──────────────────────────────────────────────────────────
|
||||
# privileged: true — required to open the binary journal on Talos (journald
|
||||
# uses file locking that non-privileged processes can't bypass).
|
||||
# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods
|
||||
# even with a read-only root filesystem. Without this, pod logs from containers
|
||||
# running as non-root UIDs would be unreadable.
|
||||
# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to
|
||||
# write to its own container filesystem.
|
||||
# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+
|
||||
# rejects privileged containers that explicitly set allowPrivilegeEscalation: false.
|
||||
containerSecurityContext:
|
||||
privileged: true
|
||||
allowPrivilegeEscalation: true
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
add:
|
||||
- DAC_READ_SEARCH
|
||||
|
||||
# runAsUser/Group: 0 (root) — needed to access journal files and pod log
|
||||
# directories that are owned by root on Talos nodes.
|
||||
podSecurityContext:
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
|
||||
# Tolerate every taint so Promtail runs on ALL nodes including the CP.
|
||||
# Without this, the control-plane node's logs (etcd, kube-apiserver) would
|
||||
# be missing from Loki entirely.
|
||||
tolerations:
|
||||
- effect: NoSchedule
|
||||
operator: Exists
|
||||
- effect: NoExecute
|
||||
operator: Exists
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
Reference in New Issue
Block a user