k8s/services: add ingress networking portainer llm and project guides

- Nginx ingress + TLS termination (homelab-ca) - Portainer container UI - CoreDNS internal DNS rewrites - DuckDNS DDNS updater - Ollama LLM inference - 8 project-usage guides (team reference)
2026-08-18 15:08:00 -07:00
parent 69ad5c371c
commit 61be24e27f
26 changed files with 3280 additions and 0 deletions
@@ -0,0 +1,67 @@
+# k8s/coredns/coredns-configmap.yaml
+# Patches the CoreDNS Corefile to rewrite homelab hostnames to internal services.
+#
+# Why this is needed:
+#   Grafana v10+ does OIDC auto-discovery by fetching
+#   /.well-known/openid-configuration from Authentik. When Grafana reaches
+#   Authentik via the external hostname (authentik.riotpiao.homelab.com), the
+#   HTTP Host header is preserved and Authentik returns external URLs in the
+#   discovery response. Without this rewrite, the hostname doesn't resolve
+#   inside the cluster and Grafana falls back to the internal service DNS,
+#   causing all OAuth redirects to go to authentik-server.iam.svc.cluster.local.
+#
+# Applied by helmfile presync hook on the ingress-nginx release.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+            lameduck 5s
+        }
+        ready
+        log . {
+            class error
+        }
+        prometheus :9153
+
+        # Forgejo: route to HTTP service (LoadBalancer handles both HTTPS:443 + SSH:2222 via same IP)
+        rewrite name forgejo.riotpiao.homelab.com forgejo-gitea-http.cicd.svc.cluster.local
+
+        # Rewrite homelab hostnames to the nginx ingress controller so in-cluster pods
+        # hit nginx TLS termination (cert-manager cert) and preserve the Host header.
+        # Routing through nginx — not directly to the backend service — is critical:
+        # direct rewrites to the backend bypass nginx TLS and expose each app's own
+        # self-signed cert, which nothing in the cluster trusts.
+        rewrite name authentik.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name grafana.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name minio.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name minio-api.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name argocd.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name vault.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name loki.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name prometheus.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name portainer.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+        rewrite name longhorn.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+            pods insecure
+            fallthrough in-addr.arpa ip6.arpa
+            ttl 30
+        }
+        forward . 8.8.8.8 1.1.1.1 {
+           max_concurrent 1000
+        }
+        cache 30 {
+           disable success cluster.local
+           disable denial cluster.local
+        }
+        loop
+        reload
+        loadbalance
+    }
@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: duckdns-updater
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: duckdns-updater
+  template:
+    metadata:
+      labels:
+        app: duckdns-updater
+    spec:
+      containers:
+        - name: updater
+          image: curlimages/curl:latest
+          command:
+            - sh
+            - -c
+            - |
+              while true; do
+                curl -fsS "https://www.duckdns.org/update?domains=riotpiao&token=${DUCKDNS_TOKEN}&ip="
+                sleep 300
+              done
+          env:
+            - name: DUCKDNS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: duckdns-token
+                  key: token
+          resources:
+            requests:
+              cpu: 5m
+              memory: 16Mi
+            limits:
+              cpu: 50m
+              memory: 32Mi
+      restartPolicy: Always
@@ -0,0 +1,297 @@
+# k8s/ingress/ingress.yaml
+# Ingress rules for all homelab services.
+# TLS is handled centrally: nginx serves the wildcard-tls cert (*.riotpiao.homelab.com)
+# as its default-ssl-certificate. No per-rule tls: blocks or cert-manager annotations
+# are needed — cert-manager manages one cert, nginx uses it for all hosts.
+#
+# DNS: *.riotpiao.homelab.com must resolve to 10.6.0.1 (WireGuard) or 192.168.1.160 (LAN).
+
+# ── Grafana ───────────────────────────────────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: grafana
+  namespace: logging
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: grafana.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: grafana
+                port:
+                  number: 80
+
+---
+# ── Loki (API access for external tools) ─────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: loki
+  namespace: logging
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: loki.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: loki
+                port:
+                  number: 3100
+
+---
+# ── Authentik ─────────────────────────────────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: authentik
+  namespace: iam
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: authentik.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: authentik-server
+                port:
+                  number: 80
+
+---
+# ── Vault ─────────────────────────────────────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: vault
+  namespace: storage
+  annotations:
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: vault.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: vault
+                port:
+                  number: 8200
+
+---
+# ── MinIO console (storage namespace) ────────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: minio
+  namespace: storage
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    nginx.ingress.kubernetes.io/affinity: "cookie"
+    nginx.ingress.kubernetes.io/session-cookie-name: "minio-console-affinity"
+    nginx.ingress.kubernetes.io/session-cookie-max-age: "3600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: minio.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: minio-console
+                port:
+                  number: 9001
+
+---
+# ── MinIO S3 API ──────────────────────────────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: minio-api
+  namespace: storage
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: minio-api.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: minio
+                port:
+                  number: 9000
+
+---
+# ── Prometheus ────────────────────────────────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: prometheus
+  namespace: monitoring
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: prometheus.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: prometheus-kube-prometheus-prometheus
+                port:
+                  number: 9090
+
+---
+# ── Portainer ─────────────────────────────────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: portainer
+  namespace: dashboard
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: portainer.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: portainer
+                port:
+                  number: 9000
+
+---
+# ── Forgejo (git forge + OCI registry UI) ────────────────────────────────────
+# nginx terminates TLS using the wildcard cert, then proxies plain HTTP to
+# Forgejo on port 3000. ROOT_URL stays https:// so Forgejo generates correct URLs.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: forgejo
+  namespace: cicd
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: forgejo.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: forgejo-gitea-http
+                port:
+                  number: 3000
+
+---
+# ── Argo CD (cicd namespace) ──────────────────────────────────────────────────
+# argocd-server runs HTTPS internally — nginx proxies via backend-protocol: HTTPS.
+# proxy-ssl-verify: off because argocd-server's pod cert is self-signed (not homelab-ca).
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: argocd
+  namespace: cicd
+  annotations:
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    nginx.ingress.kubernetes.io/proxy-ssl-verify: "off"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: argocd.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: argocd-server
+                port:
+                  number: 443
+
+---
+# ── Longhorn UI ───────────────────────────────────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: longhorn.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: longhorn-frontend
+                port:
+                  number: 80
+
+---
+# ── Temporal Web UI ────────────────────────────────────────────────────────────
+# Temporal workflow orchestration Web UI with OIDC authentication
+# TLS: wildcard cert managed by cert-manager, served by nginx default-ssl-certificate
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: temporal
+  namespace: temporal
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: temporal.riotpiao.homelab.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: temporal-web
+                port:
+                  number: 8080
@@ -0,0 +1,78 @@
+# k8s/ingress/nginx-values.yaml
+# Nginx Ingress Controller — bare-metal homelab config.
+# LoadBalancer service with Cilium LB-IPAM assigns fixed IP (192.168.1.160).
+# Access services at https://grafana.riotpiao.homelab.com (80/443 via LoadBalancer).
+
+controller:
+  kind: DaemonSet
+
+  # Single wildcard cert served for every *.riotpiao.homelab.com host.
+  # Applied by the ingress-nginx presync hook (wildcard-cert.yaml) before nginx starts.
+  # nginx hot-reloads when cert-manager renews homelab-tls — no restart needed.
+  extraArgs:
+    default-ssl-certificate: "ingress-nginx/homelab-tls"
+
+  hostPort:
+    enabled: true
+    ports:
+      http: 80
+      https: 443
+
+  # TCP proxy: forward port 2222 on every node → Forgejo SSH service.
+  # This lets `git clone [email protected]:repo` work via the
+  # same hostname as HTTPS without a separate LoadBalancer IP for SSH.
+  tcp:
+    2222: "cicd/forgejo-gitea-ssh:2222"
+
+  # Service as LoadBalancer — Cilium LB-IPAM assigns fixed IP.
+  service:
+    type: LoadBalancer
+    annotations:
+      io.cilium/lb-ipam-ips: "192.168.1.160"
+
+  # Allow the controller to land on the control-plane node.
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+
+  # Use the ingress-nginx IngressClass by default.
+  ingressClassResource:
+    default: true
+
+  # Required when using hostPort so DNS resolves correctly inside the pod.
+  dnsPolicy: ClusterFirstWithHostNet
+
+  # Reduce noise in a single-admin homelab.
+  admissionWebhooks:
+    enabled: false
+
+  # ── Connection timeouts to upstreams ──────────────────────────────────────────
+  # Increased to tolerate 5+ second pod-to-pod network latency spikes.
+  # Default: 60s for all — acceptable but explicitly set for clarity.
+  config:
+    upstream-connect-timeout: "60"
+    upstream-send-timeout: "60"
+    upstream-read-timeout: "60"
+    keepalive-timeout: "65"
+    keepalive-requests: "100"
+
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 200m
+      memory: 128Mi
+
+  # RED metrics (rate/errors/duration) for every host fronted by this controller —
+  # every exposed service in the cluster goes through here, so this single block
+  # is the cluster-wide "latency and availability" signal. Prometheus auto-discovers
+  # the ServiceMonitor (serviceMonitorSelectorNilUsesHelmValues: false in prometheus-values.yaml).
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      namespace: ingress-nginx
+      interval: 30s
+      scrapeTimeout: 60s
@@ -0,0 +1,22 @@
+# k8s/ingress/wildcard-cert.yaml
+# Single wildcard TLS certificate for all *.riotpiao.homelab.com services.
+# Lives in the ingress-nginx namespace and is set as nginx's default-ssl-certificate,
+# so every ingress host gets it automatically — no per-service TLS blocks needed.
+#
+# Renewal: cert-manager auto-renews 30 days before expiry (renewBefore: 720h).
+# nginx detects the secret update via its K8s watch and hot-reloads — no pod restart.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: homelab-tls
+  namespace: ingress-nginx
+spec:
+  secretName: homelab-tls
+  dnsNames:
+    - "*.riotpiao.homelab.com"
+    - "riotpiao.homelab.com"
+  issuerRef:
+    name: homelab-ca
+    kind: ClusterIssuer
+  duration: 8760h      # 1 year
+  renewBefore: 720h    # renew 30 days before expiry
@@ -0,0 +1,299 @@
+# Ollama LLM Inference Service
+
+CPU-only LLM inference server on talos-cp-1. Single model hot-loaded (DeepSeek-R1:70b), 42GB, 70Gi memory limit.
+
+## Quick Start
+
+### Access via port-forward
+```bash
+kubectl -n llm port-forward svc/ollama 11434:11434
+curl http://localhost:11434/api/tags
+```
+
+### Debug pod (in-cluster)
+```bash
+kubectl run debug --rm -it -n llm --image=curlimages/curl \
+  --labels="app.kubernetes.io/role=llm-debug" \
+  --serviceaccount=llm-worker -- sh
+
+# Inside pod
+TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+curl -H "Authorization: Bearer $TOKEN" \
+  http://ollama.llm.svc.cluster.local:11434/api/tags
+```
+
+## Architecture
+
+| Component | Value |
+|-----------|-------|
+| Service | ClusterIP `ollama.llm.svc.cluster.local:11434` |
+| Namespace | `llm` |
+| Node | talos-cp-1 (pinned via nodeAffinity) |
+| Memory request | 50Gi |
+| Memory limit | 70Gi |
+| Storage | 115Gi PVC (Longhorn) |
+| Model | `deepseek-r1:70b` (~42GB) |
+| Max loaded | 1 model |
+| Parallelism | 1 request at a time |
+
+## API Endpoints
+
+### List models
+```bash
+curl http://ollama.llm.svc.cluster.local:11434/api/tags
+```
+
+Response:
+```json
+{
+  "models": [
+    {"name": "deepseek-r1:70b", "size": 42000000000, ...}
+  ]
+}
+```
+
+### Generate (non-streaming)
+```bash
+curl -X POST http://ollama.llm.svc.cluster.local:11434/api/generate \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "deepseek-r1:70b",
+    "prompt": "Why is the sky blue?",
+    "stream": false
+  }'
+```
+
+### Pull model
+```bash
+curl -X POST http://ollama.llm.svc.cluster.local:11434/api/pull \
+  -H "Content-Type: application/json" \
+  -d '{"name": "deepseek-r1:70b", "stream": false}'
+```
+
+## Operations
+
+### Check pod status
+```bash
+kubectl -n llm get pod -l app.kubernetes.io/name=ollama
+kubectl -n llm describe pod -l app.kubernetes.io/name=ollama
+```
+
+### View logs
+```bash
+kubectl -n llm logs deployment/ollama -f
+```
+
+### Monitor download progress (bootstrap)
+```bash
+kubectl -n llm logs -f job/bootstrap-models -c model-download
+```
+
+### Restart deployment
+```bash
+kubectl -n llm rollout restart deployment/ollama
+```
+
+## Storage
+
+- **PVC:** `ollama-models-cache`, 115Gi, Longhorn StorageClass
+- **Mount:** `/root/.ollama/models` (Ollama model cache)
+- **Lifecycle:** RWO (Read-Write-Once), tied to talos-cp-1
+
+### Resize PVC
+⚠️ PVCs can only expand, not shrink. Edit values.yaml and redeploy:
+
+```yaml
+pvc:
+  size: 120Gi  # increase only
+```
+
+```bash
+vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama apply
+```
+
+## Networking
+
+### NetworkPolicy
+- Default-deny ingress on Ollama pods
+- Allow from pods labeled `app.kubernetes.io/name: llm-worker` (port 11434)
+- Allow from pods labeled `app.kubernetes.io/role: llm-debug` (port 11434)
+
+View policy:
+```bash
+kubectl -n llm get networkpolicy ollama
+```
+
+Test access from external pod (should fail):
+```bash
+kubectl run test --rm -it --image=curlimages/curl -- \
+  curl http://ollama.llm.svc.cluster.local:11434/
+# Connection timeout (correct)
+```
+
+Test access from debug pod (should succeed):
+```bash
+kubectl -n llm logs job/bootstrap-models  # verify bootstrap completed
+# Then run debug pod as shown above
+```
+
+## Configuration
+
+### Helm values (`k8s/llm/charts/ollama/values.yaml`)
+
+```yaml
+resources:
+  requests:
+    cpu: 8
+    memory: 50Gi
+  limits:
+    cpu: 16
+    memory: 70Gi
+
+env:
+  OLLAMA_MAX_LOADED_MODELS: "1"
+  OLLAMA_NUM_PARALLEL: "1"
+  OLLAMA_MAX_QUEUE: "32"
+  OLLAMA_KEEP_ALIVE: "-1"
+  OLLAMA_HOST: "0.0.0.0:11434"
+
+preloadJob:
+  enabled: true
+  hotModels:
+    - deepseek-r1:70b
+```
+
+### Environment variables
+
+| Variable | Value | Purpose |
+|----------|-------|---------|
+| `OLLAMA_MODELS` | `/root/.ollama/models` | Model cache dir |
+| `OLLAMA_MAX_LOADED_MODELS` | `1` | Max concurrent models in RAM |
+| `OLLAMA_NUM_PARALLEL` | `1` | Parallel request threads |
+| `OLLAMA_MAX_QUEUE` | `32` | Request queue depth |
+| `OLLAMA_KEEP_ALIVE` | `-1` | Keep model resident (never unload) |
+| `OLLAMA_HOST` | `0.0.0.0:11434` | Bind address |
+
+Tune `OLLAMA_NUM_PARALLEL` based on CPU cores. Current: 1 (conservative, CPU bottleneck).
+
+## Model Management
+
+### Current model
+- **Name:** `deepseek-r1:70b`
+- **Size:** ~42GB
+- **Quantization:** Default Ollama quant
+- **Status:** Downloaded during pod init via bootstrap job
+
+### Change model
+
+1. Edit `values.yaml`:
+```yaml
+preloadJob:
+  hotModels:
+    - deepseek-r1:32b  # or any available model
+```
+
+2. Redeploy:
+```bash
+kubectl -n llm delete job bootstrap-models --ignore-not-found
+vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama apply
+```
+
+3. Monitor:
+```bash
+kubectl -n llm logs -f job/bootstrap-models -c model-download
+```
+
+### Available models
+Ollama registry: https://ollama.com/library
+
+Examples:
+- `deepseek-r1:70b` (reasoning, 42GB)
+- `deepseek-r1:32b` (faster, 20GB)
+- `llama3.1:70b` (general, 41GB)
+- `mistral:large` (26GB)
+
+## Troubleshooting
+
+### Pod stuck in `ContainerCreating`
+```bash
+kubectl -n llm describe pod -l app.kubernetes.io/name=ollama
+# Check Events section for PVC/image pull issues
+```
+
+### Bootstrap job failing
+```bash
+kubectl -n llm logs job/bootstrap-models -c model-download --tail=50
+# Common: model not found in registry, disk full, network timeout
+```
+
+### Model pull timeout
+```bash
+# Increase pod timeout (edit deployment directly)
+kubectl -n llm edit deployment ollama
+# Change readinessProbe.initialDelaySeconds, livenessProbe.periodSeconds
+```
+
+### Out of memory
+Model size exceeds limit. Reduce `memory.limits` or choose smaller model.
+
+```bash
+kubectl top pod -n llm  # check actual usage
+```
+
+### Cannot connect from other pods
+Verify NetworkPolicy:
+```bash
+kubectl -n llm get networkpolicy
+kubectl -n llm describe networkpolicy ollama
+# Add pod label: app.kubernetes.io/name: llm-worker or app.kubernetes.io/role: llm-debug
+```
+
+## Secrets
+
+Ollama pod receives MinIO credentials via Secret `ollama-minio` (created by helmfile presync):
+
+```bash
+kubectl -n llm get secret ollama-minio -o jsonpath='{.data}' | jq
+```
+
+Keys: `endpoint`, `bucket`, `access_key`, `secret_key`
+
+Used by bootstrap job to upload model blobs to MinIO (future: auto-backup).
+
+## Metrics & Observability
+
+### Prometheus scrape (if enabled)
+ServiceMonitor: Not yet configured (see `k8s/monitoring/dashboards/services/`)
+
+Metrics to add:
+- `ollama_requests_total` (counter)
+- `ollama_request_duration_seconds` (histogram)
+- `ollama_loaded_models` (gauge)
+
+### Logs
+Pod logs via kubectl. No log aggregation to Loki yet.
+
+```bash
+kubectl -n llm logs deployment/ollama -f --timestamps
+```
+
+## Cleanup
+
+### Delete Ollama completely
+```bash
+vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama destroy
+# Keeps PVC (data safety). To delete: kubectl -n llm delete pvc ollama-models-cache
+```
+
+### Delete just the model cache (keep deployment)
+```bash
+kubectl -n llm delete pvc ollama-models-cache
+# Recreate: kubectl -n llm patch deployment ollama -p '{"spec":{"template":{"metadata":{"annotations":{"restart":"now"}}}}}'
+```
+
+## See Also
+
+- Helmfile: `helmfile.yaml.gotmpl` (llm release block)
+- Chart: `k8s/llm/charts/ollama/`
+- Namespace: `llm`
+- Bootstrap: `k8s/llm/bootstrap-models-job.yaml` (manual preload fallback)
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: ollama
+description: CPU-only Ollama LLM server with MinIO model registry
+type: application
+version: 0.1.0
+appVersion: "latest"
@@ -0,0 +1,126 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ollama
+  namespace: llm
+  labels:
+    app.kubernetes.io/name: ollama
+    app.kubernetes.io/part-of: llm
+spec:
+  replicas: {{ .Values.replicaCount }}
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ollama
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ollama
+        app.kubernetes.io/part-of: llm
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values:
+                      - {{ .Values.nodeAffinity.zone }}
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Equal
+          value: ""
+          effect: NoSchedule
+
+      initContainers:
+        - name: preload-model
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              echo "Starting Ollama server for model preload..."
+              ollama serve &
+              OLLAMA_PID=$!
+              sleep 10
+              {{- range .Values.preloadJob.hotModels }}
+              echo "Preloading {{ . }}..."
+              if ollama ls | grep -q "{{ . }}"; then
+                echo "✓ {{ . }} already cached"
+              else
+                ollama pull {{ . }}
+              fi
+              {{- end }}
+              echo "Model preload complete"
+              kill $OLLAMA_PID || true
+              wait $OLLAMA_PID 2>/dev/null || true
+          volumeMounts:
+            - name: models-cache
+              mountPath: /root/.ollama/models
+          env:
+            - name: OLLAMA_HOST
+              value: "127.0.0.1:11434"
+
+      containers:
+        - name: ollama
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - containerPort: 11434
+              name: http
+          env:
+            {{- range $key, $value := .Values.env }}
+            - name: {{ $key }}
+              value: "{{ $value }}"
+            {{- end }}
+            - name: OLLAMA_MODELS_MINIO_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: ollama-minio
+                  key: endpoint
+            - name: OLLAMA_MODELS_MINIO_BUCKET
+              valueFrom:
+                secretKeyRef:
+                  name: ollama-minio
+                  key: bucket
+            - name: OLLAMA_MODELS_MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: ollama-minio
+                  key: access_key
+            - name: OLLAMA_MODELS_MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: ollama-minio
+                  key: secret_key
+          resources:
+            requests:
+              cpu: {{ .Values.resources.requests.cpu }}
+              memory: {{ .Values.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.resources.limits.cpu }}
+              memory: {{ .Values.resources.limits.memory }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 11434
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 11434
+            initialDelaySeconds: 10
+            periodSeconds: 5
+          volumeMounts:
+            - name: models-cache
+              mountPath: /root/.ollama/models
+
+      volumes:
+        - name: models-cache
+          persistentVolumeClaim:
+            claimName: ollama-models-cache
@@ -0,0 +1,28 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ollama-access
+  namespace: llm
+  labels:
+    app.kubernetes.io/name: ollama
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: ollama
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: llm-worker
+      ports:
+        - protocol: TCP
+          port: 11434
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/role: llm-debug
+      ports:
+        - protocol: TCP
+          port: 11434
@@ -0,0 +1,92 @@
+{{- if .Values.preloadJob.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ollama-preload
+  namespace: llm
+  labels:
+    app.kubernetes.io/name: ollama-preload
+spec:
+  backoffLimit: 3
+  template:
+    spec:
+      serviceAccountName: default
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values:
+                      - az-a
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Equal
+          effect: NoSchedule
+      initContainers:
+        - name: model-cache-init
+          image: ollama/ollama:latest
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              echo "Starting Ollama server to cache models..."
+              ollama serve &
+              OLLAMA_PID=$!
+              sleep 10
+              echo "Caching hot-tier models..."
+              {{- range .Values.preloadJob.hotModels }}
+              echo "Checking if {{ . }} is cached..."
+              if ollama ls | grep -q "{{ . }}"; then
+                echo "✓ {{ . }} already cached, skipping"
+              else
+                echo "Pulling {{ . }}..."
+                ollama pull {{ . }}
+              fi
+              {{- end }}
+              echo "Model cache initialization complete"
+              kill $OLLAMA_PID || true
+              wait $OLLAMA_PID 2>/dev/null || true
+          volumeMounts:
+            - name: models
+              mountPath: /root/.ollama
+          env:
+            - name: OLLAMA_HOST
+              value: "127.0.0.1:11434"
+
+      containers:
+        - name: cache-populate
+          image: curlimages/curl:latest
+          imagePullPolicy: IfNotPresent
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              echo "Waiting for Ollama pod to be ready..."
+              until curl -f http://ollama.llm.svc.cluster.local:11434/api/tags 2>/dev/null; do
+                echo "Ollama not ready, waiting..."
+                sleep 5
+              done
+              echo "Ollama is ready, populating local cache..."
+              {{- range .Values.preloadJob.hotModels }}
+              echo "Checking if {{ . }} is already cached..."
+              if curl -s http://ollama.llm.svc.cluster.local:11434/api/tags | grep -q "{{ . }}"; then
+                echo "✓ {{ . }} already cached, skipping"
+              else
+                echo "Caching {{ . }} locally..."
+                curl -X POST http://ollama.llm.svc.cluster.local:11434/api/pull \
+                  -H "Content-Type: application/json" \
+                  -d '{"name":"{{ . }}","stream":false}'
+              fi
+              {{- end }}
+              echo "Local cache population complete"
+
+      volumes:
+        - name: models
+          emptyDir: {}
+{{- end }}
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ollama-models-cache
+  namespace: llm
+  labels:
+    app.kubernetes.io/name: ollama
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: {{ .Values.pvc.storageClassName }}
+  resources:
+    requests:
+      storage: {{ .Values.pvc.size }}
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: ollama
+  namespace: llm
+  labels:
+    app.kubernetes.io/name: ollama
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/name: ollama
@@ -0,0 +1,12 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn-llm
+  labels:
+    app.kubernetes.io/name: ollama
+provisioner: driver.longhorn.io
+parameters:
+  numberOfReplicas: "1"
+  staleReplicaTimeout: "2880"
+reclaimPolicy: Retain
+allowVolumeExpansion: true
@@ -0,0 +1,40 @@
+replicaCount: 1
+
+image:
+  repository: ollama/ollama
+  pullPolicy: IfNotPresent
+  tag: "latest"
+
+service:
+  type: ClusterIP
+  port: 11434
+
+resources:
+  requests:
+    cpu: 8
+    memory: 60Gi
+  limits:
+    cpu: 16
+    memory: 100Gi
+
+pvc:
+  enabled: true
+  size: 115Gi
+  storageClassName: longhorn-llm
+
+nodeAffinity:
+  zone: az-a
+
+env:
+  OLLAMA_MODELS: /root/.ollama/models
+  OLLAMA_MAX_LOADED_MODELS: "2"
+  OLLAMA_NUM_PARALLEL: "2"
+  OLLAMA_MAX_QUEUE: "64"
+  OLLAMA_KEEP_ALIVE: "-1"
+  OLLAMA_HOST: "0.0.0.0:11434"
+
+preloadJob:
+  enabled: false
+  hotModels:
+    - ornith:35b
+    - deepseek-r1:70b
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# setup-minio-bucket.sh
+# Creates MinIO bucket and Kubernetes secrets for Ollama LLM server
+# Runs as helmfile presync hook; all commands are idempotent
+# Dependencies: kubectl, access to minio-az-a pod in storage namespace
+# Environment: MINIO_ROOT_USER, MINIO_ROOT_PASSWORD (from Vault), AUTHENTIK_OLLAMA_CLIENT_ID, AUTHENTIK_OLLAMA_CLIENT_SECRET
+
+set -e
+
+echo "=== Step 1: Create and label llm namespace ==="
+kubectl create namespace llm --dry-run=client -o yaml | kubectl apply -f -
+kubectl label namespace llm \
+  pod-security.kubernetes.io/enforce=baseline \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite
+
+echo "✓ llm namespace created/labeled"
+
+echo ""
+echo "=== Step 2: Create MinIO bucket riotpiao-models ==="
+
+# Configure mc host inside MinIO pod
+kubectl -n storage exec deployment/minio-az-a -- \
+  mc config host add local http://localhost:9000 \
+  "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"
+
+echo "✓ mc host configured"
+
+# Create bucket (idempotent)
+kubectl -n storage exec deployment/minio-az-a -- \
+  mc mb --ignore-existing local/riotpiao-models
+
+echo "✓ MinIO bucket riotpiao-models created (or already exists)"
+
+# Enable versioning for model rollback safety
+kubectl -n storage exec deployment/minio-az-a -- \
+  mc version enable local/riotpiao-models
+
+echo "✓ Versioning enabled on riotpiao-models bucket"
+
+echo ""
+echo "=== Step 3: Create ollama-minio Secret (MinIO credentials) ==="
+
+kubectl create secret generic ollama-minio -n llm \
+  --from-literal=endpoint="http://minio-az-a.storage:9000" \
+  --from-literal=bucket="riotpiao-models" \
+  --from-literal=access_key="${MINIO_ROOT_USER}" \
+  --from-literal=secret_key="${MINIO_ROOT_PASSWORD}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+echo "✓ Secret ollama-minio created/updated"
+
+echo ""
+echo "=== Step 4: Create ollama-oidc Secret (Authentik credentials) ==="
+
+kubectl create secret generic ollama-oidc -n llm \
+  --from-literal=client_id="${AUTHENTIK_OLLAMA_CLIENT_ID}" \
+  --from-literal=client_secret="${AUTHENTIK_OLLAMA_CLIENT_SECRET}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+echo "✓ Secret ollama-oidc created/updated"
+
+echo ""
+echo "=== Verification ==="
+echo ""
+echo "Run these commands to verify:"
+echo "  kubectl -n llm get secret ollama-minio ollama-oidc"
+echo "  kubectl -n storage exec deployment/minio-az-a -- mc ls local/riotpiao-models"
+echo ""
+echo "Setup complete!"
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# portainer/bootstrap.sh
+# Deploys Portainer CE into the dashboard namespace.
+# No credentials needed — Portainer prompts you to create an admin account
+# on first browser visit.
+#
+# Prerequisites:
+#   - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
+#   - helm >= 3.x
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}"
+export KUBECONFIG
+
+# ── Namespace ─────────────────────────────────────────────────────────────────
+echo "==> Creating dashboard namespace..."
+kubectl create namespace dashboard --dry-run=client -o yaml | kubectl apply -f -
+kubectl label namespace dashboard \
+  pod-security.kubernetes.io/enforce=privileged \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite
+
+# ── Helm repo ─────────────────────────────────────────────────────────────────
+echo "==> Adding Portainer Helm repo..."
+helm repo add portainer https://portainer.github.io/k8s/
+helm repo update portainer
+
+# ── Portainer ─────────────────────────────────────────────────────────────────
+echo "==> Installing Portainer..."
+helm upgrade --install portainer portainer/portainer \
+  --namespace dashboard \
+  --values "${SCRIPT_DIR}/portainer-values.yaml" \
+  --wait \
+  --timeout 5m
+
+echo "==> Waiting for Portainer Deployment to be ready..."
+kubectl rollout status deployment/portainer -n dashboard --timeout=120s
+
+# ── Done ──────────────────────────────────────────────────────────────────────
+echo ""
+echo "==> Portainer is up."
+echo ""
+echo "Access Portainer UI:"
+echo "  make pf-portainer"
+echo "  http://localhost:9000"
+echo ""
+echo "First-time setup: Portainer will prompt you to create an admin account."
+echo "Choose 'Manage the local Kubernetes environment' when asked."
+echo ""
+echo "Node failure resilience tip:"
+echo "  For faster PVC failover on hard node failure, enable in Longhorn UI → Settings:"
+echo "    nodeDownPodDeletionPolicy = delete-deployment-pod"
@@ -0,0 +1,53 @@
+# k8s/portainer/portainer-values.yaml
+# Portainer — web UI for browsing cluster workloads, exec-ing into pods,
+# and viewing logs without kubectl. Operator-only access (ClusterIP + port-forward).
+#
+# Node failure behaviour:
+#   Portainer is a Deployment (not StatefulSet), so K8s auto-evicts and
+#   reschedules it ~5 min after a node becomes unreachable. Longhorn
+#   reattaches the PVC on the new node in ~1-2 min. Worst case: ~7-10 min.
+#
+#   To cut that down: in Longhorn UI → Settings set
+#     nodeDownPodDeletionPolicy = delete-deployment-pod
+#   Longhorn will force-delete the stuck pod immediately when the node is
+#   fenced rather than waiting for Kubernetes' eviction timeout.
+
+# ── Service ───────────────────────────────────────────────────────────────────
+# ClusterIP — no external exposure. Access via:
+#   kubectl -n dashboard port-forward svc/portainer 9000:9000
+# Portainer holds cluster-admin credentials; never expose as LoadBalancer.
+service:
+  type: ClusterIP
+
+# ── TLS ───────────────────────────────────────────────────────────────────────
+# Portainer by default redirects HTTP → HTTPS using a self-signed cert.
+# force: false disables the redirect so plain HTTP over port-forward works
+# without browser cert warnings. TLS is terminated at the ingress layer
+# if/when an ingress rule is added.
+tls:
+  force: false
+
+# ── Persistence ───────────────────────────────────────────────────────────────
+# Stores Portainer's own config: environment registrations, user accounts,
+# stack definitions, and access control settings. Longhorn provides the
+# RWO block volume. 10Gi is generous for config data but cheap on Longhorn.
+persistence:
+  enabled: true
+  storageClass: "longhorn"
+  size: 10Gi
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+# ── Scheduling ────────────────────────────────────────────────────────────────
+# Allow scheduling on talos-cp-1 (carries NoSchedule taint) so Portainer
+# keeps running even when the worker node is down.
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule