rock/homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add paperless-ngx with OIDC, CNPG db, cp-3 HDD media, MinIO backup

Browse Source 
Fixes controlplane.tftpl's install.wipe:true (should be false, live CPs already run false) and syncs coredns Corefile back to what's actually deployed (drops an unrolled-out, stale Kong-era rewrite).
This commit is contained in:
Story Crater Bot
2026-08-25 11:11:37 -07:00
parent 78e788abb3
commit e8e5acfb13
24 changed files with 660 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/infra/databases/kustomization.yaml
+4 -2
View File
@@ -1,10 +1,12 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
# Dedicated per-app CNPG clusters. NO top-level `namespace:` — each Cluster
# carries its own ns (iam / temporal / poimen); a transformer would wrongly collapse them.
# poimen ns created by poimen-root app, memory-db deployed into it.
# carries its own ns (iam / temporal / poimen / paperless); a transformer would
# wrongly collapse them. poimen ns created by poimen-root app, memory-db
# deployed into it. paperless ns declared in namespaces.yaml above.
resources:
- namespaces.yaml
- authentik-db.yaml
- temporal-db.yaml
- memory-db.yaml
- paperless-db.yaml
k8s/infra/databases/namespaces.yaml
+7 -2
View File
@@ -1,7 +1,7 @@
# DB clusters are wave 2 — their namespaces must exist first (their apps that
# would CreateNamespace run later, w3/w8). Declared here so the databases App
# creates them. authentik/vault/temporal CreateNamespace=true then no-ops.
# poimen namespace created by poimen-root app (wave 7).
# creates them. authentik/vault/temporal/paperless CreateNamespace=true then
# no-ops. poimen namespace created by poimen-root app (wave 7).
apiVersion: v1
kind: Namespace
metadata:
@@ -11,3 +11,8 @@ apiVersion: v1
kind: Namespace
metadata:
name: temporal
---
apiVersion: v1
kind: Namespace
metadata:
name: paperless
k8s/infra/databases/paperless-db.yaml
+36
View File
@@ -0,0 +1,36 @@
# Dedicated CNPG Postgres for paperless-ngx (GitOps, wave 2 — before the
# paperless app at w8). Same recipe as memory-db: default longhorn storage
# class (3 replicas), 2 instances, 20Gi.
# CNPG generates secret `paperless-db-app` + service `paperless-db-rw` in ns
# paperless; the paperless Deployment reads them locally.
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: paperless-db
namespace: paperless
annotations:
argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
spec:
instances: 2
imageName: ghcr.io/cloudnative-pg/postgresql:16.2
bootstrap:
initdb:
database: paperless
owner: app
encoding: UTF8
localeCollate: C
localeCType: C
enableSuperuserAccess: false
resources:
requests: { memory: "512Mi", cpu: "250m" }
limits: { memory: "2Gi", cpu: "1" }
storage:
size: 20Gi
storageClass: longhorn
affinity:
podAntiAffinityType: preferred
topologyKey: kubernetes.io/hostname
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
k8s/infra/iam/authentik-provision-job.yaml
+14
View File
@@ -101,6 +101,20 @@ roleRef:
name: authentik-provisioner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: authentik-provisioner
namespace: paperless
subjects:
- kind: ServiceAccount
name: authentik-provisioner
namespace: iam
roleRef:
kind: ClusterRole
name: authentik-provisioner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
k8s/infra/iam/scripts/authentik-provision.py
+37
View File
@@ -290,6 +290,17 @@ SERVICES = {
"launch_url": "https://homarr.riotpiao.com",
"display_name": "Homarr",
},
"paperless": {
# No secret exists yet for paperless - generate + store on first run.
# django-allauth's generic openid_connect provider callback path is
# /accounts/oidc/<provider_id>/login/callback/ - provider_id "authentik"
# is set in PAPERLESS_SOCIALACCOUNT_PROVIDERS (see configmap.yaml).
"client_secret_source": ("paperless", "paperless-oidc", "CLIENT_SECRET"),
"generate_if_missing": True,
"redirect_uris": ["https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"],
"launch_url": "https://paperless.riotpiao.com",
"display_name": "Paperless-ngx",
},
}
app_pks_for_binding = []
@@ -314,6 +325,32 @@ for name, cfg in SERVICES.items():
else:
print(f" {name}: using existing client secret from {ns}/{secret_name}")
if name == "paperless":
# paperless-ngx's django-allauth OIDC config takes client_id/secret
# bundled inside one JSON blob (PAPERLESS_SOCIALACCOUNT_PROVIDERS), not
# discrete env vars - compose it here and store it alongside
# CLIENT_SECRET so the Deployment can source it directly via
# secretKeyRef, no shell wrapper needed. Runs every time (not just on
# generate), so it stays in sync if the client_secret is ever rotated
# by hand.
providers_json = json.dumps({
"openid_connect": {
"APPS": [{
"provider_id": "authentik",
"name": "Authentik",
"client_id": "paperless",
"secret": client_secret,
"settings": {
"server_url": "https://authentik.riotpiao.com/application/o/paperless/.well-known/openid-configuration",
},
}],
},
})
kubectl_create_secret("paperless", "paperless-oidc", {
"CLIENT_SECRET": client_secret,
"SOCIALACCOUNT_PROVIDERS_JSON": providers_json,
})
provider = get_or_create(
"/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
f"name={name}",
k8s/infra/longhorn/kustomization.yaml
+1
View File
@@ -4,6 +4,7 @@ namespace: longhorn-system
resources:
- longhorn-storageclass.yaml
- longhorn-cnpg-storageclass.yaml # CNPG-specific with postgres UID/GID
- longhorn-paperless-storageclass.yaml # single-replica, cp-3 USB HDD only
- longhorn-servicemonitor.yaml
- longhorn-taint-toleration.yaml
- longhorn-nodes.yaml
k8s/infra/longhorn/longhorn-add-disks-job.yaml
+34
View File
@@ -1,6 +1,15 @@
# PostSync hook Job that adds extra disks to Longhorn nodes.
# talos-cp-2 has 4 extra disks mounted at /var/lib/longhorn-disk{1,2,3,4}
# that are NOT auto-discovered by Longhorn.
#
# talos-cp-3 additionally gets a tagged disk for paperless-ngx media, backed by
# the 4TB USB HDD (/dev/sdg) — tagged "paperless-media" so only the dedicated
# longhorn-paperless-media StorageClass (diskSelector match) can place replicas
# there, keeping it out of the default 3-replica pool. This patch is inert
# until the Terraform machine-config change mounts the disk at
# /var/lib/longhorn-paperless-media (pending — see terraform.tfvars.local,
# not present in this checkout); Longhorn just reports the disk not-ready
# until the path exists, no harm in applying it early.
apiVersion: batch/v1
kind: Job
metadata:
@@ -83,6 +92,31 @@ spec:
echo " ✓ talos-cp-2 already has $CURRENT_DISKS disks configured"
fi
echo "Checking talos-cp-3..."
CP3_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-3 -o json | jq -r '.spec.disks | keys | length')
echo " Current disk count: $CP3_DISKS"
if [ "$CP3_DISKS" -lt 2 ]; then
echo " Adding paperless-media disk to talos-cp-3..."
kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-3 --type merge -p '{
"spec": {
"disks": {
"paperless-media": {
"allowScheduling": true,
"diskType": "filesystem",
"evictionRequested": false,
"path": "/var/lib/longhorn-paperless-media",
"storageReserved": 0,
"tags": ["paperless-media"]
}
}
}
}'
echo " ✓ paperless-media disk added to talos-cp-3"
else
echo " ✓ talos-cp-3 already has $CP3_DISKS disks configured"
fi
echo
echo "Waiting for disks to be ready..."
sleep 10
k8s/infra/longhorn/longhorn-paperless-storageclass.yaml
+22
View File
@@ -0,0 +1,22 @@
# Dedicated StorageClass for paperless-ngx document media, backed by the 4TB
# USB HDD on talos-cp-3 (see longhorn-add-disks-job.yaml). Single disk, single
# node — no Longhorn replica is possible, so numberOfReplicas is 1 by
# necessity, not choice. diskSelector pins placement to the tagged disk only,
# so a volume from this class never lands on cp-3's regular (already
# DiskPressure) default pool. reclaimPolicy is Retain, not Delete: a PVC
# accident here has no replica to fall back on, so an accidental delete must
# not also take the underlying volume with it.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-paperless-media
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
parameters:
numberOfReplicas: "1"
diskSelector: "paperless-media"
nodeSelector: "talos-cp-3"
staleReplicaTimeout: "30"
fsType: "ext4"
k8s/infra/minio/kustomization.yaml
+7 -1
View File
@@ -1,8 +1,14 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: storage
# NOTE: no top-level `namespace:` transformer (see iam/kustomization.yaml for
# the same fix) - minio-provision-paperless-job.yaml's RoleBinding
# deliberately targets namespace paperless (least-privilege access for the
# provisioner ServiceAccount to write Secrets there); a namespace transformer
# would silently rewrite it back to storage, breaking the RBAC. Every resource
# here already sets its own explicit metadata.namespace.
resources:
- minio-tenant.yaml
- minio-provision-paperless-job.yaml
# The operator creates the minio S3/console/headless Services and the
# declarative bucket + user from the Tenant spec — no hand-rolled Service or
# Bucket/User CRs (those kinds don't exist in the operator CRD set).
k8s/infra/minio/minio-provision-paperless-job.yaml
+120
View File
@@ -0,0 +1,120 @@
# PostSync hook Job: creates a MinIO IAM user + policy scoped to only the
# `paperless` bucket (least-privilege — reuses root creds nowhere else in the
# cluster), then writes the generated access/secret key into a Secret in the
# `paperless` namespace for the nightly backup CronJob to consume.
#
# Idempotent: re-running never rotates existing credentials — if
# paperless-minio-creds already exists in ns paperless, the script reuses the
# access key it already wrote and just re-asserts the policy/user exist.
apiVersion: v1
kind: ServiceAccount
metadata:
name: minio-paperless-provisioner
namespace: storage
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: minio-paperless-provisioner
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: minio-paperless-provisioner
namespace: paperless
subjects:
- kind: ServiceAccount
name: minio-paperless-provisioner
namespace: storage
roleRef:
kind: ClusterRole
name: minio-paperless-provisioner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
name: minio-provision-paperless
namespace: storage
annotations:
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
ttlSecondsAfterFinished: 600
backoffLimit: 3
template:
spec:
serviceAccountName: minio-paperless-provisioner
restartPolicy: Never
initContainers:
- name: kubectl-copy
image: bitnami/kubectl:latest
command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
volumeMounts:
- name: shared
mountPath: /shared
containers:
- name: provision
image: minio/mc:latest
volumeMounts:
- name: shared
mountPath: /shared
- name: minio-creds
mountPath: /minio-creds
readOnly: true
command:
- /bin/sh
- -c
- |
set -e
export PATH="/shared:$PATH"
# minio-creds ships as a shell-sourceable config.env blob
# (`export MINIO_ROOT_USER=... / MINIO_ROOT_PASSWORD=...`), not
# discrete keys — source it directly rather than re-parsing.
. /minio-creds/config.env
mc alias set m http://minio-cluster-hl.storage.svc.cluster.local:9000 \
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
echo "Checking for existing paperless-minio-creds secret..."
if kubectl -n paperless get secret paperless-minio-creds >/dev/null 2>&1; then
ACCESS_KEY=$(kubectl -n paperless get secret paperless-minio-creds -o jsonpath='{.data.ACCESS_KEY}' | base64 -d)
SECRET_KEY=$(kubectl -n paperless get secret paperless-minio-creds -o jsonpath='{.data.SECRET_KEY}' | base64 -d)
echo " reusing existing credentials"
else
ACCESS_KEY="paperless"
SECRET_KEY=$(head -c 32 /dev/urandom | base64 | tr -d '/+=' | head -c 40)
echo " generated new credentials"
fi
echo "Ensuring MinIO user 'paperless' exists..."
if ! mc admin user info m "$ACCESS_KEY" >/dev/null 2>&1; then
mc admin user add m "$ACCESS_KEY" "$SECRET_KEY"
fi
echo "Writing scoped policy (paperless bucket only)..."
printf '%s' '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::paperless"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::paperless/*"]}]}' > /tmp/paperless-rw-policy.json
mc admin policy create m paperless-rw /tmp/paperless-rw-policy.json || \
mc admin policy update m paperless-rw /tmp/paperless-rw-policy.json
mc admin policy attach m paperless-rw --user "$ACCESS_KEY"
echo "Writing paperless-minio-creds secret (ns paperless)..."
kubectl -n paperless create secret generic paperless-minio-creds \
--from-literal=ACCESS_KEY="$ACCESS_KEY" \
--from-literal=SECRET_KEY="$SECRET_KEY" \
--from-literal=BUCKET=paperless \
--from-literal=ENDPOINT=http://minio-cluster-hl.storage.svc.cluster.local:9000 \
--dry-run=client -o yaml | kubectl apply -f -
echo "Done."
volumes:
- name: shared
emptyDir: {}
- name: minio-creds
secret:
secretName: minio-creds
k8s/infra/minio/minio-tenant.yaml
+1
View File
@@ -76,6 +76,7 @@ spec:
- name: loki-ruler
- name: loki-admin
- name: vault
- name: paperless
# Metrics are exposed at /minio/v2/metrics; scrape via a hand-rolled
# ServiceMonitor in the monitoring stack rather than operator auto-wiring