diff --git a/.gitignore b/.gitignore index 6ae447d..3c835e2 100644 --- a/.gitignore +++ b/.gitignore @@ -52,4 +52,5 @@ skills-lock.json secrets-plaintext.yaml skills-lock.json -CLAUDE.md \ No newline at end of file +.DS_Store +CLAUDE.md diff --git a/APPLICATION-CONSOLIDATION-PLAN.md b/APPLICATION-CONSOLIDATION-PLAN.md deleted file mode 100644 index c386b97..0000000 --- a/APPLICATION-CONSOLIDATION-PLAN.md +++ /dev/null @@ -1,301 +0,0 @@ -# ArgoCD Application Consolidation Plan - -## Executive Summary - -**Current:** 39 Applications -**Proposed:** 32 Applications (Conservative: 35) -**Benefits:** Easier management, fewer Application resources, cleaner structure - -## Phase 1: Safe & Easy Merges (Conservative - Start Here) - -These are Applications that already share the same directory and have clear parent-child relationships. - -### 1.1 Merge: wildcard-cert + homelab-ingress → "ingress-config" - -**Current State:** -``` -wildcard-cert (wave 1) → k8s/bootstrap/ingress/riotpiao-com-cert.yaml -homelab-ingress (wave 2) → k8s/bootstrap/ingress/ingress.yaml -``` - -**New State:** -``` -ingress-config (wave 1) → k8s/bootstrap/ingress/ (kustomization) -``` - -**Rationale:** Both in same directory, ingress depends on cert existing - -**Implementation:** -- Update `k8s/bootstrap/ingress/kustomization.yaml` to include both -- Create single Application pointing to k8s/bootstrap/ingress/ -- Delete wildcard-cert and homelab-ingress Applications -- Result: 2 → 1 Applications - -### 1.2 Merge: homarr + homarr-patches → "homarr" - -**Current State:** -``` -homarr (wave 8) → Helm chart (homarr-labs.github.io) -homarr-patches (wave 9) → k8s/applications/homarr/fix-probes-job.yaml -``` - -**New State:** -``` -homarr (multi-source): - - Source 1: Helm chart - - Source 2: k8s/applications/homarr/ (patches as PostSync hook) -``` - -**Rationale:** homarr-patches is literally a patch for homarr - -**Implementation:** -- Modify homarr Application to use multi-source -- Convert fix-probes-job.yaml to PostSync hook -- Delete homarr-patches Application -- Result: 2 → 1 Applications - -### 1.3 Merge: temporal-db-secret-sync → temporal (as PreSync hook) - -**Current State:** -``` -temporal-db-secret-sync (wave 7) → k8s/applications/temporal/db-secret-sync/ -temporal (wave 8) → Helm chart -``` - -**New State:** -``` -temporal (multi-source): - - Source 1: k8s/applications/temporal/db-secret-sync/ (PreSync hook) - - Source 2: Helm chart - - Source 3: k8s/applications/temporal/ (values) -``` - -**Rationale:** db-secret-sync is a bootstrap job for temporal - -**Implementation:** -- Modify temporal Application to use multi-source -- Ensure db-secret-sync Job has PreSync hook annotation -- Delete temporal-db-secret-sync Application -- Result: 2 → 1 Applications - -### 1.4 Merge: cert-manager-issuers → cert-manager (as multi-source) - -**Current State:** -``` -cert-manager (wave 0) → Helm chart -cert-manager-issuers (wave 1) → k8s/bootstrap/cert-manager/ (issuers) -``` - -**New State:** -``` -cert-manager (multi-source): - - Source 1: Helm chart - - Source 2: k8s/bootstrap/cert-manager/ (values + issuers) -``` - -**Rationale:** Issuers are part of cert-manager configuration - -**Implementation:** -- Modify cert-manager Application to use multi-source -- Keep as wave 0 (issuers can wait for CRDs to be ready) -- Delete cert-manager-issuers Application -- Result: 2 → 1 Applications - -**Phase 1 Result: 39 → 35 Applications (-4)** - ---- - -## Phase 2: Logical Consolidations (Moderate Risk) - -### 2.1 Merge: SQS Applications → "sqs-platform" - -**Current State:** -``` -strimzi-operator (wave 5) → Helm chart (operator) -kmsvc-redis (wave 5) → Helm chart (Redis) -kafka-cluster (wave 6) → k8s/applications/sqs/charts/kafka-cluster/ -queue-crd (wave 6) → k8s/applications/sqs/charts/queue-crd/ -management-service (wave 7) → k8s/applications/sqs/charts/management-service/ -``` - -**New State:** -``` -strimzi-operator (wave 5) → Keep separate (operator) -sqs-platform (wave 6) → k8s/applications/sqs/ (all services + CRDs) -``` - -**Rationale:** These 4 apps work together as the SQS platform - -**Implementation:** -- Create kustomization.yaml in k8s/applications/sqs/ -- Create single multi-source Application -- Keep strimzi-operator separate (it's infrastructure) -- Result: 5 → 2 Applications - -### 2.2 Merge: authentik + iam-jobs → "authentik" - -**Current State:** -``` -authentik (wave 3) → Helm chart -iam-jobs (wave 3) → k8s/security/iam/ (authentik helper jobs) -``` - -**New State:** -``` -authentik (multi-source): - - Source 1: Helm chart - - Source 2: k8s/security/iam/ (helper jobs) -``` - -**Rationale:** iam-jobs appears to be authentik-specific - -**Implementation:** -- Verify iam-jobs are authentik-specific -- Merge into single multi-source Application -- Result: 2 → 1 Applications - -### 2.3 Remove Duplicate: ingress-nginx Application - -**Current State:** -``` -ingress-nginx-bootstrap (wave 0, bootstrap) → Helm chart -ingress-nginx (wave 0, ArgoCD) → Helm chart (DUPLICATE!) -``` - -**New State:** -``` -ingress-nginx-bootstrap (wave 0, bootstrap) → Keep only this one -``` - -**Rationale:** These are the same thing, bootstrap one is already working - -**Implementation:** -- Delete ingress-nginx Application (redundant) -- Result: 2 → 1 Applications - -**Phase 2 Result: 35 → 31 Applications (-4)** - ---- - -## Phase 3: Advanced Consolidations (Consider Carefully) - -### 3.1 Consider: Monitoring Stack - -**Current State:** -``` -prometheus-crds (wave 0) → k8s/platform/monitoring/crds/ -prometheus (wave 1) → Helm chart -blackbox-exporter (wave 1) → Helm chart -``` - -**Option A:** Keep separate (CRDs must be wave 0) -**Option B:** Create "monitoring-stack" with prometheus + blackbox-exporter only - -**Recommendation:** Keep separate for now (CRDs are special) - -### 3.2 Consider: Logging Stack - -**Current State:** -``` -loki (wave 2) → Helm chart -promtail (wave 2) → Helm chart -grafana (wave 2) → Helm chart -``` - -**Option A:** Create "logging-stack" with all three -**Option B:** Keep separate (grafana used by multiple systems) - -**Recommendation:** Keep separate for now (grafana has multiple consumers) - ---- - -## Implementation Order - -### Sprint 1: Phase 1 (Low Risk) -1. ✅ ingress-config (wildcard-cert + homelab-ingress) -2. ✅ homarr (merge homarr-patches) -3. ✅ temporal (merge db-secret-sync) -4. ✅ cert-manager (merge issuers) - -### Sprint 2: Phase 2 (Moderate Risk) -5. ✅ Remove duplicate ingress-nginx -6. ✅ sqs-platform consolidation -7. ✅ authentik + iam-jobs - -### Sprint 3: Phase 3 (Optional) -8. ⚠️ Evaluate monitoring stack consolidation -9. ⚠️ Evaluate logging stack consolidation - ---- - -## File Changes Required - -### For each consolidation: - -1. **Update Application YAML** in `k8s/argocd/apps/*.yaml` - - Convert single-source to multi-source - - Update sync waves appropriately - - Add hook annotations where needed - -2. **Update kustomization.yaml** in target directories - - Ensure all resources are listed - - Verify namespace declarations - -3. **Test locally:** - ```bash - kubectl kustomize k8s/bootstrap/ingress/ - helm template -f values.yaml - ``` - -4. **Delete old Applications:** - ```bash - kubectl delete application -n argocd - ``` - ---- - -## Rollback Plan - -Each consolidation is independent. If issues arise: - -1. **Revert git commit** with the Application change -2. **Restore old Applications** from git history -3. **Force ArgoCD refresh:** - ```bash - kubectl delete application -n argocd - kubectl apply -f k8s/argocd/apps/.yaml - ``` - ---- - -## Success Metrics - -**Before:** 39 Applications -**After (Conservative):** 35 Applications -**After (Full):** 31-32 Applications - -**Benefits:** -- ✅ Easier to understand (related apps grouped) -- ✅ Fewer Application CRs to manage -- ✅ Cleaner ArgoCD UI -- ✅ Logical grouping by function -- ✅ Reduced sync complexity - -**Risks:** -- ⚠️ Multi-source Applications are slightly more complex -- ⚠️ Sync wave ordering must be preserved -- ⚠️ Hook timing must be correct (PreSync/PostSync) - ---- - -## Decision: Which Phase? - -**Recommendation:** Start with **Phase 1 (Conservative)** → 35 Applications - -This gives us: -- 4 fewer Applications to manage -- Low risk (same directories) -- Easy to understand merges -- Can evaluate Phase 2 later based on results - -**Do you want to proceed with Phase 1 implementation?** diff --git a/CONSOLIDATION-RESULTS.md b/CONSOLIDATION-RESULTS.md deleted file mode 100644 index 0fdbbeb..0000000 --- a/CONSOLIDATION-RESULTS.md +++ /dev/null @@ -1,188 +0,0 @@ -# ArgoCD Application Consolidation - Implementation Results - -## Summary - -**Before:** 39 Applications -**After:** 35 Applications -**Removed:** 4 Applications (-10.3%) - -## Implemented Merges - -### ✅ 1. ingress-config ← wildcard-cert + homelab-ingress - -**Before:** -- `wildcard-cert` (wave 1): k8s/bootstrap/ingress/riotpiao-com-cert.yaml -- `homelab-ingress` (wave 2): k8s/bootstrap/ingress/ingress.yaml - -**After:** -- `ingress-config` (wave 1): k8s/bootstrap/ingress/ (kustomization) - -**Changes:** -- Updated `k8s/bootstrap/ingress/kustomization.yaml` to include both files -- Merged both Applications into single `ingress-config` in `00-substrate.yaml` -- Certificate created before Ingresses (wave 1) - ---- - -### ✅ 2. homarr ← homarr + homarr-patches - -**Before:** -- `homarr` (wave 8): Helm chart + values -- `homarr-patches` (wave 9): k8s/applications/homarr/ (PostSync hook) - -**After:** -- `homarr` (wave 8): Multi-source (Helm + values + PostSync patches) - -**Changes:** -- Added third source to homarr Application -- fix-probes-job.yaml already has PostSync hook annotation -- Removed homarr-patches Application from `60-applications.yaml` - ---- - -### ✅ 3. temporal ← temporal + temporal-db-secret-sync - -**Before:** -- `temporal-db-secret-sync` (wave 7): k8s/applications/temporal/db-secret-sync/ -- `temporal` (wave 8): Helm chart + values - -**After:** -- `temporal` (wave 8): Multi-source (Helm + values + PostSync db-secret-sync) - -**Changes:** -- Added third source to temporal Application -- copy-job.yaml already has PostSync hook annotation -- Removed temporal-db-secret-sync Application from `60-applications.yaml` - ---- - -### ✅ 4. Removed Duplicate: ingress-nginx - -**Before:** -- `ingress-nginx-bootstrap` (bootstrap): Working, has LoadBalancer IP -- `ingress-nginx` (ArgoCD): Duplicate, LoadBalancer pending - -**After:** -- `ingress-nginx-bootstrap` (bootstrap): Kept - -**Changes:** -- Removed ingress-nginx Application from `00-substrate.yaml` -- Bootstrap version breaks circular dependency (ArgoCD → Forgejo → Ingress) -- Eliminated duplicate DaemonSet - ---- - -## Skipped Consolidations - -### ⏭️ cert-manager + cert-manager-issuers - -**Decision:** KEEP SEPARATE - -**Reasoning:** -- cert-manager (wave 0) installs CRDs -- cert-manager-issuers (wave 1) creates Issuers using those CRDs -- Wave separation ensures CRDs exist before Issuers -- Merging risks race condition (Issuer created before CRD ready) -- This is intentional separation for safety - ---- - -## Files Modified - -1. `k8s/bootstrap/ingress/kustomization.yaml` - Added resources list -2. `k8s/argocd/apps/00-substrate.yaml` - Merged wildcard-cert + homelab-ingress, removed ingress-nginx -3. `k8s/argocd/apps/60-applications.yaml` - Merged homarr + temporal Applications - ---- - -## Benefits Achieved - -### ✅ Easier Management -- 4 fewer Application CRs to track -- Related resources grouped together -- Clearer ownership model - -### ✅ Better Sync Behavior -- PostSync hooks ensure proper ordering -- No separate Applications for patches/hooks -- Single Application manages entire stack - -### ✅ Cleaner Structure -- Multi-source Applications are standard pattern -- Each logical service = one Application -- Hooks embedded where they belong - ---- - -## Next Steps (Optional Phase 2) - -### Potential Future Consolidations: - -1. **SQS Platform** (5 → 2 Applications) - - Keep `strimzi-operator` separate - - Merge: kmsvc-redis + kafka-cluster + queue-crd + management-service - -2. **IAM** (3 → 2 Applications) - - Keep `vault` separate - - Merge: authentik + iam-jobs - -3. **Monitoring** (3 → 2 Applications) - - Keep `prometheus-crds` separate (wave 0) - - Merge: prometheus + blackbox-exporter - -**Recommendation:** Evaluate after Forgejo push and observe current consolidations in action. - ---- - -## Validation Plan - -After pushing to Forgejo: - -```bash -# Check Applications synced successfully -kubectl get applications -n argocd - -# Verify consolidated Applications are healthy -kubectl get application ingress-config -n argocd -kubectl get application homarr -n argocd -kubectl get application temporal -n argocd - -# Check ingress-nginx duplicate removed -kubectl get application ingress-nginx -n argocd # Should be gone - -# Verify resources deployed correctly -kubectl get certificate -n ingress-nginx riotpiao-com-tls -kubectl get ingress -A -kubectl get job -n dashboard homarr-fix-probes -kubectl get job -n temporal temporal-db-secret-sync -``` - ---- - -## Rollback Procedure - -If issues arise, revert specific commits: - -```bash -# Identify commit -git log --oneline | grep consolidation - -# Revert specific merge -git revert - -# Or restore old Applications from git history -git show :k8s/argocd/apps/00-substrate.yaml > temp.yaml -kubectl apply -f temp.yaml -``` - ---- - -## Conclusion - -Successfully consolidated 4 Applications while maintaining: -- ✅ Proper sync wave ordering -- ✅ Hook execution timing -- ✅ Resource namespace separation -- ✅ GitOps best practices - -**Status:** Ready to commit and push to Forgejo for validation. diff --git a/MIGRATION-STATUS.md b/MIGRATION-STATUS.md deleted file mode 100644 index 3afa03c..0000000 --- a/MIGRATION-STATUS.md +++ /dev/null @@ -1,201 +0,0 @@ -# GitOps Migration Status - LIVE CLUSTER - -**Migration completed:** 2026-07-23 23:13:04 -**Status:** ✅ **CORE INFRASTRUCTURE HEALTHY** - ---- - -## ✅ What's Working (HEALTHY) - -### **Infrastructure (100% Healthy)** -- **ArgoCD:** 4/4 pods running -- **Cert-Manager:** Deployed, ready -- **Ingress-Nginx:** Deployed, ready -- **Reloader:** Synced, healthy - -### **Storage (3-Node HA Confirmed!)** -- **Longhorn nodes:** 3/3 ready ✅ - - talos-cp-1: Ready (4d8h) - - talos-cp-2: Ready (14h) - - talos-cp-3: Ready (14h) -- **All volumes:** 17 volumes, all with 3 replicas ✅ -- **DDB cluster:** 3/3 instances, "Cluster in healthy state" ✅ - - ddb-cluster-1: 10Gi, 3 replicas - - ddb-cluster-2: 10Gi, 3 replicas - - ddb-cluster-3: 10Gi, 3 replicas - -### **Monitoring & Logging** -- **Prometheus:** 3/3 pods running -- **Grafana:** 4/4 pods running (accessible at http://10.110.221.93) -- **Loki:** Deployed -- **Promtail:** Deployed - -### **Services** -- **Forgejo:** LoadBalancer IP 192.168.1.165 (reinitializing after update) -- **Forgejo Redis:** 1/1 running - ---- - -## ⚠️ Known Issues (Pre-Existing, Not Migration-Related) - -### **1. Kafka/SQS Pods CrashLoopBackOff** -**Status:** Pre-existing (9h old) -**Impact:** Low (SQS service-specific issue) -**Action:** Not related to migration - investigate separately - -``` -kmsvc-entity-operator: CrashLoopBackOff (9h) -kmsvc-kmsvc-pool-0: CrashLoopBackOff (9h) -kmsvc-kmsvc-pool-2: CrashLoopBackOff (9h) -``` - -**Recommendation:** Check Kafka configuration separately - -### **2. Authentik CreateContainerConfigError** -**Status:** Pre-existing (10h old) -**Impact:** Medium (IAM service affected) -**Action:** Secrets exist, may be configuration issue - -**Recommendation:** Check authentik pod describe for specific error - -### **3. Forgejo Reinitializing** -**Status:** Expected (after Application update) -**Impact:** Temporary (normal init process) -**Action:** Wait for init containers to complete (~2-5 minutes) - -``` -forgejo-gitea pods: Init:0/3 (normal) -``` - -**Recommendation:** Monitor, should auto-resolve - ---- - -## 🎯 Migration Changes Applied - -### **1. Data Schemas Path Updated** -- **Before:** `k8s/data` (included ddb-cluster - DUPLICATION) -- **After:** `k8s/data/schemas` (schemas only - no duplication) -- **Status:** ✅ Applied successfully - -### **2. Longhorn 3-Node HA Verified** -- **Before:** CLAUDE.md claimed single-node storage -- **After:** Confirmed 3-node HA with 3 replicas per volume -- **Status:** ✅ Working perfectly - -### **3. App-of-Apps Root Created** -- **Application:** homelab-root -- **Status:** ✅ Created and managing all child apps - -### **4. Wave Structure** -- All applications organized in waves 0-9 -- **Status:** ✅ Syncing properly - ---- - -## 📊 Application Status Summary - -**Total Applications:** 37 -**Synced:** 3 (reloader, kmsvc-redis, strimzi-operator) -**OutOfSync:** 2 (cnpg-operator, forgejo) - being synced -**Unknown:** 32 (normal during reconciliation) -**Degraded:** 1 (kafka-cluster) - pre-existing issue - -**Expected:** All apps will transition to "Synced" within 5-10 minutes as ArgoCD reconciles. - ---- - -## 🚀 Next Steps - -### **Immediate (Next 5 minutes)** - -1. **Monitor Forgejo initialization:** - ```bash - watch kubectl get pods -n cicd - ``` - Wait for forgejo-gitea pods to show `Running` (currently in `Init:0/3`) - -2. **Check ArgoCD sync status:** - ```bash - watch kubectl get applications -n argocd - ``` - Most apps should transition from `Unknown` → `Synced` - -### **Short-term (Next hour)** - -3. **Investigate pre-existing issues:** - - Kafka CrashLoopBackOff (not migration-related) - - Authentik CreateContainerConfigError (not migration-related) - -4. **Verify all services accessible:** - ```bash - # Test ingress - curl -k https://forgejo.riotpiao.com - curl -k https://grafana.riotpiao.com - curl -k https://argocd.riotpiao.com - ``` - -5. **Update CLAUDE.md:** - - Change topology table to reflect 3-node HA storage - - Update hard rule about node renaming (all 3 nodes, not just cp-1) - -### **Documentation Updates** - -6. **Create final migration summary:** - - Document what was changed - - Note pre-existing issues - - Update cluster architecture docs - ---- - -## 🎉 Migration Success Criteria - -| Criterion | Status | -|-----------|--------| -| Zero downtime | ✅ Achieved | -| No data loss | ✅ Confirmed (DDB cluster healthy) | -| Storage HA verified | ✅ 3 nodes, 3 replicas | -| ArgoCD managing all apps | ✅ 37 applications | -| GitOps workflow functional | ✅ Can sync via git push | -| Core services running | ✅ ArgoCD, DDB, Forgejo, monitoring | - ---- - -## 📝 Commands Reference - -### **Check overall health:** -```bash -kubectl get applications -n argocd -kubectl get pods --all-namespaces | grep -v Running | grep -v Completed -kubectl get nodes.longhorn.io -n longhorn-system -kubectl get cluster -n ddb -``` - -### **Force sync specific app:** -```bash -kubectl patch application -n argocd --type=merge \ - -p='{"operation":{"initiatedBy":{"username":"manual"},"sync":{"prune":true}}}' -``` - -### **Check logs:** -```bash -kubectl logs -n argocd deployment/argocd-application-controller --tail=50 -kubectl logs -n argocd deployment/argocd-repo-server --tail=50 -``` - ---- - -## ✅ Conclusion - -**The migration is SUCCESSFUL!** - -- Core infrastructure is fully operational -- 3-node HA storage confirmed (17 volumes with 3 replicas each) -- DDB cluster healthy (3/3 instances) -- ArgoCD managing all 37 applications -- GitOps workflow ready (future changes via git push) - -**Pre-existing issues** (Kafka, Authentik) are unrelated to the migration and should be investigated separately. - -**Your cluster is LIVE and fully operational!** 🚀 - diff --git a/REVIEW-SUMMARY.md b/REVIEW-SUMMARY.md deleted file mode 100644 index 3120753..0000000 --- a/REVIEW-SUMMARY.md +++ /dev/null @@ -1,475 +0,0 @@ -# GitOps Infrastructure Review - Summary - -**Date:** 2025-01-XX -**Objective:** Review entire k8s GitOps infrastructure, eliminate duplication, create bootstrap-from-local + GitOps-managed-future workflow. - ---- - -## 📌 What Was Done - -### 1. **Audit of Current State** - -**Findings:** -- ✅ App-of-apps pattern correctly implemented -- ✅ Wave-based deployment (0-8) mostly correct -- ⚠️ **Resource duplication:** `ddb-cluster.yaml` in both bootstrap and GitOps paths -- ⚠️ **Circular dependency:** Forgejo hosts repo → ArgoCD syncs from repo → Forgejo needs DB -- ⚠️ **Manual bootstrap steps:** Scattered, error-prone -- ⚠️ **SOPS secrets:** Some excluded from kustomization (out-of-band) -- ⚠️ **No single source of truth:** Same resources in multiple places - -### 2. **Created Bootstrap-Local Bundle** - -**New files created:** -``` -k8s/bootstrap-local/ -├── kustomization.yaml # Orchestrates bootstrap -├── 00-namespaces.yaml # All namespaces with PodSecurity labels -├── 01-argocd.yaml # ArgoCD ConfigMaps -├── 02-cnpg-operator.yaml # CNPG operator Application -├── 03-ddb-bootstrap.yaml # PostgreSQL cluster + Forgejo DB + Redis -└── 04-forgejo.yaml # Forgejo Application (inline values) -``` - -**What it does:** -- Apply once from local checkout (`kubectl apply -k k8s/bootstrap-local/`) -- Creates ArgoCD, CNPG, DDB, Forgejo (everything needed for GitOps) -- No git dependency (chicken-egg problem solved) -- Idempotent (safe to re-run) - -### 3. **Restructured k8s/data/ to Eliminate Duplication** - -**Before:** -``` -k8s/data/ -├── ddb-cluster.yaml # ❌ Deployed by data-schemas app (wrong) -├── forgejo-database.yaml # ❌ Deployed by data-schemas app (wrong) -├── authentik-database.yaml -├── temporal-database.yaml -├── ... -└── kustomization.yaml # Listed ALL resources -``` - -**After:** -``` -k8s/data/ -├── cluster/ # 🔴 Bootstrap-only (not in GitOps) -│ ├── ddb-cluster.yaml -│ ├── forgejo-database.yaml -│ └── kustomization.yaml # Reference only -└── schemas/ # 🟢 GitOps-managed (wave 6) - ├── authentik-database.yaml - ├── temporal-database.yaml - ├── schemas.yaml - ├── db-init-job.yaml - └── kustomization.yaml -``` - -**Result:** Each resource has exactly ONE source of truth. - -### 4. **Updated ArgoCD Apps to Avoid Duplication** - -**Modified:** -- `k8s/argocd/apps/40-data.yaml` → Points to `k8s/data/schemas/` (NOT `k8s/data/`) -- Wave renumbered for clarity (0-8 sequential) - -**Deleted:** -- Old `k8s/data/kustomization.yaml` (replaced by subdirectories) - -### 5. **DDB Configuration Review** - -**Created:** `DDB-REVIEW.md` with: -- Current configuration analysis -- Recommendations (resource limits, connection pooling, backups, monitoring) -- Proposed enhanced configuration -- Migration path - -**Key recommendations:** -- Increase `shared_buffers` 256MB → 512MB (multi-tenant workload) -- Add resource limits (CPU/memory) -- Enable PgBouncer pooler (Temporal has high connection count) -- Configure backups to MinIO -- Enable `enablePodMonitor: true` for Prometheus - -### 6. **Automated Bootstrap Script** - -**Created:** `bootstrap.sh` with: -- Preflight checks (kubectl, SOPS key, ArgoCD CLI) -- ArgoCD installation -- SOPS age secret creation (never in git) -- Bootstrap bundle application -- Wait loops for each component (CNPG, DDB, Forgejo) -- Secret copying (ddb to cicd namespace) -- Clear next-steps instructions - -**Usage:** -```bash -./bootstrap.sh -# ✅ ArgoCD + CNPG + DDB + Forgejo ready in <10 minutes -``` - -### 7. **Comprehensive Documentation** - -**Created:** -1. **GITOPS-REBUILD-PLAN.md** (15KB) - - Current vs proposed architecture diagrams - - Complete directory structure (single source of truth) - - Step-by-step execution plan - - Resource mapping table (no duplication) - - Testing procedures - - Day-2 operations guide - - FAQ - -2. **DDB-REVIEW.md** (6KB) - - PostgreSQL configuration review - - Performance tuning recommendations - - Backup/monitoring setup - - Enhanced configuration example - -3. **IMPLEMENTATION-CHECKLIST.md** (9KB) - - Pre-implementation tasks - - Two migration paths (fresh cluster vs incremental) - - Post-implementation verification (wave-by-wave) - - Cleanup steps - - Day-2 validation tests - - Rollback procedures - -4. **REVIEW-SUMMARY.md** (this file) - - High-level overview - - Key decisions explained - - What to review next - ---- - -## 🎯 Key Decisions Made - -### **Decision 1: DDB Cluster Stays Bootstrap-Only** - -**Rationale:** -- Circular dependency: Forgejo needs DDB → ArgoCD syncs from Forgejo → Cannot bootstrap DDB from git -- **Solution:** DDB deployed via `bootstrap-local/03-ddb-bootstrap.yaml`, never touched by ArgoCD -- **Trade-off:** DDB changes require manual `kubectl apply` (but still committed to git for record-keeping) -- **Alternative:** If Forgejo moves to external git host (GitHub), DDB could become GitOps-managed - -**ArgoCD Application for DDB:** Not created (would duplicate bootstrap). If needed later (external git), see `GITOPS-REBUILD-PLAN.md` for example. - -### **Decision 2: Forgejo Manual-Sync-Only** - -**Rationale:** -- Forgejo hosts the repo CI pushes to -- Auto-sync would let a bad CI commit break the system CI depends on -- **Solution:** `syncPolicy.automated: {}` (manual sync only) -- Application exists (`04-forgejo.yaml`) but never auto-syncs - -### **Decision 3: Wave Renumbering (0-8 Sequential)** - -**Before:** 00, 05, 10, 20, 30, 40, 50, 60 (helmfile convention, gaps for insertion) -**After:** 0, 1, 2, 3, 4, 5, 6, 7, 8 (ArgoCD native, cleaner) - -**Rationale:** -- ArgoCD sync-wave already handles ordering -- No need for gaps (can insert 2.5 if needed, or renumber) -- Easier to read/understand - -### **Decision 4: Bootstrap Script Over Manual Steps** - -**Before:** 30+ manual commands in BOOTSTRAP.md -**After:** Single `./bootstrap.sh` script - -**Rationale:** -- Reduces human error -- Idempotent (safe to re-run) -- Self-documenting (script IS the procedure) -- Faster iteration (cluster rebuild in <10 min) - -### **Decision 5: Separate k8s/data/cluster/ from k8s/data/schemas/** - -**Rationale:** -- Clear separation: bootstrap vs GitOps -- Prevents accidental deletion of cluster by ArgoCD prune -- Each directory has its own kustomization.yaml (no ambiguity) -- Easier to reason about dependencies - ---- - -## 📂 Files Created/Modified - -### **Created (New Files)** - -``` -k8s/bootstrap-local/ - kustomization.yaml - 00-namespaces.yaml - 01-argocd.yaml - 02-cnpg-operator.yaml - 03-ddb-bootstrap.yaml - 04-forgejo.yaml - -k8s/data/cluster/ - kustomization.yaml - -k8s/data/schemas/ - kustomization.yaml - -bootstrap.sh -GITOPS-REBUILD-PLAN.md -DDB-REVIEW.md -IMPLEMENTATION-CHECKLIST.md -REVIEW-SUMMARY.md (this file) -``` - -### **Modified (Updated Files)** - -``` -k8s/argocd/apps/40-data.yaml - - Changed path: k8s/data → k8s/data/schemas - - Changed sync-wave: 4 → 6 - - Updated comments -``` - -### **Deleted** - -``` -k8s/data/kustomization.yaml (replaced by subdirectories) -``` - -### **Moved** - -``` -k8s/data/ddb-cluster.yaml → k8s/data/cluster/ddb-cluster.yaml -k8s/data/forgejo-database.yaml → k8s/data/cluster/forgejo-database.yaml -k8s/data/authentik-database.yaml → k8s/data/schemas/authentik-database.yaml -k8s/data/temporal-database.yaml → k8s/data/schemas/temporal-database.yaml -k8s/data/temporal-visibility-database.yaml → k8s/data/schemas/temporal-visibility-database.yaml -k8s/data/schemas.yaml → k8s/data/schemas/schemas.yaml -k8s/data/db-init-job.yaml → k8s/data/schemas/db-init-job.yaml -``` - ---- - -## 🔍 What You Should Review - -### **1. Bootstrap Script** - -**File:** `bootstrap.sh` - -**Review for:** -- [ ] SOPS key path (`~/.sops/homelab-age.key` correct?) -- [ ] ArgoCD installation method (manifest URL vs Helm?) -- [ ] Wait timeout values (300s, 600s reasonable?) -- [ ] Error handling (should script exit or continue?) - -### **2. DDB Configuration** - -**File:** `DDB-REVIEW.md` - -**Decide:** -- [ ] Accept current config (256MB shared_buffers, no backups)? -- [ ] Implement enhanced config (512MB, PgBouncer, S3 backups)? -- [ ] When to apply changes (now vs after migration)? - -**If implementing enhanced config:** -1. Create MinIO bucket `ddb-backups` -2. Create `ddb-backup-s3` secret -3. Update `k8s/bootstrap-local/03-ddb-bootstrap.yaml` with enhanced spec -4. Test on staging cluster first - -### **3. Wave Structure** - -**Files:** `k8s/argocd/apps/*.yaml` - -**Verify:** -- [ ] Wave ordering correct? (0=substrate, 1=networking, ..., 8=apps) -- [ ] Dependencies satisfied? (e.g., schemas after secrets) -- [ ] Sync policies appropriate? (automated vs manual) - -**Current wave structure:** -``` -Wave 0: cert-manager, ingress-nginx, reloader, CNPG operator -Wave 1: Cilium policies, CoreDNS (networking) -Wave 2: MinIO, Longhorn, Prometheus (storage/observability) -Wave 3: Loki, Grafana, Promtail (logging) -Wave 4: SOPS secrets (all *.enc.yaml) -Wave 5: Vault, Authentik, Forgejo runner (IAM) -Wave 6: Database schemas (authentik-db, temporal-db, etc.) -Wave 7: Kafka, Redis, SQS (messaging) -Wave 8: Temporal, Portainer, cloudflared, etc. (applications) -``` - -### **4. Namespace Labels** - -**File:** `k8s/bootstrap-local/00-namespaces.yaml` - -**Verify PodSecurity labels correct:** -- [ ] `cicd` = privileged (Forgejo runner needs DinD) -- [ ] `ingress-nginx` = privileged (hostPort 80/443) -- [ ] `monitoring` = privileged (node-exporter hostPath) -- [ ] `logging` = privileged (promtail hostPath) -- [ ] All others = baseline (default)? - -### **5. Forgejo Configuration** - -**File:** `k8s/bootstrap-local/04-forgejo.yaml` - -**Verify inline values match:** -- [ ] Admin username/email correct? -- [ ] Domain `forgejo.riotpiao.com` correct? -- [ ] LoadBalancer IP `192.168.1.165` available? -- [ ] OAuth2 config matches Authentik setup? -- [ ] Redis connection string correct? - -**Sync with:** `k8s/security/ci-cd/forgejo-values.yaml` (keep both files in sync per comment) - -### **6. SOPS Secrets** - -**File:** `k8s/argocd/apps/04-secrets.yaml` (check if exists) - -**Verify:** -- [ ] SOPS plugin configured correctly? -- [ ] All `.enc.yaml` files decrypted successfully? -- [ ] `db-role-secrets.enc.yaml` applied before wave 6? - -**Check these secrets exist after bootstrap:** -```bash -kubectl get secret -n ddb authentik-db-role -kubectl get secret -n ddb temporal-db-role -kubectl get secret -n cicd ddb-cluster-app # Copied from ddb namespace -kubectl get secret -n argocd sops-age -``` - ---- - -## 🚀 Next Steps (Recommended Order) - -1. **Review all documents** (this file, GITOPS-REBUILD-PLAN.md, DDB-REVIEW.md) -2. **Decide on DDB config** (current vs enhanced) -3. **Review bootstrap.sh** and customize if needed -4. **Test on staging cluster first** (if available) -5. **Backup current production state** (PVCs, secrets) -6. **Choose migration path:** - - **Option A:** Fresh cluster rebuild (faster, cleaner) - - **Option B:** Incremental migration (safer, slower) -7. **Follow IMPLEMENTATION-CHECKLIST.md** step-by-step -8. **Validate each wave** before proceeding to next -9. **Document any issues** encountered -10. **Update CLAUDE.md** after successful migration - ---- - -## ⚠️ Important Notes - -### **Before You Start** - -1. **Backup PVCs** (Forgejo git repos, PostgreSQL data) - - Longhorn snapshots or Velero backup - - Export critical data (Forgejo repos, Authentik config) - -2. **Test SOPS key** works - ```bash - export SOPS_AGE_KEY_FILE=~/.sops/homelab-age.key - sops -d k8s/data/db-role-secrets.enc.yaml - # Should decrypt successfully - ``` - -3. **Verify Talos cluster healthy** - ```bash - talosctl health --nodes - kubectl get nodes - # All Ready - ``` - -### **During Implementation** - -- **Go wave-by-wave** - Don't skip verification steps -- **Watch ArgoCD UI** - https://argocd.riotpiao.com -- **Check logs** if any app stuck: - ```bash - kubectl logs -n argocd deployment/argocd-application-controller - kubectl logs -n argocd deployment/argocd-repo-server - ``` - -### **After Implementation** - -- **Test GitOps workflow** (make a change, push, verify auto-sync) -- **Test rollback** (git revert, verify auto-sync) -- **Document any deviations** from plan -- **Update runbooks** based on lessons learned - ---- - -## 📊 Resource Duplication Check (Final) - -**Bootstrap-only resources (NOT in ArgoCD GitOps):** -- ArgoCD itself -- CNPG operator (deployed as Application in bootstrap, but manual-managed) -- ddb-cluster -- forgejo-database -- Forgejo (exists as Application but manual-sync-only) -- Forgejo Redis - -**GitOps-managed resources (ArgoCD auto-syncs):** -- cert-manager, ingress-nginx, reloader -- Cilium policies, CoreDNS config -- MinIO, Longhorn config, Prometheus -- Loki, Grafana, Promtail -- SOPS secrets -- Vault, Authentik, Forgejo runner -- Database schemas (authentik-db, temporal-db, etc.) -- Kafka, Redis, SQS -- Temporal, Portainer, cloudflared, etc. - -**Terraform-managed resources:** -- Talos machine configs (controlplane.tftpl) -- No k8s resources - -**✅ No overlap - each resource has exactly ONE source of truth.** - ---- - -## 🎯 Success Criteria - -After successful implementation, you should have: - -- [x] **Single command bootstrap** (`./bootstrap.sh`) -- [x] **Zero manual kubectl apply** (except bootstrap) -- [x] **Git is source of truth** (all changes via push) -- [x] **No resource duplication** -- [x] **Clear wave ordering** (0-8) -- [x] **Fast iteration** (cluster rebuild <10 min) -- [x] **Rollback via git** (revert commit, auto-syncs) -- [x] **Well-documented** (5 comprehensive docs) - ---- - -## 📞 Questions to Resolve - -Before implementation, decide on: - -1. **Fresh cluster vs incremental migration?** - - Fresh = faster, cleaner (requires downtime) - - Incremental = safer, slower (zero downtime possible) - -2. **DDB enhanced config now or later?** - - Now = better performance from start - - Later = faster migration, can optimize afterward - -3. **Wave renumbering (00→0, 05→1, etc.)?** - - Yes = cleaner, consistent with plan - - No = keep current, less churn - -4. **Delete old k8s/argocd/bootstrap/ after migration?** - - Yes = cleaner repo - - No = keep for reference - -5. **Update CLAUDE.md immediately or after validation?** - - Immediately = stays current - - After = confirms plan actually works - ---- - -**Ready to proceed?** Start with `IMPLEMENTATION-CHECKLIST.md` and check off each step. - -**Need clarification?** Review specific sections in `GITOPS-REBUILD-PLAN.md`. - -**Performance tuning?** See `DDB-REVIEW.md` for PostgreSQL optimization. - diff --git a/STORAGE-ARCHITECTURE-CLARIFICATION.md b/STORAGE-ARCHITECTURE-CLARIFICATION.md deleted file mode 100644 index f38daee..0000000 --- a/STORAGE-ARCHITECTURE-CLARIFICATION.md +++ /dev/null @@ -1,304 +0,0 @@ -# Storage Architecture Clarification - -**Issue:** CLAUDE.md contradicts actual Longhorn configuration manifests. - ---- - -## 🚨 Contradiction Found - -### CLAUDE.md States: -``` -| Node | Storage | -|------|---------| -| talos-cp-1 (.213) | sole Longhorn node | -| talos-cp-2 (.163) | none | -| talos-cp-3 (.166) | none | - -"Only talos-cp-1 runs workloads and holds storage → - stateful services are single-instance." -``` - -### Actual Longhorn Manifests Show: - -**1. Explicit Node CRDs for ALL 3 nodes:** -```yaml -# k8s/infrastructure/longhorn/longhorn-nodes.yaml ---- -apiVersion: longhorn.io/v1beta2 -kind: Node -metadata: - name: talos-cp-2 -spec: - allowScheduling: true # ← Storage enabled! - disks: - default-disk: - allowScheduling: true - path: /var/lib/longhorn ---- -apiVersion: longhorn.io/v1beta2 -kind: Node -metadata: - name: talos-cp-3 -spec: - allowScheduling: true # ← Storage enabled! - disks: - default-disk: - allowScheduling: true - path: /var/lib/longhorn -``` - -**2. Taint toleration for control-plane:** -```yaml -# longhorn-taint-toleration.yaml -value: "node-role.kubernetes.io/control-plane:NoSchedule" -# Allows Longhorn DaemonSet on ALL control-plane nodes -``` - -**3. StorageClass with 3 replicas:** -```yaml -# longhorn-wffc-storageclass.yaml -parameters: - numberOfReplicas: "3" # ← 3-way replication! - volumeBindingMode: WaitForFirstConsumer -``` - -**4. PostSync job to expand existing volumes:** -```yaml -# expand-replicas-job.yaml -# Patches ALL volumes from 1 → 3 replicas -``` - ---- - -## 🔍 What's the Truth? - -**Need to verify cluster state:** - -```bash -# Check Longhorn nodes -kubectl get nodes.longhorn.io -n longhorn-system -o wide - -# Expected output (if 3-node setup is actually working): -# NAME READY ALLOWSCHEDULING SCHEDULABLE AGE -# talos-cp-1 True true true Xd -# talos-cp-2 True true true Xd -# talos-cp-3 True true true Xd - -# Check actual replica counts -kubectl get volumes.longhorn.io -n longhorn-system \ - -o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas,STATE:.status.state' - -# Check DDB PVCs -kubectl get pvc -n ddb -kubectl describe pvc -n ddb | grep -A 5 "Volumes:" -``` - ---- - -## 📊 Two Possible Scenarios - -### **Scenario A: 3-Node Replication is Active** ✅ - -**If the Longhorn manifests are actually deployed:** - -``` -Storage Architecture: -┌─────────────────────────────────────────────────┐ -│ DDB PVC (10Gi, Longhorn) │ -├─────────────────────────────────────────────────┤ -│ Replica 1: talos-cp-1:/var/lib/longhorn │ -│ Replica 2: talos-cp-2:/var/lib/longhorn │ -│ Replica 3: talos-cp-3:/var/lib/longhorn │ -└─────────────────────────────────────────────────┘ - -DDB PostgreSQL Pods: -┌──────────────┬──────────────┬──────────────┐ -│ ddb-cluster-1│ ddb-cluster-2│ ddb-cluster-3│ -│ (cp-1) │ (cp-2) │ (cp-3) │ -│ Primary │ Replica │ Replica │ -└──────────────┴──────────────┴──────────────┘ - ↓ ↓ ↓ - Reads all 3 Longhorn replicas locally - (dataLocality: best-effort) - -Failure Scenarios: -❌ cp-1 fails → Replica 2 & 3 still available -❌ cp-2 fails → Replica 1 & 3 still available -❌ cp-3 fails → Replica 1 & 2 still available -✅ Data survives ANY single node failure -``` - -**This is TRUE HA storage!** ✅ - -### **Scenario B: CLAUDE.md is Correct** ❌ - -**If Longhorn manifests are NOT actually deployed:** - -``` -Storage Architecture: -┌─────────────────────────────────────────────────┐ -│ DDB PVC (10Gi, Longhorn) │ -├─────────────────────────────────────────────────┤ -│ Replica 1: talos-cp-1:/var/lib/longhorn │ -│ (NO replicas on cp-2, cp-3) │ -└─────────────────────────────────────────────────┘ - -DDB PostgreSQL Pods: -┌──────────────┬──────────────┬──────────────┐ -│ ddb-cluster-1│ ddb-cluster-2│ ddb-cluster-3│ -│ (cp-1) │ (cp-2) │ (cp-3) │ -│ Primary │ Replica │ Replica │ -└──────────────┴──────────────┴──────────────┘ - ↓ ↓ ↓ - ALL pods must read from cp-1 over network - (single point of failure) - -Failure Scenarios: -❌ cp-1 disk fails → PERMANENT DATA LOSS -❌ cp-1 node fails → All PVCs inaccessible -❌ NO HA for storage at all -``` - -**This is NOT HA storage!** ❌ - ---- - -## 🎯 Action Required: Verify Cluster State - -**Run these commands to determine which scenario is true:** - -```bash -# 1. Check if Longhorn Node CRs exist -kubectl get nodes.longhorn.io -n longhorn-system - -# 2. Check if taint toleration is set -kubectl get setting taint-toleration -n longhorn-system -o yaml - -# 3. Check actual volume replica counts -kubectl get volumes.longhorn.io -n longhorn-system \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.numberOfReplicas}{"\n"}{end}' - -# 4. Check DDB PVC details -kubectl get pvc -n ddb -o yaml | grep -A 10 "volumeName:" - -# 5. Check Longhorn DaemonSet pods -kubectl get pods -n longhorn-system -o wide | grep longhorn-manager -# Should show pods on ALL 3 nodes if 3-node setup is active -``` - ---- - -## 🔧 If Scenario B (Single Node) is True - -**You need to deploy the Longhorn HA configuration:** - -```bash -# Apply the Longhorn HA manifests -kubectl apply -k k8s/infrastructure/longhorn/ - -# This will: -# 1. Create Node CRs for cp-2, cp-3 -# 2. Set taint toleration -# 3. Create 3-replica StorageClass -# 4. Run PostSync job to expand existing volumes - -# Verify expansion happened -kubectl get job longhorn-expand-replicas -n longhorn-system -kubectl logs job/longhorn-expand-replicas -n longhorn-system -``` - ---- - -## 📝 Corrected Documentation - -**If 3-node replication IS active, update CLAUDE.md:** - -```diff - | Node | IP | Zone | Scheduling | Storage | - |------|----|----|-----------|---------| --| `talos-cp-1` | .213 | az-a | schedulable (all workloads) | sole Longhorn node | --| `talos-cp-2` | .163 | az-b | dedicated (`NoSchedule`) | none | --| `talos-cp-3` | .166 | az-c | dedicated (`NoSchedule`) | none | -+| `talos-cp-1` | .213 | az-a | schedulable (all workloads) | Longhorn (replica 1/3) | -+| `talos-cp-2` | .163 | az-b | dedicated (`NoSchedule`) | Longhorn (replica 2/3) | -+| `talos-cp-3` | .166 | az-c | dedicated (`NoSchedule`) | Longhorn (replica 3/3) | - --holds storage → stateful services are single-instance. -+holds storage → stateful services are HA (3-replica volumes). -``` - -**And update the hard rule:** - -```diff --🔴 **NEVER rename or wipe `talos-cp-1` (.213).** It is the sole Longhorn storage --node — all replicas are pinned to that node name. Renaming orphans its Longhorn --node CR and faults every volume (permanent data loss). -+🔴 **NEVER rename ANY control-plane node.** Longhorn volumes have 3 replicas -+pinned to specific node names (talos-cp-1, talos-cp-2, talos-cp-3). Renaming -+ANY node orphans its Longhorn Node CR and degrades all volumes. Loss of 2+ nodes -+simultaneously = permanent data loss. -``` - ---- - -## 🎯 Impact on DDB Configuration - -**If 3-node replication is active:** - -### DDB cluster is actually HA! ✅ - -``` -Compute HA: 3 PostgreSQL pods across 3 nodes ✅ -Storage HA: 3 Longhorn replicas across 3 nodes ✅ -Network HA: 3 etcd members, Cilium IPAM ✅ - -Failure tolerance: -- 1 node failure: Cluster continues (2/3 quorum) -- 1 disk failure: Data intact (2/3 replicas) -- 2 nodes fail: ❌ Etcd loses quorum, data degrades -``` - -**Current DDB config is acceptable if 3-replica storage is confirmed.** - -### If Single-Node Storage: - -**You MUST either:** - -1. **Deploy Longhorn 3-node config** (recommended) - ```bash - kubectl apply -k k8s/infrastructure/longhorn/ - ``` - -2. **Reduce DDB to 1 instance** (match storage reality) - ```yaml - # k8s/data/cluster/ddb-cluster.yaml - instances: 1 # Single instance if single-node storage - ``` - -3. **Add external backup** (mitigate single-node risk) - ```yaml - # DDB backup to MinIO (see DDB-REVIEW.md) - backup: - barmanObjectStore: - destinationPath: s3://ddb-backups/ - ``` - ---- - -## ✅ Recommended Actions (Priority Order) - -1. **[ ] Verify cluster state** (run commands above) -2. **[ ] If single-node storage:** Deploy Longhorn 3-node config -3. **[ ] Wait for replicas to expand** (watch Longhorn UI) -4. **[ ] Update CLAUDE.md** with correct topology -5. **[ ] Update DDB-REVIEW.md** to reflect actual HA status -6. **[ ] Update GITOPS-REBUILD-PLAN.md** storage section -7. **[ ] Document findings** in TROUBLESHOOTING.md - ---- - -**Next:** Please run the verification commands and report back: -- Are there 3 Longhorn nodes? -- What are the actual replica counts on DDB PVCs? -- Is the expand-replicas job present/completed? - diff --git a/docs/adr/0001-gitops-bootstrap-and-cd.md b/docs/adr/0001-gitops-bootstrap-and-cd.md new file mode 100644 index 0000000..2e4cb1e --- /dev/null +++ b/docs/adr/0001-gitops-bootstrap-and-cd.md @@ -0,0 +1,218 @@ +# ADR 0001 — GitOps Bootstrap, CD, and Coverage + +- **Status:** Proposed +- **Date:** 2026-07-23 +- **Deciders:** Homelab platform owner +- **Context tags:** bootstrap, gitops, argocd, forgejo, cicd, supply-chain, disaster-recovery + +One delivery system, three coupled parts: + +- **Part A — Bootstrap:** bare cluster → self-hosted GitOps control plane (breaks the ArgoCD ↔ Forgejo circle). +- **Part B — Steady-state CD:** how app changes flow to the cluster once it's up. +- **Part C — Coverage & organization:** every resource captured by exactly one Application; no orphans, no duplicates. + +Single 3-control-plane Talos cluster. GitOps via ArgoCD (app-of-apps under `k8s/argocd/apps/`, waves 0–8); CI via Forgejo Actions on a self-hosted runner. + +--- + +## Part A — Bootstrap + +### Problem — the circle + +``` +ArgoCD ──syncs from──▶ git repo ──hosted by──▶ Forgejo ──deployed by──▶ (the repo) +``` + +Forgejo hosts the repo ArgoCD reconciles. Today the circle is broken by installing Forgejo + Longhorn + CNPG **by hand** (`bootstrap-local/`, `bootstrap/phase1-5/`) — imperative, off-git, never reconciled, human-ordered rebuild. + +### Decision — GitHub seed + Forgejo pull-mirror + cutover + +GitHub is the seed and permanent DR source; Forgejo is steady-state truth after cutover and stays synced via a native pull-mirror. + +**No adoption.** Longhorn, CNPG, and Forgejo are **pure Phase-0 manual bootstrap** — ArgoCD never manages them (this is the deliberate divergence from the "ArgoCD adopts hand-installed infra" pattern). They must exist *before* ArgoCD can sync from a git host at all, so making them Apps would re-introduce the circle. ArgoCD (seeded from GitHub) deploys only everything *after* the control plane. + +``` +Phase 0 (manual, once — the whole control plane): + Talos ─ Cilium CNI ─ Longhorn ─ CNPG operator ─ forgejo-db Cluster (wait Ready) + ─ Forgejo + Redis ─ push repo ─ helm install argocd ─ apply root-app→GitHub +Phase 1 (ArgoCD from GitHub seed): waves 0–8 = monitoring · logging · iam · messaging · + apps. Does NOT manage Longhorn / CNPG / Forgejo (those stay bootstrap-owned). +Phase 2 (cutover, once): flip root-app repoURL GitHub→Forgejo; Forgejo pull-mirrors + GitHub (live DR seed). +``` + +Manual order matters: CNPG operator + `forgejo-db` Cluster must be **Ready** (`.status.phase: "Cluster in healthy state"`, not just created) before `helm install forgejo`. Wait on it. `forgejo-db` = dedicated CNPG Cluster (`owner: forgejo`), preferred anti-affinity + control-plane tolerations so it can't deadlock on a partly-schedulable 3-CP. + +**Options:** (A) one-time seed + manual cutover — mirror drifts. (B) **seed + pull-mirror + cutover — chosen** — safe cutover, always-current DR. (C) external-permanent — no self-hosted control plane. Rejected. + +### Repository credentials (private GitHub seed) + +Two read-only creds, neither needs write: + +1. **ArgoCD → GitHub: SSH deploy key.** `ssh-keygen -t ed25519 -f argocd_seed -N ""`; add public key to repo → Deploy keys (read-only). Private key → ArgoCD repo-credential Secret at Phase 0 (before any secret controller): + ```yaml + apiVersion: v1 + kind: Secret + metadata: + name: seed-github-repo + namespace: argocd + labels: { argocd.argoproj.io/secret-type: repository } + stringData: + type: git + url: git@github.com:/homelab.git # SSH; must match root-app repoURL + sshPrivateKey: | + -----BEGIN OPENSSH PRIVATE KEY----- + ... + ``` +2. **Forgejo → GitHub: fine-grained PAT (Contents: Read).** HTTPS token for the pull-mirror; SOPS-encrypted in-repo (consumed at w1 once SOPS plugin is up). + +Rotation: replace GitHub-side key/token, re-apply Secret / re-encrypt SOPS. Cutover keeps both (mirror stays live). + +--- + +## Part B — Steady-state CD + +**CI does build + push + sign only.** Dedicated controllers own promotion, reconciliation, safe rollout. Git = single source; nothing mutates cluster or CD repo by string-replace. + +### Current problems + +1. **`sed` image-bump** (`ci.yml:72`) — fragile, non-atomic, races, `git revert`-only rollback. "CI mutates CD repo" anti-pattern. +2. **Registry hostname split-brain** — one Forgejo addressed 4 ways; in-cluster pods pull via public ingress name (nginx hairpin, CLAUDE.md trap). `api` defaults to `:latest`. +3. **Fragmented truth** — `project: default` vs `homelab`; `api` App points at external `rock/deploy.git`. +4. **DinD runner** — privileged; RWO cache pins it to cp-1 (SPOF). +5. **No progressive delivery / supply-chain gate** — all `prune+selfHeal`, no canary/analysis/rollback, unsigned images. + +### Decision + +| Concern | From | To | +|---|---|---| +| Image promotion | CI `sed` → deploy repo | **ArgoCD Image Updater**, git write-back, digest-pinned | +| Build | DinD (privileged, cp-1) | **rootless BuildKit** (or Kaniko) + registry cache | +| Registry addressing | 4 hostnames | in-cluster `*.svc.cluster.local:3000`; one external push name | +| Delivery | raw auto-sync | **Argo Rollouts** canary + Prometheus `AnalysisTemplate` (app tier) | +| Supply chain | none | **cosign sign + Kyverno `verifyImages`** | +| Success signal | pod `Running` | **Playwright E2E** (video/trace) gate | +| Projects/repos | `default`+`homelab`, 2 repos | one `homelab` project, one convention | + +``` +push→main → runner(BuildKit): test · build·push /@sha256 · cosign sign + → Image Updater watches registry, writes digest back to git + → app-of-apps: infra/platform = plain sync ; app tier = Rollouts canary + Kyverno gate +``` + +**Options:** promotion — Image Updater (chosen) over `kustomize edit` (still coupled) / Flux (2nd engine). Build — BuildKit (chosen) over DinD / Kaniko. Delivery — Rollouts app-tier-only (chosen) over none / Flagger (needs mesh). + +### Deployment verification (synthetic E2E) + +`Running` proves scheduling, not the user path (ingress/TLS/OIDC/session/UI). Signal ladder: readiness → synthetic HTTP → **Playwright browser E2E** (gate on the top rung). Suite hits **public ingress hostnames**, records video+trace → MinIO `e2e-artifacts/` (private, lifecycle-expired). Journeys: + +- **Portainer** — login page renders. +- **Authentik** — sign in (bootstrap `akadmin` from secret store), dashboard loads. +- **OAuth/OIDC federation** — each app (ArgoCD, Forgejo, Grafana…): click SSO → complete Authentik → land back authenticated. Catches client mis-registration / redirect-URI / broken provider that a green pod hides. + +Runs two ways: **deploy gate** (Rollouts `AnalysisTemplate` / PostSync Job → fails sync/rollout) and **continuous smoke** (`CronJob` → Alertmanager). Creds from SOPS/Vault as env, never hardcoded. Constraint: **WebKit only** (Safari engine) per requirement. Suite lives at `tests/e2e/`; manifests at `k8s/platform/e2e/` (unwired until phase 7). + +--- + +## Part C — GitOps coverage & organization + +**Principle: every deployable directory is the source of exactly one Application, grouped by wave.** No dead aggregator kustomizations, no orphan manifests, no second app-of-apps. Layer folders stay; the tree is reconciled to the map. An App uses a git **`path:`** for manifests/kustomize bases, a **`$values` valueFile** when it only parameterizes a remote Helm chart. Secrets are the exception — one `sops-secrets` App globs all `*.enc.yaml` from repo root. + +### Folder ↔ Application map (as-built) + +Consolidated to two folders — `k8s/infra` (platform/infra tier) and `k8s/apps` +(workloads) — replacing the old `infrastructure/platform/security/applications` +layer folders (deleted). + +``` +k8s/ +├── argocd/ root/ + apps/ (waves) + projects/ # engine, not an App target +├── bootstrap/ cert-manager w1 · cilium w0 · coredns w0 · ingress w1 # Phase-0 substrate +├── infra/ +│ ├── longhorn/ →longhorn-config w1 +│ ├── minio/ →minio-tenant (+minio-operator-values for operator) w1 +│ ├── monitoring/ →monitoring-config w2 · crds/ →prometheus-crds w0 +│ │ *-values →prometheus/blackbox w1 +│ ├── logging/*-values →grafana/loki/promtail w2 +│ ├── data-schemas/ →data-schemas w6 (CNPG operator + forgejo-db → Phase-0 manual bootstrap, NOT an App) +│ ├── iam/ →iam-jobs w3 (+authentik/vault values) +│ ├── forgejo-runner/ →forgejo-runner w3 +│ └── cicd/ →cicd-config w3 (Forgejo OAuth job) +└── apps/ + ├── cloudflared/ duckdns/ homarr/ portainer/ temporal/ →w8 + └── messaging/{kafka-cluster,queue-crd,management-service} →w5–7 +``` + +### Orphans — declared in git, reconciled by NOTHING + +| Path | Count | Kind | Action | +|---|---|---|---| +| `platform/monitoring/{alerts,servicemonitors,dashboards}` | 27 | PrometheusRule / ServiceMonitor / dashboard ConfigMap | wire → **`monitoring-config` App (done)** | +| `security/ci-cd/forgejo-oauth-setup-job.yaml` | 1 | Job+SA (PostSync) | wire → `cicd-config` App | +| CNPG operator + `forgejo-db` Cluster | — | CNPG install + Cluster | **Phase-0 manual bootstrap** (`bootstrap/phase2-cnpg`, `bootstrap/phase3-forgejo`), never an App — making it an App re-introduces the circle | + +Root cause (monitoring): no App used `path: k8s/platform/monitoring` — only crds + valueFiles; the aggregator kustomization chain was dead. Now fixed by one `monitoring-config` App (one kustomization, **no `namespace:` transformer** — PrometheusRules live in per-app namespaces; `ServerSideApply=true` for large dashboard JSON). + +### Stale / dead — delete, do NOT wire + +- `applications/sqs/argocd/*` — **second competing app-of-apps** (repo `rock/kafaka-management-service.git`, project `kmsvc`). +- `applications/sqs/{queues,environments,helmfile.yaml.gotmpl,kustomization.yaml}` — helmfile-era. +- **`applications/temporal/elasticsearch.yaml`** — superseded (see Temporal datastore below). +- `applications/temporal/queues/`, root `applications/kustomization.yaml` — example / legacy overlay. +- `security/iam/authentik-migrations-job.yaml` — dead/broken (server self-migrates). +- Dead aggregator kustomizations (`infrastructure/`, `platform/`, `security/`, `ci-cd/`), `logging/minio-values.yaml`, `infrastructure/minio/test/`, scripts / `.env.example`. + +**Experimental — leave unmanaged** (per `60-applications.yaml`): `applications/{dev-tools,forge,llm,shadowsocks}`. + +### 🔴 Temporal datastore = PostgreSQL (hard constraint) + +Temporal runs on the shared CNPG **PostgreSQL** (`ddb-cluster`), for BOTH the main store and **visibility**. **NOT Cassandra** (the Helm chart's default) and **NOT Elasticsearch.** `temporal-values.yaml` sets `elasticsearch.enabled: false` and points visibility at Postgres (`temporal-visibility-database`). Therefore `applications/temporal/elasticsearch.yaml` is dead → delete. When touching Temporal values, verify persistence stays on the pinned chart's SQL schema (chart schema drift silently reverts to Cassandra — see CLAUDE.md). + +### Reconcile actions (tree → map) + +1. **Restore** live-consumed paths currently staged-deleted: `infrastructure/{longhorn,minio}`, `security/iam`, `security/ci-cd/charts/forgejo-runner`, `platform/{logging,monitoring}` values + crds. Without them their Apps have no source. +2. **Add Apps:** `monitoring-config` (done), `cicd-config` (oauth job). +3. **Delete** the stale/dead list above. +4. **CNPG** (operator + `forgejo-db` Cluster) stays **Phase-0 manual bootstrap** — never an App (circular). + +⚠️ The current staged deletion is **over-broad** — it wiped live GitOps sources (longhorn, minio, iam, monitoring/logging values). Committing as-is breaks those Apps' sync. Curate: restore live, delete only confirmed orphans. + +ApplicationSet git-generator considered, rejected: the hand-tuned per-App settings (multi-source helm+values, `managedNamespaceMetadata`, `ServerSideApply`, waves) don't fit one template. + +--- + +## Consequences + +**Positive:** Forgejo/Longhorn/CNPG fully declarative; Phase-0 shrinks to "CNI + ArgoCD + root-app"; repeatable rebuild seeded from always-current GitHub (free off-site DR); CI shrinks to build/push/sign; digest-pinned auditable promotions; no privileged DinD; CI survives cp-1 loss; auto-rollback on SLO breach; only signed images admit; monitoring finally reconciled. + +**Costs:** GitHub egress + deploy key needed at Phase 0; one cutover flip remains; two read creds (deploy key + SOPS PAT); three new controllers (Image Updater, Rollouts, Kyverno) + `Deployment`→`Rollout` migration; one-time registry-hostname normalization. + +--- + +## Migration phases (one PR each, impact-over-risk) + +0. **Consolidate bootstrap + coverage.** Single-source bootstrap; convert phase1–3 to ArgoCD Apps (seed repo); Forgejo pull-mirror + SOPS PAT; script = Phase-0 + cutover only. Curate the staged deletion (Part C reconcile): restore live paths, add `monitoring-config`(done)+`cicd-config`, delete stale/dead + `data/cluster`. CNPG (operator + `forgejo-db`) stays Phase-0 manual bootstrap. +1. **Kill `sed`.** `argocd-image-updater` (~w4); annotate `image-list` + git write-back; drop ci.yml step 3. +2. **Normalize registry.** In-cluster pulls via `forgejo-gitea-http.cicd.svc.cluster.local:3000`; retire `forgejo.forge.*`; pin digests. +3. **Rootless BuildKit.** Drop `az-a` nodeSelector; `--cache-to type=registry`. +4. **Argo Rollouts.** Controller (~w4); `api`/app tier → `Rollout` + canary `AnalysisTemplate`. +5. **Supply chain.** cosign (Forgejo OIDC) + Kyverno `verifyImages`. +6. **Consolidate projects.** One `homelab` project; monorepo-only vs single `deploy` repo. +7. **Synthetic E2E.** Wire `tests/e2e` suite as Rollouts/PostSync gate + `CronJob`; Portainer + Authentik + OAuth journeys. + +--- + +## Open questions + +1. ArgoCD install — `helm install` (leaning) vs Talos `extraManifests`. +2. Post-cutover primary — push GitHub (mirror pulls) vs push Forgejo (push-mirror out). Pull-from-GitHub keeps GitHub authoritative. +3. Cutover — scripted `kubectl patch` of root repoURL vs PostSync hook gated on Forgejo health. + +--- + +## References + +- `CLAUDE.md` — ArgoCD gotchas (SkipDryRunOnMissingResource, wait-for-Ready, repoURL/ingress hairpin, Helm schema drift → Temporal/Cassandra) +- `k8s/argocd/{root,apps,projects}/` · `k8s/bootstrap/` (to be superseded) +- `k8s/security/ci-cd/{example-workflows/ci.yml,charts/forgejo-runner,deploy-scaffold}` +- `tests/e2e/` (Playwright WebKit suite) · `k8s/platform/monitoring/` (monitoring-config) diff --git a/fix-forgejo.sh b/fix-forgejo.sh deleted file mode 100755 index 019c335..0000000 --- a/fix-forgejo.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -echo "🔧 Fixing Forgejo issues..." -echo "" - -# Issue 1: Multi-Attach - old pod still holding the volume -echo "==> Issue 1: Cleaning up old Forgejo deployment" -echo "Current deployments:" -kubectl get deployment -n cicd | grep forgejo - -echo "" -OLD_DEPLOYMENT=$(kubectl get deployment -n cicd -o name | grep -E "forgejo-[0-9]" | grep -v gitea) -if [ -n "$OLD_DEPLOYMENT" ]; then - echo "Found old deployment: $OLD_DEPLOYMENT" - kubectl delete $OLD_DEPLOYMENT -n cicd --wait=true - echo " ✓ Old deployment deleted" -else - echo " No old deployment found, checking for orphaned pods..." - kubectl get pods -n cicd -l app.kubernetes.io/name=gitea -o name | while read pod; do - POD_NAME=$(echo $pod | cut -d/ -f2) - if [[ ! "$POD_NAME" =~ "forgejo-gitea" ]]; then - echo " Deleting orphaned pod: $POD_NAME" - kubectl delete pod -n cicd $POD_NAME --force --grace-period=0 - fi - done -fi - -# Issue 2: Missing homelab-ca ConfigMap -echo "" -echo "==> Issue 2: Checking homelab-ca ConfigMap" -if kubectl get configmap homelab-ca -n cicd &>/dev/null; then - echo " ✓ homelab-ca already exists" -else - echo " ⚠️ homelab-ca not found in cicd namespace" - echo " Checking if it exists elsewhere..." - - # Check common namespaces - for ns in default kube-system cert-manager; do - if kubectl get configmap homelab-ca -n $ns &>/dev/null; then - echo " Found in namespace: $ns" - echo " Copying to cicd namespace..." - kubectl get configmap homelab-ca -n $ns -o yaml | \ - sed 's/namespace: '$ns'/namespace: cicd/' | \ - kubectl apply -f - - echo " ✓ Copied homelab-ca to cicd" - break - fi - done - - # If still not found, check if we need to create it - if ! kubectl get configmap homelab-ca -n cicd &>/dev/null; then - echo " Creating empty homelab-ca ConfigMap (you may need to populate it)..." - kubectl create configmap homelab-ca -n cicd --from-literal=ca.crt="" - echo " ⚠️ Created empty ConfigMap - update with actual CA if needed" - fi -fi - -echo "" -echo "==> Waiting for volume detach (30s)..." -sleep 30 - -echo "" -echo "==> Current Forgejo pod status:" -kubectl get pods -n cicd -l app.kubernetes.io/name=gitea - -echo "" -echo "==> If still in Init or Pending, describe one pod:" -POD=$(kubectl get pods -n cicd -l app.kubernetes.io/name=gitea --no-headers | head -1 | awk '{print $1}') -if [ -n "$POD" ]; then - kubectl describe pod -n cicd $POD | grep -A 10 "Events:" | head -15 -fi - -echo "" -echo "✅ Fixes applied!" -echo "" -echo "Next: Monitor pod startup" -echo " kubectl get pods -n cicd -w" diff --git a/k8s/applications/dev-tools/Chart.yaml b/k8s/applications/dev-tools/Chart.yaml deleted file mode 100644 index 719b96e..0000000 --- a/k8s/applications/dev-tools/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: claude-terminal -description: Persistent Claude CLI terminal running in tmux with web access via gotty -type: application -version: 1.0.0 -appVersion: "1.0" diff --git a/k8s/applications/dev-tools/Dockerfile b/k8s/applications/dev-tools/Dockerfile deleted file mode 100644 index 0cab4e0..0000000 --- a/k8s/applications/dev-tools/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM --platform=linux/amd64 ubuntu:24.04 - -RUN apt-get update && apt-get install -y \ - tmux \ - curl \ - git \ - build-essential \ - nodejs \ - npm \ - bash \ - && rm -rf /var/lib/apt/lists/* - -# Install gotty (web terminal access) -RUN curl -sL https://github.com/sorenisanerd/gotty/releases/download/v1.5.0/gotty_linux_amd64.tar.gz | \ - tar xz -C /usr/local/bin && chmod +x /usr/local/bin/gotty - -# Install Claude CLI -RUN npm install -g claude-code-cli 2>&1 || echo "Note: Claude CLI will be available after NPM package is published" - -WORKDIR /root - -# Create persistent storage dir -RUN mkdir -p /root/.claude /root/.config /root/.cache - -# Entrypoint: start tmux session and gotty -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -EXPOSE 8080 - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/k8s/applications/dev-tools/README.md b/k8s/applications/dev-tools/README.md deleted file mode 100644 index e239bfc..0000000 --- a/k8s/applications/dev-tools/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Claude Terminal — Persistent Remote Dev Environment - -Runs Claude CLI in a persistent tmux session with web-based terminal access via gotty. - -## Building the Image - -Build for `linux/amd64`: - -```bash -cd homelab -docker buildx build --platform linux/amd64 \ - -t forgejo.riotpiao.com/rock/claude-terminal:latest \ - -f k8s/dev-tools/Dockerfile \ - k8s/dev-tools - -# Log in to Forgejo registry -docker login forgejo.riotpiao.com \ - --username ci-bot \ - --password "$(talos get cluster/iam/agents/ci-bot --key token)" - -# Push -docker push forgejo.riotpiao.com/rock/claude-terminal:latest -``` - -Or use the provided build script: - -```bash -./k8s/dev-tools/build.sh -``` - -## Deployment - -Update `values.yaml` if needed, then deploy via helmfile: - -```bash -helmfile apply -l name=claude-terminal -``` - -Access the terminal at: **https://claude.riotpiao.com** - -## Persistent Storage - -- All Claude configuration stored in `/root/.claude` (persistent PVC, 10Gi Longhorn) -- Survives pod restarts and node reboots -- Accessible immediately after reconnecting - -## SSH Access (Optional) - -To add SSH access, extend the Dockerfile to include openssh-server and mount the PVC as home directory. diff --git a/k8s/applications/dev-tools/build.sh b/k8s/applications/dev-tools/build.sh deleted file mode 100755 index 2658d66..0000000 --- a/k8s/applications/dev-tools/build.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -euo pipefail - -REGISTRY="forgejo.riotpiao.com" -IMAGE_NAME="rock/claude-terminal" -TAG="latest" -FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}" - -echo "🔨 Building Claude Terminal image for linux/amd64..." -docker buildx build --platform linux/amd64 \ - -t "${FULL_IMAGE}" \ - -f Dockerfile \ - . || { echo "❌ Build failed"; exit 1; } - -echo "🔓 Logging in to Forgejo registry..." -REGISTRY_TOKEN=$(talos get cluster/iam/agents/ci-bot --key token) -echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \ - --username ci-bot \ - --password-stdin || { echo "❌ Login failed"; exit 1; } - -echo "📤 Pushing image to registry..." -docker push "${FULL_IMAGE}" || { echo "❌ Push failed"; exit 1; } - -echo "✅ Successfully pushed ${FULL_IMAGE}" diff --git a/k8s/applications/dev-tools/entrypoint.sh b/k8s/applications/dev-tools/entrypoint.sh deleted file mode 100755 index bf102e3..0000000 --- a/k8s/applications/dev-tools/entrypoint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# Start tmux server in background -tmux new-session -d -s claude -c /root "bash" - -# Give tmux a moment to stabilize -sleep 1 - -# Start gotty serving the tmux session -# -w: allow write (make terminal interactive) -# -p 8080: listen on port 8080 -exec gotty -p 8080 -w tmux attach-session -t claude diff --git a/k8s/applications/dev-tools/kustomization.yaml b/k8s/applications/dev-tools/kustomization.yaml deleted file mode 100644 index 907141b..0000000 --- a/k8s/applications/dev-tools/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: dev-tools -resources: [] -# Helm chart deployed via ArgoCD Helm source diff --git a/k8s/applications/dev-tools/templates/_helpers.tpl b/k8s/applications/dev-tools/templates/_helpers.tpl deleted file mode 100644 index 933b565..0000000 --- a/k8s/applications/dev-tools/templates/_helpers.tpl +++ /dev/null @@ -1,49 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "claude-terminal.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -*/}} -{{- define "claude-terminal.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "claude-terminal.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "claude-terminal.labels" -}} -helm.sh/chart: {{ include "claude-terminal.chart" . }} -{{ include "claude-terminal.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "claude-terminal.selectorLabels" -}} -app.kubernetes.io/name: {{ include "claude-terminal.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} diff --git a/k8s/applications/dev-tools/templates/deployment.yaml b/k8s/applications/dev-tools/templates/deployment.yaml deleted file mode 100644 index 0fd322d..0000000 --- a/k8s/applications/dev-tools/templates/deployment.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "claude-terminal.fullname" . }} - labels: - {{- include "claude-terminal.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "claude-terminal.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "claude-terminal.selectorLabels" . | nindent 8 }} - spec: - containers: - - name: claude-terminal - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 8080 - protocol: TCP - livenessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: 5 - periodSeconds: 5 - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: claude-storage - mountPath: {{ .Values.persistence.mountPath }} - volumes: - - name: claude-storage - persistentVolumeClaim: - claimName: {{ include "claude-terminal.fullname" . }}-pvc - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/k8s/applications/dev-tools/templates/ingress.yaml b/k8s/applications/dev-tools/templates/ingress.yaml deleted file mode 100644 index 29f80cc..0000000 --- a/k8s/applications/dev-tools/templates/ingress.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.ingress.enabled }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ include "claude-terminal.fullname" . }} - labels: - {{- include "claude-terminal.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.className }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - pathType: {{ .pathType }} - backend: - service: - name: {{ include "claude-terminal.fullname" $ }} - port: - number: {{ $.Values.service.port }} - {{- end }} - {{- end }} -{{- end }} diff --git a/k8s/applications/dev-tools/templates/pvc.yaml b/k8s/applications/dev-tools/templates/pvc.yaml deleted file mode 100644 index 3ddbc76..0000000 --- a/k8s/applications/dev-tools/templates/pvc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "claude-terminal.fullname" . }}-pvc - labels: - {{- include "claude-terminal.labels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteOnce - storageClassName: {{ .Values.persistence.storageClass }} - resources: - requests: - storage: {{ .Values.persistence.size }} -{{- end }} diff --git a/k8s/applications/dev-tools/templates/service.yaml b/k8s/applications/dev-tools/templates/service.yaml deleted file mode 100644 index 50ea4e8..0000000 --- a/k8s/applications/dev-tools/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "claude-terminal.fullname" . }} - labels: - {{- include "claude-terminal.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "claude-terminal.selectorLabels" . | nindent 4 }} diff --git a/k8s/applications/dev-tools/values.yaml b/k8s/applications/dev-tools/values.yaml deleted file mode 100644 index a8a468b..0000000 --- a/k8s/applications/dev-tools/values.yaml +++ /dev/null @@ -1,45 +0,0 @@ -replicaCount: 1 - -image: - repository: localhost:5000/claude-terminal - pullPolicy: IfNotPresent - tag: latest - -service: - type: ClusterIP - port: 8080 - -ingress: - enabled: true - className: nginx - annotations: - cert-manager.io/cluster-issuer: homelab-ca - hosts: - - host: claude.riotpiao.com - paths: - - path: / - pathType: Prefix - tls: - - secretName: claude-terminal-tls - hosts: - - claude.riotpiao.com - -persistence: - enabled: true - storageClass: longhorn - size: 10Gi - mountPath: /root/.claude - -resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/k8s/applications/forge/kustomization.yaml b/k8s/applications/forge/kustomization.yaml deleted file mode 100644 index 01bfa2e..0000000 --- a/k8s/applications/forge/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: forge -resources: - - pki/ diff --git a/k8s/applications/forge/pki/kustomization.yaml b/k8s/applications/forge/pki/kustomization.yaml deleted file mode 100644 index 6dc0ef0..0000000 --- a/k8s/applications/forge/pki/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: [] -# PKI configuration, not K8s manifests diff --git a/k8s/applications/forge/runner-gc-cronjob.yaml b/k8s/applications/forge/runner-gc-cronjob.yaml deleted file mode 100644 index 49e21ed..0000000 --- a/k8s/applications/forge/runner-gc-cronjob.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# k8s/forge/runner-gc-cronjob.yaml -# Garbage-collects the forgejo-runner's DinD layer cache (runner-dind PVC, -# 30Gi). Every CI build/pull only adds images and build-cache layers — there -# is no automatic pruning, so without this the PVC fills up and breaks builds. -# -# Runs `docker image prune` / `docker builder prune` inside the live dind -# container via `kubectl exec`, rather than a sidecar in the runner pod itself, -# so it can run on its own schedule independent of runner restarts. - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: runner-gc - namespace: cicd - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: runner-gc - namespace: cicd -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: runner-gc - namespace: cicd -subjects: - - kind: ServiceAccount - name: runner-gc - namespace: cicd -roleRef: - kind: Role - name: runner-gc - apiGroup: rbac.authorization.k8s.io - ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: forgejo-runner-image-gc - namespace: cicd -spec: - schedule: "0 3 * * *" # daily 03:00 - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - backoffLimit: 1 - activeDeadlineSeconds: 600 - template: - spec: - serviceAccountName: runner-gc - restartPolicy: Never - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - containers: - - name: gc - image: alpine/k8s:1.31.0 - command: - - sh - - -c - - | - set -e - POD=$(kubectl -n cicd get pod -l app=forgejo-runner -o jsonpath='{.items[0].metadata.name}') - if [ -z "$POD" ]; then - echo "no forgejo-runner pod found, skipping" - exit 0 - fi - echo "before:" - kubectl -n cicd exec "$POD" -c dind -- df -h /var/lib/docker - echo "pruning images unused for >72h on $POD" - kubectl -n cicd exec "$POD" -c dind -- docker image prune -af --filter "until=72h" - echo "pruning build cache unused for >72h on $POD" - kubectl -n cicd exec "$POD" -c dind -- docker builder prune -af --filter "until=72h" - echo "after:" - kubectl -n cicd exec "$POD" -c dind -- df -h /var/lib/docker - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 250m - memory: 128Mi diff --git a/k8s/applications/forge/runner.yaml b/k8s/applications/forge/runner.yaml deleted file mode 100644 index ef406a5..0000000 --- a/k8s/applications/forge/runner.yaml +++ /dev/null @@ -1,438 +0,0 @@ -# k8s/forge/runner.yaml -# Forgejo Actions runner with Docker-in-Docker (DinD) sidecar. -# Phase 3.2 of talos_version_control.html. -# -# Pod layout (two containers, one pod): -# register initContainer — registers with Forgejo once; skips if .runner exists -# runner main container — long-running daemon that polls Forgejo for jobs -# dind sidecar — Docker daemon the runner talks to via mTLS on tcp://localhost:2376 -# -# Prerequisites (Phase 3.1): -# TOKEN=$(kubectl -n cicd exec deploy/forgejo-gitea -- \ -# gitea actions generate-runner-token 2>/dev/null | tr -d '\r\n') -# kubectl -n cicd create secret generic runner-token --from-literal=token="$TOKEN" -# # CA must come from cert-manager's homelab-ca-secret (the org-wide CA that -# # signs the live ingress cert), NOT k8s/forge/pki/ca.crt — that file is a -# # stale CA from before the "unified certificate" migration. -# kubectl get secret homelab-ca-secret -n cert-manager -o jsonpath='{.data.tls\.crt}' \ -# | base64 -d | kubectl -n cicd create secret generic homelab-ca --from-file=ca.crt=/dev/stdin -# -# CA trust for ephemeral job containers (Phase 3.3): -# The homelab-ca secret above only reaches the runner/register/dind containers. -# It does NOT reach the per-job containers DinD spawns (e.g. golangci-lint, -# node:22-bookworm) — those are fresh sibling containers with their own trust -# store. Without this, any git/curl/apk call to forgejo.riotpiao.com -# from inside a job fails with "SSL certificate problem: unable to get local -# issuer certificate". Build a merged bundle (public roots + homelab CA) and -# bind-mount it over /etc/ssl/certs/ca-certificates.crt in every job -# container via forgejo-runner's config.yaml container.options. -# -# IMPORTANT: source the CA from the live cluster secret, NOT from -# k8s/forge/pki/ca.crt — that repo file is a stale CA from before the -# "unified certificate" migration (different key, fails verification -# against the cert actually served by forgejo.riotpiao.com). The -# org-wide CA that signs the live ingress cert lives in -# cert-manager/homelab-ca-secret, and cicd/homelab-ca above is already -# synced from it. -# docker run --rm docker:27-dind cat /etc/ssl/certs/ca-certificates.crt > /tmp/ca-bundle.crt -# kubectl -n cicd get secret homelab-ca -o jsonpath='{.data.ca\.crt}' | base64 -d >> /tmp/ca-bundle.crt -# kubectl -n cicd create secret generic ca-bundle --from-file=ca-certificates.crt=/tmp/ca-bundle.crt -# Re-run this whenever the homelab CA rotates (see talos-forge-trust.yaml). -# -# Apply: -# kubectl apply -f k8s/forge/runner.yaml -# kubectl -n cicd rollout status deploy/forgejo-runner -# kubectl -n cicd logs deploy/forgejo-runner -c runner -f -# # expect: "runner: daemon started" / "connected to Forgejo" - -# ── PVCs ────────────────────────────────────────────────────────────────────── -# runner-reg — persists the .runner registration file so the runner doesn't -# re-register on every pod restart (token is one-use-per-registration) -# runner-dind — persists the Docker layer cache across pod restarts; keeps -# rebuilds fast — images don't need to be re-pulled every time -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: runner-reg - namespace: cicd -spec: - accessModes: [ReadWriteOnce] - storageClassName: longhorn - resources: - requests: - storage: 1Gi - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: runner-dind - namespace: cicd -spec: - accessModes: [ReadWriteOnce] - storageClassName: longhorn - resources: - requests: - storage: 30Gi - ---- -# ── DinD TLS certs, issued by the homelab's unified CA ─────────────────────── -# DinD's own entrypoint (dockerd-entrypoint.sh) self-generates a throwaway CA -# + server/client cert pair on every container start if none is supplied. Its -# server cert's SAN list only ever covers "docker", the pod hostname, and -# "localhost" - so anything reaching it via a stable Service DNS name (added -# below for story-crater-backend's release.yaml to build/push images) fails -# TLS hostname verification, even though the handshake itself succeeds. -# -# Fix: supply our own server+client cert pair, both issued by the same -# ClusterIssuer (homelab-ca) that already signs the live ingress cert, so -# they share one trust root. dockerd-entrypoint.sh skips its own generation -# step entirely once it finds $DOCKER_TLS_CERTDIR/server/{ca,cert,key}.pem -# already present and no CA private key alongside them (confirmed by reading -# the script directly: `kubectl exec -n cicd -c dind -- cat -# /usr/local/bin/dockerd-entrypoint.sh`) - exactly the "bring your own CA" -# path it's designed for. -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: forgejo-runner-dind-server - namespace: cicd -spec: - secretName: forgejo-runner-dind-server-tls - issuerRef: - name: homelab-ca - kind: ClusterIssuer - commonName: docker:dind server - dnsNames: - - forgejo-runner-dind.cicd.svc.cluster.local - - forgejo-runner-dind.cicd.svc - - forgejo-runner-dind - - docker - - localhost - usages: - - server auth - - digital signature - - key encipherment - ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: forgejo-runner-dind-client - namespace: cicd -spec: - secretName: forgejo-runner-dind-client-tls - issuerRef: - name: homelab-ca - kind: ClusterIssuer - commonName: docker:dind client - usages: - - client auth - - digital signature - - key encipherment - ---- -# Stable address for the dind sidecar's docker API (2376, mTLS) - lets -# CI workflows (e.g. story-crater-backend's release.yaml) build/push images -# by reaching this runner's own already-working outer dind directly, instead -# of the per-job `services:` sidecar pattern (confirmed broken: act-runner -# never registers a DNS alias for service containers - job container's -# /etc/hosts has no entry for it, `docker info` fails with a DNS lookup -# error, not a TLS/connection error). -apiVersion: v1 -kind: Service -metadata: - name: forgejo-runner-dind - namespace: cicd -spec: - selector: - app: forgejo-runner - ports: - - port: 2376 - targetPort: 2376 - ---- -# ── Runner config ───────────────────────────────────────────────────────────── -# container.options is appended to every `docker run` DinD issues for a job -# container, so this is what actually gets the merged CA bundle (ca-bundle -# secret, see header comment) trusted inside golangci-lint, node, etc. - and -# now also what propagates the homelab-CA-signed client cert (above) into -# job containers that need to talk back to dind themselves (e.g. building -# and pushing images). -# Source paths resolve against the dind container's filesystem (it's the -# daemon creating these containers), so both are mounted into dind below. -apiVersion: v1 -kind: ConfigMap -metadata: - name: forgejo-runner-config - namespace: cicd -data: - config.yaml: | - container: - options: -v /etc/forgejo-ca/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro -v /docker-certs/client:/docker-certs/client:ro - # Without this, sanitizeConfig() in forgejo/act silently drops any bind - # mount whose source isn't allowlisted here — including ones injected - # via container.options above, not just workflow-declared volumes. - valid_volumes: - - /etc/forgejo-ca/ca-certificates.crt - - /docker-certs/client - ---- -# ── Deployment ──────────────────────────────────────────────────────────────── -apiVersion: apps/v1 -kind: Deployment -metadata: - name: forgejo-runner - namespace: cicd -spec: - replicas: 1 - # RWO PVCs mean only one pod can mount them at a time. - # Recreate ensures the old pod fully terminates before the new one starts. - strategy: - type: Recreate - selector: - matchLabels: - app: forgejo-runner - template: - metadata: - labels: - app: forgejo-runner - spec: - # runner/register containers run as uid 1000 (image default); fsGroup - # makes kubelet chown+chmod the Longhorn PVC's group to 1000 with - # write access, otherwise writes to /data (.runner config) fail with - # "permission denied" since the volume is root:root 755 by default. - securityContext: - fsGroup: 1000 - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - initContainers: - # Registers the runner with Forgejo exactly once. - # test -f /data/.runner makes it idempotent — re-registration would - # consume the one-time token and break the runner. - - name: register - image: code.forgejo.org/forgejo/runner:6 - command: ["sh", "-c"] - args: - - | - test -f /data/.runner && echo "already registered, skipping" && exit 0 - forgejo-runner register --no-interactive \ - --instance https://forgejo.riotpiao.com \ - --token "$RUNNER_TOKEN" \ - --name talos-runner \ - --labels "docker:docker://node:22-bookworm" - env: - - name: RUNNER_TOKEN - valueFrom: - secretKeyRef: - name: runner-token - key: token - volumeMounts: - - name: runner-data - mountPath: /data - # CA cert so the register call can verify Forgejo's TLS cert - - name: homelab-ca - mountPath: /etc/ssl/certs/homelab-ca.pem - subPath: ca.crt - workingDir: /data - - containers: - # ── Runner daemon ──────────────────────────────────────────────────── - # Polls Forgejo for pending jobs and executes them inside DinD. - # The `until docker info` loop waits for the DinD sidecar to finish - # its TLS setup before starting the daemon — without this the runner - # starts before Docker is ready and immediately errors out. - - name: runner - image: code.forgejo.org/forgejo/runner:6 - command: ["sh", "-c"] - args: - - | - until nc -z localhost 2376 >/dev/null 2>&1; do - echo "waiting for docker daemon..."; sleep 2 - done - forgejo-runner daemon --config /data/config.yaml - workingDir: /data - env: - # Connect to the DinD sidecar via mTLS on localhost - - name: DOCKER_HOST - value: tcp://localhost:2376 - - name: DOCKER_TLS_VERIFY - value: "1" - - name: DOCKER_CERT_PATH - value: /docker-certs/client - volumeMounts: - - name: runner-data - mountPath: /data - - name: docker-certs - mountPath: /docker-certs - - name: homelab-ca - mountPath: /etc/ssl/certs/homelab-ca.pem - subPath: ca.crt - # forgejo-runner's container.options, read from this file, is what - # propagates the CA bundle into per-job containers (see ca-bundle - # secret + dind mount below) - - name: runner-config - mountPath: /data/config.yaml - subPath: config.yaml - # Homelab-CA-signed client cert (overlays whatever's in the - # docker-certs emptyDir at this subpath) - matches the server - # cert dind now presents, see Certificates above. - - name: dind-client-tls - mountPath: /docker-certs/client - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: "2" - memory: 4Gi - - # ── DinD sidecar ────────────────────────────────────────────────────── - # Full Docker daemon running inside the pod. - # privileged: true is required for DinD — the cicd namespace is labelled - # pod-security.kubernetes.io/enforce=privileged to allow this. - # DOCKER_TLS_CERTDIR causes DinD to generate mTLS certs in /docker-certs - # on startup; the runner reads the client certs from /docker-certs/client. - # runner-dind PVC mounts /var/lib/docker so the layer cache persists - # across pod restarts. - - name: dind - image: docker:27-dind - securityContext: - privileged: true - env: - - name: DOCKER_TLS_CERTDIR - value: /docker-certs - volumeMounts: - - name: docker-certs - mountPath: /docker-certs - - name: dind-storage - mountPath: /var/lib/docker - # Trust the homelab CA so DinD can pull from Forgejo's OCI registry - - name: homelab-ca - mountPath: /etc/ssl/certs/homelab-ca.pem - subPath: ca.crt - # Merged CA bundle (public roots + homelab CA), bind-mounted from - # here into every job container by container.options above — - # this path is resolved against dind's filesystem since dind is - # the daemon actually creating those containers. - - name: ca-bundle - mountPath: /etc/forgejo-ca/ca-certificates.crt - subPath: ca-certificates.crt - # Homelab-CA-signed server+client certs (see Certificates above), - # overlaying the matching subpaths of the docker-certs emptyDir. - # dockerd-entrypoint.sh detects these and skips its own - # self-signed generation entirely (no CA private key is supplied - # alongside them, so it can't regenerate even if it wanted to). - - name: dind-server-tls - mountPath: /docker-certs/server - - name: dind-client-tls - mountPath: /docker-certs/client - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: "2" - memory: 4Gi - - volumes: - - name: runner-data - persistentVolumeClaim: - claimName: runner-reg - - name: dind-storage - persistentVolumeClaim: - claimName: runner-dind - # emptyDir parent mount for /docker-certs - the server/ and client/ - # subpaths are now overlaid by the homelab-CA-signed dind-server-tls/ - # dind-client-tls secret mounts below (dockerd-entrypoint.sh no - # longer self-generates once it finds those present). This emptyDir - # just needs to exist as the parent directory; nothing writes - # directly to it anymore. - - name: docker-certs - emptyDir: {} - - name: homelab-ca - secret: - secretName: homelab-ca - - name: ca-bundle - secret: - secretName: ca-bundle - - name: runner-config - configMap: - name: forgejo-runner-config - # cert-manager issues these as tls.crt/tls.key/ca.crt - remapped to - # the ca.pem/cert.pem/key.pem filenames dockerd-entrypoint.sh expects - # under $DOCKER_TLS_CERTDIR/{server,client}/. - - name: dind-server-tls - secret: - secretName: forgejo-runner-dind-server-tls - items: - - key: ca.crt - path: ca.pem - - key: tls.crt - path: cert.pem - - key: tls.key - path: key.pem - - name: dind-client-tls - secret: - secretName: forgejo-runner-dind-client-tls - items: - - key: ca.crt - path: ca.pem - - key: tls.crt - path: cert.pem - - key: tls.key - path: key.pem - ---- -# ── NetworkPolicy ───────────────────────────────────────────────────────────── -# Restrict runner egress: it may only reach Forgejo (cicd ns), CoreDNS, and -# the public internet for action dependencies and base images. -# LAN (192.168.1.0/24) and the pod network (10.244.0.0/16) are blocked to -# prevent a compromised CI job from pivoting into the cluster or LAN. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: runner-egress - namespace: cicd -spec: - podSelector: - matchLabels: - app: forgejo-runner - policyTypes: [Egress] - egress: - # Forgejo (same namespace — git push, OCI registry push/pull) - - to: - - podSelector: {} - # ingress-nginx (the runner talks to Forgejo via its public hostname, - # https://forgejo.riotpiao.com, which resolves to the ingress - # controller's ClusterIP — a different namespace on the pod network) - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: ingress-nginx - ports: - - protocol: TCP - port: 443 - - protocol: TCP - port: 80 - # CoreDNS (DNS resolution for action deps and Forgejo hostname) - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: kube-system - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 - # Public internet for action dependencies and base images - # LAN and pod network are explicitly excluded - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 192.168.1.0/24 - - 10.244.0.0/16 diff --git a/k8s/applications/kustomization.yaml b/k8s/applications/kustomization.yaml deleted file mode 100644 index 4f4af3b..0000000 --- a/k8s/applications/kustomization.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -metadata: - name: applications - -# Layer 5: Applications — user services, workloads -# Dependencies: all previous layers (bootstrap, platform, security) -# Order: Applied fifth - -resources: - - temporal/ - - sqs/ - - llm/ - - portainer/ - - forge/ - - shadowsocks/ - - dev-tools/ - - duckdns/ - - cloudflared/ diff --git a/k8s/applications/llm/README.md b/k8s/applications/llm/README.md deleted file mode 100644 index 4056529..0000000 --- a/k8s/applications/llm/README.md +++ /dev/null @@ -1,299 +0,0 @@ -# Ollama LLM Inference Service - -CPU-only LLM inference server on talos-cp-1. Single model hot-loaded (DeepSeek-R1:70b), 42GB, 70Gi memory limit. - -## Quick Start - -### Access via port-forward -```bash -kubectl -n llm port-forward svc/ollama 11434:11434 -curl http://localhost:11434/api/tags -``` - -### Debug pod (in-cluster) -```bash -kubectl run debug --rm -it -n llm --image=curlimages/curl \ - --labels="app.kubernetes.io/role=llm-debug" \ - --serviceaccount=llm-worker -- sh - -# Inside pod -TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) -curl -H "Authorization: Bearer $TOKEN" \ - http://ollama.llm.svc.cluster.local:11434/api/tags -``` - -## Architecture - -| Component | Value | -|-----------|-------| -| Service | ClusterIP `ollama.llm.svc.cluster.local:11434` | -| Namespace | `llm` | -| Node | talos-cp-1 (pinned via nodeAffinity) | -| Memory request | 50Gi | -| Memory limit | 70Gi | -| Storage | 115Gi PVC (Longhorn) | -| Model | `deepseek-r1:70b` (~42GB) | -| Max loaded | 1 model | -| Parallelism | 1 request at a time | - -## API Endpoints - -### List models -```bash -curl http://ollama.llm.svc.cluster.local:11434/api/tags -``` - -Response: -```json -{ - "models": [ - {"name": "deepseek-r1:70b", "size": 42000000000, ...} - ] -} -``` - -### Generate (non-streaming) -```bash -curl -X POST http://ollama.llm.svc.cluster.local:11434/api/generate \ - -H "Content-Type: application/json" \ - -d '{ - "model": "deepseek-r1:70b", - "prompt": "Why is the sky blue?", - "stream": false - }' -``` - -### Pull model -```bash -curl -X POST http://ollama.llm.svc.cluster.local:11434/api/pull \ - -H "Content-Type: application/json" \ - -d '{"name": "deepseek-r1:70b", "stream": false}' -``` - -## Operations - -### Check pod status -```bash -kubectl -n llm get pod -l app.kubernetes.io/name=ollama -kubectl -n llm describe pod -l app.kubernetes.io/name=ollama -``` - -### View logs -```bash -kubectl -n llm logs deployment/ollama -f -``` - -### Monitor download progress (bootstrap) -```bash -kubectl -n llm logs -f job/bootstrap-models -c model-download -``` - -### Restart deployment -```bash -kubectl -n llm rollout restart deployment/ollama -``` - -## Storage - -- **PVC:** `ollama-models-cache`, 115Gi, Longhorn StorageClass -- **Mount:** `/root/.ollama/models` (Ollama model cache) -- **Lifecycle:** RWO (Read-Write-Once), tied to talos-cp-1 - -### Resize PVC -⚠️ PVCs can only expand, not shrink. Edit values.yaml and redeploy: - -```yaml -pvc: - size: 120Gi # increase only -``` - -```bash -vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama apply -``` - -## Networking - -### NetworkPolicy -- Default-deny ingress on Ollama pods -- Allow from pods labeled `app.kubernetes.io/name: llm-worker` (port 11434) -- Allow from pods labeled `app.kubernetes.io/role: llm-debug` (port 11434) - -View policy: -```bash -kubectl -n llm get networkpolicy ollama -``` - -Test access from external pod (should fail): -```bash -kubectl run test --rm -it --image=curlimages/curl -- \ - curl http://ollama.llm.svc.cluster.local:11434/ -# Connection timeout (correct) -``` - -Test access from debug pod (should succeed): -```bash -kubectl -n llm logs job/bootstrap-models # verify bootstrap completed -# Then run debug pod as shown above -``` - -## Configuration - -### Helm values (`k8s/llm/charts/ollama/values.yaml`) - -```yaml -resources: - requests: - cpu: 8 - memory: 50Gi - limits: - cpu: 16 - memory: 70Gi - -env: - OLLAMA_MAX_LOADED_MODELS: "1" - OLLAMA_NUM_PARALLEL: "1" - OLLAMA_MAX_QUEUE: "32" - OLLAMA_KEEP_ALIVE: "-1" - OLLAMA_HOST: "0.0.0.0:11434" - -preloadJob: - enabled: true - hotModels: - - deepseek-r1:70b -``` - -### Environment variables - -| Variable | Value | Purpose | -|----------|-------|---------| -| `OLLAMA_MODELS` | `/root/.ollama/models` | Model cache dir | -| `OLLAMA_MAX_LOADED_MODELS` | `1` | Max concurrent models in RAM | -| `OLLAMA_NUM_PARALLEL` | `1` | Parallel request threads | -| `OLLAMA_MAX_QUEUE` | `32` | Request queue depth | -| `OLLAMA_KEEP_ALIVE` | `-1` | Keep model resident (never unload) | -| `OLLAMA_HOST` | `0.0.0.0:11434` | Bind address | - -Tune `OLLAMA_NUM_PARALLEL` based on CPU cores. Current: 1 (conservative, CPU bottleneck). - -## Model Management - -### Current model -- **Name:** `deepseek-r1:70b` -- **Size:** ~42GB -- **Quantization:** Default Ollama quant -- **Status:** Downloaded during pod init via bootstrap job - -### Change model - -1. Edit `values.yaml`: -```yaml -preloadJob: - hotModels: - - deepseek-r1:32b # or any available model -``` - -2. Redeploy: -```bash -kubectl -n llm delete job bootstrap-models --ignore-not-found -vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama apply -``` - -3. Monitor: -```bash -kubectl -n llm logs -f job/bootstrap-models -c model-download -``` - -### Available models -Ollama registry: https://ollama.com/library - -Examples: -- `deepseek-r1:70b` (reasoning, 42GB) -- `deepseek-r1:32b` (faster, 20GB) -- `llama3.1:70b` (general, 41GB) -- `mistral:large` (26GB) - -## Troubleshooting - -### Pod stuck in `ContainerCreating` -```bash -kubectl -n llm describe pod -l app.kubernetes.io/name=ollama -# Check Events section for PVC/image pull issues -``` - -### Bootstrap job failing -```bash -kubectl -n llm logs job/bootstrap-models -c model-download --tail=50 -# Common: model not found in registry, disk full, network timeout -``` - -### Model pull timeout -```bash -# Increase pod timeout (edit deployment directly) -kubectl -n llm edit deployment ollama -# Change readinessProbe.initialDelaySeconds, livenessProbe.periodSeconds -``` - -### Out of memory -Model size exceeds limit. Reduce `memory.limits` or choose smaller model. - -```bash -kubectl top pod -n llm # check actual usage -``` - -### Cannot connect from other pods -Verify NetworkPolicy: -```bash -kubectl -n llm get networkpolicy -kubectl -n llm describe networkpolicy ollama -# Add pod label: app.kubernetes.io/name: llm-worker or app.kubernetes.io/role: llm-debug -``` - -## Secrets - -Ollama pod receives MinIO credentials via Secret `ollama-minio` (created by helmfile presync): - -```bash -kubectl -n llm get secret ollama-minio -o jsonpath='{.data}' | jq -``` - -Keys: `endpoint`, `bucket`, `access_key`, `secret_key` - -Used by bootstrap job to upload model blobs to MinIO (future: auto-backup). - -## Metrics & Observability - -### Prometheus scrape (if enabled) -ServiceMonitor: Not yet configured (see `k8s/monitoring/dashboards/services/`) - -Metrics to add: -- `ollama_requests_total` (counter) -- `ollama_request_duration_seconds` (histogram) -- `ollama_loaded_models` (gauge) - -### Logs -Pod logs via kubectl. No log aggregation to Loki yet. - -```bash -kubectl -n llm logs deployment/ollama -f --timestamps -``` - -## Cleanup - -### Delete Ollama completely -```bash -vsource .env && helmfile -f helmfile.yaml.gotmpl -l name=ollama destroy -# Keeps PVC (data safety). To delete: kubectl -n llm delete pvc ollama-models-cache -``` - -### Delete just the model cache (keep deployment) -```bash -kubectl -n llm delete pvc ollama-models-cache -# Recreate: kubectl -n llm patch deployment ollama -p '{"spec":{"template":{"metadata":{"annotations":{"restart":"now"}}}}}' -``` - -## See Also - -- Helmfile: `helmfile.yaml.gotmpl` (llm release block) -- Chart: `k8s/llm/charts/ollama/` -- Namespace: `llm` -- Bootstrap: `k8s/llm/bootstrap-models-job.yaml` (manual preload fallback) diff --git a/k8s/applications/llm/charts/ollama/Chart.yaml b/k8s/applications/llm/charts/ollama/Chart.yaml deleted file mode 100644 index 93c27e1..0000000 --- a/k8s/applications/llm/charts/ollama/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: ollama -description: CPU-only Ollama LLM server with MinIO model registry -type: application -version: 0.1.0 -appVersion: "latest" diff --git a/k8s/applications/llm/charts/ollama/templates/deployment.yaml b/k8s/applications/llm/charts/ollama/templates/deployment.yaml deleted file mode 100644 index 337b335..0000000 --- a/k8s/applications/llm/charts/ollama/templates/deployment.yaml +++ /dev/null @@ -1,126 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ollama - namespace: llm - labels: - app.kubernetes.io/name: ollama - app.kubernetes.io/part-of: llm -spec: - replicas: {{ .Values.replicaCount }} - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: ollama - template: - metadata: - labels: - app.kubernetes.io/name: ollama - app.kubernetes.io/part-of: llm - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: topology.kubernetes.io/zone - operator: In - values: - - {{ .Values.nodeAffinity.zone }} - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Equal - value: "" - effect: NoSchedule - - initContainers: - - name: preload-model - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - sh - - -c - - | - set -e - echo "Starting Ollama server for model preload..." - ollama serve & - OLLAMA_PID=$! - sleep 10 - {{- range .Values.preloadJob.hotModels }} - echo "Preloading {{ . }}..." - if ollama ls | grep -q "{{ . }}"; then - echo "✓ {{ . }} already cached" - else - ollama pull {{ . }} - fi - {{- end }} - echo "Model preload complete" - kill $OLLAMA_PID || true - wait $OLLAMA_PID 2>/dev/null || true - volumeMounts: - - name: models-cache - mountPath: /root/.ollama/models - env: - - name: OLLAMA_HOST - value: "127.0.0.1:11434" - - containers: - - name: ollama - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: 11434 - name: http - env: - {{- range $key, $value := .Values.env }} - - name: {{ $key }} - value: "{{ $value }}" - {{- end }} - - name: OLLAMA_MODELS_MINIO_ENDPOINT - valueFrom: - secretKeyRef: - name: ollama-minio - key: endpoint - - name: OLLAMA_MODELS_MINIO_BUCKET - valueFrom: - secretKeyRef: - name: ollama-minio - key: bucket - - name: OLLAMA_MODELS_MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - name: ollama-minio - key: access_key - - name: OLLAMA_MODELS_MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: ollama-minio - key: secret_key - resources: - requests: - cpu: {{ .Values.resources.requests.cpu }} - memory: {{ .Values.resources.requests.memory }} - limits: - cpu: {{ .Values.resources.limits.cpu }} - memory: {{ .Values.resources.limits.memory }} - livenessProbe: - httpGet: - path: / - port: 11434 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: / - port: 11434 - initialDelaySeconds: 10 - periodSeconds: 5 - volumeMounts: - - name: models-cache - mountPath: /root/.ollama/models - - volumes: - - name: models-cache - persistentVolumeClaim: - claimName: ollama-models-cache diff --git a/k8s/applications/llm/charts/ollama/templates/networkpolicy.yaml b/k8s/applications/llm/charts/ollama/templates/networkpolicy.yaml deleted file mode 100644 index c94803b..0000000 --- a/k8s/applications/llm/charts/ollama/templates/networkpolicy.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: ollama-access - namespace: llm - labels: - app.kubernetes.io/name: ollama -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: ollama - policyTypes: - - Ingress - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: llm-worker - ports: - - protocol: TCP - port: 11434 - - from: - - podSelector: - matchLabels: - app.kubernetes.io/role: llm-debug - ports: - - protocol: TCP - port: 11434 diff --git a/k8s/applications/llm/charts/ollama/templates/preload-job.yaml b/k8s/applications/llm/charts/ollama/templates/preload-job.yaml deleted file mode 100644 index 322e722..0000000 --- a/k8s/applications/llm/charts/ollama/templates/preload-job.yaml +++ /dev/null @@ -1,92 +0,0 @@ -{{- if .Values.preloadJob.enabled }} -apiVersion: batch/v1 -kind: Job -metadata: - name: ollama-preload - namespace: llm - labels: - app.kubernetes.io/name: ollama-preload -spec: - backoffLimit: 3 - template: - spec: - serviceAccountName: default - restartPolicy: Never - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: topology.kubernetes.io/zone - operator: In - values: - - az-a - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Equal - effect: NoSchedule - initContainers: - - name: model-cache-init - image: ollama/ollama:latest - imagePullPolicy: IfNotPresent - command: - - sh - - -c - - | - set -e - echo "Starting Ollama server to cache models..." - ollama serve & - OLLAMA_PID=$! - sleep 10 - echo "Caching hot-tier models..." - {{- range .Values.preloadJob.hotModels }} - echo "Checking if {{ . }} is cached..." - if ollama ls | grep -q "{{ . }}"; then - echo "✓ {{ . }} already cached, skipping" - else - echo "Pulling {{ . }}..." - ollama pull {{ . }} - fi - {{- end }} - echo "Model cache initialization complete" - kill $OLLAMA_PID || true - wait $OLLAMA_PID 2>/dev/null || true - volumeMounts: - - name: models - mountPath: /root/.ollama - env: - - name: OLLAMA_HOST - value: "127.0.0.1:11434" - - containers: - - name: cache-populate - image: curlimages/curl:latest - imagePullPolicy: IfNotPresent - command: - - sh - - -c - - | - set -e - echo "Waiting for Ollama pod to be ready..." - until curl -f http://ollama.llm.svc.cluster.local:11434/api/tags 2>/dev/null; do - echo "Ollama not ready, waiting..." - sleep 5 - done - echo "Ollama is ready, populating local cache..." - {{- range .Values.preloadJob.hotModels }} - echo "Checking if {{ . }} is already cached..." - if curl -s http://ollama.llm.svc.cluster.local:11434/api/tags | grep -q "{{ . }}"; then - echo "✓ {{ . }} already cached, skipping" - else - echo "Caching {{ . }} locally..." - curl -X POST http://ollama.llm.svc.cluster.local:11434/api/pull \ - -H "Content-Type: application/json" \ - -d '{"name":"{{ . }}","stream":false}' - fi - {{- end }} - echo "Local cache population complete" - - volumes: - - name: models - emptyDir: {} -{{- end }} diff --git a/k8s/applications/llm/charts/ollama/templates/pvc.yaml b/k8s/applications/llm/charts/ollama/templates/pvc.yaml deleted file mode 100644 index dd7de28..0000000 --- a/k8s/applications/llm/charts/ollama/templates/pvc.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ollama-models-cache - namespace: llm - labels: - app.kubernetes.io/name: ollama -spec: - accessModes: - - ReadWriteOnce - storageClassName: {{ .Values.pvc.storageClassName }} - resources: - requests: - storage: {{ .Values.pvc.size }} diff --git a/k8s/applications/llm/charts/ollama/templates/service.yaml b/k8s/applications/llm/charts/ollama/templates/service.yaml deleted file mode 100644 index 9f448a8..0000000 --- a/k8s/applications/llm/charts/ollama/templates/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: ollama - namespace: llm - labels: - app.kubernetes.io/name: ollama -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: ollama diff --git a/k8s/applications/llm/charts/ollama/templates/storageclass.yaml b/k8s/applications/llm/charts/ollama/templates/storageclass.yaml deleted file mode 100644 index 8ae0b1a..0000000 --- a/k8s/applications/llm/charts/ollama/templates/storageclass.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: longhorn-llm - labels: - app.kubernetes.io/name: ollama -provisioner: driver.longhorn.io -parameters: - numberOfReplicas: "1" - staleReplicaTimeout: "2880" -reclaimPolicy: Retain -allowVolumeExpansion: true diff --git a/k8s/applications/llm/charts/ollama/values.yaml b/k8s/applications/llm/charts/ollama/values.yaml deleted file mode 100644 index aad77fb..0000000 --- a/k8s/applications/llm/charts/ollama/values.yaml +++ /dev/null @@ -1,40 +0,0 @@ -replicaCount: 1 - -image: - repository: ollama/ollama - pullPolicy: IfNotPresent - tag: "latest" - -service: - type: ClusterIP - port: 11434 - -resources: - requests: - cpu: 8 - memory: 60Gi - limits: - cpu: 16 - memory: 100Gi - -pvc: - enabled: true - size: 115Gi - storageClassName: longhorn-llm - -nodeAffinity: - zone: az-a - -env: - OLLAMA_MODELS: /root/.ollama/models - OLLAMA_MAX_LOADED_MODELS: "2" - OLLAMA_NUM_PARALLEL: "2" - OLLAMA_MAX_QUEUE: "64" - OLLAMA_KEEP_ALIVE: "-1" - OLLAMA_HOST: "0.0.0.0:11434" - -preloadJob: - enabled: false - hotModels: - - ornith:35b - - deepseek-r1:70b diff --git a/k8s/applications/llm/kustomization.yaml b/k8s/applications/llm/kustomization.yaml deleted file mode 100644 index a5e6fe2..0000000 --- a/k8s/applications/llm/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: llm -resources: - - scripts/ -# Helm charts deployed via ArgoCD Helm source diff --git a/k8s/applications/llm/scripts/kustomization.yaml b/k8s/applications/llm/scripts/kustomization.yaml deleted file mode 100644 index 73988a5..0000000 --- a/k8s/applications/llm/scripts/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: [] -# Shell scripts, not K8s manifests diff --git a/k8s/applications/llm/scripts/setup-minio-bucket.sh b/k8s/applications/llm/scripts/setup-minio-bucket.sh deleted file mode 100644 index ef67011..0000000 --- a/k8s/applications/llm/scripts/setup-minio-bucket.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# setup-minio-bucket.sh -# Creates MinIO bucket and Kubernetes secrets for Ollama LLM server -# Runs as helmfile presync hook; all commands are idempotent -# Dependencies: kubectl, access to minio-az-a pod in storage namespace -# Environment: MINIO_ROOT_USER, MINIO_ROOT_PASSWORD (from Vault), AUTHENTIK_OLLAMA_CLIENT_ID, AUTHENTIK_OLLAMA_CLIENT_SECRET - -set -e - -echo "=== Step 1: Create and label llm namespace ===" -kubectl create namespace llm --dry-run=client -o yaml | kubectl apply -f - -kubectl label namespace llm \ - pod-security.kubernetes.io/enforce=baseline \ - pod-security.kubernetes.io/enforce-version=latest \ - --overwrite - -echo "✓ llm namespace created/labeled" - -echo "" -echo "=== Step 2: Create MinIO bucket riotpiao-models ===" - -# Configure mc host inside MinIO pod -kubectl -n storage exec deployment/minio-az-a -- \ - mc config host add local http://localhost:9000 \ - "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" - -echo "✓ mc host configured" - -# Create bucket (idempotent) -kubectl -n storage exec deployment/minio-az-a -- \ - mc mb --ignore-existing local/riotpiao-models - -echo "✓ MinIO bucket riotpiao-models created (or already exists)" - -# Enable versioning for model rollback safety -kubectl -n storage exec deployment/minio-az-a -- \ - mc version enable local/riotpiao-models - -echo "✓ Versioning enabled on riotpiao-models bucket" - -echo "" -echo "=== Step 3: Create ollama-minio Secret (MinIO credentials) ===" - -kubectl create secret generic ollama-minio -n llm \ - --from-literal=endpoint="http://minio-az-a.storage:9000" \ - --from-literal=bucket="riotpiao-models" \ - --from-literal=access_key="${MINIO_ROOT_USER}" \ - --from-literal=secret_key="${MINIO_ROOT_PASSWORD}" \ - --dry-run=client -o yaml | kubectl apply -f - - -echo "✓ Secret ollama-minio created/updated" - -echo "" -echo "=== Step 4: Create ollama-oidc Secret (Authentik credentials) ===" - -kubectl create secret generic ollama-oidc -n llm \ - --from-literal=client_id="${AUTHENTIK_OLLAMA_CLIENT_ID}" \ - --from-literal=client_secret="${AUTHENTIK_OLLAMA_CLIENT_SECRET}" \ - --dry-run=client -o yaml | kubectl apply -f - - -echo "✓ Secret ollama-oidc created/updated" - -echo "" -echo "=== Verification ===" -echo "" -echo "Run these commands to verify:" -echo " kubectl -n llm get secret ollama-minio ollama-oidc" -echo " kubectl -n storage exec deployment/minio-az-a -- mc ls local/riotpiao-models" -echo "" -echo "Setup complete!" diff --git a/k8s/applications/portainer/bootstrap.sh b/k8s/applications/portainer/bootstrap.sh deleted file mode 100755 index ee173d7..0000000 --- a/k8s/applications/portainer/bootstrap.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# portainer/bootstrap.sh -# Deploys Portainer CE into the dashboard namespace. -# No credentials needed — Portainer prompts you to create an admin account -# on first browser visit. -# -# Prerequisites: -# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig) -# - helm >= 3.x -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}" -export KUBECONFIG - -# ── Namespace ───────────────────────────────────────────────────────────────── -echo "==> Creating dashboard namespace..." -kubectl create namespace dashboard --dry-run=client -o yaml | kubectl apply -f - -kubectl label namespace dashboard \ - pod-security.kubernetes.io/enforce=privileged \ - pod-security.kubernetes.io/enforce-version=latest \ - --overwrite - -# ── Helm repo ───────────────────────────────────────────────────────────────── -echo "==> Adding Portainer Helm repo..." -helm repo add portainer https://portainer.github.io/k8s/ -helm repo update portainer - -# ── Portainer ───────────────────────────────────────────────────────────────── -echo "==> Installing Portainer..." -helm upgrade --install portainer portainer/portainer \ - --namespace dashboard \ - --values "${SCRIPT_DIR}/portainer-values.yaml" \ - --wait \ - --timeout 5m - -echo "==> Waiting for Portainer Deployment to be ready..." -kubectl rollout status deployment/portainer -n dashboard --timeout=120s - -# ── Done ────────────────────────────────────────────────────────────────────── -echo "" -echo "==> Portainer is up." -echo "" -echo "Access Portainer UI:" -echo " make pf-portainer" -echo " http://localhost:9000" -echo "" -echo "First-time setup: Portainer will prompt you to create an admin account." -echo "Choose 'Manage the local Kubernetes environment' when asked." -echo "" -echo "Node failure resilience tip:" -echo " For faster PVC failover on hard node failure, enable in Longhorn UI → Settings:" -echo " nodeDownPodDeletionPolicy = delete-deployment-pod" diff --git a/k8s/applications/shadowsocks/kustomization.yaml b/k8s/applications/shadowsocks/kustomization.yaml deleted file mode 100644 index 5a57675..0000000 --- a/k8s/applications/shadowsocks/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: shadowsocks -resources: [] diff --git a/k8s/applications/shadowsocks/shadowsocks.yaml b/k8s/applications/shadowsocks/shadowsocks.yaml deleted file mode 100644 index e86fcb0..0000000 --- a/k8s/applications/shadowsocks/shadowsocks.yaml +++ /dev/null @@ -1,159 +0,0 @@ -# k8s/shadowsocks/shadowsocks.yaml -# Personal Shadowsocks proxy (for Shadowrocket/other SS clients) — an -# alternative tunnel to the WireGuard setup in cluster-config/, useful when -# a network blocks/throttles WireGuard but not generic TLS-looking traffic. -# -# Everything that varies between deployments (port, password, method) lives -# in the shadowsocks-config Secret below — the Deployment/Service never -# hardcode a value, so re-pointing this at a new port or rotating the -# password is a Secret edit + rollout restart, no YAML edit. -# -# Prerequisites: -# talos put cluster/SHADOWSOCKS_PASSWORD SHADOWSOCKS_PASSWORD="$(openssl rand -base64 24)" -# talos put cluster/SHADOWSOCKS_PORT SHADOWSOCKS_PORT="8388" -# -# Apply: -# kubectl create namespace vpn --dry-run=client -o yaml | kubectl apply -f - -# kubectl -n vpn create secret generic shadowsocks-config \ -# --from-literal=SERVER_PORT="$(talos get cluster/SHADOWSOCKS_PORT --key SHADOWSOCKS_PORT)" \ -# --from-literal=PASSWORD="$(talos get cluster/SHADOWSOCKS_PASSWORD --key SHADOWSOCKS_PASSWORD)" \ -# --from-literal=METHOD="aes-256-gcm" \ -# --from-literal=TIMEOUT="300" -# kubectl apply -f k8s/shadowsocks/shadowsocks.yaml -# -# Rotate password (or change port) later: -# kubectl -n vpn delete secret shadowsocks-config && -# kubectl -n vpn rollout restart deploy/shadowsocks -# -# Client config: SERVER_PORT/METHOD/PASSWORD above feed directly into the -# Shadowrocket/SS client's server, method, and password fields. SERVER_ADDR -# for the client is the LB IP below (192.168.1.166), or your router's WAN -# address/DDNS hostname (riotpiao.duckdns.org) with port-forwarding to it — -# same pattern as the wg1 WireGuard peer in cluster-config/phone_config.conf. -apiVersion: v1 -kind: Namespace -metadata: - name: vpn - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: shadowsocks - namespace: vpn -spec: - replicas: 1 - selector: - matchLabels: - app: shadowsocks - template: - metadata: - labels: - app: shadowsocks - spec: - containers: - - name: shadowsocks - image: shadowsocks/shadowsocks-libev:latest - env: - - name: SERVER_ADDR - value: "0.0.0.0" - - name: SERVER_PORT - valueFrom: - secretKeyRef: - name: shadowsocks-config - key: SERVER_PORT - - name: PASSWORD - valueFrom: - secretKeyRef: - name: shadowsocks-config - key: PASSWORD - - name: METHOD - valueFrom: - secretKeyRef: - name: shadowsocks-config - key: METHOD - - name: TIMEOUT - valueFrom: - secretKeyRef: - name: shadowsocks-config - key: TIMEOUT - # containerPort is informational only (no portRange support for - # env-driven SERVER_PORT) — the Service below is what actually - # routes traffic, matched on the same Secret key via downward API - # isn't available for Service ports, so targetPort uses the literal - # port name instead; see Service ports comment. - ports: - - containerPort: 8388 - protocol: TCP - - containerPort: 8388 - protocol: UDP - resources: - requests: - cpu: 50m - memory: 32Mi - limits: - cpu: 500m - memory: 128Mi - ---- -# LoadBalancer via Cilium LB-IPAM (see k8s/cilium/lb-ipam-pool.yaml) — pinned -# to .166 so router port-forwarding and the DDNS hostname stay stable across -# pod/service recreates, same pattern forgejo uses at .165. -# -# NOTE: SERVER_PORT in the Secret must match port/targetPort/nodePort here. -# If you change the port, update both the Secret and this Service together. -apiVersion: v1 -kind: Service -metadata: - name: shadowsocks - namespace: vpn - annotations: - io.cilium/lb-ipam-ips: "192.168.1.166" -spec: - type: LoadBalancer - selector: - app: shadowsocks - ports: - - name: tcp - protocol: TCP - port: 8388 - targetPort: 8388 - - name: udp - protocol: UDP - port: 8388 - targetPort: 8388 - ---- -# Restrict egress like the forgejo-runner pattern (k8s/forge/runner.yaml) — -# a proxy server is, by design, an open relay to the internet for whoever -# holds the password; LAN/pod-network egress is blocked so a compromised -# password can't be used to pivot into the cluster or LAN. CoreDNS is -# explicitly allowed — shadowsocks-libev resolves client-requested hostnames -# itself, so blanket-blocking the service subnet would break that. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: shadowsocks-egress - namespace: vpn -spec: - podSelector: - matchLabels: - app: shadowsocks - policyTypes: [Egress] - egress: - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: kube-system - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 192.168.1.0/24 - - 10.244.0.0/16 - - 10.96.0.0/12 diff --git a/k8s/applications/sqs/argocd/apps/00-strimzi-operator.yaml b/k8s/applications/sqs/argocd/apps/00-strimzi-operator.yaml deleted file mode 100644 index 48d92d5..0000000 --- a/k8s/applications/sqs/argocd/apps/00-strimzi-operator.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: strimzi-operator - namespace: cicd - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - project: kmsvc - source: - repoURL: https://strimzi.io/charts/ - chart: strimzi-kafka-operator - targetRevision: 0.46.0 - helm: - values: | - watchNamespaces: ["sqs"] - destination: - server: https://kubernetes.default.svc - namespace: sqs - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/k8s/applications/sqs/argocd/apps/01-kafka-cluster.yaml b/k8s/applications/sqs/argocd/apps/01-kafka-cluster.yaml deleted file mode 100644 index ef90331..0000000 --- a/k8s/applications/sqs/argocd/apps/01-kafka-cluster.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: kafka-cluster - namespace: cicd - annotations: - argocd.argoproj.io/sync-wave: "1" -spec: - project: kmsvc - source: - repoURL: https://forgejo.riotpiao.com/rock/kafaka-management-service.git - targetRevision: main - path: k8s/charts/kafka-cluster - helm: - values: | - namespace: sqs - nodePool: - replicas: 3 - storage: - class: longhorn - sizeGi: 50 - resources: - memory: 5Gi - cpu: "2" - destination: - server: https://kubernetes.default.svc - namespace: sqs - syncPolicy: - automated: - prune: true - selfHeal: true diff --git a/k8s/applications/sqs/argocd/apps/02-redis.yaml b/k8s/applications/sqs/argocd/apps/02-redis.yaml deleted file mode 100644 index e1076a3..0000000 --- a/k8s/applications/sqs/argocd/apps/02-redis.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: kmsvc-redis - namespace: cicd - annotations: - argocd.argoproj.io/sync-wave: "1" -spec: - project: kmsvc - source: - repoURL: https://charts.bitnami.com/bitnami - chart: redis - targetRevision: 20.6.0 - helm: - values: | - architecture: standalone - # docker.io/bitnami stopped publishing version-pinned tags; bitnamilegacy - # mirrors them for free. allowInsecureImages silences the chart's - # container-image allowlist check, which doesn't know about that mirror. - global: - security: - allowInsecureImages: true - image: - repository: bitnamilegacy/redis - auth: - enabled: false - master: - persistence: - enabled: true - storageClass: longhorn - size: 2Gi - resources: - limits: - memory: 1Gi - requests: - memory: 1Gi - destination: - server: https://kubernetes.default.svc - namespace: sqs - syncPolicy: - automated: - prune: true - selfHeal: true diff --git a/k8s/applications/sqs/argocd/apps/03-queue-crd.yaml b/k8s/applications/sqs/argocd/apps/03-queue-crd.yaml deleted file mode 100644 index 874ced0..0000000 --- a/k8s/applications/sqs/argocd/apps/03-queue-crd.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: queue-crd - namespace: cicd - annotations: - argocd.argoproj.io/sync-wave: "2" -spec: - project: kmsvc - source: - repoURL: https://forgejo.riotpiao.com/rock/kafaka-management-service.git - targetRevision: main - path: k8s/charts/queue-crd - helm: - values: | - namespace: sqs - kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092" - redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379" - image: - repository: forgejo.riotpiao.com/rock/kafka-management-service-queue-operator - # CI (.forgejo/workflows/release.yaml) writes the released git tag - # here and pushes the commit -- ArgoCD picks it up on its next sync. - tag: latest - destination: - server: https://kubernetes.default.svc - namespace: sqs - syncPolicy: - automated: - prune: true - selfHeal: true diff --git a/k8s/applications/sqs/argocd/apps/04-management-service.yaml b/k8s/applications/sqs/argocd/apps/04-management-service.yaml deleted file mode 100644 index 9e56a39..0000000 --- a/k8s/applications/sqs/argocd/apps/04-management-service.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: management-service - namespace: cicd - annotations: - argocd.argoproj.io/sync-wave: "2" -spec: - project: kmsvc - source: - repoURL: https://forgejo.riotpiao.com/rock/kafaka-management-service.git - targetRevision: main - path: k8s/charts/management-service - helm: - values: | - namespace: sqs - image: - repository: forgejo.riotpiao.com/rock/kafka-management-service - # CI (.forgejo/workflows/release.yaml) writes the released git tag - # here and pushes the commit -- ArgoCD picks it up on its next sync. - tag: latest - env: - kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092" - redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379" - authentikIssuerURL: "https://authentik.riotpiao.com/application/o/kafaka/" - authentikAudience: "QI0gPtR99ar8VvhK8Tqox4SDkTKzbNU7lbgwBNSc" - ingress: - enabled: true - host: kmsvc.riotpiao.com - clusterIssuer: homelab-ca - destination: - server: https://kubernetes.default.svc - namespace: sqs - syncPolicy: - automated: - prune: true - selfHeal: true diff --git a/k8s/applications/sqs/argocd/apps/kustomization.yaml b/k8s/applications/sqs/argocd/apps/kustomization.yaml deleted file mode 100644 index b0588a9..0000000 --- a/k8s/applications/sqs/argocd/apps/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: sqs -resources: - - 02-redis.yaml - - 03-queue-crd.yaml - - 04-management-service.yaml diff --git a/k8s/applications/sqs/argocd/kustomization.yaml b/k8s/applications/sqs/argocd/kustomization.yaml deleted file mode 100644 index d867168..0000000 --- a/k8s/applications/sqs/argocd/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: sqs -resources: - - project.yaml - - root.yaml - - apps/ diff --git a/k8s/applications/sqs/argocd/project.yaml b/k8s/applications/sqs/argocd/project.yaml deleted file mode 100644 index 73c8bb3..0000000 --- a/k8s/applications/sqs/argocd/project.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: AppProject -metadata: - name: kmsvc - namespace: cicd -spec: - description: Kafka Management Service (design.md) -- Strimzi/Kafka, Redis, queue-operator, message-plane server - sourceRepos: - - https://forgejo.riotpiao.com/rock/kafaka-management-service.git - - https://strimzi.io/charts/ - - https://charts.bitnami.com/bitnami - destinations: - - namespace: sqs - server: https://kubernetes.default.svc - - namespace: cicd - server: https://kubernetes.default.svc - clusterResourceWhitelist: - - group: "apiextensions.k8s.io" - kind: CustomResourceDefinition - - group: "rbac.authorization.k8s.io" - kind: ClusterRole - - group: "rbac.authorization.k8s.io" - kind: ClusterRoleBinding diff --git a/k8s/applications/sqs/argocd/root.yaml b/k8s/applications/sqs/argocd/root.yaml deleted file mode 100644 index 1a7d14e..0000000 --- a/k8s/applications/sqs/argocd/root.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: kmsvc-root - namespace: cicd -spec: - project: kmsvc - source: - repoURL: https://forgejo.riotpiao.com/rock/kafaka-management-service.git - targetRevision: main - path: k8s/argocd/apps - directory: - recurse: false - destination: - server: https://kubernetes.default.svc - namespace: cicd - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/k8s/applications/sqs/environments/homelab.yaml b/k8s/applications/sqs/environments/homelab.yaml deleted file mode 100644 index 74afced..0000000 --- a/k8s/applications/sqs/environments/homelab.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# design.md §7b: cluster-specific values for the homelab environment. -# No secrets here — Authentik client secret etc. flow through the existing -# Vault/talos-cli pattern, referenced at deploy time, not inlined. -namespace: sqs - -kafkaCluster: - nodePool: - replicas: 3 - storage: - # Use unified longhorn StorageClass (3 replicas, Immediate binding) - class: longhorn - # Per-node headroom: with 3 nodes and existing PVCs, estimate ~100+ Gi total - # available. Each node hosts one replica of all 3 broker volumes, so 3 * - # sizeGi must fit. Monitor usage during Kafka deployment. - sizeGi: 10 - resources: - memory: 5Gi - cpu: "2" - -redis: - storageClass: longhorn - memoryLimit: 1Gi - -managementService: - ingress: - host: kmsvc.riotpiao.com - clusterIssuer: homelab-ca - authentikIssuerURL: "https://authentik.riotpiao.com/application/o/kafaka/" - authentikAudience: "QI0gPtR99ar8VvhK8Tqox4SDkTKzbNU7lbgwBNSc" diff --git a/k8s/applications/sqs/environments/kustomization.yaml b/k8s/applications/sqs/environments/kustomization.yaml deleted file mode 100644 index f8c3667..0000000 --- a/k8s/applications/sqs/environments/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: sqs -resources: [] -# homelab.yaml is configuration, not a K8s manifest diff --git a/k8s/applications/sqs/helmfile.yaml.gotmpl b/k8s/applications/sqs/helmfile.yaml.gotmpl deleted file mode 100644 index 27e4655..0000000 --- a/k8s/applications/sqs/helmfile.yaml.gotmpl +++ /dev/null @@ -1,99 +0,0 @@ -environments: - default: - values: - - environments/homelab.yaml - homelab: - values: - - environments/homelab.yaml ---- -helmDefaults: - wait: true - timeout: 600 - -repositories: - - name: strimzi - url: https://strimzi.io/charts/ - - name: bitnami - url: https://charts.bitnami.com/bitnami - -releases: - - name: strimzi-operator - namespace: {{ .Values.namespace }} - chart: strimzi/strimzi-kafka-operator - version: 0.46.0 - values: - - watchNamespaces: ["{{ .Values.namespace }}"] - - - name: kafka-cluster - namespace: {{ .Values.namespace }} - chart: charts/kafka-cluster - needs: - - {{ .Values.namespace }}/strimzi-operator - values: - - namespace: {{ .Values.namespace }} - nodePool: - replicas: {{ .Values.kafkaCluster.nodePool.replicas }} - storage: - class: {{ .Values.kafkaCluster.nodePool.storage.class }} - sizeGi: {{ .Values.kafkaCluster.nodePool.storage.sizeGi }} - resources: - memory: {{ .Values.kafkaCluster.nodePool.resources.memory }} - cpu: {{ .Values.kafkaCluster.nodePool.resources.cpu | quote }} - - - name: kmsvc-redis - namespace: {{ .Values.namespace }} - chart: bitnami/redis - version: 20.6.0 - values: - - architecture: standalone - # Bitnami stopped publishing version-pinned tags under docker.io/bitnami - # (only `latest` remains there); bitnamilegacy/* mirrors the old - # versioned tags for free, so pin there instead of floating on `latest`. - # The chart's container-image allowlist check doesn't know about the - # legacy mirror, hence allowInsecureImages. - global: - security: - allowInsecureImages: true - image: - repository: bitnamilegacy/redis - auth: - enabled: false - master: - persistence: - enabled: true - storageClass: {{ .Values.redis.storageClass }} - size: 2Gi - resources: - limits: - memory: {{ .Values.redis.memoryLimit }} - requests: - memory: {{ .Values.redis.memoryLimit }} - - - name: queue-crd - namespace: {{ .Values.namespace }} - chart: charts/queue-crd - needs: - - {{ .Values.namespace }}/kafka-cluster - - {{ .Values.namespace }}/kmsvc-redis - values: - - namespace: {{ .Values.namespace }} - kafkaBrokers: "kmsvc-kafka-bootstrap.{{ .Values.namespace }}.svc.cluster.local:9092" - redisAddr: "kmsvc-redis-master.{{ .Values.namespace }}.svc.cluster.local:6379" - - - name: management-service - namespace: {{ .Values.namespace }} - chart: charts/management-service - needs: - - {{ .Values.namespace }}/kafka-cluster - - {{ .Values.namespace }}/kmsvc-redis - values: - - namespace: {{ .Values.namespace }} - env: - kafkaBrokers: "kmsvc-kafka-bootstrap.{{ .Values.namespace }}.svc.cluster.local:9092" - redisAddr: "kmsvc-redis-master.{{ .Values.namespace }}.svc.cluster.local:6379" - authentikIssuerURL: {{ .Values.managementService.authentikIssuerURL | quote }} - authentikAudience: {{ .Values.managementService.authentikAudience | quote }} - ingress: - enabled: true - host: {{ .Values.managementService.ingress.host | quote }} - clusterIssuer: {{ .Values.managementService.ingress.clusterIssuer | quote }} diff --git a/k8s/applications/sqs/kustomization.yaml b/k8s/applications/sqs/kustomization.yaml deleted file mode 100644 index c0a5fd9..0000000 --- a/k8s/applications/sqs/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: sqs -resources: - - argocd/ - - environments/ - - queues/ -# Helm charts deployed via ArgoCD Helm source diff --git a/k8s/applications/sqs/queues/kustomization.yaml b/k8s/applications/sqs/queues/kustomization.yaml deleted file mode 100644 index 587b59f..0000000 --- a/k8s/applications/sqs/queues/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - orders-fifo.yaml diff --git a/k8s/applications/sqs/queues/orders-fifo.yaml b/k8s/applications/sqs/queues/orders-fifo.yaml deleted file mode 100644 index a58b487..0000000 --- a/k8s/applications/sqs/queues/orders-fifo.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: kmsvc.io/v1 -kind: Queue -metadata: - name: orders-fifo - namespace: sqs -spec: - fifoQueue: true - visibilityTimeoutSeconds: 30 - messageRetentionPeriodSeconds: 345600 - maxReceiveCount: 5 - deadLetterTargetQueue: orders-fifo-dlq - delaySeconds: 0 - partitionsPerShard: 6 - minShards: 1 - maxShards: 8 - shardSplitThresholdBytesPerSec: 5242880 - shardSplitCooldownSeconds: 300 ---- -apiVersion: kmsvc.io/v1 -kind: Queue -metadata: - name: orders-fifo-dlq - namespace: sqs -spec: - fifoQueue: true - isDLQ: true - visibilityTimeoutSeconds: 30 - messageRetentionPeriodSeconds: 1209600 - maxReceiveCount: 5 - partitionsPerShard: 6 - minShards: 1 - maxShards: 1 diff --git a/k8s/applications/temporal/PHASE_2_TESTING.md b/k8s/applications/temporal/PHASE_2_TESTING.md deleted file mode 100644 index 2531c66..0000000 --- a/k8s/applications/temporal/PHASE_2_TESTING.md +++ /dev/null @@ -1,237 +0,0 @@ -# Phase 2: Namespace-Scoped Auto-Provisioning Testing Guide - -## Overview - -Phase 2 implements **namespace-scoped automatic TemporalWorker provisioning** (Kafka broker model). - -One TemporalWorker per Temporal namespace processes ALL task queues in that namespace. When multiple Queues share the same `temporal.io/namespace` label, they trigger creation of a single TemporalWorker that handles all of them. - -``` -Queues (labeled temporal.io/namespace: "production") - ├── orders-fifo - ├── payments - └── notifications - ↓ -queue-operator creates 1 TemporalWorker (worker-production) - ↓ -TemporalWorker controller creates 1 Deployment - ↓ -Worker pod(s) connect to Temporal namespace "production" - ↓ -Process ALL task queues in that namespace (scale horizontally by replicas) -``` - -## Implementation Changes - -### 1. TemporalWorker CRD (`apis/kmsvc/v1/temporalworker_types.go`) -- New Kubernetes resource type to manage namespace-scoped workers -- Fields: Namespace (required), Image, Replicas, Resources, NodeSelector, Affinity, Tolerations -- Status: Phase (Pending/Ready/Failed), Replicas, ReadyReplicas, Conditions -- Model: 1 TemporalWorker per Temporal namespace (not per queue) - -### 2. QueueReconciler Extension (`internal/operator/queue_controller.go`) -- New method: `reconcileTemporalWorker()` -- Logic: If Queue has `temporal.io/namespace` label, create TemporalWorker for that namespace -- Idempotent: multiple queues with same namespace label create same TemporalWorker (no duplicates) - -### 3. TemporalWorkerReconciler (`internal/operator/temporal_worker_controller.go`) -- New controller watching TemporalWorker objects -- Creates/updates Kubernetes Deployment with: - - Pod spec: container image, env vars (TEMPORAL_FRONTEND_ADDRESS, TEMPORAL_TASK_QUEUE) - - Replicas, resources, node selector, affinity, tolerations from TemporalWorker spec -- Updates TemporalWorker status with deployment replica counts and phase - -### 4. Operator Main (`cmd/queue-operator/main.go`) -- Registers TemporalWorker CRD in scheme -- Registers TemporalWorkerReconciler controller -- Controller watches TemporalWorker objects; owns Deployment objects - -## Testing Procedure - -### Prerequisites -- kmsvc queue-operator must be running (built and deployed) -- Temporal cluster must be ready (temporal-frontend service available at `temporal-frontend.temporal.svc.cluster.local:7233`) -- story-crater-backend Docker image must exist (used as default worker image) - -### Step 1: Build and Deploy kmsvc Operator -```bash -cd /Users/rockliang/workplace/kmsvc-manage -make build # builds queue-operator binary -make docker-build # builds Docker image -make deploy # deploys to cluster (requires Helm chart) -``` - -Or manually: -```bash -cd /Users/rockliang/workplace/kmsvc-manage -go build -o bin/queue-operator ./cmd/queue-operator -kubectl apply -f k8s/queue-operator-rbac.yaml -kubectl apply -f k8s/queue-operator-deployment.yaml -``` - -### Step 2: Create Queues with Temporal Namespace Labels -```bash -kubectl apply -f /Users/rockliang/workplace/homelab/k8s/temporal/queues/example-queue.yaml -``` - -Verify Queues are Ready: -```bash -kubectl get queue -n sqs -l temporal.io/namespace=production -kubectl describe queue -n sqs story-crater-tasks -``` - -Expected: -``` -NAME FIFO PHASE AGE -story-crater-tasks false Ready 5s -story-crater-notifications false Ready 5s -``` - -### Step 3: Verify TemporalWorker CRD Auto-Created (1 per namespace) -```bash -kubectl get temporalworker -n temporal -kubectl describe temporalworker -n temporal worker-production -``` - -Expected: -``` -NAME PHASE READY DESIRED AGE -worker-production Pending 0 1 5s -``` - -Only ONE TemporalWorker for all queues in "production" namespace! - -### Step 4: Verify Deployment Auto-Created -```bash -kubectl get deploy -n temporal -l app.kubernetes.io/managed-by=kmsvc-temporal-operator -kubectl get pods -n temporal -l app.kubernetes.io/instance=worker-production -``` - -Expected: -``` -NAME READY UP-TO-DATE AVAILABLE AGE -worker-production 1/1 1 1 10s - -NAME READY STATUS RESTARTS AGE -worker-production-5f8b4c... 1/1 Running 0 10s -``` - -### Step 5: Verify Worker Connected to Temporal Namespace -Check Temporal UI for namespace "production": -```bash -open https://temporal.riotpiao.com/namespaces/production/task-queues -``` - -Look for all task queues with worker count > 0: -- `story-crater-tasks` -- `story-crater-notifications` -- (worker processes all of them) - -Or via CLI: -```bash -kubectl port-forward -n temporal svc/temporal-frontend 7233 & -curl http://localhost:7233/api/v1/task-queues?namespace=production -``` - -### Step 6: Verify TemporalWorker Status Updated -```bash -kubectl get temporalworker -n temporal -kubectl describe temporalworker -n temporal worker-production -``` - -Expected: -``` -NAME PHASE READY DESIRED AGE -worker-production Ready 1 1 15s - -Status: - Phase: Ready - Ready Replicas: 1 - Replicas: 1 -``` - -### Step 7: Test Namespace-Level Scaling -Create more queues in the same namespace: -```yaml -apiVersion: kmsvc.io/v1 -kind: Queue -metadata: - name: story-crater-llm-processing - namespace: sqs - labels: - temporal.io/namespace: "production" # same namespace -``` - -Verify: No new TemporalWorker created (same worker handles all 3 queues): -```bash -kubectl get temporalworker -n temporal # still just 1 worker-production -kubectl get deploy -n temporal worker-production # same deployment -``` - -Worker auto-discovers new task queue in namespace and processes it. - -### Step 8: Test Cascading Deletion -Delete a Queue; worker should remain (other queues still need it): -```bash -kubectl delete queue -n sqs story-crater-notifications -``` - -Verify: -```bash -kubectl get temporalworker -n temporal # worker-production still exists -kubectl get pods -n temporal worker-production # still running -``` - -Delete all queues in namespace: -```bash -kubectl delete queue -n sqs -l temporal.io/namespace=production -``` - -Verify: TemporalWorker now has no owner (not cascade-deleted; manual cleanup needed): -```bash -kubectl get temporalworker -n temporal # worker-production still there (manual cleanup) -kubectl delete temporalworker -n temporal worker-production # cleanup manually -``` - -## Debugging - -### Queue stuck in Pending -Check queue-operator logs: -```bash -kubectl logs -n sqs deploy/kmsvc-queue-operator -f -kubectl logs -n sqs deploy/kmsvc-queue-operator --tail=50 | grep -i error -``` - -### TemporalWorker not created -- Verify Queue has the label: `kubectl get queue -o yaml | grep temporal.io` -- Check queue-operator logs for "reconcileTemporalWorker" errors - -### Deployment not created -- Check TemporalWorker controller logs: `kubectl logs -n sqs deploy/kmsvc-queue-operator -f` -- Verify TemporalWorker exists: `kubectl get temporalworker -n temporal` -- Check Deployment errors: `kubectl describe deploy -n temporal worker-story-crater-tasks` - -### Worker not showing in Temporal UI -- Check pod logs: `kubectl logs -n temporal deploy/worker-story-crater-tasks` -- Verify env vars: `kubectl set env pod -n temporal --list | grep TEMPORAL` -- Test connectivity: `kubectl exec -n temporal -- nc -zv temporal-frontend.temporal.svc.cluster.local 7233` - -## Next Steps - -Once Phase 2 is working: -1. **Phase 3 (Future):** Implement autoscaling based on queue depth metrics -2. **Production Hardening:** - - Add QueueRef validation (ensure Queue exists in sqs namespace) - - Add image validation/defaults from ConfigMap - - Add worker readiness probe configuration - - Add graceful shutdown/drain behavior - -## Files Modified/Created - -| File | Change | -|------|--------| -| `apis/kmsvc/v1/temporalworker_types.go` | NEW: CRD type definitions | -| `internal/operator/queue_controller.go` | MODIFIED: Added reconcileTemporalWorker() | -| `internal/operator/temporal_worker_controller.go` | NEW: TemporalWorker → Deployment reconciler | -| `cmd/queue-operator/main.go` | MODIFIED: Register TemporalWorker CRD + controller | -| `k8s/temporal/queues/example-queue.yaml` | NEW: Example Queue with label | diff --git a/k8s/applications/temporal/TEMPORAL_OAUTH2_SETUP.md b/k8s/applications/temporal/TEMPORAL_OAUTH2_SETUP.md deleted file mode 100644 index e292132..0000000 --- a/k8s/applications/temporal/TEMPORAL_OAUTH2_SETUP.md +++ /dev/null @@ -1,220 +0,0 @@ -# Temporal OAuth2-Proxy Setup (Authentik OIDC) - -## Overview - -Protects Temporal UI with Authentik OIDC authentication. Traffic flow: - -``` -Browser → Ingress (TLS) → oauth2-proxy (OIDC check) → temporal-web (internal) - ↓ - Redirects to Authentik login - ↓ - JWT cookie issued - ↓ - Forwards to temporal-web -``` - -## Prerequisites - -✅ Authentik OIDC provider `temporal` already exists with: -- Client ID: `temporal` -- Client Secret: stored in Kubernetes secret `temporal-oidc` (key: `clientSecret`) -- Redirect URI: `https://temporal.riotpiao.com/oauth2/callback` - -## Secrets - -The `temporal-oidc` secret must contain: - -| Key | Value | Source | -|-----|-------|--------| -| `clientSecret` | OAuth2 client secret from Authentik | Authentik → Applications → temporal | -| `cookieSecret` | Session encryption key (base64 32-byte) | Generate: `openssl rand -base64 32` | - -### Check existing secret: - -```bash -kubectl get secret -n temporal temporal-oidc -kubectl describe secret -n temporal temporal-oidc -``` - -### If missing, create it: - -```bash -# Get client secret from Authentik UI -# Applications → temporal → copy "Client Secret" -CLIENT_SECRET="..." - -# Generate cookie secret -COOKIE_SECRET=$(openssl rand -base64 32) - -# Create secret -kubectl create secret generic temporal-oidc \ - -n temporal \ - --from-literal=clientSecret="${CLIENT_SECRET}" \ - --from-literal=cookieSecret="${COOKIE_SECRET}" -``` - -## Deployment Steps - -### Step 1: Apply OAuth2-Proxy Manifests -```bash -kubectl apply -f k8s/temporal/oauth2-proxy.yaml -``` - -Verify: -```bash -kubectl get deploy -n temporal oauth2-proxy -kubectl logs -n temporal deploy/oauth2-proxy -``` - -Expected log: -``` -[] [oauthproxy.go:...] Listening on 0.0.0.0:4180 -``` - -### Step 2: Apply OAuth2-Proxy Ingress -```bash -kubectl apply -f k8s/temporal/temporal-ingress-oauth2.yaml -``` - -Verify: -```bash -kubectl get ingress -n temporal -``` - -Expected: -``` -NAME CLASS HOSTS ADDRESS PORTS AGE -temporal nginx temporal.riotpiao.com ... 80, 443 10s -``` - -### Step 3: Test Access - -1. **Open Temporal UI (unauthenticated):** - ```bash - open https://temporal.riotpiao.com - ``` - - Expected: Redirects to Authentik login page - -2. **Login with Authentik credentials** - - Username/email - - Password - - Should redirect back to `temporal.riotpiao.com` and display UI - -3. **Verify auth:** - ```bash - # Check for oauth2_proxy cookie - curl -v https://temporal.riotpiao.com 2>&1 | grep -i cookie - ``` - -4. **Check oauth2-proxy logs:** - ```bash - kubectl logs -n temporal deploy/oauth2-proxy -f - ``` - - Look for: - ``` - [timestamp] [auth_test.go:...] Authentication successful - ``` - -## Troubleshooting - -### Redirect URI mismatch -Error in oauth2-proxy logs: -``` -redirect_uri_mismatch: The redirect_uri does not match the one registered in Authentik -``` - -Fix: -- Verify Authentik application (Applications → temporal) has redirect URI: `https://temporal.riotpiao.com/oauth2/callback` -- Ensure HTTPS (not HTTP) - -### Missing secret -Error: -``` -clientSecret: key not found in temporal-oidc secret -``` - -Fix: -```bash -kubectl get secret -n temporal temporal-oidc -o yaml -# If missing, create per "Secrets" section above -``` - -### Cookie secret expiration -OAuth2-Proxy won't start if `cookieSecret` is empty or invalid. - -Fix: -```bash -COOKIE_SECRET=$(openssl rand -base64 32) -kubectl patch secret temporal-oidc -n temporal \ - -p "{\"data\":{\"cookieSecret\":\"$(echo -n $COOKIE_SECRET | base64)\"}}}" -kubectl rollout restart deploy/oauth2-proxy -n temporal -``` - -### oauth2-proxy crashes with "connection refused" -Error in logs: -``` -upstream connect error or disconnect/reset before headers -``` - -Likely cause: `temporal-web` service not accessible. - -Check: -```bash -kubectl get svc -n temporal temporal-web -kubectl exec -n temporal deploy/oauth2-proxy -- curl http://temporal-web:8080 -``` - -## File Structure - -``` -k8s/temporal/ -├── oauth2-proxy.yaml # oauth2-proxy Deployment + Service + SA -├── temporal-ingress-oauth2.yaml # Ingress routing to oauth2-proxy -├── oauth2-proxy-values.yaml # Helm values (reference only) -└── temporal-values.yaml # Modified: ingress.enabled=false -``` - -## Next: Add to Helmfile - -If integrating with helmfile.yaml.gotmpl: - -```yaml -releases: - - name: temporal - # ... existing config ... - hooks: - postSync: - - events: ["success"] - showlogs: true - command: "sh" - args: - - -c - - | - kubectl apply -f k8s/temporal/oauth2-proxy.yaml - kubectl apply -f k8s/temporal/temporal-ingress-oauth2.yaml -``` - -Or add separate releases: - -```yaml - - name: oauth2-proxy-temporal - namespace: temporal - chart: oauth2-proxy/oauth2-proxy - version: "6.x.x" - values: - - k8s/temporal/oauth2-proxy-values.yaml - set: - - name: config.clientSecret - value: "{{ (env "TEMPORAL_OIDC_CLIENT_SECRET") }}" - - name: config.cookieSecret - value: "{{ (env "TEMPORAL_OIDC_COOKIE_SECRET") }}" -``` - -Then add to `.env`: -```bash -TEMPORAL_OIDC_CLIENT_SECRET= -TEMPORAL_OIDC_COOKIE_SECRET=$(openssl rand -base64 32) -``` diff --git a/k8s/applications/temporal/elasticsearch.yaml b/k8s/applications/temporal/elasticsearch.yaml deleted file mode 100644 index 5c73b1f..0000000 --- a/k8s/applications/temporal/elasticsearch.yaml +++ /dev/null @@ -1,102 +0,0 @@ -# Elasticsearch 7.17.0 for Temporal visibility store -# Deployed to worker nodes (not control plane to save CP resources for LLM work) -# 2Gi heap + 4Gi memory limit for stable operation - -apiVersion: v1 -kind: ConfigMap -metadata: - name: elasticsearch-config - namespace: temporal -data: - elasticsearch.yml: | - cluster.name: temporal-elasticsearch - node.name: temporal-elasticsearch-0 - discovery.type: single-node - network.host: 0.0.0.0 - http.host: 0.0.0.0 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: temporal-elasticsearch - namespace: temporal -spec: - replicas: 1 - selector: - matchLabels: - app: temporal-elasticsearch - template: - metadata: - labels: - app: temporal-elasticsearch - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/worker - operator: Exists - containers: - - name: elasticsearch - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 - env: - - name: discovery.type - value: single-node - - name: "ES_JAVA_OPTS" - value: "-Xms2g -Xmx2g" - ports: - - containerPort: 9200 - name: http - - containerPort: 9300 - name: transport - livenessProbe: - httpGet: - path: /_cluster/health - port: 9200 - initialDelaySeconds: 180 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /_cluster/health?local=true - port: 9200 - initialDelaySeconds: 150 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 5 - resources: - requests: - cpu: 500m - memory: 2Gi - limits: - cpu: 2000m - memory: 4Gi - volumeMounts: - - name: config - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml - subPath: elasticsearch.yml - volumes: - - name: config - configMap: - name: elasticsearch-config - ---- -apiVersion: v1 -kind: Service -metadata: - name: temporal-elasticsearch - namespace: temporal -spec: - selector: - app: temporal-elasticsearch - ports: - - port: 9200 - targetPort: 9200 - name: http - - port: 9300 - targetPort: 9300 - name: transport - type: ClusterIP diff --git a/k8s/applications/temporal/kustomization.yaml b/k8s/applications/temporal/kustomization.yaml deleted file mode 100644 index 5dadecf..0000000 --- a/k8s/applications/temporal/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: temporal -resources: - - elasticsearch.yaml - - queues/ -# SOPS secrets (*.enc.yaml) handled by ArgoCD SOPS plugin at sync time -# temporal deployed via ArgoCD Helm source diff --git a/k8s/applications/temporal/queues/example-queue.yaml b/k8s/applications/temporal/queues/example-queue.yaml deleted file mode 100644 index 794ee41..0000000 --- a/k8s/applications/temporal/queues/example-queue.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Example Queues for "production" Temporal namespace -# When applied, queue-operator creates 1 TemporalWorker (worker-production) -# that processes ALL queues in the "production" namespace ---- -apiVersion: kmsvc.io/v1 -kind: Queue -metadata: - name: story-crater-tasks - namespace: sqs - labels: - temporal.io/namespace: "production" -spec: - fifoQueue: false - visibilityTimeoutSeconds: 30 - messageRetentionPeriodSeconds: 345600 - maxReceiveCount: 5 - partitionsPerShard: 6 - minShards: 1 - maxShards: 8 - shardSplitThresholdBytesPerSec: 5242880 - ---- -apiVersion: kmsvc.io/v1 -kind: Queue -metadata: - name: story-crater-notifications - namespace: sqs - labels: - temporal.io/namespace: "production" -spec: - fifoQueue: false - visibilityTimeoutSeconds: 60 - messageRetentionPeriodSeconds: 345600 - maxReceiveCount: 3 - partitionsPerShard: 3 - minShards: 1 - maxShards: 4 - shardSplitThresholdBytesPerSec: 2621440 diff --git a/k8s/applications/temporal/queues/kustomization.yaml b/k8s/applications/temporal/queues/kustomization.yaml deleted file mode 100644 index c5741b6..0000000 --- a/k8s/applications/temporal/queues/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - example-queue.yaml diff --git a/k8s/applications/cloudflared/cloudflared-secrets.enc.yaml b/k8s/apps/cloudflared/cloudflared-secrets.enc.yaml similarity index 100% rename from k8s/applications/cloudflared/cloudflared-secrets.enc.yaml rename to k8s/apps/cloudflared/cloudflared-secrets.enc.yaml diff --git a/k8s/applications/cloudflared/deployment.yaml b/k8s/apps/cloudflared/deployment.yaml similarity index 100% rename from k8s/applications/cloudflared/deployment.yaml rename to k8s/apps/cloudflared/deployment.yaml diff --git a/k8s/applications/cloudflared/kustomization.yaml b/k8s/apps/cloudflared/kustomization.yaml similarity index 100% rename from k8s/applications/cloudflared/kustomization.yaml rename to k8s/apps/cloudflared/kustomization.yaml diff --git a/k8s/applications/duckdns/duckdns-corn.yaml b/k8s/apps/duckdns/duckdns-corn.yaml similarity index 100% rename from k8s/applications/duckdns/duckdns-corn.yaml rename to k8s/apps/duckdns/duckdns-corn.yaml diff --git a/k8s/applications/duckdns/kustomization.yaml b/k8s/apps/duckdns/kustomization.yaml similarity index 100% rename from k8s/applications/duckdns/kustomization.yaml rename to k8s/apps/duckdns/kustomization.yaml diff --git a/k8s/applications/homarr/fix-probes-job.yaml b/k8s/apps/homarr/fix-probes-job.yaml similarity index 100% rename from k8s/applications/homarr/fix-probes-job.yaml rename to k8s/apps/homarr/fix-probes-job.yaml diff --git a/k8s/applications/homarr/homarr-secrets.enc.yaml b/k8s/apps/homarr/homarr-secrets.enc.yaml similarity index 100% rename from k8s/applications/homarr/homarr-secrets.enc.yaml rename to k8s/apps/homarr/homarr-secrets.enc.yaml diff --git a/k8s/applications/homarr/homarr-values.yaml b/k8s/apps/homarr/homarr-values.yaml similarity index 100% rename from k8s/applications/homarr/homarr-values.yaml rename to k8s/apps/homarr/homarr-values.yaml diff --git a/k8s/applications/homarr/kustomization.yaml b/k8s/apps/homarr/kustomization.yaml similarity index 100% rename from k8s/applications/homarr/kustomization.yaml rename to k8s/apps/homarr/kustomization.yaml diff --git a/k8s/applications/sqs/charts/kafka-cluster/Chart.yaml b/k8s/apps/messaging/kafka-cluster/Chart.yaml similarity index 100% rename from k8s/applications/sqs/charts/kafka-cluster/Chart.yaml rename to k8s/apps/messaging/kafka-cluster/Chart.yaml diff --git a/k8s/applications/sqs/charts/kafka-cluster/templates/kafka-cluster.yaml b/k8s/apps/messaging/kafka-cluster/templates/kafka-cluster.yaml similarity index 100% rename from k8s/applications/sqs/charts/kafka-cluster/templates/kafka-cluster.yaml rename to k8s/apps/messaging/kafka-cluster/templates/kafka-cluster.yaml diff --git a/k8s/applications/sqs/charts/kafka-cluster/templates/kafka-nodepool.yaml b/k8s/apps/messaging/kafka-cluster/templates/kafka-nodepool.yaml similarity index 100% rename from k8s/applications/sqs/charts/kafka-cluster/templates/kafka-nodepool.yaml rename to k8s/apps/messaging/kafka-cluster/templates/kafka-nodepool.yaml diff --git a/k8s/applications/sqs/charts/kafka-cluster/values.yaml b/k8s/apps/messaging/kafka-cluster/values.yaml similarity index 100% rename from k8s/applications/sqs/charts/kafka-cluster/values.yaml rename to k8s/apps/messaging/kafka-cluster/values.yaml diff --git a/k8s/applications/sqs/charts/management-service/Chart.yaml b/k8s/apps/messaging/management-service/Chart.yaml similarity index 100% rename from k8s/applications/sqs/charts/management-service/Chart.yaml rename to k8s/apps/messaging/management-service/Chart.yaml diff --git a/k8s/applications/sqs/charts/management-service/templates/configmap.yaml b/k8s/apps/messaging/management-service/templates/configmap.yaml similarity index 100% rename from k8s/applications/sqs/charts/management-service/templates/configmap.yaml rename to k8s/apps/messaging/management-service/templates/configmap.yaml diff --git a/k8s/applications/sqs/charts/management-service/templates/deployment.yaml b/k8s/apps/messaging/management-service/templates/deployment.yaml similarity index 100% rename from k8s/applications/sqs/charts/management-service/templates/deployment.yaml rename to k8s/apps/messaging/management-service/templates/deployment.yaml diff --git a/k8s/applications/sqs/charts/management-service/templates/hpa.yaml b/k8s/apps/messaging/management-service/templates/hpa.yaml similarity index 100% rename from k8s/applications/sqs/charts/management-service/templates/hpa.yaml rename to k8s/apps/messaging/management-service/templates/hpa.yaml diff --git a/k8s/applications/sqs/charts/management-service/templates/ingress-grpc.yaml b/k8s/apps/messaging/management-service/templates/ingress-grpc.yaml similarity index 100% rename from k8s/applications/sqs/charts/management-service/templates/ingress-grpc.yaml rename to k8s/apps/messaging/management-service/templates/ingress-grpc.yaml diff --git a/k8s/applications/sqs/charts/management-service/templates/ingress.yaml b/k8s/apps/messaging/management-service/templates/ingress.yaml similarity index 100% rename from k8s/applications/sqs/charts/management-service/templates/ingress.yaml rename to k8s/apps/messaging/management-service/templates/ingress.yaml diff --git a/k8s/applications/sqs/charts/management-service/templates/service.yaml b/k8s/apps/messaging/management-service/templates/service.yaml similarity index 100% rename from k8s/applications/sqs/charts/management-service/templates/service.yaml rename to k8s/apps/messaging/management-service/templates/service.yaml diff --git a/k8s/applications/sqs/charts/management-service/values.yaml b/k8s/apps/messaging/management-service/values.yaml similarity index 100% rename from k8s/applications/sqs/charts/management-service/values.yaml rename to k8s/apps/messaging/management-service/values.yaml diff --git a/k8s/applications/sqs/charts/queue-crd/Chart.yaml b/k8s/apps/messaging/queue-crd/Chart.yaml similarity index 100% rename from k8s/applications/sqs/charts/queue-crd/Chart.yaml rename to k8s/apps/messaging/queue-crd/Chart.yaml diff --git a/k8s/applications/sqs/charts/queue-crd/templates/crd.yaml b/k8s/apps/messaging/queue-crd/templates/crd.yaml similarity index 100% rename from k8s/applications/sqs/charts/queue-crd/templates/crd.yaml rename to k8s/apps/messaging/queue-crd/templates/crd.yaml diff --git a/k8s/applications/sqs/charts/queue-crd/templates/operator-deployment.yaml b/k8s/apps/messaging/queue-crd/templates/operator-deployment.yaml similarity index 100% rename from k8s/applications/sqs/charts/queue-crd/templates/operator-deployment.yaml rename to k8s/apps/messaging/queue-crd/templates/operator-deployment.yaml diff --git a/k8s/applications/sqs/charts/queue-crd/templates/rbac.yaml b/k8s/apps/messaging/queue-crd/templates/rbac.yaml similarity index 100% rename from k8s/applications/sqs/charts/queue-crd/templates/rbac.yaml rename to k8s/apps/messaging/queue-crd/templates/rbac.yaml diff --git a/k8s/applications/sqs/charts/queue-crd/templates/temporalworker-crd.yaml b/k8s/apps/messaging/queue-crd/templates/temporalworker-crd.yaml similarity index 100% rename from k8s/applications/sqs/charts/queue-crd/templates/temporalworker-crd.yaml rename to k8s/apps/messaging/queue-crd/templates/temporalworker-crd.yaml diff --git a/k8s/applications/sqs/charts/queue-crd/values.yaml b/k8s/apps/messaging/queue-crd/values.yaml similarity index 100% rename from k8s/applications/sqs/charts/queue-crd/values.yaml rename to k8s/apps/messaging/queue-crd/values.yaml diff --git a/k8s/applications/portainer/kustomization.yaml b/k8s/apps/portainer/kustomization.yaml similarity index 100% rename from k8s/applications/portainer/kustomization.yaml rename to k8s/apps/portainer/kustomization.yaml diff --git a/k8s/applications/portainer/portainer-values.yaml b/k8s/apps/portainer/portainer-values.yaml similarity index 100% rename from k8s/applications/portainer/portainer-values.yaml rename to k8s/apps/portainer/portainer-values.yaml diff --git a/k8s/applications/temporal/temporal-secrets.enc.yaml b/k8s/apps/temporal/temporal-secrets.enc.yaml similarity index 100% rename from k8s/applications/temporal/temporal-secrets.enc.yaml rename to k8s/apps/temporal/temporal-secrets.enc.yaml diff --git a/k8s/applications/temporal/temporal-values.yaml b/k8s/apps/temporal/temporal-values.yaml similarity index 87% rename from k8s/applications/temporal/temporal-values.yaml rename to k8s/apps/temporal/temporal-values.yaml index 4bb5dcc..6b666ab 100644 --- a/k8s/applications/temporal/temporal-values.yaml +++ b/k8s/apps/temporal/temporal-values.yaml @@ -1,6 +1,6 @@ # k8s/temporal/temporal-values.yaml # Temporal — workflow engine -# Uses external CNPG PostgreSQL for persistence (ddb-cluster) +# Uses external CNPG PostgreSQL for persistence (temporal-db) # Visibility via same PostgreSQL instance, separate database. # # IMPORTANT — chart schema note (root-caused after Postgres never actually @@ -43,11 +43,11 @@ grafana: enabled: false # ── Schema setup/update Jobs ────────────────────────────────────────────────── -# The `temporal` and `temporal_visibility` databases are provisioned -# declaratively by CNPG Database CRs (k8s/data/temporal-database.yaml, -# temporal-visibility-database.yaml), so createDatabase stays disabled (the -# `temporal` role also lacks CREATEDB). setup/update run temporal-sql-tool as -# the `temporal` owner against those existing DBs to install and migrate the +# The `temporal` DB is created by the dedicated temporal-db cluster's initdb and +# `temporal_visibility` by a CNPG Database CR — both in +# k8s/infra/databases/temporal-db.yaml — so createDatabase stays disabled. +# setup/update run temporal-sql-tool as the `app` owner against those existing +# DBs to install and migrate the # Temporal server schema — without them both DBs have zero tables and the # server dies on "no usable database connection found" (no schema_version row). schema: @@ -92,7 +92,7 @@ server: driver: "sql" sql: driver: "postgres12" - host: "ddb-cluster-rw.ddb.svc.cluster.local" + host: "temporal-db-rw.temporal.svc.cluster.local" port: 5432 database: "temporal" user: "app" @@ -102,8 +102,8 @@ server: # existingSecret is set the chart's own server-secret.yaml Secret # template is skipped entirely (see templates/server-secret.yaml: # `not $driverConfig.existingSecret` guards its creation). - # Use unified ddb-cluster-app secret (copied to temporal namespace) - existingSecret: "ddb-cluster-app" + # Use unified temporal-db-app secret (generated in temporal namespace) + existingSecret: "temporal-db-app" secretKey: "password" maxConns: 20 maxIdleConns: 10 @@ -118,12 +118,12 @@ server: driver: "sql" sql: driver: "postgres12" - host: "ddb-cluster-rw.ddb.svc.cluster.local" + host: "temporal-db-rw.temporal.svc.cluster.local" port: 5432 database: "temporal_visibility" user: "app" - # Use unified ddb-cluster-app secret (copied to temporal namespace) - existingSecret: "ddb-cluster-app" + # Use unified temporal-db-app secret (generated in temporal namespace) + existingSecret: "temporal-db-app" secretKey: "password" maxConns: 20 maxIdleConns: 10 diff --git a/k8s/argocd/apps/10-storage-observability.yaml b/k8s/argocd/apps/10-storage-observability.yaml index 6070433..2cecff3 100644 --- a/k8s/argocd/apps/10-storage-observability.yaml +++ b/k8s/argocd/apps/10-storage-observability.yaml @@ -16,7 +16,7 @@ spec: targetRevision: "5.0.18" helm: valueFiles: - - $values/k8s/infrastructure/minio/minio-operator-values.yaml + - $values/k8s/infra/minio/minio-operator-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values @@ -43,7 +43,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/infrastructure/minio + path: k8s/infra/minio destination: server: https://kubernetes.default.svc namespace: storage @@ -68,7 +68,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/infrastructure/longhorn + path: k8s/infra/longhorn destination: server: https://kubernetes.default.svc namespace: longhorn-system @@ -93,7 +93,7 @@ spec: helm: skipCrds: true valueFiles: - - $values/k8s/platform/monitoring/prometheus-values.yaml + - $values/k8s/infra/monitoring/prometheus-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values @@ -122,7 +122,7 @@ spec: # metadata conflict since CRDs are cluster-scoped). --- # CRDs only, extracted to plain YAML (`helm show crds kube-prometheus-stack`) -# and committed to git under k8s/platform/monitoring/crds/, applied via Server- +# and committed to git under k8s/infra/monitoring/crds/, applied via Server- # Side Apply to avoid the etcd 262144-byte last-applied-configuration # annotation limit that client-side apply hits on these very large CRDs # (prometheuses, alertmanagers, scrapeconfigs, etc). A plain git path source @@ -131,7 +131,7 @@ spec: # means from a Helm chart. Split out from the main `prometheus` Application # (helm.skipCrds: true there) because ServerSideApply conflicts with that # app's managedNamespaceMetadata. -# NOTE: bump k8s/platform/monitoring/crds/kube-prometheus-stack-crds.yaml +# NOTE: bump k8s/infra/monitoring/crds/kube-prometheus-stack-crds.yaml # whenever the kube-prometheus-stack chart version changes materially # (`helm show crds prometheus-community/kube-prometheus-stack > ...`). apiVersion: argoproj.io/v1alpha1 @@ -146,7 +146,38 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/platform/monitoring/crds + path: k8s/infra/monitoring/crds + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true +--- +# Cluster monitoring config: custom PrometheusRules (per-app namespaces), +# ServiceMonitors (monitoring ns), and Grafana dashboard ConfigMaps (logging ns, +# grafana sidecar-discovered). Single source = k8s/infra/monitoring (one +# kustomization, no namespace transformer so per-app rule namespaces are kept). +# Wave 2: after prometheus-operator CRDs (wave 0) + stack (wave 1) and grafana +# (wave 2, logging). ServerSideApply avoids the etcd last-applied annotation +# limit on the large dashboard ConfigMap JSON. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: monitoring-config + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + project: homelab + source: + repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git + targetRevision: main + path: k8s/infra/monitoring destination: server: https://kubernetes.default.svc namespace: monitoring @@ -173,7 +204,7 @@ spec: targetRevision: "~11" helm: valueFiles: - - $values/k8s/platform/monitoring/blackbox-exporter-values.yaml + - $values/k8s/infra/monitoring/blackbox-exporter-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values diff --git a/k8s/argocd/apps/20-logging.yaml b/k8s/argocd/apps/20-logging.yaml index d8f7613..b2bffac 100644 --- a/k8s/argocd/apps/20-logging.yaml +++ b/k8s/argocd/apps/20-logging.yaml @@ -18,7 +18,7 @@ spec: targetRevision: "*" helm: valueFiles: - - $values/k8s/platform/logging/loki-values.yaml + - $values/k8s/infra/logging/loki-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values @@ -52,7 +52,7 @@ spec: targetRevision: "*" helm: valueFiles: - - $values/k8s/platform/logging/grafana-values.yaml + - $values/k8s/infra/logging/grafana-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values @@ -86,7 +86,7 @@ spec: targetRevision: "*" helm: valueFiles: - - $values/k8s/platform/logging/promtail-values.yaml + - $values/k8s/infra/logging/promtail-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values diff --git a/k8s/argocd/apps/30-security.yaml b/k8s/argocd/apps/30-security.yaml index 20e8084..c3329a8 100644 --- a/k8s/argocd/apps/30-security.yaml +++ b/k8s/argocd/apps/30-security.yaml @@ -1,6 +1,6 @@ # Wave 3 — Vault + Authentik (identity), plus IAM raw jobs and the Forgejo # runner. Authentik/Vault values reference SOPS-managed secrets (see *.enc.yaml -# in k8s/security/iam) resolved by the ArgoCD SOPS plugin at sync time. +# in k8s/infra/iam) resolved by the ArgoCD SOPS plugin at sync time. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: @@ -16,7 +16,7 @@ spec: targetRevision: "*" helm: valueFiles: - - $values/k8s/security/iam/vault-values.yaml + - $values/k8s/infra/iam/vault-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values @@ -45,7 +45,7 @@ spec: targetRevision: "*" helm: valueFiles: - - $values/k8s/security/iam/authentik-values.yaml + - $values/k8s/infra/iam/authentik-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values @@ -70,7 +70,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/security/iam + path: k8s/infra/iam destination: server: https://kubernetes.default.svc namespace: iam @@ -92,7 +92,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/security/ci-cd/charts/forgejo-runner + path: k8s/infra/forgejo-runner destination: server: https://kubernetes.default.svc namespace: cicd diff --git a/k8s/argocd/apps/40-data.yaml b/k8s/argocd/apps/40-data.yaml index 60eab27..c2bd7c1 100644 --- a/k8s/argocd/apps/40-data.yaml +++ b/k8s/argocd/apps/40-data.yaml @@ -1,24 +1,28 @@ -# Wave 6 — Database schemas + init jobs. -# CNPG operator and ddb-cluster are bootstrap-only (k8s/bootstrap-local/). -# This app manages ONLY the per-app databases and schema initialization. -# Dependencies: ddb-cluster (bootstrap wave 0), SOPS secrets (wave 4) +# Wave 2 — dedicated per-app CNPG Postgres clusters (authentik-db → ns iam, +# temporal-db + visibility → ns temporal). ONE App, ONE folder (k8s/infra/databases). +# CNPG operator is Phase-0 bootstrap; these Cluster CRs are GitOps — no circular +# dep (they run after ArgoCD is up, before their apps at w3/w8). CNPG generates +# each cluster's `-app` secret + `-rw` service in-namespace; the apps +# read them locally. Forgejo's DB stays separate (bootstrap/circular). apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: data-schemas + name: databases namespace: argocd annotations: - argocd.argoproj.io/sync-wave: "6" + argocd.argoproj.io/sync-wave: "2" spec: project: homelab source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/data/schemas # CHANGED from k8s/data to avoid ddb-cluster duplication + path: k8s/infra/databases destination: server: https://kubernetes.default.svc - namespace: ddb + namespace: default syncPolicy: automated: prune: true selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/k8s/argocd/apps/50-messaging.yaml b/k8s/argocd/apps/50-messaging.yaml index 7e2bd83..6d857c3 100644 --- a/k8s/argocd/apps/50-messaging.yaml +++ b/k8s/argocd/apps/50-messaging.yaml @@ -70,7 +70,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/applications/sqs/charts/kafka-cluster + path: k8s/apps/messaging/kafka-cluster destination: server: https://kubernetes.default.svc namespace: sqs @@ -91,7 +91,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/applications/sqs/charts/queue-crd + path: k8s/apps/messaging/queue-crd destination: server: https://kubernetes.default.svc namespace: sqs @@ -112,7 +112,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/applications/sqs/charts/management-service + path: k8s/apps/messaging/management-service destination: server: https://kubernetes.default.svc namespace: sqs diff --git a/k8s/argocd/apps/60-applications.yaml b/k8s/argocd/apps/60-applications.yaml index 8434033..59a4daa 100644 --- a/k8s/argocd/apps/60-applications.yaml +++ b/k8s/argocd/apps/60-applications.yaml @@ -2,7 +2,7 @@ # helpers (cloudflared tunnel, duckdns updater) that are already running. # Experimental dirs (llm, forge, dev-tools, shadowsocks) are intentionally # NOT included yet — add them here once they're production-ready. -# temporal using unified CNPG pattern (app user, ddb-cluster-app secret) +# temporal using unified CNPG pattern (app user, temporal-db-app secret) # Secret copied by bootstrap.sh (like cicd/iam namespaces) apiVersion: argoproj.io/v1alpha1 kind: Application @@ -19,7 +19,7 @@ spec: targetRevision: "0.74.0" helm: valueFiles: - - $values/k8s/applications/temporal/temporal-values.yaml + - $values/k8s/apps/temporal/temporal-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values @@ -48,7 +48,7 @@ spec: targetRevision: "*" helm: valueFiles: - - $values/k8s/applications/portainer/portainer-values.yaml + - $values/k8s/apps/portainer/portainer-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values @@ -74,7 +74,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/applications/cloudflared + path: k8s/apps/cloudflared destination: server: https://kubernetes.default.svc namespace: cloudflared @@ -97,7 +97,7 @@ spec: source: repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/applications/duckdns + path: k8s/apps/duckdns destination: server: https://kubernetes.default.svc namespace: duckdns @@ -125,13 +125,13 @@ spec: targetRevision: "*" helm: valueFiles: - - $values/k8s/applications/homarr/homarr-values.yaml + - $values/k8s/apps/homarr/homarr-values.yaml - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main - path: k8s/applications/homarr # PostSync hook: fix-probes-job.yaml + path: k8s/apps/homarr # PostSync hook: fix-probes-job.yaml destination: server: https://kubernetes.default.svc namespace: dashboard diff --git a/k8s/bootstrap/README.md b/k8s/bootstrap/README.md index fac7724..b54da74 100644 --- a/k8s/bootstrap/README.md +++ b/k8s/bootstrap/README.md @@ -115,7 +115,7 @@ git push forgejo main - **Phase stuck?** Check `kubectl get events -n --sort-by='.lastTimestamp'` - **ArgoCD duplicating?** Verify manifests match exactly (Helm values ↔ ArgoCD Application) -- **Forgejo won't start?** Check CNPG cluster Ready: `kubectl get cluster forgejo-db -n forgejo` +- **Forgejo won't start?** Check CNPG cluster Ready: `kubectl get cluster forgejo-db -n cicd` - **Can't push to Forgejo?** Verify ingress-nginx healthy, DNS resolves `forgejo.riotpiao.com` ## Migration from Old Bootstrap diff --git a/k8s/bootstrap/phase3-forgejo/forgejo-db.yaml b/k8s/bootstrap/phase3-forgejo/forgejo-db.yaml index c2c4060..53bb0bc 100644 --- a/k8s/bootstrap/phase3-forgejo/forgejo-db.yaml +++ b/k8s/bootstrap/phase3-forgejo/forgejo-db.yaml @@ -5,7 +5,7 @@ apiVersion: v1 kind: Namespace metadata: - name: forgejo + name: cicd annotations: argocd.argoproj.io/sync-options: Prune=false --- @@ -13,7 +13,7 @@ apiVersion: postgresql.cnpg.io/v1 kind: Cluster metadata: name: forgejo-db - namespace: forgejo + namespace: cicd annotations: argocd.argoproj.io/sync-options: Prune=false # Let ArgoCD adopt, don't delete labels: @@ -71,15 +71,21 @@ spec: enablePodMonitor: true affinity: - podAntiAffinityType: required + # preferred (not required) so it can't deadlock if fewer than 3 nodes are + # schedulable; tolerations let CNPG pods land on control-plane nodes. + podAntiAffinityType: preferred topologyKey: kubernetes.io/hostname + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule --- # Forgejo Redis (cache, session, queue) apiVersion: v1 kind: Service metadata: name: forgejo-redis - namespace: forgejo + namespace: cicd annotations: argocd.argoproj.io/sync-options: Prune=false spec: @@ -95,7 +101,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: forgejo-redis - namespace: forgejo + namespace: cicd annotations: argocd.argoproj.io/sync-options: Prune=false spec: @@ -108,6 +114,10 @@ spec: labels: app: forgejo-redis spec: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule containers: - name: redis image: redis:7-alpine diff --git a/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml b/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml index 3303699..cf57cdf 100644 --- a/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml +++ b/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml @@ -16,7 +16,7 @@ gitea: database: DB_TYPE: postgres - HOST: forgejo-db-rw.forgejo.svc.cluster.local:5432 + HOST: forgejo-db-rw.cicd.svc.cluster.local:5432 NAME: forgejo # User/password from CNPG-generated secret USER: @@ -32,15 +32,15 @@ gitea: cache: ADAPTER: redis - HOST: redis://forgejo-redis.forgejo.svc.cluster.local:6379/0 + HOST: redis://forgejo-redis.cicd.svc.cluster.local:6379/0 session: PROVIDER: redis - PROVIDER_CONFIG: redis://forgejo-redis.forgejo.svc.cluster.local:6379/1 + PROVIDER_CONFIG: redis://forgejo-redis.cicd.svc.cluster.local:6379/1 queue: TYPE: redis - CONN_STR: redis://forgejo-redis.forgejo.svc.cluster.local:6379/2 + CONN_STR: redis://forgejo-redis.cicd.svc.cluster.local:6379/2 # Persistence (shared storage for repos) persistence: diff --git a/k8s/data/cluster/ddb-cluster.yaml b/k8s/data/cluster/ddb-cluster.yaml deleted file mode 100644 index e2c2650..0000000 --- a/k8s/data/cluster/ddb-cluster.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: Cluster -metadata: - name: ddb-cluster - namespace: ddb - labels: - app: postgresql - layer: data -spec: - # 3-replica cluster — distributed across control-plane nodes (cp-1, cp-2, cp-3) - # Provides HA for Forgejo and other stateful apps using shared DDB - instances: 3 - - # PostgreSQL 16.2 - imageName: ghcr.io/cloudnative-pg/postgresql:16.2 - - # Bootstrap: create app database + extensions - bootstrap: - initdb: - database: app - owner: app - encoding: UTF8 - localeCollate: C - localeCType: C - postInitApplicationSQL: - - CREATE EXTENSION IF NOT EXISTS vector; - - CREATE EXTENSION IF NOT EXISTS pgcrypto; - - CREATE EXTENSION IF NOT EXISTS pg_trgm; - - # Simple ownership model: all apps use 'app' bootstrap user. - # Isolation via separate database names, not separate roles. - # Aligns with CNPG design (single cluster, multiple databases). - # managed.roles removed - no per-app roles needed. - - # Disable superuser (security) - enableSuperuserAccess: false - - # PostgreSQL configuration - postgresql: - parameters: - shared_buffers: "256MB" - max_parallel_workers: "4" - max_parallel_workers_per_gather: "4" - # WAL archiving for backups - archive_mode: "on" - archive_timeout: "5min" - log_destination: "csvlog" - log_directory: "/controller/log" - log_filename: "postgres" - log_rotation_age: "0" - dynamic_shared_memory_type: "posix" - - # Storage on Longhorn - storage: - size: 10Gi - storageClass: longhorn - - # Monitoring - monitoring: - enablePodMonitor: false - disableDefaultQueries: false - customQueriesConfigMap: - - name: cnpg-default-monitoring - key: queries - - # Pod anti-affinity for spreading replicas - affinity: - podAntiAffinityType: preferred diff --git a/k8s/data/cluster/forgejo-database.yaml b/k8s/data/cluster/forgejo-database.yaml deleted file mode 100644 index 1814887..0000000 --- a/k8s/data/cluster/forgejo-database.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: Database -metadata: - name: forgejo - namespace: ddb -spec: - name: forgejo - owner: app - cluster: - name: ddb-cluster diff --git a/k8s/data/cluster/kustomization.yaml b/k8s/data/cluster/kustomization.yaml deleted file mode 100644 index 79e3c4c..0000000 --- a/k8s/data/cluster/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -metadata: - name: ddb-cluster-bootstrap - annotations: - description: | - Bootstrap-only resources (NOT managed by ArgoCD GitOps). - These are applied via k8s/bootstrap-local/ and never touched afterward. - The actual deployment is in bootstrap-local/03-ddb-bootstrap.yaml. - -namespace: ddb - -# IMPORTANT: These files are duplicated in k8s/bootstrap-local/03-ddb-bootstrap.yaml -# DO NOT reference this kustomization from any ArgoCD Application. -resources: - - ddb-cluster.yaml - - forgejo-database.yaml diff --git a/k8s/data/db-role-secrets.enc.yaml b/k8s/data/db-role-secrets.enc.yaml deleted file mode 100644 index 9355fc0..0000000 --- a/k8s/data/db-role-secrets.enc.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: ENC[AES256_GCM,data:u74=,iv:vPf+N0mpxoTTHn8t3W6kiIbNrXGOFngfnc5vk3gzq+Y=,tag:OlJWpHdVqQHeA96OdJoA7A==,type:str] -kind: ENC[AES256_GCM,data:9f6uPGLQ,iv:V5KjMaBQoQRwMxHRoStLIlSYW0zqgcORLX5GKYKZ7hY=,tag:zLozLmSNHOd2/lFxFfnxtw==,type:str] -metadata: - name: ENC[AES256_GCM,data:uNU+lZyIdW0bwLQHoecYqdU=,iv:ov6MQQ9e7s+dRoQbZQPbjtB5j42dckyg27nx+LWJw0I=,tag:74nJSTgy9pZ5LH/O3xpwSA==,type:str] - namespace: ENC[AES256_GCM,data:xIre,iv:Gv//YoAeIoFgFbW9OJkHPv1Fz8NEAzB+qSpNYXm1008=,tag:Vsj/V3wLX8mUnHrMGZZxFw==,type:str] -type: ENC[AES256_GCM,data:pkm2nd/50FTigAJyEW2mEcgKVR0Y61VA,iv:DQ3k8Xp54nySnVEJvbiMXYJ+0ssUC8oSDLsoRut+qMg=,tag:ZwVH9ivA5hBCCkiGjXPbUw==,type:str] -stringData: - username: ENC[AES256_GCM,data:TeIYIIDS+RWp,iv:feJ2lRnPWnDKIMgSEz4A070k2ZrDFz7VzF7vthxoQuM=,tag:QYeI+6qeuyL3hIZSH7eQWg==,type:str] - password: ENC[AES256_GCM,data:ACIyp9eAiQtuzcN8piX5XX/UzHEjobxT1M6usFV6jwQ=,iv:AnmrpNo9JqtWYMwh4ghWXZfmeCTXFqzkfN69CPbWkpM=,tag:kZ2jfUAvtG39+Og/Km2fwA==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSaFFueUY2MUZ1dzRkelRL - aklvRytMSVBJQjA4UXdqSjlRQUo2R0lyTTFVCktUNWl1cWttZDdVd2ZyR0lPVERj - T1ZpMkRrZGgvWFJVRnZOdlZJc3RNcTAKLS0tIHVwY0JNMXVWeTZnWW9BQnM0TCtK - bVJ0eU05N3hIT3Z6TWdSOUxxeGh1b1UKuQ23KqCTNlEk3c8gbQsFGQwkN4gkI7Z2 - HauqbNHQoU64smns+HPDoHq7z87xRlQ8c8efNpIGhfYgS/nf1eGXbg== - -----END AGE ENCRYPTED FILE----- - recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh - lastmodified: "2026-07-21T06:35:41Z" - mac: ENC[AES256_GCM,data:VeVffJWYwoqNe/irAsWrl1rAOr2xMnsPgSCqiszaoF1807iU7FSpuvSOCAbaOJXSKgJ/MBflnegCIsDb7a1W4BcwhwuMGoGl2C2vBs2YPZvqTzfnfXAGrWCbZdyCh16Dg8nZQlwRe2QwokqW+t3j1qAYfq4KVu8MLi648adRIt0=,iv:mCMQXdfVQNDgY1ZHEQGBC4YaHwkCsyo+oADMJ7p23B8=,tag:7c9VkImsgkjSxOkM1Q3qgg==,type:str] - unencrypted_suffix: _unencrypted - version: 3.13.2 ---- -apiVersion: ENC[AES256_GCM,data:DxA=,iv:eWizGuPMW/RfEkLvL4pei42sov2EUwSDHKrtdraSxWw=,tag:E1w/dzFGqUIr+W+wY4YKNA==,type:str] -kind: ENC[AES256_GCM,data:amkGISvV,iv:13KTCVXg9EKQLfz4cCCPxfHAnPHZ0AheDmn8HIm87rw=,tag:wHfOOTJsrN2iiOTsSN507A==,type:str] -metadata: - name: ENC[AES256_GCM,data:lpkz7+GPScHzbUsBL276Gw==,iv:ce+pCqVqXZD7G6LZzFGMCK58k2icd07cE9DTJYOid5I=,tag:ug6gUmLuTkyy0eRsTGJqOw==,type:str] - namespace: ENC[AES256_GCM,data:sCqW,iv:5rvJSGBshQsjrUkizUekMpS0Fpyfkf7IcNL8aZR3lks=,tag:cFtIZO6F9eAdzE+IVv1wyg==,type:str] -type: ENC[AES256_GCM,data:aUYmzPoRlh2uDcjhZ5OD82ScAtBxd1Fm,iv:ovGv+qPiiLBC1mVSEX+QuJOkAjqPAUO/e+Nux2xuVKg=,tag:gaZwWSH7LF1FCKZr+P+R1A==,type:str] -stringData: - username: ENC[AES256_GCM,data:kvKdlAd1ox4=,iv:NcwxeRHzyRECQ7cLUwOeEiKkHMG7FhFMZRsfUfdtBmM=,tag:zfS+ZOFMHA6Q3LCkM2GSqA==,type:str] - password: ENC[AES256_GCM,data:7/ZJxWr39GrKA+i50S7wLBjt2Kaz9aG1ML44VGyRWi0=,iv:y0KlJ5ogdBDGzL6IPIjqD6Ir/S6K6v1dGTg8wxxp8n4=,tag:zwnCvi7RPqQdLJvfk69KgA==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSaFFueUY2MUZ1dzRkelRL - aklvRytMSVBJQjA4UXdqSjlRQUo2R0lyTTFVCktUNWl1cWttZDdVd2ZyR0lPVERj - T1ZpMkRrZGgvWFJVRnZOdlZJc3RNcTAKLS0tIHVwY0JNMXVWeTZnWW9BQnM0TCtK - bVJ0eU05N3hIT3Z6TWdSOUxxeGh1b1UKuQ23KqCTNlEk3c8gbQsFGQwkN4gkI7Z2 - HauqbNHQoU64smns+HPDoHq7z87xRlQ8c8efNpIGhfYgS/nf1eGXbg== - -----END AGE ENCRYPTED FILE----- - recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh - lastmodified: "2026-07-21T06:35:41Z" - mac: ENC[AES256_GCM,data:VeVffJWYwoqNe/irAsWrl1rAOr2xMnsPgSCqiszaoF1807iU7FSpuvSOCAbaOJXSKgJ/MBflnegCIsDb7a1W4BcwhwuMGoGl2C2vBs2YPZvqTzfnfXAGrWCbZdyCh16Dg8nZQlwRe2QwokqW+t3j1qAYfq4KVu8MLi648adRIt0=,iv:mCMQXdfVQNDgY1ZHEQGBC4YaHwkCsyo+oADMJ7p23B8=,tag:7c9VkImsgkjSxOkM1Q3qgg==,type:str] - unencrypted_suffix: _unencrypted - version: 3.13.2 diff --git a/k8s/data/schemas/authentik-database.yaml b/k8s/data/schemas/authentik-database.yaml deleted file mode 100644 index fb3d791..0000000 --- a/k8s/data/schemas/authentik-database.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: Database -metadata: - name: authentik - namespace: ddb -spec: - name: authentik - owner: app # All apps use shared 'app' user (CNPG design pattern) - cluster: - name: ddb-cluster diff --git a/k8s/data/schemas/db-init-job.yaml b/k8s/data/schemas/db-init-job.yaml deleted file mode 100644 index 08b9d4f..0000000 --- a/k8s/data/schemas/db-init-job.yaml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: db-schema-init - namespace: ddb - labels: - app: postgresql-init - layer: data -spec: - backoffLimit: 3 - ttlSecondsAfterFinished: 3600 # Keep Job for 1 hour after completion - template: - metadata: - labels: - app: postgresql-init - spec: - serviceAccountName: default - restartPolicy: Never - containers: - - name: schema-init - image: postgres:16.2-alpine - env: - - name: PGHOST - value: ddb-cluster-rw.ddb.svc.cluster.local - - name: PGPORT - value: "5432" - - name: PGDATABASE - value: app - - name: PGUSER - value: app - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: ddb-cluster-app - key: password - command: - - /bin/sh - - -c - - | - # Wait for cluster to be ready - echo "Waiting for PostgreSQL to be ready..." - until pg_isready -h $PGHOST -p $PGPORT -U $PGUSER; do - echo "Waiting..." - sleep 5 - done - - echo "Creating schemas..." - - # Create schemas from ConfigMap - psql -h $PGHOST -p $PGPORT -U $PGUSER -d $PGDATABASE << 'EOF' - CREATE SCHEMA IF NOT EXISTS authentik; - GRANT USAGE ON SCHEMA authentik TO app; - GRANT CREATE ON SCHEMA authentik TO app; - - CREATE SCHEMA IF NOT EXISTS temporal; - GRANT USAGE ON SCHEMA temporal TO app; - GRANT CREATE ON SCHEMA temporal TO app; - - CREATE SCHEMA IF NOT EXISTS vault; - GRANT USAGE ON SCHEMA vault TO app; - GRANT CREATE ON SCHEMA vault TO app; - - GRANT USAGE ON SCHEMA public TO app; - GRANT CREATE ON SCHEMA public TO app; - EOF - - echo "✓ Schema initialization complete" diff --git a/k8s/data/schemas/kustomization.yaml b/k8s/data/schemas/kustomization.yaml deleted file mode 100644 index 415fabf..0000000 --- a/k8s/data/schemas/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -metadata: - name: data-schemas - -namespace: ddb - -# GitOps-managed database schemas (ArgoCD wave 6). -# These depend on ddb-cluster existing (bootstrap wave 0). -resources: - - authentik-database.yaml - - temporal-database.yaml - - temporal-visibility-database.yaml - - schemas.yaml - - db-init-job.yaml - -# db-role-secrets.enc.yaml handled by SOPS secrets Application (wave 4) diff --git a/k8s/data/schemas/schemas.yaml b/k8s/data/schemas/schemas.yaml deleted file mode 100644 index 7a10bfe..0000000 --- a/k8s/data/schemas/schemas.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# Schema initialization ConfigMap for PostgreSQL services -# Services: Authentik, Vault (optional), Temporal, others -# Applied via initdb Jobs after CNPG cluster is ready - -apiVersion: v1 -kind: ConfigMap -metadata: - name: db-schemas - namespace: ddb - labels: - app: postgresql-schemas -data: - # Authentik schema (minimal — Authentik creates most tables via Django ORM) - authentik-init.sql: | - CREATE SCHEMA IF NOT EXISTS authentik; - GRANT USAGE ON SCHEMA authentik TO app; - GRANT CREATE ON SCHEMA authentik TO app; - - # Temporal schema - temporal-init.sql: | - CREATE SCHEMA IF NOT EXISTS temporal; - GRANT USAGE ON SCHEMA temporal TO app; - GRANT CREATE ON SCHEMA temporal TO app; - - # Vault schema (if using PostgreSQL backend) - vault-init.sql: | - CREATE SCHEMA IF NOT EXISTS vault; - GRANT USAGE ON SCHEMA vault TO app; - GRANT CREATE ON SCHEMA vault TO app; - - # Application schema (generic) - app-init.sql: | - -- Default app schema (public) - GRANT USAGE ON SCHEMA public TO app; - GRANT CREATE ON SCHEMA public TO app; diff --git a/k8s/data/schemas/temporal-database.yaml b/k8s/data/schemas/temporal-database.yaml deleted file mode 100644 index 10a61aa..0000000 --- a/k8s/data/schemas/temporal-database.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: Database -metadata: - name: temporal - namespace: ddb -spec: - name: temporal - owner: app # All apps use shared 'app' user (CNPG design pattern) - cluster: - name: ddb-cluster diff --git a/k8s/data/schemas/temporal-visibility-database.yaml b/k8s/data/schemas/temporal-visibility-database.yaml deleted file mode 100644 index f7cb6b6..0000000 --- a/k8s/data/schemas/temporal-visibility-database.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: postgresql.cnpg.io/v1 -kind: Database -metadata: - name: temporal-visibility - namespace: ddb -spec: - name: temporal_visibility - owner: app # All apps use shared 'app' user (CNPG design pattern) - cluster: - name: ddb-cluster diff --git a/k8s/infra/databases/authentik-db.yaml b/k8s/infra/databases/authentik-db.yaml new file mode 100644 index 0000000..9b7e54a --- /dev/null +++ b/k8s/infra/databases/authentik-db.yaml @@ -0,0 +1,38 @@ +# Dedicated CNPG Postgres for Authentik (GitOps, wave 2 — before authentik w3). +# CNPG generates secret `authentik-db-app` + service `authentik-db-rw` in ns iam; +# authentik-values reads them locally (no cross-ns secret copy). +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: authentik-db + namespace: iam + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + instances: 3 + imageName: ghcr.io/cloudnative-pg/postgresql:16.2 + bootstrap: + initdb: + database: authentik + owner: app + encoding: UTF8 + localeCollate: C + localeCType: C + enableSuperuserAccess: false + resources: + requests: { memory: "512Mi", cpu: "250m" } + limits: { memory: "2Gi", cpu: "1" } + storage: + size: 10Gi + storageClass: longhorn-cnpg + monitoring: + enablePodMonitor: true + affinity: + # preferred + tolerations: HA across nodes without deadlocking a partly- + # schedulable 3-CP (same as forgejo-db). + podAntiAffinityType: preferred + topologyKey: kubernetes.io/hostname + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule diff --git a/k8s/infra/databases/kustomization.yaml b/k8s/infra/databases/kustomization.yaml new file mode 100644 index 0000000..40312a9 --- /dev/null +++ b/k8s/infra/databases/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Dedicated per-app CNPG clusters. NO top-level `namespace:` — each Cluster +# carries its own ns (iam / temporal); a transformer would wrongly collapse them. +resources: + - namespaces.yaml + - authentik-db.yaml + - temporal-db.yaml diff --git a/k8s/infra/databases/namespaces.yaml b/k8s/infra/databases/namespaces.yaml new file mode 100644 index 0000000..a1da89d --- /dev/null +++ b/k8s/infra/databases/namespaces.yaml @@ -0,0 +1,12 @@ +# DB clusters are wave 2 — their namespaces must exist first (their apps that +# would CreateNamespace run later, w3/w8). Declared here so the databases App +# creates them. authentik/vault/temporal CreateNamespace=true then no-ops. +apiVersion: v1 +kind: Namespace +metadata: + name: iam +--- +apiVersion: v1 +kind: Namespace +metadata: + name: temporal diff --git a/k8s/infra/databases/temporal-db.yaml b/k8s/infra/databases/temporal-db.yaml new file mode 100644 index 0000000..443002a --- /dev/null +++ b/k8s/infra/databases/temporal-db.yaml @@ -0,0 +1,49 @@ +# Dedicated CNPG Postgres for Temporal (GitOps, wave 2 — before temporal w8). +# Temporal needs TWO databases: `temporal` (initdb) + `temporal_visibility` +# (Database CR). CNPG generates secret `temporal-db-app` + service +# `temporal-db-rw` in ns temporal; temporal-values reads them locally. +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: temporal-db + namespace: temporal + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + instances: 3 + imageName: ghcr.io/cloudnative-pg/postgresql:16.2 + bootstrap: + initdb: + database: temporal + owner: app + encoding: UTF8 + localeCollate: C + localeCType: C + enableSuperuserAccess: false + resources: + requests: { memory: "512Mi", cpu: "250m" } + limits: { memory: "2Gi", cpu: "1" } + storage: + size: 10Gi + storageClass: longhorn-cnpg + monitoring: + enablePodMonitor: true + affinity: + podAntiAffinityType: preferred + topologyKey: kubernetes.io/hostname + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule +--- +# Second database Temporal requires (visibility store), on the same cluster. +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: temporal-visibility + namespace: temporal +spec: + name: temporal_visibility + owner: app + cluster: + name: temporal-db diff --git a/k8s/security/ci-cd/charts/forgejo-runner/Chart.yaml b/k8s/infra/forgejo-runner/Chart.yaml similarity index 100% rename from k8s/security/ci-cd/charts/forgejo-runner/Chart.yaml rename to k8s/infra/forgejo-runner/Chart.yaml diff --git a/k8s/security/ci-cd/charts/forgejo-runner/templates/deployment.yaml b/k8s/infra/forgejo-runner/templates/deployment.yaml similarity index 100% rename from k8s/security/ci-cd/charts/forgejo-runner/templates/deployment.yaml rename to k8s/infra/forgejo-runner/templates/deployment.yaml diff --git a/k8s/security/ci-cd/charts/forgejo-runner/templates/networkpolicy.yaml b/k8s/infra/forgejo-runner/templates/networkpolicy.yaml similarity index 100% rename from k8s/security/ci-cd/charts/forgejo-runner/templates/networkpolicy.yaml rename to k8s/infra/forgejo-runner/templates/networkpolicy.yaml diff --git a/k8s/security/ci-cd/charts/forgejo-runner/templates/pvc.yaml b/k8s/infra/forgejo-runner/templates/pvc.yaml similarity index 100% rename from k8s/security/ci-cd/charts/forgejo-runner/templates/pvc.yaml rename to k8s/infra/forgejo-runner/templates/pvc.yaml diff --git a/k8s/security/ci-cd/charts/forgejo-runner/values.yaml b/k8s/infra/forgejo-runner/values.yaml similarity index 100% rename from k8s/security/ci-cd/charts/forgejo-runner/values.yaml rename to k8s/infra/forgejo-runner/values.yaml diff --git a/k8s/security/iam/authentik-provision-job.yaml b/k8s/infra/iam/authentik-provision-job.yaml similarity index 100% rename from k8s/security/iam/authentik-provision-job.yaml rename to k8s/infra/iam/authentik-provision-job.yaml diff --git a/k8s/security/iam/authentik-secrets.enc.yaml b/k8s/infra/iam/authentik-secrets.enc.yaml similarity index 100% rename from k8s/security/iam/authentik-secrets.enc.yaml rename to k8s/infra/iam/authentik-secrets.enc.yaml diff --git a/k8s/security/iam/authentik-values.yaml b/k8s/infra/iam/authentik-values.yaml similarity index 95% rename from k8s/security/iam/authentik-values.yaml rename to k8s/infra/iam/authentik-values.yaml index e20a816..37c1324 100644 --- a/k8s/security/iam/authentik-values.yaml +++ b/k8s/infra/iam/authentik-values.yaml @@ -24,9 +24,9 @@ authentik: # PostgreSQL connection — points at CloudNativePG cluster in ddb namespace. # Uses 'app' bootstrap user (CNPG simple pattern, same as Forgejo). - # Credentials injected from ddb-cluster-app secret via env vars below. + # Credentials injected from authentik-db-app secret via env vars below. postgresql: - host: ddb-cluster-rw.ddb.svc.cluster.local + host: authentik-db-rw.iam.svc.cluster.local port: 5432 name: authentik user: app # All apps use shared 'app' user (CNPG design pattern) @@ -130,16 +130,16 @@ server: value: /merged/ca-bundle.crt - name: SSL_CERT_FILE value: /merged/ca-bundle.crt - # Override database credentials to use 'app' from ddb-cluster-app + # Override database credentials to use 'app' from authentik-db-app - name: AUTHENTIK_POSTGRESQL__USER valueFrom: secretKeyRef: - name: ddb-cluster-app + name: authentik-db-app key: username - name: AUTHENTIK_POSTGRESQL__PASSWORD valueFrom: secretKeyRef: - name: ddb-cluster-app + name: authentik-db-app key: password podAnnotations: configmap.reloader.stakater.com/reload: "homelab-ca" @@ -201,16 +201,16 @@ worker: value: /merged/ca-bundle.crt - name: SSL_CERT_FILE value: /merged/ca-bundle.crt - # Override database credentials to use 'app' from ddb-cluster-app + # Override database credentials to use 'app' from authentik-db-app - name: AUTHENTIK_POSTGRESQL__USER valueFrom: secretKeyRef: - name: ddb-cluster-app + name: authentik-db-app key: username - name: AUTHENTIK_POSTGRESQL__PASSWORD valueFrom: secretKeyRef: - name: ddb-cluster-app + name: authentik-db-app key: password podAnnotations: configmap.reloader.stakater.com/reload: "homelab-ca" @@ -222,7 +222,7 @@ worker: scrapeTimeout: 30s # ── PostgreSQL (external: CloudNativePG cluster in ddb namespace) ───────────── -# Authentik connects to the centralized ddb-cluster (1 primary + 2 replicas with pgvector). +# Authentik connects to the dedicated authentik-db (1 primary + 2 replicas). # Do not use the bundled Bitnami subchart — CNPG is already running. postgresql: enabled: false diff --git a/k8s/security/iam/kustomization.yaml b/k8s/infra/iam/kustomization.yaml similarity index 98% rename from k8s/security/iam/kustomization.yaml rename to k8s/infra/iam/kustomization.yaml index 57f060c..de471d8 100644 --- a/k8s/security/iam/kustomization.yaml +++ b/k8s/infra/iam/kustomization.yaml @@ -12,7 +12,6 @@ kind: Kustomization # metadata.namespace, so dropping the transformer changes nothing for the # existing resources/. resources: - - key-rotation-cronjob.yaml - authentik-provision-job.yaml - rbac-dashboard-rolebinding.yaml diff --git a/k8s/security/iam/rbac-dashboard-rolebinding.yaml b/k8s/infra/iam/rbac-dashboard-rolebinding.yaml similarity index 100% rename from k8s/security/iam/rbac-dashboard-rolebinding.yaml rename to k8s/infra/iam/rbac-dashboard-rolebinding.yaml diff --git a/k8s/security/iam/scripts/authentik-provision.py b/k8s/infra/iam/scripts/authentik-provision.py similarity index 100% rename from k8s/security/iam/scripts/authentik-provision.py rename to k8s/infra/iam/scripts/authentik-provision.py diff --git a/k8s/security/iam/vault-values.yaml b/k8s/infra/iam/vault-values.yaml similarity index 100% rename from k8s/security/iam/vault-values.yaml rename to k8s/infra/iam/vault-values.yaml diff --git a/k8s/platform/logging/grafana-secrets.enc.yaml b/k8s/infra/logging/grafana-secrets.enc.yaml similarity index 100% rename from k8s/platform/logging/grafana-secrets.enc.yaml rename to k8s/infra/logging/grafana-secrets.enc.yaml diff --git a/k8s/platform/logging/grafana-values.yaml b/k8s/infra/logging/grafana-values.yaml similarity index 100% rename from k8s/platform/logging/grafana-values.yaml rename to k8s/infra/logging/grafana-values.yaml diff --git a/k8s/platform/logging/loki-secrets.enc.yaml b/k8s/infra/logging/loki-secrets.enc.yaml similarity index 100% rename from k8s/platform/logging/loki-secrets.enc.yaml rename to k8s/infra/logging/loki-secrets.enc.yaml diff --git a/k8s/platform/logging/loki-values-secrets.enc.yaml b/k8s/infra/logging/loki-values-secrets.enc.yaml similarity index 100% rename from k8s/platform/logging/loki-values-secrets.enc.yaml rename to k8s/infra/logging/loki-values-secrets.enc.yaml diff --git a/k8s/platform/logging/loki-values.yaml b/k8s/infra/logging/loki-values.yaml similarity index 100% rename from k8s/platform/logging/loki-values.yaml rename to k8s/infra/logging/loki-values.yaml diff --git a/k8s/platform/logging/promtail-values.yaml b/k8s/infra/logging/promtail-values.yaml similarity index 100% rename from k8s/platform/logging/promtail-values.yaml rename to k8s/infra/logging/promtail-values.yaml diff --git a/k8s/infrastructure/longhorn/expand-replicas-job.yaml b/k8s/infra/longhorn/expand-replicas-job.yaml similarity index 100% rename from k8s/infrastructure/longhorn/expand-replicas-job.yaml rename to k8s/infra/longhorn/expand-replicas-job.yaml diff --git a/k8s/infrastructure/longhorn/kustomization.yaml b/k8s/infra/longhorn/kustomization.yaml similarity index 100% rename from k8s/infrastructure/longhorn/kustomization.yaml rename to k8s/infra/longhorn/kustomization.yaml diff --git a/k8s/infrastructure/longhorn/longhorn-cnpg-storageclass.yaml b/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml similarity index 100% rename from k8s/infrastructure/longhorn/longhorn-cnpg-storageclass.yaml rename to k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml diff --git a/k8s/infrastructure/longhorn/longhorn-nodes.yaml b/k8s/infra/longhorn/longhorn-nodes.yaml similarity index 100% rename from k8s/infrastructure/longhorn/longhorn-nodes.yaml rename to k8s/infra/longhorn/longhorn-nodes.yaml diff --git a/k8s/infrastructure/longhorn/longhorn-servicemonitor.yaml b/k8s/infra/longhorn/longhorn-servicemonitor.yaml similarity index 100% rename from k8s/infrastructure/longhorn/longhorn-servicemonitor.yaml rename to k8s/infra/longhorn/longhorn-servicemonitor.yaml diff --git a/k8s/infrastructure/longhorn/longhorn-storageclass.yaml b/k8s/infra/longhorn/longhorn-storageclass.yaml similarity index 100% rename from k8s/infrastructure/longhorn/longhorn-storageclass.yaml rename to k8s/infra/longhorn/longhorn-storageclass.yaml diff --git a/k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml b/k8s/infra/longhorn/longhorn-taint-toleration.yaml similarity index 100% rename from k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml rename to k8s/infra/longhorn/longhorn-taint-toleration.yaml diff --git a/k8s/infrastructure/longhorn/patch-csi-tolerations-job.yaml b/k8s/infra/longhorn/patch-csi-tolerations-job.yaml similarity index 100% rename from k8s/infrastructure/longhorn/patch-csi-tolerations-job.yaml rename to k8s/infra/longhorn/patch-csi-tolerations-job.yaml diff --git a/k8s/infrastructure/minio/kustomization.yaml b/k8s/infra/minio/kustomization.yaml similarity index 92% rename from k8s/infrastructure/minio/kustomization.yaml rename to k8s/infra/minio/kustomization.yaml index fa50943..1622999 100644 --- a/k8s/infrastructure/minio/kustomization.yaml +++ b/k8s/infra/minio/kustomization.yaml @@ -3,7 +3,6 @@ kind: Kustomization namespace: storage resources: - minio-tenant.yaml - - minio-policy-job.yaml # The operator creates the minio S3/console/headless Services and the # declarative bucket + user from the Tenant spec — no hand-rolled Service or # Bucket/User CRs (those kinds don't exist in the operator CRD set). diff --git a/k8s/infrastructure/minio/minio-operator-values.yaml b/k8s/infra/minio/minio-operator-values.yaml similarity index 100% rename from k8s/infrastructure/minio/minio-operator-values.yaml rename to k8s/infra/minio/minio-operator-values.yaml diff --git a/k8s/infrastructure/minio/minio-secrets.enc.yaml b/k8s/infra/minio/minio-secrets.enc.yaml similarity index 100% rename from k8s/infrastructure/minio/minio-secrets.enc.yaml rename to k8s/infra/minio/minio-secrets.enc.yaml diff --git a/k8s/infrastructure/minio/minio-tenant.yaml b/k8s/infra/minio/minio-tenant.yaml similarity index 93% rename from k8s/infrastructure/minio/minio-tenant.yaml rename to k8s/infra/minio/minio-tenant.yaml index 15b6422..190d0de 100644 --- a/k8s/infrastructure/minio/minio-tenant.yaml +++ b/k8s/infra/minio/minio-tenant.yaml @@ -72,15 +72,10 @@ spec: # ── Declarative buckets (operator creates on first boot) ──────────────────── buckets: - - name: riotpiao-models - name: loki-chunks - name: loki-ruler - name: loki-admin - name: vault - # ── Declarative users (each references a Secret of the same name holding - # CONSOLE_ACCESS_KEY / CONSOLE_SECRET_KEY) ───────────────────────────── - users: - - name: minio-user-ollama # Metrics are exposed at /minio/v2/metrics; scrape via a hand-rolled # ServiceMonitor in the monitoring stack rather than operator auto-wiring diff --git a/k8s/platform/monitoring/alerts/ingress-alerts.yaml b/k8s/infra/monitoring/alerts/ingress-alerts.yaml similarity index 100% rename from k8s/platform/monitoring/alerts/ingress-alerts.yaml rename to k8s/infra/monitoring/alerts/ingress-alerts.yaml diff --git a/k8s/platform/monitoring/alerts/svc-argocd-rules.yaml b/k8s/infra/monitoring/alerts/svc-argocd-rules.yaml similarity index 100% rename from k8s/platform/monitoring/alerts/svc-argocd-rules.yaml rename to k8s/infra/monitoring/alerts/svc-argocd-rules.yaml diff --git a/k8s/platform/monitoring/alerts/svc-authentik-rules.yaml b/k8s/infra/monitoring/alerts/svc-authentik-rules.yaml similarity index 100% rename from k8s/platform/monitoring/alerts/svc-authentik-rules.yaml rename to k8s/infra/monitoring/alerts/svc-authentik-rules.yaml diff --git a/k8s/platform/monitoring/alerts/svc-forgejo-rules.yaml b/k8s/infra/monitoring/alerts/svc-forgejo-rules.yaml similarity index 100% rename from k8s/platform/monitoring/alerts/svc-forgejo-rules.yaml rename to k8s/infra/monitoring/alerts/svc-forgejo-rules.yaml diff --git a/k8s/platform/monitoring/alerts/svc-grafana-rules.yaml b/k8s/infra/monitoring/alerts/svc-grafana-rules.yaml similarity index 100% rename from k8s/platform/monitoring/alerts/svc-grafana-rules.yaml rename to k8s/infra/monitoring/alerts/svc-grafana-rules.yaml diff --git a/k8s/platform/monitoring/alerts/svc-minio-rules.yaml b/k8s/infra/monitoring/alerts/svc-minio-rules.yaml similarity index 100% rename from k8s/platform/monitoring/alerts/svc-minio-rules.yaml rename to k8s/infra/monitoring/alerts/svc-minio-rules.yaml diff --git a/k8s/platform/monitoring/alerts/svc-vault-rules.yaml b/k8s/infra/monitoring/alerts/svc-vault-rules.yaml similarity index 100% rename from k8s/platform/monitoring/alerts/svc-vault-rules.yaml rename to k8s/infra/monitoring/alerts/svc-vault-rules.yaml diff --git a/k8s/platform/monitoring/blackbox-exporter-values.yaml b/k8s/infra/monitoring/blackbox-exporter-values.yaml similarity index 100% rename from k8s/platform/monitoring/blackbox-exporter-values.yaml rename to k8s/infra/monitoring/blackbox-exporter-values.yaml diff --git a/k8s/platform/monitoring/crds/kube-prometheus-stack-crds.yaml b/k8s/infra/monitoring/crds/kube-prometheus-stack-crds.yaml similarity index 100% rename from k8s/platform/monitoring/crds/kube-prometheus-stack-crds.yaml rename to k8s/infra/monitoring/crds/kube-prometheus-stack-crds.yaml diff --git a/k8s/platform/monitoring/dashboards/control-plane-logs.yaml b/k8s/infra/monitoring/dashboards/control-plane-logs.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/control-plane-logs.yaml rename to k8s/infra/monitoring/dashboards/control-plane-logs.yaml diff --git a/k8s/platform/monitoring/dashboards/hardware-overview.yaml b/k8s/infra/monitoring/dashboards/hardware-overview.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/hardware-overview.yaml rename to k8s/infra/monitoring/dashboards/hardware-overview.yaml diff --git a/k8s/platform/monitoring/dashboards/kube-controller-health.yaml b/k8s/infra/monitoring/dashboards/kube-controller-health.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/kube-controller-health.yaml rename to k8s/infra/monitoring/dashboards/kube-controller-health.yaml diff --git a/k8s/platform/monitoring/dashboards/service-availability.yaml b/k8s/infra/monitoring/dashboards/service-availability.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/service-availability.yaml rename to k8s/infra/monitoring/dashboards/service-availability.yaml diff --git a/k8s/platform/monitoring/dashboards/service-golden-signals.yaml b/k8s/infra/monitoring/dashboards/service-golden-signals.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/service-golden-signals.yaml rename to k8s/infra/monitoring/dashboards/service-golden-signals.yaml diff --git a/k8s/platform/monitoring/dashboards/service-internals.yaml b/k8s/infra/monitoring/dashboards/service-internals.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/service-internals.yaml rename to k8s/infra/monitoring/dashboards/service-internals.yaml diff --git a/k8s/platform/monitoring/dashboards/svc-argocd.yaml b/k8s/infra/monitoring/dashboards/svc-argocd.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/svc-argocd.yaml rename to k8s/infra/monitoring/dashboards/svc-argocd.yaml diff --git a/k8s/platform/monitoring/dashboards/svc-authentik.yaml b/k8s/infra/monitoring/dashboards/svc-authentik.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/svc-authentik.yaml rename to k8s/infra/monitoring/dashboards/svc-authentik.yaml diff --git a/k8s/platform/monitoring/dashboards/svc-forgejo.yaml b/k8s/infra/monitoring/dashboards/svc-forgejo.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/svc-forgejo.yaml rename to k8s/infra/monitoring/dashboards/svc-forgejo.yaml diff --git a/k8s/platform/monitoring/dashboards/svc-grafana.yaml b/k8s/infra/monitoring/dashboards/svc-grafana.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/svc-grafana.yaml rename to k8s/infra/monitoring/dashboards/svc-grafana.yaml diff --git a/k8s/platform/monitoring/dashboards/svc-minio.yaml b/k8s/infra/monitoring/dashboards/svc-minio.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/svc-minio.yaml rename to k8s/infra/monitoring/dashboards/svc-minio.yaml diff --git a/k8s/platform/monitoring/dashboards/svc-vault.yaml b/k8s/infra/monitoring/dashboards/svc-vault.yaml similarity index 100% rename from k8s/platform/monitoring/dashboards/svc-vault.yaml rename to k8s/infra/monitoring/dashboards/svc-vault.yaml diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml new file mode 100644 index 0000000..eb14f2c --- /dev/null +++ b/k8s/infra/monitoring/kustomization.yaml @@ -0,0 +1,30 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Single source for cluster monitoring config (alerts, servicemonitors, dashboards). +# NO top-level `namespace:` transformer — PrometheusRules deliberately live in +# per-app namespaces (forgejo, iam, argocd, storage, ...); a transformer would +# wrongly rewrite them all to one ns. +resources: + - alerts/ingress-alerts.yaml + - alerts/svc-argocd-rules.yaml + - alerts/svc-authentik-rules.yaml + - alerts/svc-forgejo-rules.yaml + - alerts/svc-grafana-rules.yaml + - alerts/svc-minio-rules.yaml + - alerts/svc-vault-rules.yaml + - servicemonitors/argocd.yaml + - servicemonitors/authentik.yaml + - servicemonitors/forgejo.yaml + - servicemonitors/minio.yaml + - dashboards/control-plane-logs.yaml + - dashboards/hardware-overview.yaml + - dashboards/kube-controller-health.yaml + - dashboards/service-availability.yaml + - dashboards/service-golden-signals.yaml + - dashboards/service-internals.yaml + - dashboards/svc-argocd.yaml + - dashboards/svc-authentik.yaml + - dashboards/svc-forgejo.yaml + - dashboards/svc-grafana.yaml + - dashboards/svc-minio.yaml + - dashboards/svc-vault.yaml diff --git a/k8s/platform/monitoring/prometheus-values.yaml b/k8s/infra/monitoring/prometheus-values.yaml similarity index 100% rename from k8s/platform/monitoring/prometheus-values.yaml rename to k8s/infra/monitoring/prometheus-values.yaml diff --git a/k8s/platform/monitoring/servicemonitors/argocd.yaml b/k8s/infra/monitoring/servicemonitors/argocd.yaml similarity index 100% rename from k8s/platform/monitoring/servicemonitors/argocd.yaml rename to k8s/infra/monitoring/servicemonitors/argocd.yaml diff --git a/k8s/platform/monitoring/servicemonitors/authentik.yaml b/k8s/infra/monitoring/servicemonitors/authentik.yaml similarity index 100% rename from k8s/platform/monitoring/servicemonitors/authentik.yaml rename to k8s/infra/monitoring/servicemonitors/authentik.yaml diff --git a/k8s/platform/monitoring/servicemonitors/forgejo.yaml b/k8s/infra/monitoring/servicemonitors/forgejo.yaml similarity index 100% rename from k8s/platform/monitoring/servicemonitors/forgejo.yaml rename to k8s/infra/monitoring/servicemonitors/forgejo.yaml diff --git a/k8s/platform/monitoring/servicemonitors/minio.yaml b/k8s/infra/monitoring/servicemonitors/minio.yaml similarity index 100% rename from k8s/platform/monitoring/servicemonitors/minio.yaml rename to k8s/infra/monitoring/servicemonitors/minio.yaml diff --git a/k8s/infrastructure/kustomization.yaml b/k8s/infrastructure/kustomization.yaml deleted file mode 100644 index 51d5656..0000000 --- a/k8s/infrastructure/kustomization.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -metadata: - name: infrastructure - -namespace: infrastructure - -# Layer 1: Infrastructure — storage, persistent volumes -# Dependencies: None (base layer) -# Order: Applied first - -resources: - - minio/ - - longhorn/ diff --git a/k8s/infrastructure/minio/.env.example b/k8s/infrastructure/minio/.env.example deleted file mode 100644 index 211f6f0..0000000 --- a/k8s/infrastructure/minio/.env.example +++ /dev/null @@ -1,9 +0,0 @@ -# storage/.env.example -# Copy to storage/.env and fill in real values. -# The real .env is gitignored — never commit it. -# Used by: k8s/storage/bootstrap.sh - -# MinIO root credentials (S3-compatible object store) -# openssl rand -base64 24 -MINIO_ROOT_USER= -MINIO_ROOT_PASSWORD= diff --git a/k8s/infrastructure/minio/bootstrap.sh b/k8s/infrastructure/minio/bootstrap.sh deleted file mode 100755 index 63e5887..0000000 --- a/k8s/infrastructure/minio/bootstrap.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -# k8s/storage/bootstrap.sh -# Deploys the MinIO multi-AZ object store into the storage namespace. -# -# Prerequisites: -# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig) -# - helm >= 3.x installed -# - k8s/storage/.env file containing: -# MINIO_ROOT_USER=... -# MINIO_ROOT_PASSWORD=... -# OR those variables already exported in the calling shell. -# -# Idempotent: safe to re-run — helm upgrade --install and kubectl apply are both idempotent. -# Disaster recovery order: run this before k8s/logging/bootstrap.sh (Loki depends on MinIO). -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../../cluster-config/kubeconfig}" -export KUBECONFIG - -# ── Load credentials ────────────────────────────────────────────────────────── -if [[ -f "${SCRIPT_DIR}/.env" ]]; then - # shellcheck source=/dev/null - source "${SCRIPT_DIR}/.env" -fi - -for var in MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do - if [[ -z "${!var:-}" ]]; then - echo "ERROR: ${var} is not set. Export it or place it in k8s/storage/.env" - exit 1 - fi -done - -# ── Namespace ───────────────────────────────────────────────────────────────── -echo "==> Creating storage namespace with privileged pod security..." -kubectl create namespace storage --dry-run=client -o yaml | kubectl apply -f - -kubectl label namespace storage \ - pod-security.kubernetes.io/enforce=privileged \ - pod-security.kubernetes.io/enforce-version=latest \ - --overwrite - -# ── Helm repos ──────────────────────────────────────────────────────────────── -echo "==> Adding Helm repositories..." -helm repo add minio https://charts.min.io -helm repo update minio - -# ── PVC for az-a (adopts existing volume) ──────────────────────────────────── -echo "==> Applying az-a PVC (adopts pre-existing Longhorn volume)..." -kubectl apply -f "${SCRIPT_DIR}/minio-az-a-pvc.yaml" - -# ── MinIO az-a (control-plane node, zone az-a) ─────────────────────────────── -echo "==> Installing MinIO az-a..." -helm upgrade --install minio-az-a minio/minio \ - --namespace storage \ - --values "${SCRIPT_DIR}/minio-az-a-values.yaml" \ - --set rootUser="${MINIO_ROOT_USER}" \ - --set rootPassword="${MINIO_ROOT_PASSWORD}" \ - --wait \ - --timeout 5m - -echo "==> Waiting for minio-az-a Deployment to be ready..." -kubectl rollout status deployment/minio-az-a -n storage --timeout=120s - -# ── MinIO az-b (worker node, zone az-b) ────────────────────────────────────── -echo "==> Installing MinIO az-b..." -helm upgrade --install minio-az-b minio/minio \ - --namespace storage \ - --values "${SCRIPT_DIR}/minio-az-b-values.yaml" \ - --set rootUser="${MINIO_ROOT_USER}" \ - --set rootPassword="${MINIO_ROOT_PASSWORD}" \ - --wait \ - --timeout 5m - -echo "==> Waiting for minio-az-b Deployment to be ready..." -kubectl rollout status deployment/minio-az-b -n storage --timeout=120s - -# ── Universal frontend service + legacy alias ───────────────────────────────── -echo "==> Applying universal frontend service and legacy ExternalName alias..." -kubectl apply -f "${SCRIPT_DIR}/minio-service.yaml" -kubectl apply -f "${SCRIPT_DIR}/minio-legacy-alias.yaml" - -# ── Site replication ────────────────────────────────────────────────────────── -echo "==> Running site replication setup job..." -# Delete any leftover completed job first (kubectl apply on Jobs is not idempotent) -kubectl delete job minio-site-replication-setup -n storage --ignore-not-found -kubectl apply -f "${SCRIPT_DIR}/minio-replication-job.yaml" - -echo "==> Waiting for replication job to complete..." -kubectl wait --for=condition=complete job/minio-site-replication-setup \ - -n storage --timeout=120s - -echo "==> Replication job output:" -kubectl logs -n storage \ - -l job-name=minio-site-replication-setup \ - --tail=20 - -# ── Done ────────────────────────────────────────────────────────────────────── -echo "" -echo "==> MinIO multi-AZ stack is up." -echo "" -echo "MinIO console (S3 browser):" -echo " kubectl port-forward -n storage svc/minio 9001:9001" -echo " http://localhost:9001 (${MINIO_ROOT_USER} / )" -echo "" -echo "MinIO S3 endpoint for apps (active-active frontend):" -echo " http://minio.storage.svc.cluster.local:9000" -echo "" -echo "Verify site replication:" -echo " kubectl logs -n storage -l job-name=minio-site-replication-setup" diff --git a/k8s/infrastructure/minio/minio-bucket-init.sh b/k8s/infrastructure/minio/minio-bucket-init.sh deleted file mode 100755 index a38cfec..0000000 --- a/k8s/infrastructure/minio/minio-bucket-init.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Safe MinIO bucket initialization via Job (credentials in Secret, not env) -# Usage: ./minio-bucket-init.sh [bucket2] ... -set -euo pipefail - -NAMESPACE="${1:?Missing namespace}" -shift -BUCKETS=("$@") - -if [ ${#BUCKETS[@]} -eq 0 ]; then - echo "Usage: $0 [bucket2] ..." >&2 - exit 1 -fi - -# Create Secret with Minio credentials (safe: sourced from env, not exposed in pod) -kubectl create secret generic minio-creds -n "$NAMESPACE" \ - --from-literal=MINIO_ROOT_USER="${MINIO_ROOT_USER:?Missing MINIO_ROOT_USER}" \ - --from-literal=MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:?Missing MINIO_ROOT_PASSWORD}" \ - --dry-run=client -o yaml | kubectl apply -f - - -# Create init Job that mounts Secret as volume, preventing env exposure -BUCKET_ARGS=$(printf '"%s", ' "${BUCKETS[@]}" | sed 's/, $//') -kubectl apply -f - </dev/null; do - echo "waiting for minio..."; sleep 5 - done - - # Scoped policy for the ollama service user: read/write its bucket only. - cat >/tmp/policy-ollama.json <<'JSON' - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:GetObject", "s3:PutObject"], - "Resource": ["arn:aws:s3:::riotpiao-models/*"] - }, - { - "Effect": "Allow", - "Action": ["s3:ListBucket"], - "Resource": ["arn:aws:s3:::riotpiao-models"] - } - ] - } - JSON - - mc admin policy create local policy-ollama /tmp/policy-ollama.json || true - - # Attach the policy to the declaratively-created user. - mc admin policy attach local policy-ollama --user "$OLLAMA_ACCESS_KEY" || true - - echo "ollama policy configured" - env: - # mc writes its config to $HOME/.mc; runAsUser 1000 has HOME=/ which - # isn't writable, so `mc alias set` failed on `mkdir /.mc` and the - # wait-loop spun forever. Point HOME at the always-writable /tmp. - - name: HOME - value: /tmp - - name: ROOT_USER - valueFrom: - secretKeyRef: - name: minio-creds - key: accesskey - - name: ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio-creds - key: secretkey - - name: OLLAMA_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-user-ollama - key: CONSOLE_ACCESS_KEY diff --git a/k8s/infrastructure/minio/minio-values.yaml b/k8s/infrastructure/minio/minio-values.yaml deleted file mode 100644 index b61f34d..0000000 --- a/k8s/infrastructure/minio/minio-values.yaml +++ /dev/null @@ -1,64 +0,0 @@ -mode: standalone - -rootUser: "" -rootPassword: "" - -persistence: - enabled: true - storageClass: longhorn - size: 100Gi - -deploymentUpdate: - type: Recreate - -resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - -service: - type: ClusterIP - port: 9000 - -consoleService: - type: ClusterIP - port: 9001 - -ingress: - enabled: false - -consoleIngress: - enabled: false - -metrics: - serviceMonitor: - enabled: true - -envFrom: - - secretRef: - name: minio-oidc - -environment: - MINIO_IDENTITY_OPENID_CONFIG_URL: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration" - MINIO_IDENTITY_OPENID_CLIENT_ID: "minio" - MINIO_IDENTITY_OPENID_CLAIM_NAME: "policy" - MINIO_IDENTITY_OPENID_SCOPES: "openid,profile,email,minio" - MINIO_IDENTITY_OPENID_REDIRECT_URI: "https://minio.riotpiao.com/oauth_callback" - MINIO_IDENTITY_OPENID_DISPLAY_NAME: "Authentik" - -podAnnotations: - secret.reloader.stakater.com/reload: "minio-oidc" - configmap.reloader.stakater.com/reload: "homelab-ca" - -extraVolumes: - - name: homelab-ca - configMap: - name: homelab-ca - -extraVolumeMounts: - - name: homelab-ca - mountPath: /etc/minio/certs/CAs - readOnly: true diff --git a/k8s/infrastructure/minio/test/README.md b/k8s/infrastructure/minio/test/README.md deleted file mode 100644 index f8d2a85..0000000 --- a/k8s/infrastructure/minio/test/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# MinIO CRUD Example (Go) - -A minimal Go program that exercises the cluster's object storage through the -**universal storage frontend** — `minio.storage.svc.cluster.local:9000` — the -single DNS name that load-balances across both node-pinned MinIO instances -(`minio-az-a` on talos-cp-1, `minio-az-b` on talos-worker-1). - -It runs a full CRUD cycle with a random text file: - -| Step | S3 call | What it proves | -|------|---------|----------------| -| Ensure bucket | `BucketExists` / `MakeBucket` | bucket `crud-test` exists (idempotent) | -| **C**reate | `PutObject` | write path through the frontend | -| **R**ead | `GetObject` + byte compare | content round-trips intact | -| **U**pdate | `PutObject` (overwrite) | S3 update semantics (objects are replaced, not edited) | -| List | `ListObjects` prefix `demo/` | enumeration | -| **D**elete | `RemoveObject` + `StatObject` | object gone (`NoSuchKey` confirmed) | - -Every operation emits `[SERVICE_METRIC] s3..latency_ms= ms`; any failure -emits `[APP_METRIC] ERROR s3. failed ... | trace=...` and exits non-zero. - -## Run it - -```bash -# 1. expose the frontend locally (leave running in another terminal) -kubectl port-forward svc/minio -n storage 9000:9000 - -# 2. credentials — same root creds used by both MinIO sites -source logging/.env # exports MINIO_ROOT_USER / MINIO_ROOT_PASSWORD - -# 3. run -cd storage/test -go mod tidy && go run . -``` - -In-cluster (e.g. from a Job), skip the port-forward and set -`MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000`. - -Expected output: - -``` -[SERVICE_METRIC] s3.ensure_bucket.latency_ms=145 ms -[SERVICE_METRIC] s3.put.latency_ms=19 ms -created crud-test/demo/.txt (256 bytes of random text) -[SERVICE_METRIC] s3.get.latency_ms=9 ms -read back and verified content -[SERVICE_METRIC] s3.update.latency_ms=70 ms -updated (overwrote) object - demo/.txt 140 bytes -[SERVICE_METRIC] s3.list.latency_ms=11 ms -[SERVICE_METRIC] s3.delete.latency_ms=68 ms -deleted and verified gone — CRUD cycle complete -``` - -## Validating each state with kubectl - -The program verifies itself in-process (read-back compare, post-delete stat), -but every state is also independently observable from outside with `kubectl`. -The helper below drops you into a throwaway `mc` shell wired to both sites — -all subsequent checks use it: - -```bash -source logging/.env -kubectl run mc-shell --rm -it --restart=Never --image=minio/mc -n storage \ - --env="U=$MINIO_ROOT_USER" --env="P=$MINIO_ROOT_PASSWORD" \ - --command -- /bin/sh -c ' -mc alias set front http://minio.storage.svc.cluster.local:9000 "$U" "$P" -mc alias set az-a http://minio-az-a.storage.svc.cluster.local:9000 "$U" "$P" -mc alias set az-b http://minio-az-b.storage.svc.cluster.local:9000 "$U" "$P" -exec /bin/sh' -``` - -> The demo deletes its object at the end, so to inspect the CREATE/UPDATE -> states at your own pace, comment out the `// DELETE` block in `main.go` -> and re-run (the delete is idempotent to re-apply later). - -**0. Frontend is healthy (before running anything)** - -```bash -kubectl get endpoints minio -n storage # expect TWO pod IPs on :9000 -kubectl get pods -n storage -o wide # az-a on talos-cp-1, az-b on talos-worker-1 -``` - -**1. Bucket created** — and replicated to BOTH sites - -```bash -# inside mc-shell — the bucket must appear on each site individually -mc ls az-a | grep crud-test -mc ls az-b | grep crud-test # proves site replication propagated it -``` - -**2. Object created (CREATE)** — 256 bytes, present on both nodes - -```bash -mc ls az-a/crud-test/demo/ # .txt, 256 B -mc ls az-b/crud-test/demo/ # same object, replicated (allow ~seconds of lag) -mc cat front/crud-test/demo/.txt # the random text itself -``` - -**3. Object updated (UPDATE)** — size changed 256 → 140 bytes, content starts with `UPDATED ---` - -```bash -mc stat az-a/crud-test/demo/.txt # Size: 140 B, fresh LastModified -mc cat az-b/crud-test/demo/.txt | head -1 # "UPDATED ---" (replicated overwrite) -``` - -**4. Object deleted (DELETE)** — gone from both sites - -```bash -mc ls az-a/crud-test/demo/ # empty -mc ls az-b/crud-test/demo/ # empty — deletes replicate too -mc stat front/crud-test/demo/.txt # error: Object does not exist -``` - -**5. Replication layer itself** - -```bash -# inside mc-shell -mc admin replicate status az-a # buckets/policies/users "in sync" -``` - -**6. Storage layer under it** - -```bash -kubectl get volumes.longhorn.io -n longhorn-system # both volumes attached / healthy -``` - -## Troubleshooting - -- `connection refused` on localhost:9000 → the port-forward isn't running. -- `[APP_METRIC] ERROR config missing` → `source logging/.env` first. -- Object visible on az-a but not az-b → check `mc admin replicate status az-a`; - replication is near-synchronous, not instant. Persistent divergence: - `mc admin replicate resync start az-a az-b`. -- Frontend has one endpoint instead of two → a MinIO pod is unready; - `kubectl describe pod -n storage `. Traffic still flows via the - surviving pod (that's the failover design — see `minio_migration.html`). diff --git a/k8s/infrastructure/minio/test/go.mod b/k8s/infrastructure/minio/test/go.mod deleted file mode 100644 index 291a942..0000000 --- a/k8s/infrastructure/minio/test/go.mod +++ /dev/null @@ -1,23 +0,0 @@ -module homelab/storage/test - -go 1.22 - -require github.com/minio/minio-go/v7 v7.0.80 - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.3 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rs/xid v1.6.0 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/k8s/infrastructure/minio/test/go.sum b/k8s/infrastructure/minio/test/go.sum deleted file mode 100644 index 3cdb8bd..0000000 --- a/k8s/infrastructure/minio/test/go.sum +++ /dev/null @@ -1,37 +0,0 @@ -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.80 h1:2mdUHXEykRdY/BigLt3Iuu1otL0JTogT0Nmltg0wujk= -github.com/minio/minio-go/v7 v7.0.80/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/k8s/infrastructure/minio/test/main.go b/k8s/infrastructure/minio/test/main.go deleted file mode 100644 index 33475fb..0000000 --- a/k8s/infrastructure/minio/test/main.go +++ /dev/null @@ -1,152 +0,0 @@ -// CRUD demo against the universal MinIO storage frontend -// (minio.storage.svc.cluster.local:9000). -// -// Run from outside the cluster via a port-forward: -// -// kubectl port-forward svc/minio -n storage 9000:9000 & -// source logging/.env -// cd storage/test && go mod tidy && go run . -// -// In-cluster, set MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000. -// -// Every S3 call emits [SERVICE_METRIC] op latency; every failure emits -// [APP_METRIC] ERROR with context and aborts (no silent catches). -package main - -import ( - "bytes" - "context" - "fmt" - "io" - "math/rand" - "os" - "time" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -const bucket = "crud-test" - -func getenv(key, fallback string) string { - if v := os.Getenv(key); v != "" { - return v - } - return fallback -} - -// timed wraps an S3 operation: emits a serviceMetric on success, -// an applicationMetric and exit(1) on failure. -func timed(op string, fn func() error) { - start := time.Now() - if err := fn(); err != nil { - fmt.Printf("[APP_METRIC] ERROR s3.%s failed bucket=%s | trace=%v\n", op, bucket, err) - os.Exit(1) - } - fmt.Printf("[SERVICE_METRIC] s3.%s.latency_ms=%d ms\n", op, time.Since(start).Milliseconds()) -} - -func randomText(n int) []byte { - const letters = "abcdefghijklmnopqrstuvwxyz \n" - b := make([]byte, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] - } - return b -} - -func main() { - endpoint := getenv("MINIO_ENDPOINT", "localhost:9000") - user := os.Getenv("MINIO_ROOT_USER") - pass := os.Getenv("MINIO_ROOT_PASSWORD") - if user == "" || pass == "" { - fmt.Println("[APP_METRIC] ERROR config missing | trace=MINIO_ROOT_USER / MINIO_ROOT_PASSWORD not set (source storage/.env)") - os.Exit(1) - } - - ctx := context.Background() - client, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(user, pass, ""), - Secure: false, // in-cluster traffic, no TLS - }) - if err != nil { - fmt.Printf("[APP_METRIC] ERROR s3.connect failed endpoint=%s | trace=%v\n", endpoint, err) - os.Exit(1) - } - - key := fmt.Sprintf("demo/%d.txt", time.Now().Unix()) - original := randomText(256) - updated := append([]byte("UPDATED ---\n"), randomText(128)...) - - // Ensure bucket (idempotent). Site replication propagates it to az-b. - timed("ensure_bucket", func() error { - exists, err := client.BucketExists(ctx, bucket) - if err != nil || exists { - return err - } - return client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) - }) - - // CREATE - timed("put", func() error { - _, err := client.PutObject(ctx, bucket, key, - bytes.NewReader(original), int64(len(original)), - minio.PutObjectOptions{ContentType: "text/plain"}) - return err - }) - fmt.Printf("created %s/%s (%d bytes of random text)\n", bucket, key, len(original)) - - // READ — and verify content round-trips - timed("get", func() error { - obj, err := client.GetObject(ctx, bucket, key, minio.GetObjectOptions{}) - if err != nil { - return err - } - defer obj.Close() - got, err := io.ReadAll(obj) - if err != nil { - return err - } - if !bytes.Equal(got, original) { - return fmt.Errorf("read-back mismatch: want %d bytes, got %d", len(original), len(got)) - } - return nil - }) - fmt.Println("read back and verified content") - - // UPDATE — S3 semantics: overwrite the object in place - timed("update", func() error { - _, err := client.PutObject(ctx, bucket, key, - bytes.NewReader(updated), int64(len(updated)), - minio.PutObjectOptions{ContentType: "text/plain"}) - return err - }) - fmt.Println("updated (overwrote) object") - - // LIST the demo/ prefix - timed("list", func() error { - for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: "demo/", Recursive: true}) { - if obj.Err != nil { - return obj.Err - } - fmt.Printf(" %s %d bytes %s\n", obj.Key, obj.Size, obj.LastModified.Format(time.RFC3339)) - } - return nil - }) - - // DELETE — and verify it is gone - timed("delete", func() error { - if err := client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}); err != nil { - return err - } - _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) - if err == nil { - return fmt.Errorf("object %s still exists after delete", key) - } - if minio.ToErrorResponse(err).Code != "NoSuchKey" { - return err - } - return nil - }) - fmt.Println("deleted and verified gone — CRUD cycle complete") -} diff --git a/k8s/infrastructure/minio/test/test b/k8s/infrastructure/minio/test/test deleted file mode 100755 index d107195..0000000 Binary files a/k8s/infrastructure/minio/test/test and /dev/null differ diff --git a/k8s/platform/e2e/cronjob-smoke.yaml b/k8s/platform/e2e/cronjob-smoke.yaml deleted file mode 100644 index 8b4e07d..0000000 --- a/k8s/platform/e2e/cronjob-smoke.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# Continuous smoke — runs the full WebKit suite every 15 min. A failed run means -# something users touch broke; alert on it (Job failure → kube-state-metrics -# kube_job_status_failed → Alertmanager). Same image/env as the deploy-gate Job. -apiVersion: batch/v1 -kind: CronJob -metadata: - name: e2e-smoke - namespace: platform - annotations: - argocd.argoproj.io/sync-options: Prune=false -spec: - schedule: "*/15 * * * *" - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 5 - jobTemplate: - spec: - backoffLimit: 1 - ttlSecondsAfterFinished: 86400 - template: - spec: - restartPolicy: Never - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - containers: - - name: e2e - image: forgejo-gitea-http.cicd.svc.cluster.local:3000/riotpiao.com/homelab-e2e:latest - imagePullPolicy: Always - env: - - name: BASE_DOMAIN - value: riotpiao.com - - name: E2E_IGNORE_TLS - value: "0" - - name: CI - value: "1" - - name: AK_ADMIN_USER - value: akadmin - - name: AK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: { name: e2e-credentials, key: authentik-admin-password } - - name: MINIO_ENDPOINT - value: http://minio.storage.svc.cluster.local:9000 - - name: MINIO_BUCKET - value: e2e-artifacts - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: { name: e2e-credentials, key: minio-access-key } - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: { name: e2e-credentials, key: minio-secret-key } - resources: - requests: { cpu: 200m, memory: 512Mi } - limits: { cpu: "1", memory: 2Gi } diff --git a/k8s/platform/e2e/job-smoke.yaml b/k8s/platform/e2e/job-smoke.yaml deleted file mode 100644 index bfb820b..0000000 --- a/k8s/platform/e2e/job-smoke.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# One-shot E2E smoke — run as a deploy verification gate. -# Wire as an ArgoCD PostSync hook (annotation below) OR call from an Argo Rollouts -# AnalysisTemplate. Job success == the app is actually viewable in Safari's engine; -# failure fails the sync/rollout. -# -# NOTE: not yet added to any kustomization / app-of-apps. Wiring = migration phase 7. -apiVersion: batch/v1 -kind: Job -metadata: - name: e2e-smoke - namespace: platform - annotations: - # Uncomment to make this a deploy gate on the owning Application: - # argocd.argoproj.io/hook: PostSync - # argocd.argoproj.io/hook-delete-policy: BeforeHookCreation - argocd.argoproj.io/sync-options: Prune=false -spec: - backoffLimit: 1 - ttlSecondsAfterFinished: 86400 - template: - spec: - restartPolicy: Never - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - containers: - - name: e2e - # Built + pushed by CI from tests/e2e/Dockerfile. Pin a digest in prod. - image: forgejo-gitea-http.cicd.svc.cluster.local:3000/riotpiao.com/homelab-e2e:latest - imagePullPolicy: Always - env: - - name: BASE_DOMAIN - value: riotpiao.com - # strict TLS by default; set "1" only during staging-cert bootstrap - - name: E2E_IGNORE_TLS - value: "0" - - name: CI - value: "1" - - name: AK_ADMIN_USER - value: akadmin - - name: AK_ADMIN_PASSWORD - valueFrom: - secretKeyRef: - name: e2e-credentials - key: authentik-admin-password - - name: MINIO_ENDPOINT - value: http://minio.storage.svc.cluster.local:9000 - - name: MINIO_BUCKET - value: e2e-artifacts - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: { name: e2e-credentials, key: minio-access-key } - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: { name: e2e-credentials, key: minio-secret-key } - resources: - requests: { cpu: 200m, memory: 512Mi } - limits: { cpu: "1", memory: 2Gi } diff --git a/k8s/platform/kustomization.yaml b/k8s/platform/kustomization.yaml deleted file mode 100644 index d38c599..0000000 --- a/k8s/platform/kustomization.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -metadata: - name: platform - -# Layer 3: Platform — observability, monitoring, logging -# Dependencies: bootstrap (networking), infrastructure (storage) -# Order: Applied third - -resources: - - monitoring/ - - logging/ diff --git a/k8s/platform/logging/.env.example b/k8s/platform/logging/.env.example deleted file mode 100644 index a1ad07e..0000000 --- a/k8s/platform/logging/.env.example +++ /dev/null @@ -1,13 +0,0 @@ -# logging/.env.example -# Copy to logging/.env and fill in real values. -# The real .env is gitignored — never commit it. -# Used by: k8s/logging/bootstrap.sh and k8s/monitoring/bootstrap.sh (Grafana upgrade step) - -# Grafana admin UI password -# openssl rand -base64 24 -GRAFANA_ADMIN_PASSWORD= - -# MinIO credentials — must match the values used in k8s/storage/.env -# Loki uses these to authenticate to the MinIO S3 backend -MINIO_ROOT_USER= -MINIO_ROOT_PASSWORD= diff --git a/k8s/platform/logging/bootstrap.sh b/k8s/platform/logging/bootstrap.sh deleted file mode 100755 index d82d397..0000000 --- a/k8s/platform/logging/bootstrap.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash -# logging/bootstrap.sh -# Deploys MinIO + Loki + Promtail + Grafana into the logging namespace. -# -# Prerequisites: -# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig) -# - helm >= 3.x installed -# - logging/.env file containing: -# GRAFANA_ADMIN_PASSWORD=... -# MINIO_ROOT_USER=... -# MINIO_ROOT_PASSWORD=... -# OR those variables already exported in the calling shell. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../cluster-config/kubeconfig}" -export KUBECONFIG - -# ── Load credentials ────────────────────────────────────────────────────────── -if [[ -f "${SCRIPT_DIR}/.env" ]]; then - # shellcheck source=/dev/null - source "${SCRIPT_DIR}/.env" -fi - -for var in GRAFANA_ADMIN_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do - if [[ -z "${!var:-}" ]]; then - echo "ERROR: ${var} is not set. Export it or place it in logging/.env" - exit 1 - fi -done - -# ── Namespace ───────────────────────────────────────────────────────────────── -echo "==> Creating logging namespace with privileged pod security..." -kubectl create namespace logging --dry-run=client -o yaml | kubectl apply -f - -kubectl label namespace logging \ - pod-security.kubernetes.io/enforce=privileged \ - pod-security.kubernetes.io/enforce-version=latest \ - --overwrite - -# ── Helm repos ──────────────────────────────────────────────────────────────── -echo "==> Adding Helm repositories..." -helm repo add minio https://charts.min.io -helm repo add grafana https://grafana.github.io/helm-charts -helm repo update minio grafana - -# ── MinIO ───────────────────────────────────────────────────────────────────── -echo "==> Installing MinIO..." -helm upgrade --install minio minio/minio \ - --namespace logging \ - --values "${SCRIPT_DIR}/minio-values.yaml" \ - --set auth.rootUser="${MINIO_ROOT_USER}" \ - --set auth.rootPassword="${MINIO_ROOT_PASSWORD}" \ - --wait \ - --timeout 5m - -echo "==> Waiting for MinIO Deployment to be ready..." -kubectl rollout status deployment/minio -n logging --timeout=120s - -# ── Create Loki buckets in MinIO ────────────────────────────────────────────── -echo "==> Creating Loki buckets in MinIO..." -MINIO_POD=$(kubectl get pod -n logging -l app=minio,release=minio -o jsonpath='{.items[0].metadata.name}') - -# Read the actual credentials MinIO is running with from the cluster secret -ACTUAL_MINIO_USER=$(kubectl get secret minio -n logging -o jsonpath='{.data.rootUser}' | base64 --decode) -ACTUAL_MINIO_PASS=$(kubectl get secret minio -n logging -o jsonpath='{.data.rootPassword}' | base64 --decode) - -kubectl exec -n logging "${MINIO_POD}" -- \ - mc alias set local http://localhost:9000 "${ACTUAL_MINIO_USER}" "${ACTUAL_MINIO_PASS}" -for bucket in loki-chunks loki-ruler loki-admin; do - kubectl exec -n logging "${MINIO_POD}" -- \ - mc mb --ignore-existing "local/${bucket}" - echo " bucket: ${bucket} ready" -done - -# ── Loki ────────────────────────────────────────────────────────────────────── -echo "==> Installing Loki (SingleBinary + MinIO backend)..." -helm upgrade --install loki grafana/loki \ - --namespace logging \ - --values "${SCRIPT_DIR}/loki-values.yaml" \ - --set loki.storage.s3.access_key_id="${ACTUAL_MINIO_USER}" \ - --set loki.storage.s3.secret_access_key="${ACTUAL_MINIO_PASS}" \ - --wait \ - --timeout 5m - -echo "==> Waiting for Loki StatefulSet to be ready..." -kubectl rollout status statefulset/loki -n logging --timeout=120s - -# ── Promtail ────────────────────────────────────────────────────────────────── -echo "==> Installing Promtail..." -helm upgrade --install promtail grafana/promtail \ - --namespace logging \ - --values "${SCRIPT_DIR}/promtail-values.yaml" \ - --wait \ - --timeout 3m - -echo "==> Waiting for Promtail DaemonSet to be ready..." -kubectl rollout status daemonset/promtail -n logging --timeout=60s - -# ── Grafana ─────────────────────────────────────────────────────────────────── -echo "==> Installing Grafana..." -helm upgrade --install grafana grafana/grafana \ - --namespace logging \ - --values "${SCRIPT_DIR}/grafana-values.yaml" \ - --set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \ - --wait \ - --timeout 5m - -echo "==> Waiting for Grafana Deployment to be ready..." -kubectl rollout status deployment/grafana -n logging --timeout=120s - -# ── Done ────────────────────────────────────────────────────────────────────── -echo "" -echo "==> Logging stack is up." -echo "" -echo "Grafana (log explorer UI):" -echo " kubectl port-forward -n logging svc/grafana 3000:80" -echo " http://localhost:3000 (admin / )" -echo "" -echo "MinIO console (S3 object browser):" -echo " kubectl port-forward -n logging svc/minio 9001:9001" -echo " http://localhost:9001 (${MINIO_ROOT_USER} / )" -echo "" -echo "MinIO S3 endpoint for other apps:" -echo " http://minio.logging.svc.cluster.local:9000" diff --git a/k8s/platform/logging/kustomization.yaml b/k8s/platform/logging/kustomization.yaml deleted file mode 100644 index 3c6d481..0000000 --- a/k8s/platform/logging/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: logging -resources: [] -# Loki/Promtail deployed via Helm chart diff --git a/k8s/platform/logging/minio-values.yaml b/k8s/platform/logging/minio-values.yaml deleted file mode 100644 index 529737b..0000000 --- a/k8s/platform/logging/minio-values.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# logging/minio-values.yaml -# Official MinIO chart (minio/minio from https://charts.min.io). -# Single pod with console built-in on port 9001. -# rootUser and rootPassword are injected via --set at install time from .env. - -mode: standalone - -rootUser: "" # injected via --set -rootPassword: "" # injected via --set - -persistence: - enabled: true - storageClass: longhorn - accessMode: ReadWriteOnce - size: 100Gi - -resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - -service: - type: ClusterIP - port: 9000 - -consoleService: - type: ClusterIP - port: 9001 - -# Disable ingress — access via kubectl port-forward -ingress: - enabled: false - -consoleIngress: - enabled: false - -metrics: - serviceMonitor: - enabled: false diff --git a/k8s/platform/monitoring/alerts/kustomization.yaml b/k8s/platform/monitoring/alerts/kustomization.yaml deleted file mode 100644 index eb85ebd..0000000 --- a/k8s/platform/monitoring/alerts/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ingress-alerts.yaml - - svc-argocd-rules.yaml - - svc-authentik-rules.yaml - - svc-forgejo-rules.yaml - - svc-grafana-rules.yaml - - svc-minio-rules.yaml - - svc-story-crater-backend-rules.yaml - - svc-story-crater-frontend-rules.yaml - - svc-vault-rules.yaml diff --git a/k8s/platform/monitoring/alerts/svc-story-crater-backend-rules.yaml b/k8s/platform/monitoring/alerts/svc-story-crater-backend-rules.yaml deleted file mode 100644 index ed90c69..0000000 --- a/k8s/platform/monitoring/alerts/svc-story-crater-backend-rules.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: story-crater-backend-rules - namespace: story-crater-backend -spec: - groups: - - name: story-crater-backend.rules - interval: 15s - rules: - - alert: StoryCraterBackendHighErrorRate - expr: | - ( - sum(rate(story_crater_app_metric_total{severity="error"}[5m])) - / - sum(rate(story_crater_messages_handled_total[5m])) - ) > 0.05 - for: 10m - labels: - severity: warning - annotations: - summary: "High application error rate on story-crater-backend" - description: "Error events exceeded 5% of message volume for 10 minutes. Value: {{ $value | humanizePercentage }}" - - - alert: StoryCraterBackendCheckLatencySLOBreach - expr: | - histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le)) > 1200 - for: 10m - labels: - severity: critical - annotations: - summary: "CheckScene p95 latency breaching NFR-01 (1200ms SLO)" - description: "p95 CheckScene latency has exceeded 1200ms for 10 minutes. Value: {{ $value }}ms" diff --git a/k8s/platform/monitoring/alerts/svc-story-crater-frontend-rules.yaml b/k8s/platform/monitoring/alerts/svc-story-crater-frontend-rules.yaml deleted file mode 100644 index b2dc55c..0000000 --- a/k8s/platform/monitoring/alerts/svc-story-crater-frontend-rules.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: story-crater-frontend-rules - namespace: story-crater-frontend -spec: - groups: - - name: story-crater-frontend.rules - interval: 15s - rules: - - alert: StoryCraterFrontendHighErrorRate - expr: | - ( - sum(rate(story_crater_frontend_http_requests_total{status=~"5.."}[5m])) - / - sum(rate(story_crater_frontend_http_requests_total[5m])) - ) > 0.05 - for: 10m - labels: - severity: warning - annotations: - summary: "High error rate on story-crater-frontend" - description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}" - - - alert: StoryCraterFrontendWebVitalsLCPRegression - expr: | - histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms) > 2500 - for: 10m - labels: - severity: warning - annotations: - summary: "LCP (Largest Contentful Paint) regression on story-crater-frontend" - description: "LCP p75 exceeded 2500ms for 10 minutes. Value: {{ $value }}ms" diff --git a/k8s/platform/monitoring/bootstrap.sh b/k8s/platform/monitoring/bootstrap.sh deleted file mode 100755 index cffe4d7..0000000 --- a/k8s/platform/monitoring/bootstrap.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -# monitoring/bootstrap.sh -# Deploys kube-prometheus-stack into the monitoring namespace, then upgrades -# Grafana (in the logging namespace) to add the Prometheus datasource and -# pre-built dashboards. -# -# Prerequisites: -# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig) -# - helm >= 3.x -# - GRAFANA_ADMIN_PASSWORD set, or present in k8s/logging/.env -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}" -export KUBECONFIG - -# ── Namespace ───────────────────────────────────────────────────────────────── -echo "==> Creating monitoring namespace..." -kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - -kubectl label namespace monitoring \ - pod-security.kubernetes.io/enforce=privileged \ - pod-security.kubernetes.io/enforce-version=latest \ - --overwrite - -# ── Helm repo ───────────────────────────────────────────────────────────────── -echo "==> Adding prometheus-community Helm repo..." -helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -helm repo update prometheus-community - -# ── kube-prometheus-stack ───────────────────────────────────────────────────── -echo "==> Installing kube-prometheus-stack..." -helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \ - --namespace monitoring \ - --values "${SCRIPT_DIR}/prometheus-values.yaml" \ - --wait \ - --timeout 10m - -echo "==> Waiting for Prometheus StatefulSet..." -kubectl rollout status \ - statefulset/prometheus-prometheus-kube-prometheus-prometheus \ - -n monitoring --timeout=180s - -echo "==> Waiting for node-exporter DaemonSet..." -kubectl rollout status \ - daemonset/prometheus-prometheus-node-exporter \ - -n monitoring --timeout=60s - -echo "==> Waiting for kube-state-metrics..." -kubectl rollout status \ - deployment/prometheus-kube-state-metrics \ - -n monitoring --timeout=60s - -# ── Upgrade Grafana with Prometheus datasource + dashboards ─────────────────── -echo "" -echo "==> Upgrading Grafana to wire in Prometheus datasource + dashboards..." -LOGGING_DIR="${REPO_ROOT}/k8s/logging" - -if [[ -f "${LOGGING_DIR}/.env" ]]; then - # shellcheck source=/dev/null - source "${LOGGING_DIR}/.env" -fi - -if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then - echo "ERROR: GRAFANA_ADMIN_PASSWORD is not set. Export it or place it in k8s/logging/.env" - exit 1 -fi - -helm repo add grafana https://grafana.github.io/helm-charts -helm repo update grafana - -helm upgrade grafana grafana/grafana \ - --namespace logging \ - --values "${LOGGING_DIR}/grafana-values.yaml" \ - --set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \ - --wait \ - --timeout 5m - -echo "==> Waiting for Grafana rollout..." -kubectl rollout status deployment/grafana -n logging --timeout=120s - -# ── Done ────────────────────────────────────────────────────────────────────── -echo "" -echo "==> Monitoring stack is up." -echo "" -echo "Prometheus UI:" -echo " kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090" -echo " http://localhost:9090 (Targets page shows node-exporter + kube-state-metrics)" -echo "" -echo "Grafana:" -echo " kubectl port-forward -n logging svc/grafana 3000:80" -echo " http://localhost:3000" -echo " Datasources: Loki (logs, default) + Prometheus (metrics)" -echo " Dashboards → General:" -echo " - Node Exporter Full (per-node CPU, RAM, disk, network)" -echo " - Kubernetes Cluster (pod resource usage across namespaces)" diff --git a/k8s/platform/monitoring/dashboards/services/argocd.descriptor.yaml b/k8s/platform/monitoring/dashboards/services/argocd.descriptor.yaml deleted file mode 100644 index b31dcee..0000000 --- a/k8s/platform/monitoring/dashboards/services/argocd.descriptor.yaml +++ /dev/null @@ -1,48 +0,0 @@ -service: argocd -display_name: "Argo CD" -namespace: argocd -service_type: web-app -folder: "Argo CD" -jobs: [argocd-server, argocd-repo-server] -metrics: - prefix: argocd - up_selector: 'job=~"argocd-.*"' - rate: - metric: argocd_http_request_total - labels: [method, path, status] - duration: - metric: argocd_http_request_duration_seconds - type: histogram - unit: s - error: - metric: argocd_http_request_total - filter: 'status=~"5.."' - domain: - - metric: argocd_app_total - kind: gauge - - metric: argocd_app_sync_total - kind: counter - labels: [sync_status] - - metric: argocd_app_health_degraded_total - kind: counter - - metric: argocd_git_sync_total - kind: counter - labels: [git_operation, git_status] - - metric: argocd_reconcile_total - kind: counter - labels: [app_name] -correlation_ids: [] -logs: - loki_namespace_selector: 'namespace="argocd"' -alerts: - - name: HighErrorRate - expr_template: rate_error_ratio - threshold: 0.05 - for: 10m - severity: warning - - name: AppSyncFailure - expr_template: gauge_above_threshold - metric: argocd_app_health_degraded_total - threshold: 1 - for: 10m - severity: warning diff --git a/k8s/platform/monitoring/dashboards/services/authentik.descriptor.yaml b/k8s/platform/monitoring/dashboards/services/authentik.descriptor.yaml deleted file mode 100644 index 6095e89..0000000 --- a/k8s/platform/monitoring/dashboards/services/authentik.descriptor.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# authentik.descriptor.yaml -# Build input for generating svc-authentik.yaml dashboard ConfigMap -# and svc-authentik-rules.yaml PrometheusRule. -# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation. - -service: authentik -display_name: "Authentik" -namespace: iam -service_type: identity-provider -folder: "Authentik" - -jobs: - - authentik-server - - authentik-worker - -metrics: - prefix: authentik - up_selector: 'job="authentik-server"' - - rate: - metric: authentik_flows_execution_stage_time_count - labels: [flow_slug, stage_name] - - duration: - metric: authentik_main_request_duration_seconds - type: histogram - unit: s - - error: - metric: authentik_flows_cached - filter: null - - domain: - - metric: authentik_outpost_connection - kind: gauge - group_by: [outpost_name, outpost_type] - - metric: authentik_outposts_connected - kind: gauge - - metric: authentik_flows_cached - kind: gauge - - metric: authentik_policies_cached - kind: gauge - - metric: authentik_tasks_queued - kind: gauge - - metric: authentik_admin_workers - kind: gauge - -correlation_ids: [] - -logs: - loki_namespace_selector: 'namespace="iam"' - -alerts: - - name: HighErrorRate - expr_template: rate_error_ratio - threshold: 0.05 - for: 10m - severity: warning - - name: OutpostDown - expr_template: gauge_below_threshold - metric: authentik_outpost_total_up - threshold: 1 - for: 5m - severity: warning diff --git a/k8s/platform/monitoring/dashboards/services/forgejo.descriptor.yaml b/k8s/platform/monitoring/dashboards/services/forgejo.descriptor.yaml deleted file mode 100644 index b8242d9..0000000 --- a/k8s/platform/monitoring/dashboards/services/forgejo.descriptor.yaml +++ /dev/null @@ -1,39 +0,0 @@ -service: forgejo -display_name: "Forgejo" -namespace: forgejo -service_type: web-app -folder: "Forgejo" -jobs: [forgejo] -metrics: - prefix: forgejo - up_selector: 'job="forgejo"' - rate: - metric: forgejo_http_request_total - labels: [method, status] - duration: - metric: forgejo_http_request_duration_seconds - type: histogram - unit: s - error: - metric: forgejo_http_request_total - filter: 'status=~"5.."' - domain: - - metric: forgejo_repositories_total - kind: gauge - - metric: forgejo_users_total - kind: gauge - - metric: forgejo_git_operations_total - kind: counter - labels: [operation_type] - - metric: forgejo_runner_tasks_total - kind: counter - labels: [status] -correlation_ids: [] -logs: - loki_namespace_selector: 'namespace="forgejo"' -alerts: - - name: HighErrorRate - expr_template: rate_error_ratio - threshold: 0.05 - for: 10m - severity: warning diff --git a/k8s/platform/monitoring/dashboards/services/grafana.descriptor.yaml b/k8s/platform/monitoring/dashboards/services/grafana.descriptor.yaml deleted file mode 100644 index d04ee39..0000000 --- a/k8s/platform/monitoring/dashboards/services/grafana.descriptor.yaml +++ /dev/null @@ -1,35 +0,0 @@ -service: grafana -display_name: "Grafana" -namespace: logging -service_type: web-app -folder: "Grafana" -jobs: [grafana] -metrics: - prefix: grafana - up_selector: 'job="grafana"' - rate: - metric: grafana_http_request_total - labels: [handler, status] - duration: - metric: grafana_http_request_duration_seconds - type: histogram - unit: s - error: - metric: grafana_http_request_total - filter: 'status=~"5.."' - domain: - - metric: grafana_dashboard_total - kind: gauge - - metric: grafana_user_total - kind: gauge - - metric: grafana_alerts_total - kind: gauge -correlation_ids: [] -logs: - loki_namespace_selector: 'namespace="logging"' -alerts: - - name: HighErrorRate - expr_template: rate_error_ratio - threshold: 0.05 - for: 10m - severity: warning diff --git a/k8s/platform/monitoring/dashboards/services/minio.descriptor.yaml b/k8s/platform/monitoring/dashboards/services/minio.descriptor.yaml deleted file mode 100644 index e948d4d..0000000 --- a/k8s/platform/monitoring/dashboards/services/minio.descriptor.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# minio.descriptor.yaml -# Build input for generating svc-minio.yaml dashboard ConfigMap -# and svc-minio-rules.yaml PrometheusRule. - -service: minio -display_name: "MinIO" -namespace: storage -service_type: stateful-store -folder: "MinIO" - -jobs: - - minio - -metrics: - prefix: minio - up_selector: 'job="minio"' - - rate: - metric: minio_s3_requests_total - labels: [method, bucket] - - duration: - metric: minio_s3_requests_duration_seconds - type: histogram - unit: s - - error: - metric: minio_s3_requests_total - filter: 'error="true"' - - domain: - - metric: minio_cluster_capacity_usable_bytes - kind: gauge - - metric: minio_cluster_capacity_raw_total_bytes - kind: gauge - - metric: minio_replication_metrics_failed_byte_count - kind: gauge - - metric: minio_replication_metrics_replicating_byte_count - kind: gauge - - metric: minio_cluster_health_drives_online - kind: gauge - - metric: minio_cluster_health_drives_offline - kind: gauge - -correlation_ids: [] - -logs: - loki_namespace_selector: 'namespace="storage"' - -alerts: - - name: HighErrorRate - expr_template: rate_error_ratio - threshold: 0.05 - for: 10m - severity: warning - - name: DiskSpaceLow - expr_template: gauge_below_percentage - metric: minio_cluster_capacity_usable_bytes - threshold: 0.1 - base_metric: minio_cluster_capacity_raw_total_bytes - for: 5m - severity: critical - - name: ReplicationLag - expr_template: gauge_above_threshold - metric: minio_replication_metrics_replicating_byte_count - threshold: 1073741824 - for: 15m - severity: warning diff --git a/k8s/platform/monitoring/dashboards/services/story-crater-backend.descriptor.yaml b/k8s/platform/monitoring/dashboards/services/story-crater-backend.descriptor.yaml deleted file mode 100644 index b64b998..0000000 --- a/k8s/platform/monitoring/dashboards/services/story-crater-backend.descriptor.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# story-crater-backend.descriptor.yaml -# Build input for generating svc-story-crater-backend.yaml dashboard ConfigMap -# and svc-story-crater-backend-rules.yaml PrometheusRule. -# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation. - -service: story-crater-backend -display_name: "Story Crater Backend" -namespace: story-crater-backend -service_type: message-worker -folder: "Story Crater Backend" - -jobs: - - agent-worker - - check - - edit-collab - - canary - -metrics: - prefix: story_crater - up_selector: 'job=~"agent-worker|check|edit-collab|canary"' - - rate: - metric: story_crater_messages_handled_total - labels: [agent, queue, status] # status: succeeded | failed - - duration: - metric: story_crater_message_handle_duration_ms - type: histogram - unit: ms - - error: - metric: story_crater_app_metric_total - filter: 'severity="error"' - - domain: - - metric: story_crater_queue_depth - kind: gauge - group_by: [queue] - - metric: story_crater_dedup_redeliveries_total - kind: counter - group_by: [agent, queue] - - metric: story_crater_outbox_publish_lag_ms - kind: histogram - - metric: story_crater_llm_tokens_used_total - kind: counter - group_by: [agent, kind] - - metric: story_crater_llm_call_duration_seconds - kind: histogram - group_by: [agent, model_provider] - - metric: story_crater_check_latency_ms - kind: histogram - slo_ms: 1200 - - metric: story_crater_degrade_state - kind: gauge - group_by: [service] - -correlation_ids: [session_id, trace_id, tenant] - -logs: - loki_namespace_selector: 'namespace="story-crater-backend"' - -alerts: - - name: HighErrorRate - expr_template: rate_error_ratio - threshold: 0.05 - for: 10m - severity: warning - - name: CheckLatencySLOBreach - expr_template: histogram_quantile_over_threshold - metric: story_crater_check_latency_ms - quantile: 0.95 - threshold: 1200 - for: 10m - severity: critical diff --git a/k8s/platform/monitoring/dashboards/services/story-crater-frontend.descriptor.yaml b/k8s/platform/monitoring/dashboards/services/story-crater-frontend.descriptor.yaml deleted file mode 100644 index 9196d7f..0000000 --- a/k8s/platform/monitoring/dashboards/services/story-crater-frontend.descriptor.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# story-crater-frontend.descriptor.yaml -# Build input for generating svc-story-crater-frontend.yaml dashboard ConfigMap -# and svc-story-crater-frontend-rules.yaml PrometheusRule. -# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation. - -service: story-crater-frontend -display_name: "Story Crater Frontend" -namespace: story-crater-frontend -service_type: web-app -folder: "Story Crater Frontend" - -jobs: - - story-crater-frontend - -metrics: - prefix: story_crater_frontend - up_selector: 'job="story-crater-frontend"' - - rate: - metric: story_crater_frontend_http_requests_total - labels: [method, route, status] - - duration: - metric: story_crater_frontend_http_request_duration_ms - type: histogram - unit: ms - - error: - metric: story_crater_frontend_http_requests_total - filter: 'status=~"5.."' - - domain: - - metric: story_crater_frontend_web_vitals_lcp_ms - kind: gauge - group_by: [page] - - metric: story_crater_frontend_web_vitals_fid_ms - kind: gauge - group_by: [page] - - metric: story_crater_frontend_web_vitals_cls - kind: gauge - group_by: [page] - - metric: story_crater_frontend_web_vitals_inp_ms - kind: gauge - group_by: [page] - -correlation_ids: [] # Frontend doesn't have session_id/trace_id in metrics - -logs: - loki_namespace_selector: 'namespace="story-crater-frontend"' - -alerts: - - name: HighErrorRate - expr_template: rate_error_ratio - threshold: 0.05 - for: 10m - severity: warning - - name: WebVitalsLCPRegression - expr_template: histogram_quantile_over_threshold - metric: story_crater_frontend_web_vitals_lcp_ms - quantile: 0.75 - threshold: 2500 - for: 10m - severity: warning diff --git a/k8s/platform/monitoring/dashboards/services/vault.descriptor.yaml b/k8s/platform/monitoring/dashboards/services/vault.descriptor.yaml deleted file mode 100644 index cc76479..0000000 --- a/k8s/platform/monitoring/dashboards/services/vault.descriptor.yaml +++ /dev/null @@ -1,45 +0,0 @@ -service: vault -display_name: "Vault" -namespace: storage -service_type: stateful-store -folder: "Vault" -jobs: [vault] -metrics: - prefix: vault - up_selector: 'job="vault"' - rate: - metric: vault_core_handle_request_total - labels: [method, path] - duration: - metric: vault_core_handle_request_duration_seconds - type: histogram - unit: s - error: - metric: vault_core_handle_request_total - filter: 'error="true"' - domain: - - metric: vault_core_unsealed - kind: gauge - - metric: vault_core_active - kind: gauge - - metric: vault_core_replication_primary - kind: gauge - - metric: vault_token_total - kind: gauge - - metric: vault_database_connection_close_total - kind: counter -correlation_ids: [] -logs: - loki_namespace_selector: 'namespace="iam"' -alerts: - - name: HighErrorRate - expr_template: rate_error_ratio - threshold: 0.05 - for: 10m - severity: warning - - name: VaultSealed - expr_template: gauge_below_threshold - metric: vault_core_unsealed - threshold: 1 - for: 1m - severity: critical diff --git a/k8s/platform/monitoring/dashboards/svc-story-crater-backend.yaml b/k8s/platform/monitoring/dashboards/svc-story-crater-backend.yaml deleted file mode 100644 index 88d3c72..0000000 --- a/k8s/platform/monitoring/dashboards/svc-story-crater-backend.yaml +++ /dev/null @@ -1,191 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: svc-story-crater-backend-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "Story Crater Backend" -data: - svc-story-crater-backend.json: | - { - "title": "Story Crater Backend — Service Overview", - "uid": "svc-story-crater-backend", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-6h", "to": "now" }, - "refresh": "30s", - "panels": [ - { - "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "panels": [ - { - "id": 2, "title": "Up", "type": "stat", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } - ], - "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } - } - }, - "targets": [{ "expr": "min(up{job=~\"agent-worker|check|edit-collab|canary\"})" }] - }, - { - "id": 3, "title": "Message rate by status", "type": "timeseries", - "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(story_crater_messages_handled_total[5m])) by (status)", "legendFormat": "{{status}}" }] - }, - { - "id": 4, "title": "Error rate %", "type": "timeseries", - "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, - "fieldConfig": { "defaults": { "unit": "percent" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(story_crater_app_metric_total{severity=\"error\"}[5m])) / sum(rate(story_crater_messages_handled_total[5m])) * 100" }] - }, - { - "id": 5, "title": "Duration p50/p95/p99", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "fieldConfig": { "defaults": { "unit": "ms" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "histogram_quantile(0.50, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" }, - { "expr": "histogram_quantile(0.95, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" }, - { "expr": "histogram_quantile(0.99, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" } - ] - } - ] - }, - { - "id": 10, "title": "Row: Resource Usage", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, - "panels": [ - { - "id": 11, "title": "CPU by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-backend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] - }, - { - "id": 12, "title": "Memory by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, - "fieldConfig": { "defaults": { "unit": "bytes" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-backend\"}) by (pod)", "legendFormat": "{{pod}}" }] - }, - { - "id": 13, "title": "Restart rate by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-backend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] - } - ] - }, - { - "id": 20, "title": "Row: Broker / Outbox / Dedup", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, - "panels": [ - { - "id": 21, "title": "Queue depth", "type": "timeseries", - "gridPos": { "h": 7, "w": 8, "x": 0, "y": 3 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "story_crater_queue_depth", "legendFormat": "{{queue}}" }] - }, - { - "id": 22, "title": "Outbox publish lag p95", "type": "timeseries", - "gridPos": { "h": 7, "w": 8, "x": 8, "y": 3 }, - "fieldConfig": { "defaults": { "unit": "ms" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_outbox_publish_lag_ms_bucket[5m])) by (le))" }] - }, - { - "id": 23, "title": "Dedup redeliveries rate", "type": "timeseries", - "gridPos": { "h": 7, "w": 8, "x": 16, "y": 3 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(story_crater_dedup_redeliveries_total[5m])) by (agent, queue)", "legendFormat": "{{agent}}-{{queue}}" }] - } - ] - }, - { - "id": 30, "title": "Row: LLM / Inference", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, - "panels": [ - { - "id": 31, "title": "Token usage rate", "type": "timeseries", - "gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(story_crater_llm_tokens_used_total[5m])) by (agent, kind)", "legendFormat": "{{agent}}-{{kind}}" }] - }, - { - "id": 32, "title": "LLM call duration p95", "type": "timeseries", - "gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 }, - "fieldConfig": { "defaults": { "unit": "s" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_llm_call_duration_seconds_bucket[5m])) by (le, agent))", "legendFormat": "{{agent}}" }] - }, - { - "id": 33, "title": "LLM call errors rate", "type": "timeseries", - "gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(story_crater_llm_call_errors_total[5m])) by (agent, error_class)", "legendFormat": "{{agent}}-{{error_class}}" }] - } - ] - }, - { - "id": 40, "title": "Row: SLOs / Degradation", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, - "panels": [ - { - "id": 41, "title": "CheckScene p95 latency (NFR-01: <1200ms)", "type": "timeseries", - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 }, - "fieldConfig": { - "defaults": { - "unit": "ms", - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": null, "color": "green" }, - { "value": 1200, "color": "red" } - ] - } - } - }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le))" }] - }, - { - "id": 42, "title": "Degrade state by service", "type": "stat", - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 }, - "fieldConfig": { - "defaults": { - "mappings": [ - { "type": "value", "options": { "0": { "text": "NORMAL", "color": "green" } } }, - { "type": "value", "options": { "1": { "text": "SHEDDING", "color": "red" } } } - ] - } - }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "story_crater_degrade_state", "legendFormat": "{{service}}" }] - } - ] - }, - { - "id": 50, "title": "Row: Logs", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, - "panels": [ - { - "id": 51, "title": "Recent logs (session_id / trace_id searchable)", "type": "logs", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 }, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [{ "expr": "{namespace=\"story-crater-backend\"} | json" }] - } - ] - } - ] - } diff --git a/k8s/platform/monitoring/dashboards/svc-story-crater-frontend.yaml b/k8s/platform/monitoring/dashboards/svc-story-crater-frontend.yaml deleted file mode 100644 index ebf672c..0000000 --- a/k8s/platform/monitoring/dashboards/svc-story-crater-frontend.yaml +++ /dev/null @@ -1,184 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: svc-story-crater-frontend-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "Story Crater Frontend" -data: - svc-story-crater-frontend.json: | - { - "title": "Story Crater Frontend — Service Overview", - "uid": "svc-story-crater-frontend", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-6h", "to": "now" }, - "refresh": "30s", - "panels": [ - { - "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "panels": [ - { - "id": 2, "title": "Up", "type": "stat", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } - ], - "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } - } - }, - "targets": [{ "expr": "min(up{job=\"story-crater-frontend\"})" }] - }, - { - "id": 3, "title": "HTTP request rate by status", "type": "timeseries", - "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(story_crater_frontend_http_requests_total[5m])) by (status)", "legendFormat": "{{status}}" }] - }, - { - "id": 4, "title": "Error rate % (5xx)", "type": "timeseries", - "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, - "fieldConfig": { "defaults": { "unit": "percent" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(story_crater_frontend_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(story_crater_frontend_http_requests_total[5m])) * 100" }] - }, - { - "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "fieldConfig": { "defaults": { "unit": "ms" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "histogram_quantile(0.50, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" }, - { "expr": "histogram_quantile(0.95, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" }, - { "expr": "histogram_quantile(0.99, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" } - ] - } - ] - }, - { - "id": 10, "title": "Row: Resource Usage", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, - "panels": [ - { - "id": 11, "title": "CPU by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-frontend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] - }, - { - "id": 12, "title": "Memory by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, - "fieldConfig": { "defaults": { "unit": "bytes" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-frontend\"}) by (pod)", "legendFormat": "{{pod}}" }] - }, - { - "id": 13, "title": "Restart rate by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-frontend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] - } - ] - }, - { - "id": 20, "title": "Row: Web Vitals", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, - "panels": [ - { - "id": 21, "title": "LCP p75 (Largest Contentful Paint)", "type": "timeseries", - "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 }, - "fieldConfig": { - "defaults": { - "unit": "ms", - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": null, "color": "green" }, - { "value": 2500, "color": "orange" }, - { "value": 4000, "color": "red" } - ] - } - } - }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms)", "legendFormat": "LCP p75" }] - }, - { - "id": 22, "title": "FID p75 (First Input Delay)", "type": "timeseries", - "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 }, - "fieldConfig": { - "defaults": { - "unit": "ms", - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": null, "color": "green" }, - { "value": 100, "color": "orange" }, - { "value": 300, "color": "red" } - ] - } - } - }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_fid_ms)", "legendFormat": "FID p75" }] - }, - { - "id": 23, "title": "CLS (Cumulative Layout Shift)", "type": "timeseries", - "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": null, "color": "green" }, - { "value": 0.1, "color": "orange" }, - { "value": 0.25, "color": "red" } - ] - } - } - }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "max(story_crater_frontend_web_vitals_cls)", "legendFormat": "CLS" }] - }, - { - "id": 24, "title": "INP p75 (Interaction to Next Paint)", "type": "timeseries", - "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 }, - "fieldConfig": { - "defaults": { - "unit": "ms", - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": null, "color": "green" }, - { "value": 200, "color": "orange" }, - { "value": 500, "color": "red" } - ] - } - } - }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_inp_ms)", "legendFormat": "INP p75" }] - } - ] - }, - { - "id": 30, "title": "Row: Logs", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, - "panels": [ - { - "id": 31, "title": "Recent logs", "type": "logs", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 }, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [{ "expr": "{namespace=\"story-crater-frontend\"}" }] - } - ] - } - ] - } diff --git a/k8s/platform/monitoring/kustomization.yaml b/k8s/platform/monitoring/kustomization.yaml deleted file mode 100644 index a711a13..0000000 --- a/k8s/platform/monitoring/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: monitoring -resources: - - alerts/ -# prometheus/blackbox deployed via ArgoCD Helm source diff --git a/k8s/security/ci-cd/argocd-values.yaml b/k8s/security/ci-cd/argocd-values.yaml deleted file mode 100644 index 3251ea8..0000000 --- a/k8s/security/ci-cd/argocd-values.yaml +++ /dev/null @@ -1,206 +0,0 @@ -# k8s/talos-ci-cd/argocd-values.yaml -# Argo CD — single-replica homelab install (fits the 4-CPU / 32 GB RAM budget). -# Deployed via helmfile (name=argocd) so it sits alongside Authentik, MinIO, etc. -# -# UI is exposed via the cluster Ingress (k8s/ingress/ingress.yaml, -# argocd.riotpiao.com) over the WireGuard/LAN-only nginx ingress — -# never as a LoadBalancer, since argocd-server holds cluster-admin -# credentials and that Ingress isn't reachable from the WAN. -# -# OIDC + RBAC declared in configs.cm / configs.rbac below — applied by Helm directly. -# Requires: oidc-secret K8s secret (created by helmfile argocd presync hook from env vars). -# CA trust for Forgejo repo clones injected into argocd-tls-certs-cm by postsync hook. - -global: - domain: argocd.riotpiao.com - -configs: - params: - server.insecure: false # keep TLS on argocd-server even behind the ingress - - cm: - # Must match the Ingress host above and the redirect URI registered in - # Authentik (provision_oidc.py argocd_url) exactly — ArgoCD builds its - # OIDC redirect_uri as "{url}/auth/callback", so any mismatch here is - # what Authentik's "Invalid redirect URL" error means. - url: "https://argocd.riotpiao.com" - oidc.config: | - name: Authentik - issuer: https://authentik.riotpiao.com/application/o/argocd/ - clientID: $oidc-secret:client-id - clientSecret: $oidc-secret:client-secret - requestedScopes: [openid, profile, email, groups] - insecureSkipVerify: true - rootCA: | - -----BEGIN CERTIFICATE----- - MIIBbTCCARSgAwIBAgIUNa409I6cGHye4YqeiphmWDaCEXUwCgYIKoZIzj0EAwIw - FTETMBEGA1UEAxMKaG9tZWxhYi1jYTAeFw0yNjA2MTcxNjUzMjVaFw0zNjA2MTQx - NjUzMjVaMBUxEzARBgNVBAMTCmhvbWVsYWItY2EwWTATBgcqhkjOPQIBBggqhkjO - PQMBBwNCAARwkubJPPdhgKcqr+3AEO2tr5I7MhC3zzeAqpmv8glngsweiGznaDhi - Dbf8JFfilbrLEJBSuwHZQPuoNx+3fbvYo0IwQDAOBgNVHQ8BAf8EBAMCAqQwDwYD - VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUod8iYq0+QyetnxfKDprIf3XbWkEwCgYI - KoZIzj0EAwIDRwAwRAIgQ4HOLs5DOqcfAMv8NSImxoYN7TyebnlQAQXSARnIqBMC - IB9RycFvG/rpJuz/LIKi4rf6RARjLcHM/zqhXQJvBw53 - -----END CERTIFICATE----- - - rbac: - policy.csv: | - g, homelab-admins, role:admin - policy.default: role:readonly - -server: - replicas: 1 - deploymentStrategy: - type: Recreate - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - # No Service of type LoadBalancer — port-forward only - service: - type: ClusterIP - podAnnotations: - secret.reloader.stakater.com/reload: "oidc-secret" - configmap.reloader.stakater.com/reload: "argocd-tls-certs-cm,argocd-cm" - metrics: - enabled: true - serviceMonitor: - enabled: true - -repoServer: - replicas: 1 - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - cpu: 500m - # 512Mi OOM-killed the container under CMP + Helm rendering of ~30 apps, - # causing chronic restarts → not-ready → dropped Cilium backend → every - # app's manifest comparison failed with "no route to host". 1Gi headroom. - memory: 1Gi - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - metrics: - enabled: true - serviceMonitor: - enabled: true - - # ── SOPS ConfigManagementPlugin ───────────────────────────────────────────── - # initContainer fetches sops+yq into a shared volume; the sidecar runs - # argocd-cmp-server with the plugin.yaml from the sops-cmp-plugin ConfigMap and - # decrypts *.enc.yaml with the age key from the sops-age Secret. - initContainers: - - name: install-sops-tools - image: alpine:3.20 - command: [sh, -c] - args: - - | - set -e - apk add --no-cache curl - curl -sSfL https://github.com/getsops/sops/releases/download/v3.9.0/sops-v3.9.0.linux.amd64 -o /custom-tools/sops - curl -sSfL https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -o /custom-tools/yq - chmod +x /custom-tools/sops /custom-tools/yq - volumeMounts: - - name: custom-tools - mountPath: /custom-tools - extraContainers: - - name: sops-cmp - image: quay.io/argoproj/argocd:v3.4.5 - command: [/var/run/argocd/argocd-cmp-server] - env: - - name: PATH - value: /custom-tools:/usr/local/bin:/usr/bin:/bin - securityContext: - runAsNonRoot: true - runAsUser: 999 - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /var/run/argocd - name: var-files - - mountPath: /home/argocd/cmp-server/plugins - name: plugins - - mountPath: /home/argocd/cmp-server/config/plugin.yaml - subPath: plugin.yaml - name: sops-cmp-plugin - - mountPath: /home/argocd/plugins/generate.sh - subPath: generate.sh - name: sops-cmp-plugin - - mountPath: /custom-tools - name: custom-tools - - mountPath: /sops-age - name: sops-age - - mountPath: /tmp - name: cmp-tmp - volumes: - - name: custom-tools - emptyDir: {} - - name: cmp-tmp - emptyDir: {} - - name: sops-cmp-plugin - configMap: - name: sops-cmp-plugin - defaultMode: 0555 - - name: sops-age - secret: - secretName: sops-age - -applicationSet: - replicas: 1 - resources: - requests: - cpu: 50m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - -controller: - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 1000m - memory: 1Gi - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - metrics: - enabled: true - serviceMonitor: - enabled: true - -redis: - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 128Mi - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - -notifications: - enabled: false # add back later if alert routing is needed diff --git a/k8s/security/ci-cd/charts/kustomization.yaml b/k8s/security/ci-cd/charts/kustomization.yaml deleted file mode 100644 index 393243b..0000000 --- a/k8s/security/ci-cd/charts/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - forgejo-runner/ diff --git a/k8s/security/ci-cd/deploy-scaffold/api/deployment.yaml b/k8s/security/ci-cd/deploy-scaffold/api/deployment.yaml deleted file mode 100644 index 158b8e5..0000000 --- a/k8s/security/ci-cd/deploy-scaffold/api/deployment.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: api - namespace: api -spec: - replicas: 1 - selector: - matchLabels: - app: api - template: - metadata: - labels: - app: api - spec: - containers: - - name: api - # CI bumps this tag on every push to main (ci.yml step "bump deploy repo") - image: forgejo.riotpiao.com/rock/api:latest - ports: - - containerPort: 8080 - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi - readinessProbe: - httpGet: - path: /healthz - port: 8080 - initialDelaySeconds: 5 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /healthz - port: 8080 - initialDelaySeconds: 15 - periodSeconds: 30 diff --git a/k8s/security/ci-cd/deploy-scaffold/api/service.yaml b/k8s/security/ci-cd/deploy-scaffold/api/service.yaml deleted file mode 100644 index 4e52894..0000000 --- a/k8s/security/ci-cd/deploy-scaffold/api/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: api - namespace: api -spec: - selector: - app: api - ports: - - name: http - port: 80 - targetPort: 8080 diff --git a/k8s/security/ci-cd/deploy-scaffold/apps/api.yaml b/k8s/security/ci-cd/deploy-scaffold/apps/api.yaml deleted file mode 100644 index 5a5534e..0000000 --- a/k8s/security/ci-cd/deploy-scaffold/apps/api.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: api - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "10" -spec: - project: default - source: - repoURL: https://forgejo.forge.riotpiao.com/rock/deploy.git - targetRevision: main - path: api - destination: - server: https://kubernetes.default.svc - namespace: api - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/k8s/security/ci-cd/deploy-scaffold/apps/argocd.yaml b/k8s/security/ci-cd/deploy-scaffold/apps/argocd.yaml deleted file mode 100644 index e376d90..0000000 --- a/k8s/security/ci-cd/deploy-scaffold/apps/argocd.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: argocd - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - project: default - source: - repoURL: https://forgejo.forge.riotpiao.com/rock/deploy.git - targetRevision: main - path: argocd - destination: - server: https://kubernetes.default.svc - namespace: argocd - # NO syncPolicy.automated — manual sync required. - # Argo CD managing itself auto-synced is a footgun: a misconfigured commit could - # take down the CD system before anyone can intervene. Approve manually. diff --git a/k8s/security/ci-cd/deploy-scaffold/apps/forge.yaml b/k8s/security/ci-cd/deploy-scaffold/apps/forge.yaml deleted file mode 100644 index be10f48..0000000 --- a/k8s/security/ci-cd/deploy-scaffold/apps/forge.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: forge - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - project: default - source: - repoURL: https://forgejo.forge.riotpiao.com/rock/deploy.git - targetRevision: main - path: forge - destination: - server: https://kubernetes.default.svc - namespace: forge - # NO syncPolicy.automated — manual sync required. - # Forgejo is what CI uses to push commits; auto-sync would let a bad CI commit - # break the very system CI depends on. Approve syncs manually in the Argo CD UI. diff --git a/k8s/security/ci-cd/example-workflows/ci.yml b/k8s/security/ci-cd/example-workflows/ci.yml deleted file mode 100644 index b40feeb..0000000 --- a/k8s/security/ci-cd/example-workflows/ci.yml +++ /dev/null @@ -1,81 +0,0 @@ -# .forgejo/workflows/ci.yml -# -# Copy to YOUR APPLICATION REPO at .forgejo/workflows/ci.yml -# (not this infra repo — this is a template). -# -# What this does (on every push to main): -# 1. Run tests — fail here and nothing ships. -# 2. Build a Docker image and push it to the Forgejo built-in OCI registry. -# 3. Clone rock/deploy, bump the image tag in api/deployment.yaml, push the commit. -# 4. Argo CD sees the commit within 3 minutes and rolls out the new version. -# -# Required repo secrets (Forgejo UI → repo → Settings → Actions → Secrets): -# REGISTRY_TOKEN — ci-bot's package:write Forgejo API token -# DEPLOY_TOKEN — ci-bot's repo:write Forgejo API token (scoped to rock/deploy only) -# -# Prerequisites: -# - ci-bot user created in Forgejo (see IAM section §11.5 of talos_version_control.html) -# - ci-bot added as collaborator on rock/deploy with Write access -# - Runner (cicd ns) is online and registered (Block 3 of build runbook) -# - Talos nodes trust the homelab CA (Block 2 of build runbook) - -on: - push: - branches: [main] - -jobs: - build-push-deploy: - runs-on: docker - - steps: - - uses: actions/checkout@v4 - - # ── 1. Tests ──────────────────────────────────────────────────────────── - - name: test - run: make test # replace with your test command; failure stops the pipeline - - # ── 2. Build + push OCI image ─────────────────────────────────────────── - - name: build and push image - env: - REGISTRY: forgejo.forge.riotpiao.com - OWNER: rock - run: | - REPO_NAME=${{ github.event.repository.name }} - TAG=$(git rev-parse --short HEAD) - IMAGE="${REGISTRY}/${OWNER}/${REPO_NAME}:${TAG}" - - echo "${{ secrets.REGISTRY_TOKEN }}" \ - | docker login "${REGISTRY}" -u "${OWNER}" --password-stdin - - docker build -t "${IMAGE}" . - docker push "${IMAGE}" - - # Pass values to subsequent steps - echo "TAG=${TAG}" >> "$GITHUB_ENV" - echo "IMAGE=${IMAGE}" >> "$GITHUB_ENV" - echo "REPO_NAME=${REPO_NAME}" >> "$GITHUB_ENV" - - # ── 3. Bump image tag in the deploy repo ──────────────────────────────── - # This is the ONLY write operation CI has on the cluster side. - # Argo CD notices the commit and rolls out the new image. - - name: bump deploy repo - env: - REGISTRY: forgejo.forge.riotpiao.com - OWNER: rock - run: | - git clone \ - "https://ci-bot:${{ secrets.DEPLOY_TOKEN }}@${REGISTRY}/${OWNER}/deploy.git" \ - /tmp/deploy - - TARGET_FILE="/tmp/deploy/${REPO_NAME}/deployment.yaml" - - sed -i \ - "s|${REGISTRY}/${OWNER}/${REPO_NAME}:.*|${IMAGE}|" \ - "${TARGET_FILE}" - - git -C /tmp/deploy \ - -c user.name="ci-bot" \ - -c user.email="ci-bot@forgejo.forge.riotpiao.com" \ - commit -am "${REPO_NAME}: deploy ${TAG}" - - git -C /tmp/deploy push diff --git a/k8s/security/ci-cd/forgejo-oauth-setup-job.yaml b/k8s/security/ci-cd/forgejo-oauth-setup-job.yaml deleted file mode 100644 index 27acf2a..0000000 --- a/k8s/security/ci-cd/forgejo-oauth-setup-job.yaml +++ /dev/null @@ -1,111 +0,0 @@ -# PostSync hook to register Authentik as OAuth login source in Forgejo -apiVersion: batch/v1 -kind: Job -metadata: - name: forgejo-oauth-setup - namespace: cicd - annotations: - argocd.argoproj.io/hook: PostSync - argocd.argoproj.io/hook-delete-policy: BeforeHookCreation -spec: - backoffLimit: 3 - template: - spec: - restartPolicy: Never - serviceAccountName: forgejo-oauth-setup - containers: - - name: setup - image: python:3.12-alpine - command: - - /bin/sh - - -c - - | - set -e - - # Download kubectl - echo "Installing kubectl..." - wget -q -O /tmp/kubectl https://dl.k8s.io/release/v1.28.0/bin/linux/amd64/kubectl - chmod +x /tmp/kubectl - export PATH=/tmp:$PATH - - echo "Waiting for Forgejo to be ready..." - for i in {1..30}; do - if wget -q -O- http://forgejo-gitea-http.cicd.svc:3000/api/healthz 2>/dev/null; then - echo "Forgejo is ready" - break - fi - echo " Waiting... ($i/30)" - sleep 5 - done - - # Get admin credentials - ADMIN_USER=$(kubectl -n cicd get secret forgejo-admin-secret -o jsonpath='{.data.username}' 2>/dev/null | base64 -d || echo "rock") - ADMIN_PASS=$(kubectl -n cicd get secret forgejo-admin-secret -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") - - if [ -z "$ADMIN_PASS" ]; then - echo "ERROR: No admin password found. Cannot configure OAuth." - exit 1 - fi - - # Get OAuth credentials - CLIENT_SECRET=$(kubectl -n cicd get secret forgejo-oidc -o jsonpath='{.data.CLIENT_SECRET}' | base64 -d) - - echo "Checking if OAuth source already exists..." - SOURCES=$(wget -q -O- \ - --header="Content-Type: application/json" \ - --user="$ADMIN_USER:$ADMIN_PASS" \ - http://forgejo-gitea-http.cicd.svc:3000/api/v1/admin/auth) - - if echo "$SOURCES" | grep -q "authentik"; then - echo "OAuth source 'authentik' already exists" - exit 0 - fi - - echo "Creating OAuth authentication source..." - wget -q -O- \ - --header="Content-Type: application/json" \ - --user="$ADMIN_USER:$ADMIN_PASS" \ - --post-data='{ - "type": "oauth2", - "name": "authentik", - "is_active": true, - "oauth2_provider": "openidConnect", - "client_id": "forgejo", - "client_secret": "'"$CLIENT_SECRET"'", - "openid_connect_auto_discovery_url": "https://authentik.riotpiao.com/application/o/forgejo/.well-known/openid-configuration", - "scopes": ["openid", "profile", "email", "groups"] - }' \ - http://forgejo-gitea-http.cicd.svc:3000/api/v1/admin/auth - - echo - echo "OAuth source configured successfully" ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: forgejo-oauth-setup - namespace: cicd ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: forgejo-oauth-setup - namespace: cicd -rules: -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: forgejo-oauth-setup - namespace: cicd -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: forgejo-oauth-setup -subjects: -- kind: ServiceAccount - name: forgejo-oauth-setup - namespace: cicd diff --git a/k8s/security/ci-cd/forgejo-redis.yaml b/k8s/security/ci-cd/forgejo-redis.yaml deleted file mode 100644 index 8ca2342..0000000 --- a/k8s/security/ci-cd/forgejo-redis.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: forgejo-redis - namespace: cicd -spec: - replicas: 1 - selector: - matchLabels: - app: forgejo-redis - template: - metadata: - labels: - app: forgejo-redis - spec: - containers: - - name: redis - image: redis:7-alpine - ports: - - containerPort: 6379 - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 128Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: forgejo-redis - namespace: cicd -spec: - selector: - app: forgejo-redis - ports: - - port: 6379 - targetPort: 6379 diff --git a/k8s/security/ci-cd/forgejo-secrets.enc.yaml b/k8s/security/ci-cd/forgejo-secrets.enc.yaml deleted file mode 100644 index a567b83..0000000 --- a/k8s/security/ci-cd/forgejo-secrets.enc.yaml +++ /dev/null @@ -1,21 +0,0 @@ -gitea: - admin: - password: ENC[AES256_GCM,data:8ZURcU/Zsq9visv81heAsrFN+y1mRCkvpOYgW4jiUO0=,iv:D0vr89m2wby10ogjs0Io6vBfdxW1OP2qJHgJObD4u9g=,tag:8okzx98V3zd8zHB6rK7Tkg==,type:str] - config: - oauth2: - client_secret: ENC[AES256_GCM,data:YbLBR21+4ZtsLwHhNQYj/gUIv5hNvabrsckmDKyYuOCgpR4LmUVdyWxFF2QycoofxHc7dIjZ9a1VMCN4umzeMQ==,iv:VrXWlcC4j5xOF+9cPVsphARdal9guE3NMtxM16AKZa0=,tag:1ch/k8pFJJ/LW/hJR+WNkA==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAya3ZaRmlhVlV5cEJnSDRJ - cjZwN25xRUhsVkdlT0R4cy9pcmdnUVJLczJNCm0yY0w1c1Bhd2ZaRjhoQVJUOUox - UHRIUjBKalM3S3N4b29xUDVnS1NVbDgKLS0tIEN3NUZYdFExdm0vU3dMenZLTlFD - UWZlZWN6TXhvazhQN0c1Q1lFV1VHMkEKknyf4XuNt6FXZFFlKR0tGvd1qA83TTY+ - olMucdjBGEFO4bylbuOO1Z8l7xyfPiFPNpadLtNb9vbxauXXn/ISfA== - -----END AGE ENCRYPTED FILE----- - recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh - lastmodified: "2026-07-15T23:02:08Z" - mac: ENC[AES256_GCM,data:LGRfU9VM87dbY5FCif2UWHxk0AuUsOEU2+t4+8wMgGCFzUCxPqnNd2vt5SF9SGoegfIGAZJw0uGQk89AfH+RNn8JTkU6a2hu4B+Zp7ebdCwDh8KSgUftYLLRu4BWf8Of+hpFfWVbZx4zw4bXGERt9nGfYyfTJM6RV+kcEgth1co=,iv:k9y3vOJsWLbqWXh8cbQFxD414dOMU4xKisgx4Xb1yb0=,tag:dN++eeYLfBReHJ2Mz9PRwQ==,type:str] - unencrypted_suffix: _unencrypted - version: 3.13.2 diff --git a/k8s/security/ci-cd/forgejo-values.yaml b/k8s/security/ci-cd/forgejo-values.yaml deleted file mode 100644 index ddaffde..0000000 --- a/k8s/security/ci-cd/forgejo-values.yaml +++ /dev/null @@ -1,189 +0,0 @@ -# k8s/security/ci-cd/forgejo-values.yaml -# Forgejo deployed via the gitea-charts/gitea Helm chart with image override, -# as an ArgoCD Application with an inline Helm source (k8s/argocd/apps/forgejo.yaml). -# Admin password set via --set at install time (FORGEJO_ADMIN_PASSWORD in .env). -# DB: CNPG (k8s/data/ddb-cluster.yaml + forgejo-database.yaml), not SQLite. -# Cache/session/queue: dedicated Redis (forgejo-redis.yaml), not in-memory — -# required once replicaCount > 1, since in-memory state isn't shared across pods. -# -# Chart docs: https://gitea.com/gitea/helm-chart - -# ── Image (Forgejo replaces Gitea — drop-in compatible) ────────────────────── -image: - repository: codeberg.org/forgejo/forgejo - tag: "13" # pin exact release — check codeberg.org/forgejo/forgejo/releases - pullPolicy: IfNotPresent - -# ── Bootstrap admin (provisioned by a post-install Job inside the chart) ────── -gitea: - admin: - username: rock - email: locartrock@gmail.com - # password: injected via helmfile --set (FORGEJO_ADMIN_PASSWORD from .env) - - config: - server: - PROTOCOL: http # nginx ingress handles TLS; pod serves plain HTTP - DOMAIN: forgejo.riotpiao.com - ROOT_URL: https://forgejo.riotpiao.com/ - HTTP_PORT: 3000 - START_SSH_SERVER: true - SSH_DOMAIN: forgejo.riotpiao.com - SSH_PORT: 2222 - SSH_LISTEN_PORT: 2222 - database: - DB_TYPE: postgres - HOST: ddb-cluster-rw.ddb.svc:5432 - NAME: forgejo - USER: app - repository: - ROOT: /data/git - ENABLE_PUSH_CREATE_USER: true - ENABLE_PUSH_CREATE_ORG: true - actions: - ENABLED: true - packages: - ENABLED: true # built-in OCI registry - metrics: - ENABLED: true # Prometheus at /metrics - service: - DISABLE_REGISTRATION: true # no self-signup; Authentik OAuth2 auto-creates accounts - oauth2: - ENABLED: true - PROVIDER: openidconnect - OPENID_CONNECT_DISCOVERY_URL: https://authentik.riotpiao.com/application/o/forgejo/.well-known/openid-configuration - CLIENT_ID: forgejo - AUTO_DISCOVER_URL: https://authentik.riotpiao.com/application/o/forgejo/.well-known/openid-configuration - # CLIENT_SECRET was missing entirely before - oauth2 login could never - # have worked. Value comes from the forgejo-oidc Secret (created by the - # authentik-provision PostSync hook, see k8s/security/iam/authentik- - # provision-job.yaml) via extraEnv below, since this oauth2: map is - # rendered directly into plain env vars and can't reference a Secret - # inline the way envFromSecret/extraEnv can. - cache: - ADAPTER: redis - HOST: redis://forgejo-redis.cicd.svc:6379/0 - session: - PROVIDER: redis - PROVIDER_CONFIG: redis://forgejo-redis.cicd.svc:6379/1 - queue: - TYPE: redis - CONN_STR: redis://forgejo-redis.cicd.svc:6379/2 - - metrics: - enabled: true - serviceMonitor: - enabled: true # kube-prometheus-stack discovers ServiceMonitors cluster-wide - -# ── Persistence (local/emptyDir — single pod only; no replicas) ──────── -# Longhorn RWX volume was corrupted. Rebuild with local storage. -# Once stable, can re-enable RWX and scale to replicas. -persistence: - enabled: true - storageClass: "" # empty string = local/emptyDir, no storage backend - size: 20Gi - accessModes: - - ReadWriteOnce - -# ── Replicas ────────────────────────────────────────────────────────────────── -# Temporary: single pod with local storage while volume recovers. -# Once stable, scale back to 2+ with RWX Longhorn storage. -replicaCount: 1 - -# ── Deployment strategy ──────────────────────────────────────────────────────── -deployment: - strategy: - type: RollingUpdate - env: - - name: SSL_CERT_DIR - value: /homelab-ca - - name: GITEA__database__PASSWD - valueFrom: - secretKeyRef: - name: ddb-cluster-app - key: password - -# ── Cert / CA auto-reload ───────────────────────────────────────────────────── -# nginx serves the wildcard-tls cert — Forgejo itself never reads a TLS secret. -# The only reload trigger is homelab-ca: if the root CA rotates, the mounted -# ConfigMap changes and Forgejo must restart to pick up the new CA bundle for OIDC. -podAnnotations: - configmap.reloader.stakater.com/reload: "homelab-ca" - -# ── Services (Cilium LB-IPAM pins both to 192.168.1.165) ───────────────────── -service: - http: - type: LoadBalancer - port: 3000 - targetPort: 3000 - annotations: - io.cilium/lb-ipam-ips: "192.168.1.165" - io.cilium/lb-ipam-sharing-key: "forgejo" - ssh: - type: LoadBalancer - port: 2222 - targetPort: 2222 - annotations: - io.cilium/lb-ipam-ips: "192.168.1.165" - io.cilium/lb-ipam-sharing-key: "forgejo" - -# ── Resources ───────────────────────────────────────────────────────────────── -resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: "1" - memory: 1Gi - -# ── Node resilience ─────────────────────────────────────────────────────────── -tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - -# ── CA trust ───────────────────────────────────────────────────────────────── -# Go reads SSL_CERT_DIR as an additional cert directory ON TOP OF the default -# cert files (ca-certificates.crt stays intact — no init container needed). -# Setting SSL_CERT_DIR=/homelab-ca makes Go also read homelab-ca.crt from there, -# trusting both the standard Mozilla bundle and our homelab CA. -# Required for OIDC: Forgejo fetches Authentik's discovery endpoint which -# presents a cert signed by homelab-ca. -# CLIENT_SECRET for the oauth2 block above - gitea's own config map (oauth2:) -# can only hold plain values, so the Secret-backed one is injected as -# GITEA__oauth2__CLIENT_SECRET, following the chart's standard -# GITEA__
__ env-var-to-ini-config convention. -extraEnv: - - name: GITEA__oauth2__CLIENT_SECRET - valueFrom: - secretKeyRef: - name: forgejo-oidc - key: CLIENT_SECRET - -extraVolumes: - - name: homelab-ca - configMap: - name: homelab-ca - -extraVolumeMounts: - - name: homelab-ca - mountPath: /homelab-ca - readOnly: true - -# ── Ingress: disabled — rule lives in k8s/ingress/ingress.yaml ─────────────── -ingress: - enabled: false - -# ── Bundled databases + cache: all disabled — SQLite + memory is the chosen backend ── -postgresql: - enabled: false -postgresql-ha: - enabled: false -mysql: - enabled: false -redis-cluster: - enabled: false # 6-node cluster is overkill for single-replica SQLite Forgejo - -# ── Act runner subchart: disabled — managed by the forgejo-runner helmfile release -act_runner: - enabled: false diff --git a/k8s/security/ci-cd/kustomization.yaml b/k8s/security/ci-cd/kustomization.yaml deleted file mode 100644 index e8ed7b0..0000000 --- a/k8s/security/ci-cd/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: ci-cd -resources: - - forgejo-redis.yaml -# Helm charts (forgejo, forgejo-runner) deployed via standalone ArgoCD -# Applications with Helm sources (k8s/argocd/apps/forgejo.yaml) — not through -# this kustomization, since layer-4-security syncs from Forgejo's own repo -# (chicken-egg: Forgejo can't be the source of truth for deploying itself). diff --git a/k8s/security/ci-cd/runner-token-secrets.enc.yaml b/k8s/security/ci-cd/runner-token-secrets.enc.yaml deleted file mode 100644 index 085374f..0000000 --- a/k8s/security/ci-cd/runner-token-secrets.enc.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: ENC[AES256_GCM,data:10E=,iv:d9f42/CflQcsg9slEoL5WMpmA+suJWT3Tu0KBnwWj2s=,tag:iGN51CHt8MkuNPW7cQZ4ng==,type:str] -kind: ENC[AES256_GCM,data:iKd9XU1h,iv:VW9s86R+IhrmNK7QprHEI/ATUedVtNq8WokmvKk2pc8=,tag:BeZtca7GrakMQl0ukPA1Cw==,type:str] -metadata: - name: ENC[AES256_GCM,data:52roCFUzAcgFp32i,iv:RcXd+WLfOQVrMp4ELnb4v4ZxfUOmht5GNslnw85rveM=,tag:N/7NnhksSA+sxapqrADcVw==,type:str] - namespace: ENC[AES256_GCM,data:6PJz/A==,iv:xubNR2zOYM8tNw6MfpQlgSCdXij1Wl2NQg2GB5sqspU=,tag:cGEtLUxBlIoz3edxru2iEQ==,type:str] -type: ENC[AES256_GCM,data:sTmLx272,iv:vJyc7Hp7Zg41BzYueL+QfNo6at0m8QPoMMMwJf+D7AU=,tag:2TjuW+l6BAZISN/B102glA==,type:str] -stringData: - token: ENC[AES256_GCM,data:WMTIJva3oyj67TTPuAByOWxcIrxRP1SSGTAGC77p5pUHR3JTxORuzA==,iv:9ymwOyW+z1rskwoCqRG6mvkEX+EwcUEdqqWJ7XrSg38=,tag:7CwsSgpvooufdC46xCq4cQ==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLZDd5Q3dkT3hrT3B2QXpr - ZWRaUmhMY244RXFsUEtLZ0dYU2hkYU9sTVRzCjJZQnlzWi9udG9GL3F4U2F2WEFh - L0FMWFQ4anFmOUFHZ1Jib2tmYzhodGMKLS0tIHZONTQ0R1NOYjBSWGV0S3ZpMUFL - WmtNdEdFd1B5WkNEME9rNk02VHJiMzAKjhs40RQ8S0ecP4yVqgZA7i1OP733vIwE - QwbR75V+OQSN+lo6ymPcf0OeV6mi4E2n/sOZl7XGMYCa/XjONh6mzQ== - -----END AGE ENCRYPTED FILE----- - recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh - lastmodified: "2026-07-21T16:38:15Z" - mac: ENC[AES256_GCM,data:MMdgv5MeDSgtXpe1RApsPnHmEKPSobid1IwoDhZplVibybLyD+zN7j1L/cDDHKtwfrOVqKWkI5OnQ4FAoZ/6Re8vfX3k4Y9FqJVxf9qbLHbxKD/YSUQmasPPnGY+MxIKBB5/nrUpMugFCZwLht8z6C7HiPwNNBTloPUA/u9cGXU=,iv:/NV+QhIl4lv7PFr3EcNlvBMew9AGkYn05JzdnSZ8h6g=,tag:i5918QAWTBbmrFXSskvUyA==,type:str] - unencrypted_suffix: _unencrypted - version: 3.13.2 diff --git a/k8s/security/ci-cd/sops-cmp-plugin.yaml b/k8s/security/ci-cd/sops-cmp-plugin.yaml deleted file mode 100644 index 993bf7d..0000000 --- a/k8s/security/ci-cd/sops-cmp-plugin.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# ConfigMap holding the SOPS CMP plugin spec + generate script. Mounted into the -# repo-server sidecar at /home/argocd/cmp-server/config/plugin.yaml (the path the -# argocd-cmp-server binary reads) and /home/argocd/plugins/generate.sh. -# -# The plugin decrypts every k8s/**/*secrets*.enc.yaml Helm-values fragment and -# emits correctly-keyed Kubernetes Secrets to stdout — no helm template inside -# the plugin. Applied to the argocd namespace (bootstrap resource). -apiVersion: v1 -kind: ConfigMap -metadata: - name: sops-cmp-plugin - namespace: argocd -data: - plugin.yaml: | - apiVersion: argoproj.io/v1alpha1 - kind: ConfigManagementPlugin - metadata: - name: sops-secrets - spec: - version: v1.0 - generate: - command: [sh, -c] - args: - - /home/argocd/plugins/generate.sh - generate.sh: | - #!/bin/sh - set -eu - export SOPS_AGE_KEY_FILE=/sops-age/keys.txt - # CMP runs with cwd = the app source path; sops-secrets app points at repo - # root, so enc files resolve from the current directory. - REPO_ROOT="$(pwd)" - - emit_secret() { - # $1 ns $2 name then key=jqpath pairs read from decrypted $ENC - ns="$1"; name="$2"; shift 2 - printf 'apiVersion: v1\nkind: Secret\nmetadata:\n name: %s\n namespace: %s\ntype: Opaque\ndata:\n' "$name" "$ns" - for kv in "$@"; do - k="${kv%%=*}"; path="${kv#*=}" - val="$(echo "$DEC" | yq -r "$path")" - printf ' %s: %s\n' "$k" "$(printf '%s' "$val" | base64 -w0)" - done - printf -- '---\n' - } - - # ── authentik (iam) ──────────────────────────────────────────────── - # Separate secret (authentik-secrets) merged via envFrom AFTER the chart's - # own `authentik` config secret — avoids two ArgoCD apps owning `authentik`. - DEC="$(sops -d "$REPO_ROOT/k8s/security/iam/authentik-secrets.enc.yaml")" - emit_secret iam authentik-secrets \ - AUTHENTIK_SECRET_KEY=.authentik.secret_key \ - AUTHENTIK_BOOTSTRAP_PASSWORD=.authentik.bootstrap_password \ - AUTHENTIK_BOOTSTRAP_TOKEN=.authentik.bootstrap_token \ - AUTHENTIK_POSTGRESQL__PASSWORD=.authentik.postgresql_password - - # ── loki S3 (logging) ────────────────────────────────────────────── - DEC="$(sops -d "$REPO_ROOT/k8s/platform/logging/loki-secrets.enc.yaml")" - emit_secret logging loki-s3-creds \ - access_key_id=.loki.storage.s3.accessKeyId \ - secret_access_key=.loki.storage.s3.secretAccessKey - - # ── grafana (logging) ────────────────────────────────────────────── - DEC="$(sops -d "$REPO_ROOT/k8s/platform/logging/grafana-secrets.enc.yaml")" - # grafana chart's admin.existingSecret needs BOTH admin-user and admin-password. - printf 'apiVersion: v1\nkind: Secret\nmetadata:\n name: grafana-admin\n namespace: logging\ntype: Opaque\ndata:\n admin-user: %s\n admin-password: %s\n---\n' \ - "$(printf 'admin' | base64 -w0)" \ - "$(echo "$DEC" | yq -r '.adminPassword' | base64 -w0)" - emit_secret logging grafana-oidc \ - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=.env.GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET - - # ── vault S3 backend creds (iam) ─────────────────────────────────── - # Vault's s3 storage stanza reads AWS_ACCESS_KEY_ID/SECRET from vault-minio-creds - # (keys access_key/secret_key). MinIO root creds live in minio-creds' config.env - # shell exports — parse them out. - MINIO_ENV="$(sops -d "$REPO_ROOT/k8s/infrastructure/minio/minio-secrets.enc.yaml" \ - | yq -r 'select(.metadata.name=="minio-creds") | .stringData."config.env"')" - MINIO_AK="$(printf '%s' "$MINIO_ENV" | sed -n 's/.*MINIO_ROOT_USER="\([^"]*\)".*/\1/p')" - MINIO_SK="$(printf '%s' "$MINIO_ENV" | sed -n 's/.*MINIO_ROOT_PASSWORD="\([^"]*\)".*/\1/p')" - printf 'apiVersion: v1\nkind: Secret\nmetadata:\n name: vault-minio-creds\n namespace: iam\ntype: Opaque\ndata:\n access_key: %s\n secret_key: %s\n---\n' \ - "$(printf '%s' "$MINIO_AK" | base64 -w0)" \ - "$(printf '%s' "$MINIO_SK" | base64 -w0)" diff --git a/k8s/security/ci-cd/talos-cli.sh b/k8s/security/ci-cd/talos-cli.sh deleted file mode 100755 index 351973b..0000000 --- a/k8s/security/ci-cd/talos-cli.sh +++ /dev/null @@ -1,49 +0,0 @@ -# 0. Source your env (Authentik bootstrap token + Vault addr) -set -a && source ~/.authentik/.env && set +a -VAULT_ADDR="http://vault.riotpiao.com" -AUTHENTIK_URL="http://authentik.riotpiao.com" - -# 1. Fetch the real client_id + client_secret from Authentik -PROVIDER_PK=$(curl -s \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_URL}/api/v3/providers/oauth2/?name=talos-cli-shell" \ - | python3 -c "import json,sys; r=json.load(sys.stdin)['results']; print(r[0]['pk'] if r else 'NOT_FOUND')") - -echo "Provider PK: ${PROVIDER_PK}" - -CLIENT_ID=$(curl -s \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_URL}/api/v3/providers/oauth2/${PROVIDER_PK}/" \ - | python3 -c "import json,sys; p=json.load(sys.stdin); print(p['client_id'])") - -CLIENT_SECRET=$(curl -s \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_URL}/api/v3/providers/oauth2/${PROVIDER_PK}/" \ - | python3 -c "import json,sys; p=json.load(sys.stdin); print(p['client_secret'])") - -echo "client_id: ${CLIENT_ID}" - -# 2. Get a JWT from Authentik (client credentials flow) -# JWT=$(curl -s -X POST \ -# "${AUTHENTIK_URL}/application/o/talos-cli-shell/token/" \ -# -d "grant_type=client_credentials&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&scope=openid" \ -# | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('access_token', d))") -# echo "JWT: ${JWT}..." - -curl -v -s -X POST \ - "${AUTHENTIK_URL}/application/o/homelab-mac-cli/token/" \ - -d "grant_type=client_credentials&client_id=${CLIENT_ID}&client_secret=${CLIENT_SECRET}&scope=openid" - -# # 3. Exchange JWT for a Vault token -# VAULT_TOKEN=$(curl -s -X POST \ -# "${VAULT_ADDR}/v1/auth/jwt/login" \ -# -H "Content-Type: application/json" \ -# -d "{\"jwt\": \"${JWT}\", \"role\": \"shell\"}" \ -# | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('auth',{}).get('client_token', d))") - -# echo "Vault token: ${VAULT_TOKEN:0:20}..." - -# # 4. Verify the token works -# curl -s -H "X-Vault-Token: ${VAULT_TOKEN}" \ -# "${VAULT_ADDR}/v1/auth/token/lookup-self" \ -# | python3 -c "import json,sys; d=json.load(sys.stdin); print('policies:', d['data']['policies'])" diff --git a/k8s/security/iam/.env.example b/k8s/security/iam/.env.example deleted file mode 100644 index ef4d79a..0000000 --- a/k8s/security/iam/.env.example +++ /dev/null @@ -1,27 +0,0 @@ -# talos-iam/.env.example -# Copy to talos-iam/.env and fill in. The real .env is gitignored — never commit it. -# These are ADMIN credentials for the homelab identity provider; use strong values. - -# Authentik's signing/encryption key. SET ONCE — rotating it invalidates all -# existing sessions, tokens, and encrypted fields. Generate: -# openssl rand -base64 60 | tr -d '\n' -AUTHENTIK_SECRET_KEY= - -# Initial password for the built-in admin user 'akadmin'. Change after first login. -# openssl rand -base64 24 -AUTHENTIK_BOOTSTRAP_PASSWORD= - -# Initial API token for 'akadmin' (used for automation / blueprints). -# openssl rand -hex 32 -AUTHENTIK_BOOTSTRAP_TOKEN= - -# Password for the bundled PostgreSQL 'authentik' user (system of record). -# openssl rand -base64 24 -PG_PASSWORD= - -# OAuth2 client secrets for downstream OIDC integrations. -# These are registered with Authentik and injected as K8s secrets -# into the logging and storage namespaces by this bootstrap script. -# openssl rand -hex 32 -GRAFANA_OIDC_CLIENT_SECRET= -MINIO_OIDC_CLIENT_SECRET= diff --git a/k8s/security/iam/OAUTH_APP_SETUP.md b/k8s/security/iam/OAUTH_APP_SETUP.md deleted file mode 100644 index d86700c..0000000 --- a/k8s/security/iam/OAUTH_APP_SETUP.md +++ /dev/null @@ -1,479 +0,0 @@ -# OAuth App Setup & Management Guide - -This guide provides a standardized workflow for registering new services with the homelab OIDC provider (Authentik) and wiring them for Vault JWT authentication. - -> **TL;DR:** Run `register_oauth_app.py` with your app's details; it handles Authentik + Vault + K8s secrets automatically. - ---- - -## Architecture Overview - -The homelab uses a three-tier authentication stack: - -``` -User / Service - ↓ -Authentik (OIDC IdP) - ├─ Grafana - ├─ MinIO - ├─ Forgejo - ├─ Argo CD - ├─ Your New Service - └─ ... - ↓ (device code / authorization code flow) -Vault (KV + JWT auth) - ├─ JWT role mapping (sub/groups → policies) - ├─ OIDC browser login (vault-browser provider) - └─ KV secrets (secret/cluster/*, secret/mcp/*) -``` - -**Key concepts:** -- **Authentik** is the OIDC identity provider — it issues tokens and manages users/groups. -- **Vault** validates Authentik's JWT tokens and maps them to policies & secret access. -- **Kubernetes secrets** store OAuth credentials that services read at startup (ConfigMap-based). -- **Groups** in Authentik control access policies and Vault role assignment. - ---- - -## Prerequisites - -Before registering a new OAuth service, ensure: - -1. **Authentik is running** (in `iam` namespace) - ```bash - kubectl get pods -n iam | grep authentik-server - ``` - -2. **Vault is running** (in `storage` namespace) - ```bash - kubectl get pods -n storage | grep vault-0 - ``` - -3. **SSH/API access to Authentik** — port-forward available - ```bash - kubectl port-forward -n iam svc/authentik-server 7000:80 - ``` - -4. **Vault bootstrap is complete** (run `setup_vault.sh` if not already done) - -5. **Required environment variables** — populate these in `~/.authentik/.env`: - ```bash - # Token for Authentik API (from bootstrap) - AUTHENTIK_BOOTSTRAP_TOKEN= - - # Service-specific client secret (generate via openssl rand -hex 32) - # Example: MY_SERVICE_OIDC_CLIENT_SECRET= - ``` - ---- - -## One-Line Setup (Recommended) - -For most services, use the automated registration script: - -```bash -cd /Users/rockliang/workplace/homelab - -# Register an OAuth app with automatic Vault JWT wiring -python3 k8s/talos-iam/register_oauth_app.py \ - --service-name my-app \ - --namespace my-namespace \ - --redirect-uri "https://my-app.riotpiao.com/oauth2/callback" \ - --service-name-in-vault "MY_SERVICE" \ - --vault-jwt-policy "shell-secrets" -``` - -The script will: -1. ✅ Create an OAuth2 provider in Authentik (from credentials in Vault) -2. ✅ Create an Application in Authentik -3. ✅ Create/bind groups for RBAC (optional) -4. ✅ Create a Kubernetes secret with the client credentials -5. ✅ Create a Vault JWT role for the service's JWT auth method -6. ✅ Write policies for the service's scope in Vault - ---- - -## Manual Setup (Step-by-Step) - -If you prefer manual control, or if the script doesn't fit your use case: - -### Step 1: Generate Credentials - -```bash -# Generate a new client secret -CLIENT_SECRET=$(openssl rand -hex 32) -echo "CLIENT_SECRET=$CLIENT_SECRET" - -# Store in Vault (required for service initialization) -# Convention: YOURSERVICE_OIDC_CLIENT_SECRET -talos put cluster/YOURSERVICE_OIDC_CLIENT_SECRET YOURSERVICE_OIDC_CLIENT_SECRET="$CLIENT_SECRET" - -# Or set in .env temporarily -export YOURSERVICE_OIDC_CLIENT_SECRET="$CLIENT_SECRET" -``` - -### Step 2: Create Authentik OAuth2 Provider - -Access Authentik UI via port-forward: - -```bash -kubectl port-forward -n iam svc/authentik-server 7000:80 & -# http://localhost:7000/if/admin/ → login with akadmin -``` - -**In Authentik UI:** -1. Navigate to **Applications → Providers → Create → OpenID Connect (OAuth2) Provider** -2. Fill in: - - **Name:** `yourservice` - - **Client ID:** `yourservice` (or custom) - - **Client Secret:** (paste from `$CLIENT_SECRET` above) - - **Redirect URIs:** `https://your-app.riotpiao.com/oauth2/callback` (or your app's callback URL) - - **Sub Mode:** `Hashed User ID` - - **Include claims in ID Token:** ✓ Enabled - - **Grant Types:** `Authorization Code`, `Refresh Token` - - **Signing Key:** (select the homelab-oidc key) - -3. **Save** and note the **provider slug** (usually auto-generated from Name). - -### Step 3: Create Authentik Application - -**In Authentik UI:** -1. Navigate to **Applications → Applications → Create** -2. Fill in: - - **Name:** `Your App Display Name` - - **Slug:** `yourservice` (must match provider slug) - - **Provider:** (select the provider created above) - - **Meta Launch URL:** `https://your-app.riotpiao.com` (optional, for app launcher) - -3. **Save** - -### Step 4: Configure OIDC in Your Service - -Pass the following environment variables to your service: - -```bash -# OIDC endpoint (in-cluster: authentik-server.iam.svc.cluster.local) -OIDC_ISSUER_URL=http://authentik-server.iam.svc.cluster.local/application/o/yourservice/ - -# OAuth2 credentials (from step 1-2) -OIDC_CLIENT_ID=yourservice -OIDC_CLIENT_SECRET=$CLIENT_SECRET - -# Redirect URI (must match what you configured in step 2) -OIDC_REDIRECT_URI=https://your-app.riotpiao.com/oauth2/callback - -# Optionally, token validation endpoint -OIDC_TOKEN_URL=http://authentik-server.iam.svc.cluster.local/application/o/yourservice/token/ -OIDC_USERINFO_URL=http://authentik-server.iam.svc.cluster.local/application/o/yourservice/userinfo/ - -# Optional: JWKS endpoint for offline token validation -OIDC_JWKS_URL=http://authentik-server.iam.svc.cluster.local/application/o/yourservice/jwks/ -``` - -**Store secrets in Vault:** -```bash -# Store the client secret for runtime retrieval -talos put cluster/YOURSERVICE_OIDC_CLIENT_SECRET YOURSERVICE_OIDC_CLIENT_SECRET="$CLIENT_SECRET" - -# Store other config if needed -talos put cluster/yourservice-oidc \ - client_id=yourservice \ - issuer_url="http://authentik-server.iam.svc.cluster.local/application/o/yourservice/" \ - callback_uri="https://your-app.riotpiao.com/oauth2/callback" -``` - -### Step 5: Mount Credentials in Your Service - -**Option A: Kubernetes Secret (ConfigMap-based)** - -Create a Kubernetes secret with the credentials: - -```bash -kubectl create secret generic yourservice-oidc \ - --from-literal=client_id=yourservice \ - --from-literal=client_secret="$CLIENT_SECRET" \ - -n your-namespace -``` - -Reference in your Helm values or Pod spec: - -```yaml -# In Helm values -env: - - name: OIDC_CLIENT_ID - valueFrom: - secretKeyRef: - name: yourservice-oidc - key: client_id - - name: OIDC_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: yourservice-oidc - key: client_secret -``` - -**Option B: Vault KV Secret (Runtime)** - -Store in Vault and retrieve at startup: - -```bash -# Store full config -vault kv put secret/services/yourservice \ - client_id=yourservice \ - client_secret="$CLIENT_SECRET" \ - issuer_url="http://authentik-server.iam.svc.cluster.local/application/o/yourservice/" - -# Service reads at startup: -# curl -H "Authorization: Bearer $VAULT_TOKEN" \ -# http://vault.storage.svc.cluster.local:8200/v1/secret/data/services/yourservice -``` - ---- - -## Vault JWT Authentication (For Services) - -If your service needs to authenticate to Vault directly (beyond just reading secrets), create a JWT role: - -### 1. Create Vault JWT Role - -```bash -# Authenticate to Vault (as cluster-admin or root) -kubectl port-forward -n storage svc/vault 8200:8200 & -export VAULT_ADDR=http://127.0.0.1:8200 -vault login -method=oidc role=homelab - -# Create a service-specific JWT role -vault write auth/jwt/role/yourservice \ - role_type=jwt \ - bound_audiences="vault" \ - user_claim="sub" \ - bound_claims='{"client_id":["yourservice"]}' \ - policies="shell-secrets,service-read-secrets" \ - ttl=4h \ - max_ttl=8h -``` - -### 2. Create Service Policy in Vault - -```bash -# Policy that allows the service to read its own secrets -vault policy write service-read-yourservice - <<'EOF' -path "secret/data/services/yourservice" { - capabilities = ["read"] -} -path "secret/data/cluster/minio" { - capabilities = ["read"] -} -EOF - -# Update the role to use this policy -vault write auth/jwt/role/yourservice \ - policies="service-read-secrets,service-read-yourservice" -``` - -### 3. Service JWT Authentication Flow - -Your service obtains a Vault token from Authentik's JWT: - -```python -# Example service code (Python) -import requests -import json - -# 1. Authenticate to Authentik OIDC (get ID token) -oidc_response = requests.post( - "http://authentik-server.iam.svc.cluster.local/application/o/yourservice/token/", - data={ - "grant_type": "client_credentials", - "client_id": "yourservice", - "client_secret": os.environ["OIDC_CLIENT_SECRET"], - "audience": "vault" # Important: request JWT for Vault - } -) -id_token = oidc_response.json()["access_token"] - -# 2. Authenticate to Vault using the JWT -vault_response = requests.post( - "http://vault.storage.svc.cluster.local:8200/v1/auth/jwt/login", - data=json.dumps({ - "role": "yourservice", - "jwt": id_token - }), - headers={"Content-Type": "application/json"} -) -vault_token = vault_response.json()["auth"]["client_token"] - -# 3. Use Vault token to read secrets -secret_response = requests.get( - "http://vault.storage.svc.cluster.local:8200/v1/secret/data/services/yourservice", - headers={"X-Vault-Token": vault_token} -) -secrets = secret_response.json()["data"]["data"] -``` - ---- - -## RBAC & Groups - -### Adding Group-Based Access Control - -Groups in Authentik map to Vault policies and app bindings: - -```bash -# Create a group for your service's administrators -python3 k8s/talos-iam/provision_oidc.py --add-group "yourservice-admins" - -# Add users to the group (Authentik UI) -# Applications → Groups → yourservice-admins → Users - -# Bind group to app (Authentik UI) -# Applications → Applications → yourservice → Policies → Binding -# Select the group and enable the binding -``` - -### Group-Based Vault Policy Mapping - -Create a Vault policy that checks Authentik group membership: - -```bash -vault policy write service-admin-yourservice - <<'EOF' -# Full access for admins -path "secret/data/services/yourservice/*" { - capabilities = ["read", "create", "update", "delete"] -} -path "secret/data/mcp/*" { - capabilities = ["read"] -} -EOF - -# Create a separate JWT role for admins -vault write auth/jwt/role/yourservice-admin \ - role_type=jwt \ - bound_audiences="vault" \ - user_claim="sub" \ - bound_claims='{"groups":["yourservice-admins"]}' \ - policies="shell-secrets,service-admin-yourservice" \ - ttl=4h -``` - ---- - -## Verification Checklist - -After registration, verify everything works: - -```bash -# 1. Verify Authentik provider exists -curl -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \ - http://localhost:7000/api/v3/providers/oauth2/?name=yourservice - -# 2. Verify Authentik application exists -curl -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \ - http://localhost:7000/api/v3/core/applications/?slug=yourservice - -# 3. Verify K8s secret is mounted -kubectl get secret yourservice-oidc -n your-namespace -o jsonpath='{.data}' | base64 -d - -# 4. Verify Vault JWT role -vault read auth/jwt/role/yourservice - -# 5. Test OAuth login (requires port-forward to your app) -# Open http://your-app.riotpiao.com/login -# Should redirect to Authentik, then back to your app - -# 6. Test JWT auth to Vault (if applicable) -# Service obtains ID token and authenticates to Vault -vault login -method=jwt role=yourservice jwt=$ID_TOKEN -``` - ---- - -## Troubleshooting - -### "JWKS URL not found" error in Vault - -**Cause:** Authentik is not reachable from Vault pod at the configured URL. - -**Fix:** -1. Check Authentik is running: `kubectl get pods -n iam` -2. Verify URL uses in-cluster address: `http://authentik-server.iam.svc.cluster.local` -3. Test DNS from Vault pod: - ```bash - kubectl exec -n storage vault-0 -- nslookup authentik-server.iam.svc.cluster.local - ``` - -### "OAuth callback failed" or redirect loop - -**Cause:** Redirect URI in Authentik does not match what the service sends. - -**Fix:** -1. Check Authentik provider's Redirect URIs: UI → Applications → Providers → Edit -2. Must exactly match the URI your service redirects to (protocol, domain, port, path) -3. Common mistake: `https://` in service but `http://` in Authentik - -### Service can't read K8s secret - -**Cause:** Secret is in wrong namespace or service account lacks permissions. - -**Fix:** -1. Verify secret exists: `kubectl get secret yourservice-oidc -n your-namespace` -2. Check RBAC for service account: - ```bash - kubectl describe sa yourservice -n your-namespace - ``` - -### "client_id mismatch" or "client authentication failed" - -**Cause:** Client secret in K8s secret does not match what's in Authentik. - -**Fix:** -1. Rotate the secret in Authentik (delete and recreate) -2. Update the K8s secret with the new value -3. Restart the service pod(s) - ---- - -## Advanced: Custom Property Mappings - -For services that need custom JWT claims (e.g., MinIO's `policy` claim), use property mappings: - -```bash -# Create a custom scope with expression -curl -X POST http://localhost:7000/api/v3/propertymappings/provider/scope/ \ - -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "name": "yourservice-claims", - "scope_name": "yourservice-custom", - "expression": "return {\"department\": request.user.attributes.get(\"department\", \"unknown\")}" - }' - -# Add to provider's property_mappings: -# Authentik UI → Applications → Providers → Edit yourservice -# Select the custom scope in the Property Mappings list -``` - ---- - -## Integration Examples - -### Integrate with Grafana - -See `k8s/logging/grafana-values.yaml` for an example of configuring generic OAuth in Grafana. - -### Integrate with MinIO - -See `provision_oidc.py` for MinIO's group-based policy claim injection. - -### Integrate with Custom Go App - -See `k8s/talos-iam/go-example-oidc/` for a minimal Authorization Code flow example. - ---- - -## Related Documentation - -- [`README.md`](README.md) — Authentik deployment & key rotation -- [`provision_oidc.py`](provision_oidc.py) — Automated OIDC provisioning for core apps -- [`setup_vault.sh`](setup_vault.sh) — Vault initialization & JWT auth wiring -- [`CLAUDE.md`](../CLAUDE.md) — Homelab architecture & secrets management diff --git a/k8s/security/iam/README.md b/k8s/security/iam/README.md deleted file mode 100644 index 09c76e2..0000000 --- a/k8s/security/iam/README.md +++ /dev/null @@ -1,302 +0,0 @@ -# Talos IAM — Authentik - -Standalone **SSO / Identity Provider** for the homelab. -Authentik gives every homelab app one login (OIDC / OAuth2 / SAML / forward-auth). It is an -**identity provider, not a secret vault** — its system of record is PostgreSQL (users, apps, -tokens, policies) with Redis for cache/queue. There is no MinIO/S3 involvement. - -> Want a Vault-style secret store instead? That's a different tool — OpenBao / HashiCorp Vault / -> Infisical — and a separate setup. This folder is SSO only. - -## Architecture - -``` - ┌─────────────────────────────────────┐ - │ namespace: iam │ - │ │ - Browser / CLI ───────▶│ authentik-server (UI + API :80) │ - │ authentik-worker (tasks / flows) │ - │ postgresql (Longhorn 8Gi) │ - │ redis (ephemeral) │ - └───────────────┬─────────────────────┘ - │ OIDC / OAuth2 - ┌─────────────────┼─────────────────┐ - ▼ ▼ ▼ - ns: logging ns: storage ns: iam - Grafana MinIO (CronJob) - client_id=grafana client_id=minio key-rotation -``` - -**OIDC clients provisioned:** - -| App | Namespace | Client ID | Redirect URI | -|---|---|---|---| -| Grafana | `logging` | `grafana` | `http://localhost:3000/login/generic_oauth` | -| MinIO | `storage` | `minio` | `http://localhost:9001/oauth_callback` | -| Portainer | `portainer` | — | placeholder (CE has no OIDC) | - -**Groups:** - -| Group | Maps to | -|---|---| -| `grafana-admins` | Grafana Admin role | -| `grafana-viewers` | Grafana Viewer role | -| `minio-admins` | MinIO `readwrite` policy | -| `minio-readonly` | MinIO `readonly` policy | - ---- - -## Layout - -``` -talos-iam/ -├── setup_talos_iam.sh # one-shot deploy (namespace → helm → verify → provision) -├── authentik-values.yaml # Helm values (Postgres on Longhorn, Redis ephemeral, tolerations) -├── provision_oidc.py # idempotent OIDC provisioner (providers, groups, K8s secrets) -├── register_oauth_app.py # 🆕 register new services with OAuth + Vault JWT (recommended) -├── example_register_dashboard.py # 🆕 example: register 'dashboard-service' with full RBAC -├── verify_existing_oauth_integrations.sh # 🆕 verify Grafana/MinIO/Forgejo/Argo CD OIDC still work -├── OAUTH_APP_SETUP.md # 🆕 comprehensive guide: manual & automated OAuth setup -├── key_rotate.rs # stdlib-only Rust script — rotates OIDC signing key -├── key-rotation-cronjob.yaml # K8s CronJob running key_rotate.rs quarterly -├── setup_vault.sh # Vault initialization & JWT auth wiring (run after Authentik) -├── go-example-oidc/ # Go Authorization Code flow example against Authentik -│ ├── main.go -│ ├── go.mod -│ └── .env.example -├── .env.example # required secrets — copy to ~/.authentik/.env and fill in -└── README.md # you are here -``` - ---- - -## Quickstart - -```bash -cp k8s/talos-iam/.env.example ~/.authentik/.env -# fill in the secrets (generators are in .env.example) - -bash k8s/talos-iam/setup_talos_iam.sh -``` - -The script creates the `iam` namespace, installs the Authentik Helm chart (server, worker, bundled -PostgreSQL + Redis), waits for rollout, probes the readiness endpoint, then calls -`provision_oidc.py` to wire up Grafana and MinIO as OIDC clients. - ---- - -## Access - -```bash -kubectl port-forward svc/authentik-server -n iam 7000:80 -# Admin UI: http://localhost:7000/if/admin/ -# Login: akadmin / -``` - ---- - -## OIDC Provisioning - -`provision_oidc.py` idempotently creates all Authentik resources from the API — safe to re-run. - -```bash -# port-forward must be active (localhost:7000) -source ~/.authentik/.env -python k8s/talos-iam/provision_oidc.py -``` - -What it provisions: -- RSA-4096 signing certificate `homelab-oidc` -- OAuth2 providers for Grafana and MinIO (with the signing cert attached) -- Property mapping that injects a `policy` JWT claim for MinIO access control -- Groups: `grafana-admins`, `grafana-viewers`, `minio-admins`, `minio-readonly` -- K8s secrets `grafana-oidc` (ns: `logging`) and `minio-oidc` (ns: `storage`) - ---- - -## Key Rotation - -OIDC signing keys should be rotated periodically. The `key_rotate.rs` script generates a new -RSA-4096 cert in Authentik and patches all providers to use it. The old cert stays in the JWKS -endpoint until you delete it — existing tokens remain valid through their TTL (default 5 min). - -**Manual rotation** (port-forward must be active): -```bash -source ~/.authentik/.env -python k8s/talos-iam/provision_oidc.py --rotate -``` - -**Automated rotation** (quarterly CronJob in-cluster): -```bash -# One-time setup -kubectl create configmap key-rotation-script \ - --from-file=rotate_key.rs=k8s/talos-iam/key_rotate.rs \ - -n iam --dry-run=client -o yaml | kubectl apply -f - - -kubectl create secret generic authentik-key-rotation-token \ - --from-literal=AUTHENTIK_BOOTSTRAP_TOKEN="${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -n iam --dry-run=client -o yaml | kubectl apply -f - - -kubectl apply -f k8s/talos-iam/key-rotation-cronjob.yaml - -# Test the job immediately -kubectl create job --from=cronjob/authentik-key-rotation test-rotation -n iam -kubectl logs -n iam -l job-name=test-rotation -f -``` - -**Validate rotation:** -```bash -# Confirm signing_key is set and changed -curl -s -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \ - "http://localhost:7000/api/v3/providers/oauth2/?name=grafana" \ - | python3 -c "import json,sys; p=json.load(sys.stdin)['results'][0]; print('signing_key:', p['signing_key'])" - -# Confirm JWKS shows both old and new key during transition -curl -s http://localhost:7000/application/o/grafana/.well-known/jwks.json \ - | python3 -c "import json,sys; [print('kid:', k['kid']) for k in json.load(sys.stdin)['keys']]" -``` - -> After rotation, nothing in `~/.authentik/.env` changes. Client secrets, the bootstrap token, -> and `AUTHENTIK_SECRET_KEY` are all separate from the OIDC signing keypair. - ---- - -## Go OIDC Example - -A minimal Authorization Code flow demo against Authentik — useful for verifying the IdP -end-to-end or as a starting point for a new OIDC client. - -```bash -cp k8s/talos-iam/go-example-oidc/.env.example ~/.authentik/.env -# add OIDC_CLIENT_ID and OIDC_CLIENT_SECRET for an app you register in Authentik - -cd k8s/talos-iam/go-example-oidc -go mod tidy && go run . -# open http://localhost:8080/login -``` - -The callback prints the verified ID token claims as JSON — `email`, `name`, `sub`, and any -custom claims (e.g. the MinIO `policy` claim). - ---- - -## Node Resilience — Auto-Start on Reboot - -All four components (server, worker, PostgreSQL, Redis) carry the control-plane toleration: - -```yaml -tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule -``` - -- **Any node goes down** → Kubernetes reschedules all Authentik pods onto the surviving node. -- **PostgreSQL PVC** (8Gi Longhorn `ReadWriteOnce`) → Longhorn reattaches automatically (~2 min). All user/app/token data is preserved. -- **Redis** is ephemeral (no PVC) — restarts clean, which is correct (cache/queue only). -- **Full cluster reboot** → cp-1 comes up first; Kubernetes reconciles Deployments; Longhorn reattaches. Zero manual action needed. - -**Startup order after reboot:** -1. `kubelet` starts on both nodes -2. `etcd` + API server on cp-1 -3. Controllers reconcile Authentik Deployments and PostgreSQL StatefulSet -4. PostgreSQL starts (Authentik server/worker wait via init probes) -5. Redis starts -6. Authentik server + worker become ready - ---- - -## Verify - -```bash -kubectl get pods -n iam -# authentik-server, authentik-worker, authentik-postgresql-0, authentik-redis-master-0 → Running - -curl -fsS -o /dev/null -w '%{http_code}\n' http://localhost:7000/-/health/ready/ -# 204 -``` - ---- - -## Registering New Services (OAuth App Workflow) - -The homelab provides **automated OAuth registration** for new services via `register_oauth_app.py`: - -```bash -# Register a new service with OAuth + Vault JWT auth (recommended) -python3 k8s/talos-iam/register_oauth_app.py \ - --service-name my-service \ - --namespace my-ns \ - --redirect-uri "https://my-service.riotpiao.com/oauth2/callback" \ - --service-name-in-vault MY_SERVICE \ - --vault-jwt-policy shell-secrets \ - --add-group my-service-admins -``` - -This automates: -1. ✅ Authentik OAuth2 provider creation (credentials from Vault) -2. ✅ Authentik application binding -3. ✅ Kubernetes secret provisioning (client ID/secret) -4. ✅ Vault JWT role creation (for service → Vault auth) -5. ✅ Group-based RBAC setup (optional) - -**For manual control or step-by-step guidance,** see [`OAUTH_APP_SETUP.md`](OAUTH_APP_SETUP.md) which covers both automated and manual workflows. - -### Example: Register "dashboard-service" - -```bash -# 1. Generate client secret and store in Vault -DASHBOARD_OIDC_CLIENT_SECRET=$(openssl rand -hex 32) -talos put cluster/DASHBOARD_OIDC_CLIENT_SECRET DASHBOARD_OIDC_CLIENT_SECRET="$DASHBOARD_OIDC_CLIENT_SECRET" - -# 2. Register with automation -export DASHBOARD_OIDC_CLIENT_SECRET -python3 k8s/talos-iam/register_oauth_app.py \ - --service-name dashboard-service \ - --namespace apps \ - --redirect-uri "https://dashboard.riotpiao.com/oauth2/callback" \ - --service-name-in-vault DASHBOARD \ - --vault-jwt-policy shell-secrets \ - --add-group dashboard-admins \ - --vault-jwt-bound-claims '{"groups":["dashboard-admins"]}' - -# 3. Service now has: -# - Authentik provider (dashboard-service) -# - K8s secret (dashboard-service-oidc) in 'apps' namespace -# - Vault JWT role (dashboard-service) with group-based access -``` - -### Verification - -After registration, verify the integration: - -```bash -# Check Authentik provider and app -curl -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \ - http://localhost:7000/api/v3/core/applications/?slug=my-service | jq . - -# Check K8s secret -kubectl get secret my-service-oidc -n my-ns -o yaml - -# Check Vault JWT role -vault read auth/jwt/role/my-service - -# Test OAuth login -# Browser: https://my-service.riotpiao.com/login -# Should redirect to Authentik → back to service with session -``` - -**Full verification script:** -```bash -bash k8s/talos-iam/verify_existing_oauth_integrations.sh -``` - ---- - -## Notes - -- **`AUTHENTIK_SECRET_KEY` is set-once.** Rotating it invalidates all sessions, tokens, and encrypted fields in the database. Keep `~/.authentik/.env` safe and backed up. -- **PostgreSQL holds everything that matters** — users, providers, groups, tokens, certificates. It lives on an 8Gi Longhorn PVC. Redis is ephemeral by design. -- **No ingress.** Access is via port-forward, matching the rest of the homelab. -- **OIDC signing keys** (rotated by `provision_oidc.py --rotate`) are separate from all `.env` credentials. Rotation requires no client reconfiguration — Grafana and MinIO pick up the new public key from the JWKS endpoint automatically. -- **New app registration** is automated via `register_oauth_app.py` and fully documented in `OAUTH_APP_SETUP.md`. Both manual and automated workflows are supported. diff --git a/k8s/security/iam/authentik-migrations-job.yaml b/k8s/security/iam/authentik-migrations-job.yaml deleted file mode 100644 index 4b0150d..0000000 --- a/k8s/security/iam/authentik-migrations-job.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: authentik-migrations - namespace: iam -spec: - ttlSecondsAfterFinished: 600 - backoffLimit: 3 - template: - spec: - serviceAccountName: default - restartPolicy: Never - containers: - - name: migrate - image: ghcr.io/goauthentik/server:2026.5.4 - imagePullPolicy: IfNotPresent - envFrom: - - secretRef: - name: authentik - command: - - /bin/sh - - -c - - | - python -m manage migrate --noinput && echo "✓ Migrations complete" diff --git a/k8s/security/iam/example_register_dashboard.py b/k8s/security/iam/example_register_dashboard.py deleted file mode 100755 index 54c13e5..0000000 --- a/k8s/security/iam/example_register_dashboard.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python3 -""" -Example: Register 'dashboard-service' with OAuth + Vault JWT auth. - -This is a complete working example that demonstrates: -1. Creating an Authentik OAuth2 provider and application -2. Creating Kubernetes secrets for OIDC credentials -3. Creating a Vault JWT role for service authentication -4. Group-based RBAC - -To use this example: - - # 1. Ensure prerequisites - kubectl port-forward -n iam svc/authentik-server 7000:80 & - kubectl port-forward -n storage svc/vault 8200:8200 & - - # 2. Generate and store credentials - DASHBOARD_OIDC_CLIENT_SECRET=$(openssl rand -hex 32) - export DASHBOARD_OIDC_CLIENT_SECRET="$DASHBOARD_OIDC_CLIENT_SECRET" - talos put cluster/DASHBOARD_OIDC_CLIENT_SECRET DASHBOARD_OIDC_CLIENT_SECRET="$DASHBOARD_OIDC_CLIENT_SECRET" - - # 3. Run registration - python3 register_oauth_app.py \\ - --service-name dashboard-service \\ - --namespace apps \\ - --redirect-uri "https://dashboard.riotpiao.com/oauth2/callback" \\ - --service-name-in-vault DASHBOARD \\ - --vault-jwt-policy shell-secrets \\ - --add-group dashboard-admins \\ - --vault-jwt-bound-claims '{"groups":["dashboard-admins"]}' - - # 4. Verify registration - curl -s -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \\ - http://localhost:7000/api/v3/core/applications/?slug=dashboard-service | jq . - - # 5. Test OAuth callback (requires app running) - # Browser: http://dashboard.riotpiao.com/login - # Should redirect to Authentik → back to dashboard with session -""" - -import subprocess -import sys -import os - - -def example_dashboard_service(): - """Register the example 'dashboard-service' with full RBAC.""" - - # Check prerequisites - print("Checking prerequisites...") - - required_env = [ - "AUTHENTIK_BOOTSTRAP_TOKEN", - "DASHBOARD_OIDC_CLIENT_SECRET", - ] - - missing = [v for v in required_env if not os.environ.get(v)] - if missing: - print(f"❌ Missing environment variables: {', '.join(missing)}") - print("\nSet them:") - print(" talos get cluster/AUTHENTIK_BOOTSTRAP_TOKEN --key AUTHENTIK_BOOTSTRAP_TOKEN | source") - print(" talos get cluster/DASHBOARD_OIDC_CLIENT_SECRET --key DASHBOARD_OIDC_CLIENT_SECRET | source") - return False - - # Call the registration script - cmd = [ - "python3", - "register_oauth_app.py", - "--service-name", "dashboard-service", - "--namespace", "apps", - "--redirect-uri", "https://dashboard.riotpiao.com/oauth2/callback", - "--service-name-in-vault", "DASHBOARD", - "--vault-jwt-policy", "shell-secrets", - "--add-group", "dashboard-admins", - "--vault-jwt-bound-claims", '{"groups":["dashboard-admins"]}', - ] - - print(f"\nRunning: {' '.join(cmd)}\n") - - result = subprocess.run(cmd, check=False) - return result.returncode == 0 - - -def verify_example(): - """Verify that the dashboard-service was registered correctly.""" - print("\n" + "="*70) - print("Verification Steps") - print("="*70 + "\n") - - print("✅ Check Authentik provider:") - print(" curl -s -H 'Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN' \\") - print(" http://localhost:7000/api/v3/providers/oauth2/?name=dashboard-service \\") - print(" | jq '.results[0] | {name, client_id, redirect_uris}'") - print() - - print("✅ Check Authentik application:") - print(" curl -s -H 'Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN' \\") - print(" http://localhost:7000/api/v3/core/applications/?slug=dashboard-service \\") - print(" | jq '.results[0] | {name, slug, provider}'") - print() - - print("✅ Check Kubernetes secret:") - print(" kubectl get secret dashboard-service-oidc -n apps -o jsonpath='{.data}' \\") - print(" | base64 -d | jq .") - print() - - print("✅ Check Vault JWT role:") - print(" vault read auth/jwt/role/dashboard-service") - print() - - print("✅ Test Vault JWT authentication:") - print(" # Get ID token from Authentik") - print(" ID_TOKEN=$(curl -s -X POST \\") - print(" http://localhost:7000/application/o/dashboard-service/token/ \\") - print(" -d 'grant_type=client_credentials&client_id=dashboard-service&client_secret=...' \\") - print(" | jq -r '.access_token')") - print() - print(" # Authenticate to Vault with JWT") - print(" vault write auth/jwt/login \\") - print(" role=dashboard-service \\") - print(" jwt=\"$ID_TOKEN\"") - print() - - print("✅ Test OAuth redirect (requires app running):") - print(" curl -L https://dashboard.riotpiao.com/login") - print(" # Should redirect to Authentik, then back to dashboard") - print() - - -if __name__ == "__main__": - if not example_dashboard_service(): - sys.exit(1) - - verify_example() diff --git a/k8s/security/iam/go-example-oidc/.env.example b/k8s/security/iam/go-example-oidc/.env.example deleted file mode 100644 index 3ed72f5..0000000 --- a/k8s/security/iam/go-example-oidc/.env.example +++ /dev/null @@ -1,8 +0,0 @@ -# Copy to ~/.authentik/.env and fill in your values. -# Port-forward Authentik first: kubectl port-forward svc/authentik-server -n iam 7000:80 - -AUTHENTIK_BASE_URL=http://localhost:7000 -APP_SLUG=go-example -OIDC_CLIENT_ID=go-example -OIDC_CLIENT_SECRET=your-client-secret-here -REDIRECT_URL=http://localhost:8080/callback diff --git a/k8s/security/iam/go-example-oidc/go.mod b/k8s/security/iam/go-example-oidc/go.mod deleted file mode 100644 index 2c20ecb..0000000 --- a/k8s/security/iam/go-example-oidc/go.mod +++ /dev/null @@ -1,14 +0,0 @@ -module homelab/go-example-oidc - -go 1.22 - -require ( - github.com/coreos/go-oidc/v3 v3.11.0 - github.com/joho/godotenv v1.5.1 - golang.org/x/oauth2 v0.24.0 -) - -require ( - github.com/go-jose/go-jose/v4 v4.0.2 // indirect - golang.org/x/crypto v0.25.0 // indirect -) diff --git a/k8s/security/iam/go-example-oidc/go.sum b/k8s/security/iam/go-example-oidc/go.sum deleted file mode 100644 index d7bbc33..0000000 --- a/k8s/security/iam/go-example-oidc/go.sum +++ /dev/null @@ -1,20 +0,0 @@ -github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI= -github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-jose/go-jose/v4 v4.0.2 h1:R3l3kkBds16bO7ZFAEEcofK0MkrAJt3jlJznWZG0nvk= -github.com/go-jose/go-jose/v4 v4.0.2/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/k8s/security/iam/go-example-oidc/main.go b/k8s/security/iam/go-example-oidc/main.go deleted file mode 100644 index b15d04b..0000000 --- a/k8s/security/iam/go-example-oidc/main.go +++ /dev/null @@ -1,90 +0,0 @@ -// Minimal OIDC Authorization Code flow against Authentik. -// -// Setup: -// 1. In Authentik create a provider + app (slug "go-example", redirect URI http://localhost:8080/callback). -// 2. Fill in ~/.authentik/.env (see .env.example). -// 3. go mod tidy && go run . -// 4. Open http://localhost:8080/login -package main - -import ( - "context" - "encoding/json" - "fmt" - "log" - "net/http" - "os" - "path/filepath" - - "github.com/coreos/go-oidc/v3/oidc" - "github.com/joho/godotenv" - "golang.org/x/oauth2" -) - -// oauthState is a fixed random value for this process — good enough for a local demo. -// In production, generate a per-request random state and store it in a cookie. -var oauthState = "homelab-oidc-example" - -func main() { - // Load ~/.authentik/.env; shell env vars already set take precedence. - home, _ := os.UserHomeDir() - godotenv.Load(filepath.Join(home, ".authentik", ".env")) - - ctx := context.Background() - - // go-oidc discovers the token endpoint, auth endpoint, and JWKS URI automatically - // from Authentik's /.well-known/openid-configuration. - issuer := os.Getenv("AUTHENTIK_BASE_URL") + "/application/o/" + os.Getenv("APP_SLUG") - provider, err := oidc.NewProvider(ctx, issuer) - if err != nil { - log.Fatalf("OIDC discovery failed (%s): %v", issuer, err) - } - - cfg := &oauth2.Config{ - ClientID: os.Getenv("OIDC_CLIENT_ID"), - ClientSecret: os.Getenv("OIDC_CLIENT_SECRET"), - RedirectURL: os.Getenv("REDIRECT_URL"), - Endpoint: provider.Endpoint(), - Scopes: []string{oidc.ScopeOpenID, "email", "profile"}, - } - verifier := provider.Verifier(&oidc.Config{ClientID: cfg.ClientID}) - - // /login — redirect the browser to Authentik's authorization endpoint - http.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, cfg.AuthCodeURL(oauthState), http.StatusFound) - }) - - // /callback — Authentik redirects here with ?code=...&state=... - http.HandleFunc("/callback", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("state") != oauthState { - http.Error(w, "state mismatch", http.StatusBadRequest) - return - } - - // Exchange the authorization code for tokens - token, err := cfg.Exchange(ctx, r.URL.Query().Get("code")) - if err != nil { - http.Error(w, "token exchange: "+err.Error(), http.StatusInternalServerError) - return - } - - // Verify the ID token signature against Authentik's JWKS, then extract claims - rawID, _ := token.Extra("id_token").(string) - idToken, err := verifier.Verify(ctx, rawID) - if err != nil { - http.Error(w, "id_token verify: "+err.Error(), http.StatusInternalServerError) - return - } - - var claims map[string]any - idToken.Claims(&claims) - - w.Header().Set("Content-Type", "application/json") - enc := json.NewEncoder(w) - enc.SetIndent("", " ") - enc.Encode(claims) - }) - - fmt.Println("open http://localhost:8080/login") - log.Fatal(http.ListenAndServe(":8080", nil)) -} diff --git a/k8s/security/iam/key-rotation-cronjob.yaml b/k8s/security/iam/key-rotation-cronjob.yaml deleted file mode 100644 index 5efa3cf..0000000 --- a/k8s/security/iam/key-rotation-cronjob.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Rotates the Authentik OIDC signing key quarterly for all homelab providers. -# -# Prerequisites (run once): -# # 1. ConfigMap from the Rust source file -# kubectl create configmap key-rotation-script \ -# --from-file=rotate_key.rs=key_rotate.rs \ -# -n iam --dry-run=client -o yaml | kubectl apply -f - -# -# # 2. Token secret — fill in your value, never commit it -# kubectl create secret generic authentik-key-rotation-token \ -# --from-literal=AUTHENTIK_BOOTSTRAP_TOKEN="${AUTHENTIK_BOOTSTRAP_TOKEN}" \ -# -n iam --dry-run=client -o yaml | kubectl apply -f - -# -# Apply: kubectl apply -f key-rotation-cronjob.yaml -# Test: kubectl create job --from=cronjob/authentik-key-rotation test-rotation -n iam -# Logs: kubectl logs -n iam -l job-name=test-rotation -f - -apiVersion: batch/v1 -kind: CronJob -metadata: - name: authentik-key-rotation - namespace: iam -spec: - schedule: "0 0 1 */3 *" # 00:00 UTC on the 1st of Jan, Apr, Jul, Oct - concurrencyPolicy: Forbid # skip if a previous job is still running - jobTemplate: - spec: - template: - spec: - restartPolicy: OnFailure - volumes: - - name: script - configMap: - name: key-rotation-script - containers: - - name: rotate - image: rust:1.82-slim - command: - - sh - - -c - - rustc /scripts/rotate_key.rs -o /tmp/rotate_key && /tmp/rotate_key - volumeMounts: - - name: script - mountPath: /scripts - env: - - name: AUTHENTIK_BASE_URL - value: "http://authentik-server.iam.svc.cluster.local" - - name: AUTHENTIK_BOOTSTRAP_TOKEN - valueFrom: - secretKeyRef: - name: authentik-key-rotation-token - key: AUTHENTIK_BOOTSTRAP_TOKEN diff --git a/k8s/security/iam/key_rotate.rs b/k8s/security/iam/key_rotate.rs deleted file mode 100644 index 8d346ab..0000000 --- a/k8s/security/iam/key_rotate.rs +++ /dev/null @@ -1,114 +0,0 @@ -use std::env; -use std::io::{Read, Write}; -use std::net::TcpStream; -use std::time::{SystemTime, UNIX_EPOCH}; - -const SIGNING_CERT_NAME: &str = "homelab-oidc"; -const PROVIDERS: &[&str] = &["grafana", "minio"]; - -fn main() { - let base_url = env::var("AUTHENTIK_BASE_URL") - .unwrap_or_else(|_| "http://localhost:7000".into()); - let token = env::var("AUTHENTIK_BOOTSTRAP_TOKEN") - .expect("AUTHENTIK_BOOTSTRAP_TOKEN must be set"); - - let host = parse_host(&base_url); - - let ts = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); - let new_name = format!("{}-{}", SIGNING_CERT_NAME, ts); - - let body = format!( - r#"{{"common_name":"{}","validity_days":365,"key_size":4096}}"#, - new_name - ); - let resp = http(&host, &token, "POST", "/api/v3/crypto/certificatekeypairs/generate/", Some(&body)); - let new_pk = extract_str(&resp, "pk").expect("no pk in generate response"); - println!("created '{}' pk={}", new_name, new_pk); - - for provider in PROVIDERS { - let list = http( - &host, &token, "GET", - &format!("/api/v3/providers/oauth2/?name={}", provider), - None, - ); - let provider_pk = match extract_str(&list, "pk") { - Some(pk) => pk, - None => { - eprintln!("WARN: provider '{}' not found — skipping", provider); - continue; - } - }; - let patch = format!(r#"{{"signing_key":"{}"}}"#, new_pk); - http(&host, &token, "PATCH", - &format!("/api/v3/providers/oauth2/{}/", provider_pk), - Some(&patch), - ); - println!("rotated '{}' → signing_key={}", provider, new_pk); - } - - println!("rotation complete"); -} - -fn parse_host(base_url: &str) -> String { - let stripped = base_url - .trim_start_matches("http://") - .trim_start_matches("https://"); - let host = stripped.split('/').next().unwrap_or(stripped); - if host.contains(':') { - host.to_string() - } else { - format!("{}:80", host) - } -} - -fn http(host: &str, token: &str, method: &str, path: &str, body: Option<&str>) -> String { - let mut stream = TcpStream::connect(host) - .unwrap_or_else(|e| panic!("connect {}: {}", host, e)); - - let body_str = body.unwrap_or(""); - let hostname = host.split(':').next().unwrap_or(host); - let req = format!( - "{method} {path} HTTP/1.1\r\n\ - Host: {hostname}\r\n\ - Authorization: Bearer {token}\r\n\ - Content-Type: application/json\r\n\ - Content-Length: {len}\r\n\ - Connection: close\r\n\ - \r\n\ - {body_str}", - len = body_str.len(), - ); - stream.write_all(req.as_bytes()).unwrap(); - - let mut raw = String::new(); - stream.read_to_string(&mut raw).unwrap(); - - let (head, resp_body) = raw.split_once("\r\n\r\n").unwrap_or((&raw, "")); - - let status: u16 = head.lines().next() - .and_then(|l| l.split_whitespace().nth(1)) - .and_then(|s| s.parse().ok()) - .unwrap_or(0); - if status >= 400 { - panic!("{} {} → HTTP {} — {}", method, path, status, resp_body.trim()); - } - - resp_body.to_string() -} - -// Extracts the value of the first `"key": ` match in raw JSON. -// Handles both quoted strings ("pk": "uuid") and bare numbers ("pk": 5). -fn extract_str(json: &str, key: &str) -> Option { - let needle = format!("\"{}\":", key); - let after_colon = json.find(&needle)? + needle.len(); - let rest = json[after_colon..].trim_start(); - if let Some(inner) = rest.strip_prefix('"') { - Some(inner[..inner.find('"')?].to_string()) - } else { - let end = rest.find(|c: char| c == ',' || c == '}' || c.is_ascii_whitespace())?; - Some(rest[..end].to_string()) - } -} diff --git a/k8s/security/iam/register_oauth_app.py b/k8s/security/iam/register_oauth_app.py deleted file mode 100755 index affd9ff..0000000 --- a/k8s/security/iam/register_oauth_app.py +++ /dev/null @@ -1,512 +0,0 @@ -#!/usr/bin/env python3 -""" -register_oauth_app.py -Automated OAuth app registration & Vault JWT wiring for homelab services. - -This script automates the full workflow of registering a new service with -Authentik OIDC and wiring it for Vault JWT authentication. - -Usage: - python3 register_oauth_app.py \\ - --service-name my-service \\ - --namespace my-ns \\ - --redirect-uri "https://my-service.riotpiao.com/oauth2/callback" \\ - [--service-name-in-vault MY_SERVICE] \\ - [--vault-jwt-policy shell-secrets] - -Requirements: - - Authentik running in iam namespace (port-forward 7000:80 active) - - Vault running in storage namespace - - kubectl configured (KUBECONFIG → cluster-config/kubeconfig) - - Environment variables set: - AUTHENTIK_BOOTSTRAP_TOKEN — Authentik API token - {SERVICE_NAME}_OIDC_CLIENT_SECRET — OAuth client secret (generated or from Vault) - -Examples: - - # Simple: register a web app with default settings - python3 register_oauth_app.py \\ - --service-name myapp \\ - --namespace apps \\ - --redirect-uri "https://myapp.riotpiao.com/callback" - - # Advanced: full service with Vault JWT auth - python3 register_oauth_app.py \\ - --service-name myservice \\ - --namespace my-namespace \\ - --redirect-uri "https://myservice.riotpiao.com/oauth2/callback" \\ - --service-name-in-vault MYSERVICE \\ - --vault-jwt-policy shell-secrets \\ - --add-group myservice-admins \\ - --vault-jwt-bound-claims '{"groups":["myservice-admins"]}' -""" - -import argparse -import json -import os -import subprocess -import sys -import urllib.error -import urllib.parse -import urllib.request -from dataclasses import dataclass -from typing import Optional - - -BASE_AUTHENTIK_URL = "http://localhost:7000/api/v3" -VAULT_NAMESPACE = "storage" -IAM_NAMESPACE = "iam" - - -def _auth_headers() -> dict: - """Return HTTP headers for Authentik API requests.""" - token = os.environ.get("AUTHENTIK_BOOTSTRAP_TOKEN", "") - if not token: - sys.exit( - "ERROR: AUTHENTIK_BOOTSTRAP_TOKEN not set\n" - " Set it in ~/.authentik/.env or export it:\n" - " talos get cluster/AUTHENTIK_BOOTSTRAP_TOKEN --key AUTHENTIK_BOOTSTRAP_TOKEN" - ) - return { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - } - - -def _request(method: str, path: str, body: Optional[dict] = None, **params) -> dict: - """HTTP request helper for Authentik API.""" - url = BASE_AUTHENTIK_URL + path - if params: - url += "?" + urllib.parse.urlencode(params) - - req = urllib.request.Request( - url, - data=json.dumps(body).encode() if body else None, - headers=_auth_headers(), - method=method, - ) - - try: - with urllib.request.urlopen(req) as resp: - raw = resp.read() - return json.loads(raw) if raw else {} - except urllib.error.HTTPError as exc: - detail = exc.read().decode(errors="replace") - print(f"ERROR {method} {path}: HTTP {exc.code} — {detail}", file=sys.stderr) - raise - - -def get_pk(path: str, **filters) -> Optional[str]: - """Fetch the PK of the first result matching filters.""" - results = _request("GET", path, **filters).get("results", []) - return results[0]["pk"] if results else None - - -def get_or_create(path: str, filters: dict, data: dict) -> str: - """Create or fetch a resource; returns PK.""" - pk = get_pk(path, **filters) - if pk is None: - pk = _request("POST", path, data)["pk"] - print(f"✓ created {path.rstrip('/')} {filters}") - else: - print(f"✓ exists {path.rstrip('/')} {filters}") - return pk - - -def fetch_flows_and_signing_key() -> tuple[str, str, str]: - """Fetch required Authentik resource PKs.""" - auth_pk = get_pk( - "/flows/instances/", - slug="default-provider-authorization-implicit-consent" - ) - inval_pk = get_pk( - "/flows/instances/", - slug="default-provider-invalidation-flow" - ) - - if not auth_pk or not inval_pk: - sys.exit( - "ERROR: required Authentik flows not found\n" - " Run provision_oidc.py first to initialize defaults" - ) - - results = _request("GET", "/crypto/certificatekeypairs/", has_key=True).get("results", []) - signing_pk = results[0]["pk"] if results else None - - if not signing_pk: - sys.exit( - "ERROR: no signing key found in Authentik\n" - " Create one: Authentik UI → System → Certificates" - ) - - return auth_pk, inval_pk, signing_pk - - -def apply_k8s_secret(name: str, namespace: str, literals: dict) -> None: - """Create or update a Kubernetes secret.""" - env = {**os.environ} - manifest = subprocess.run( - [ - "kubectl", "create", "secret", "generic", name, - "-n", namespace, "--dry-run=client", "-o", "yaml", - *[f"--from-literal={k}={v}" for k, v in literals.items()], - ], - capture_output=True, text=True, check=True, env=env, - ).stdout - - subprocess.run( - ["kubectl", "apply", "-f", "-"], - input=manifest, text=True, check=True, env=env - ) - print(f"✓ secret {name} → ns/{namespace}") - - -def create_authentik_provider( - service_name: str, - client_id: str, - client_secret: str, - redirect_uris: list[str], - auth_flow_pk: str, - inval_flow_pk: str, - signing_key_pk: str, -) -> str: - """Create OAuth2 provider in Authentik.""" - data = { - "name": service_name, - "client_id": client_id, - "client_secret": client_secret, - "authorization_flow": auth_flow_pk, - "invalidation_flow": inval_flow_pk, - "redirect_uris": [{"matching_mode": "strict", "url": u} for u in redirect_uris], - "sub_mode": "hashed_user_id", - "include_claims_in_id_token": True, - "property_mappings": [], # Will be populated by caller if needed - "grant_types": ["authorization_code", "refresh_token"], - "signing_key": signing_key_pk, - } - - pk = get_pk("/providers/oauth2/", name=service_name) - if pk is None: - pk = _request("POST", "/providers/oauth2/", data)["pk"] - print(f"✓ created provider={service_name} client_id={client_id}") - else: - _request("PATCH", f"/providers/oauth2/{pk}/", data) - print(f"✓ synced provider={service_name} (credentials from Vault)") - - return pk - - -def create_authentik_application( - display_name: str, - slug: str, - provider_pk: str, - launch_url: Optional[str] = None, -) -> str: - """Create application in Authentik.""" - data = {"name": display_name, "slug": slug, "provider": provider_pk} - if launch_url: - data["meta_launch_url"] = launch_url - - return get_or_create( - "/core/applications/", - filters={"slug": slug}, - data=data - ) - - -def create_authentik_group(group_name: str) -> str: - """Create a group in Authentik.""" - return get_or_create( - "/core/groups/", - filters={"name": group_name}, - data={"name": group_name, "is_superuser": False} - ) - - -def bind_group_to_app(app_pk: str, group_pk: str) -> None: - """Bind a group to an application (allow-list policy).""" - existing = _request( - "GET", "/policies/bindings/", - target=app_pk, group=group_pk - ).get("results", []) - - if existing: - print(f"✓ exists binding group={group_pk} → app={app_pk}") - else: - _request("POST", "/policies/bindings/", { - "target": app_pk, - "group": group_pk, - "enabled": True, - "order": 0, - }) - print(f"✓ created binding group={group_pk} → app={app_pk}") - - -def create_vault_jwt_policy( - service_name: str, - vault_addr: str, -) -> str: - """Create a Vault policy for the service.""" - policy_name = f"service-read-{service_name}" - - # Policy definition: service can read its own secrets - policy_rules = f""" -path "secret/data/services/{service_name}/*" {{ - capabilities = ["read"] -}} -path "secret/data/cluster/*" {{ - capabilities = ["read"] -}} -""" - - # Write policy via vault CLI (requires auth) - try: - subprocess.run( - ["vault", "policy", "write", policy_name, "-"], - input=policy_rules, - text=True, - check=True, - env={**os.environ, "VAULT_ADDR": vault_addr} - ) - print(f"✓ created Vault policy={policy_name}") - except subprocess.CalledProcessError as e: - print(f"⚠ warning Failed to create Vault policy: {e}", file=sys.stderr) - return "" - - return policy_name - - -def create_vault_jwt_role( - service_name: str, - vault_addr: str, - policies: list[str], - bound_claims: Optional[dict] = None, -) -> None: - """Create a Vault JWT auth role for the service.""" - bound_claims_json = json.dumps(bound_claims) if bound_claims else "{}" - - role_config = { - "role_type": "jwt", - "bound_audiences": ["vault"], - "user_claim": "sub", - "bound_claims": bound_claims, - "token_policies": policies, - "token_ttl": "4h", - "token_max_ttl": "8h", - } - - try: - subprocess.run( - ["vault", "write", f"auth/jwt/role/{service_name}", "-"], - input=json.dumps(role_config), - text=True, - check=True, - env={**os.environ, "VAULT_ADDR": vault_addr} - ) - print(f"✓ created Vault JWT role={service_name} policies={policies}") - except subprocess.CalledProcessError as e: - print(f"⚠ warning Failed to create Vault JWT role: {e}", file=sys.stderr) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Register a new OAuth service with Authentik + Vault", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__, - ) - - parser.add_argument( - "--service-name", - required=True, - help="Service name (used as Authentik provider/app slug)" - ) - parser.add_argument( - "--namespace", - required=True, - help="Kubernetes namespace where service runs" - ) - parser.add_argument( - "--redirect-uri", - required=True, - help="OAuth2 redirect URI (e.g., https://myapp.riotpiao.com/callback)" - ) - parser.add_argument( - "--service-name-in-vault", - default=None, - help="Service name for Vault (defaults to uppercase service-name)" - ) - parser.add_argument( - "--vault-jwt-policy", - default="shell-secrets", - help="Vault policy to attach to JWT role (comma-separated for multiple)" - ) - parser.add_argument( - "--add-group", - default=None, - help="Create and bind a group to the app (e.g., 'myservice-admins')" - ) - parser.add_argument( - "--vault-jwt-bound-claims", - default=None, - help='JSON string of claims to bind JWT role (e.g., \'{"groups":["myservice-admins"]}\')' - ) - parser.add_argument( - "--no-vault", - action="store_true", - help="Skip Vault JWT role creation (just register with Authentik)" - ) - parser.add_argument( - "--vault-addr", - default=None, - help="Vault address (defaults to localhost:8200 via port-forward)" - ) - - args = parser.parse_args() - - service_name = args.service_name.lower() - namespace = args.namespace - redirect_uri = args.redirect_uri - vault_service_name = (args.service_name_in_vault or service_name).upper() - vault_addr = args.vault_addr or "http://127.0.0.1:8200" - vault_policies = [p.strip() for p in args.vault_jwt_policy.split(",")] - - # Fetch or generate client secret from environment / Vault - client_secret_env = f"{vault_service_name}_OIDC_CLIENT_SECRET" - client_secret = os.environ.get(client_secret_env, "") - - if not client_secret: - sys.exit( - f"ERROR: {client_secret_env} not set\n" - f" Generate and store in Vault:\n" - f" talos put cluster/{client_secret_env} " - f"{client_secret_env}=$(openssl rand -hex 32)" - ) - - print(f"\n{'='*70}") - print(f"OAuth App Registration: {service_name}") - print(f"{'='*70}\n") - - print(f"📋 Configuration:") - print(f" Service Name: {service_name}") - print(f" Namespace: {namespace}") - print(f" Redirect URI: {redirect_uri}") - print(f" Vault Service: {vault_service_name}") - print(f" Vault Policies: {', '.join(vault_policies)}") - if args.add_group: - print(f" Group: {args.add_group}") - print() - - # Fetch Authentik resources - print("🔍 Fetching Authentik configuration...") - auth_flow_pk, inval_flow_pk, signing_key_pk = fetch_flows_and_signing_key() - print(f"✓ flows & signing key found\n") - - # Create Authentik provider - print("🔐 Creating Authentik OAuth2 Provider...") - provider_pk = create_authentik_provider( - service_name=service_name, - client_id=service_name, - client_secret=client_secret, - redirect_uris=[redirect_uri], - auth_flow_pk=auth_flow_pk, - inval_flow_pk=inval_flow_pk, - signing_key_pk=signing_key_pk, - ) - print() - - # Create Authentik application - print("📱 Creating Authentik Application...") - app_pk = create_authentik_application( - display_name=service_name.replace("-", " ").title(), - slug=service_name, - provider_pk=provider_pk, - launch_url=f"https://{service_name}.riotpiao.com" - ) - print() - - # Create and bind group (if requested) - if args.add_group: - print(f"👥 Creating Group: {args.add_group}") - group_pk = create_authentik_group(args.add_group) - bind_group_to_app(app_pk, group_pk) - print() - - # Create K8s secret - print("🔑 Creating Kubernetes Secret...") - apply_k8s_secret( - f"{service_name}-oidc", - namespace, - { - "client_id": service_name, - "client_secret": client_secret, - "issuer_url": f"http://authentik-server.iam.svc.cluster.local/application/o/{service_name}/", - "redirect_uri": redirect_uri, - } - ) - print() - - # Create Vault resources (if not disabled) - if not args.no_vault: - print("🔐 Configuring Vault JWT Authentication...") - - # Parse bound claims if provided - bound_claims = None - if args.vault_jwt_bound_claims: - try: - bound_claims = json.loads(args.vault_jwt_bound_claims) - except json.JSONDecodeError as e: - print(f"⚠ warning Invalid JSON for --vault-jwt-bound-claims: {e}", file=sys.stderr) - - # Create Vault policy - policy_name = create_vault_jwt_policy(service_name, vault_addr) - - # Create Vault JWT role - if policy_name: - all_policies = list(vault_policies) + [policy_name] - create_vault_jwt_role( - service_name, - vault_addr, - all_policies, - bound_claims - ) - print() - - # Summary - print(f"{'='*70}") - print(f"✅ Registration Complete!") - print(f"{'='*70}\n") - - print("📝 Next Steps:\n") - - print(f"1. Configure your service with these environment variables:") - print(f" export OIDC_ISSUER_URL='http://authentik-server.iam.svc.cluster.local/application/o/{service_name}/'") - print(f" export OIDC_CLIENT_ID='{service_name}'") - print(f" export OIDC_CLIENT_SECRET='$({vault_service_name}_OIDC_CLIENT_SECRET)'") - print(f" export OIDC_REDIRECT_URI='{redirect_uri}'") - print() - - print(f"2. Mount the Kubernetes secret in your Helm values:") - print(f" env:") - print(f" - name: OIDC_CLIENT_SECRET") - print(f" valueFrom:") - print(f" secretKeyRef:") - print(f" name: {service_name}-oidc") - print(f" key: client_secret") - print() - - if not args.no_vault: - print(f"3. Test Vault JWT authentication:") - print(f" kubectl port-forward -n {VAULT_NAMESPACE} svc/vault 8200:8200 &") - print(f" vault login -method=oidc role=homelab") - print(f" vault read auth/jwt/role/{service_name}") - print() - - print(f"4. Verify OAuth flow:") - print(f" kubectl port-forward -n {IAM_NAMESPACE} svc/authentik-server 7000:80 &") - print(f" # Open Authentik UI: http://localhost:7000/if/admin/") - print(f" # Check: Applications → {service_name}") - print() - - -if __name__ == "__main__": - main() diff --git a/k8s/security/iam/setup_talos_iam.sh b/k8s/security/iam/setup_talos_iam.sh deleted file mode 100755 index 23c3d52..0000000 --- a/k8s/security/iam/setup_talos_iam.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# k8s/talos-iam/setup_talos_iam.sh -# Validates required env vars are set, then delegates to helmfile. -# All provisioning logic lives in helmfile hooks + provision_oidc.py. -# -# Usage: -# vsource .env && bash k8s/talos-iam/setup_talos_iam.sh -# -# To re-provision OIDC without redeploying Authentik: -# vsource .env && helmfile apply -l name=authentik -# -# To patch existing providers (update redirect URIs, scopes): -# vsource .env && helmfile apply -l name=authentik # postsync runs provision_oidc.py --patch implicitly -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../cluster-config/kubeconfig}" -export KUBECONFIG - -# ── Validate all required secrets are loaded from Vault ────────────────────── -MISSING=() -for var in \ - AUTHENTIK_SECRET_KEY \ - AUTHENTIK_BOOTSTRAP_PASSWORD \ - AUTHENTIK_BOOTSTRAP_TOKEN \ - AUTHENTIK_PG_PASSWORD \ - GRAFANA_OIDC_CLIENT_SECRET \ - MINIO_OIDC_CLIENT_SECRET \ - AUTHENTIK_FORGEJO_CLIENT_SECRET \ - AUTHENTIK_ARGOCD_CLIENT_SECRET; do - [[ -z "${!var:-}" ]] && MISSING+=("$var") -done - -if [[ ${#MISSING[@]} -gt 0 ]]; then - echo "ERROR: the following vars are not set — run 'vsource .env' first:" >&2 - printf ' %s\n' "${MISSING[@]}" >&2 - echo "" >&2 - echo "If a var is missing from Vault, store it first:" >&2 - echo " talos put cluster/VAR_NAME VAR_NAME=\"\$(openssl rand -hex 32)\"" >&2 - exit 1 -fi - -echo "All required secrets present." -echo "" -echo "Provider credential mapping (client_id → client_secret):" -printf ' %-10s client_id=%-40s secret=%s\n' \ - "grafana" "${GRAFANA_OIDC_CLIENT_ID:-"grafana (default)"}" "GRAFANA_OIDC_CLIENT_SECRET" \ - "minio" "${MINIO_OIDC_CLIENT_ID:-"minio (default)"}" "MINIO_OIDC_CLIENT_SECRET" \ - "forgejo" "${AUTHENTIK_FORGEJO_CLIENT_ID:-"forgejo (default)"}" "AUTHENTIK_FORGEJO_CLIENT_SECRET" \ - "argocd" "${AUTHENTIK_ARGOCD_CLIENT_ID:-"argocd (default)"}" "AUTHENTIK_ARGOCD_CLIENT_SECRET" -echo "" -echo "Delegating to helmfile..." -cd "${SCRIPT_DIR}/../.." -helmfile apply -l name=authentik diff --git a/k8s/security/iam/setup_vault.sh b/k8s/security/iam/setup_vault.sh deleted file mode 100755 index acd8a7b..0000000 --- a/k8s/security/iam/setup_vault.sh +++ /dev/null @@ -1,527 +0,0 @@ -#!/usr/bin/env bash -# k8s/talos-iam/setup_vault.sh -# Deploys HashiCorp Vault into the iam namespace, initialises it, and wires -# the JWT auth backend to Authentik so the talos-cli secrets subcommand works. -# -# Prerequisites: -# - Authentik already running in the iam namespace (run setup_talos_iam.sh first) -# - kubectl configured (KUBECONFIG → cluster-config/kubeconfig) -# - helm >= 3.x -# - vault CLI installed locally (https://developer.hashicorp.com/vault/downloads) -# - talos-iam/.env containing (see .env.example): -# MINIO_ROOT_USER=... -# MINIO_ROOT_PASSWORD=... -# The script seeds initial secrets and needs read access to MinIO credentials. -# -# What this script does (in order): -# 1. Create vault-minio-creds K8s Secret (MinIO creds for the S3 backend) -# 2. Helm install hashicorp/vault -# 3. Wait for vault pod to be Running -# 4. vault operator init → capture unseal keys + root token -# 5. vault operator unseal (3 of 5 key shares) -# 6. Login with root token -# 7. Enable KV v2 secret engine at secret/ -# 8. Enable JWT auth backend, configure with Authentik JWKS -# 9. Write policies (shell-secrets, mcp-readonly, cluster-admin) -# 10. Create JWT roles (shell, mcp, cluster) -# 11. Seed initial secrets from .env -# 12. Register talos-cli-shell OIDC app in Authentik via API -# 13. Print next steps -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}" -export KUBECONFIG - -NAMESPACE=storage -RELEASE=vault -AUTHENTIK_URL="http://authentik-server.iam.svc.cluster.local" -VAULT_ADDR_CLUSTER="http://vault.storage.svc.cluster.local:8200" -# Local access via port-forward during setup -VAULT_PORT_FWD_ADDR="http://127.0.0.1:8200" -UNSEAL_KEYS_FILE="$HOME/.vault-data/.vault-init.json" # KEEP OFFLINE after setup - -# ── Failure surfacing ───────────────────────────────────────────────────────── -CURRENT_STEP="init" -on_err() { - local rc=$? - echo "[ERROR] step '${CURRENT_STEP}' failed (exit ${rc} at line ${BASH_LINENO[0]})" >&2 - exit "${rc}" -} -trap on_err ERR - -step() { CURRENT_STEP="$1"; echo ""; echo "==> $2"; } - -# ── Load credentials ────────────────────────────────────────────────────────── -step load_env "Loading credentials from talos-iam/.env..." -if [[ -f "$HOME/.authentik/.env" ]]; then - # shellcheck source=/dev/null - source "$HOME/.authentik/.env" -fi - -for var in MINIO_ROOT_USER MINIO_ROOT_PASSWORD AUTHENTIK_BOOTSTRAP_TOKEN; do - if [[ -z "${!var:-}" ]]; then - echo "ERROR: ${var} is not set — export it or add it to talos-iam/.env" >&2 - exit 1 - fi -done - -# ── 1. MinIO credentials secret for Vault S3 backend ───────────────────────── -step minio_secret "Creating vault-minio-creds secret..." -kubectl create secret generic vault-minio-creds \ - --namespace="${NAMESPACE}" \ - --from-literal=access_key="${MINIO_ROOT_USER}" \ - --from-literal=secret_key="${MINIO_ROOT_PASSWORD}" \ - --dry-run=client -o yaml | kubectl apply -f - - -# ── 1b. Placeholder vault-unseal-keys secret (required before Helm install) ─── -# extraSecretEnvironmentVars references vault-unseal-keys at pod start time, -# but the real keys only exist after vault operator init. Create empty placeholders -# so the pod starts; step vault_unseal_secret overwrites them with real values. -step vault_unseal_placeholder "Creating placeholder vault-unseal-keys secret..." -kubectl create secret generic vault-unseal-keys \ - --namespace="${NAMESPACE}" \ - --from-literal=key1="" \ - --from-literal=key2="" \ - --from-literal=key3="" \ - --dry-run=client -o yaml | kubectl apply -f - - -# ── 1c. Register talos-cli-shell OIDC app in Authentik ─────────────────────── -# Must exist before Helm install so Vault can validate the JWKS URL at jwt/config time. -step authentik_app "Registering talos-cli-shell OIDC provider in Authentik..." -AUTHENTIK_API="http://127.0.0.1:7000/api/v3" - -kubectl port-forward -n "${NAMESPACE}" svc/authentik-server 7000:80 & -AK_PF_PID=$! -trap 'kill ${AK_PF_PID} 2>/dev/null' EXIT -sleep 3 - -# Fetch required flow PKs from Authentik -_auth_flow_pk=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API}/flows/instances/?slug=default-provider-authorization-implicit-consent" \ - | jq -r '.results[0].pk // empty') -_inval_flow_pk=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API}/flows/instances/?slug=default-provider-invalidation-flow" \ - | jq -r '.results[0].pk // empty') - -if [[ -z "${_auth_flow_pk}" || -z "${_inval_flow_pk}" ]]; then - echo "ERROR: required Authentik flows not found — is provision_oidc.py already run?" >&2 - exit 1 -fi - -_signing_key_pk=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API}/crypto/certificatekeypairs/?has_key=true&page_size=1" \ - | jq -r '.results[0].pk // empty') - -if [[ -z "${_signing_key_pk}" ]]; then - echo "ERROR: no signing key found in Authentik — create one under System → Certificates" >&2 - exit 1 -fi - -_provider_payload=$(jq -n \ - --arg auth_flow "${_auth_flow_pk}" \ - --arg inval_flow "${_inval_flow_pk}" \ - --arg signing_key "${_signing_key_pk}" \ - '{ - name: "talos-cli-shell", - client_type: "confidential", - grant_types: ["client_credentials"], - token_validity: "minutes=5", - sub_mode: "hashed_user_id", - include_claims_in_id_token: true, - audience: "vault", - authorization_flow: $auth_flow, - invalidation_flow: $inval_flow, - signing_key: $signing_key, - redirect_uris: [] - }') - -_existing_provider=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API}/providers/oauth2/?name=talos-cli-shell" | jq -r '.results[0].pk // empty') - -if [[ -n "${_existing_provider}" ]]; then - echo "Provider talos-cli-shell exists (pk=${_existing_provider}) — patching config..." - PROVIDER_PK="${_existing_provider}" - curl -sf -X PATCH "${AUTHENTIK_API}/providers/oauth2/${PROVIDER_PK}/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "${_provider_payload}" > /dev/null - CLIENT_SECRET="(unchanged — retrieve from Authentik admin UI if needed)" -else - PROVIDER_PK=$(curl -sf -X POST "${AUTHENTIK_API}/providers/oauth2/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "${_provider_payload}" | jq -r '.pk') - - CLIENT_SECRET=$(curl -sf -X POST \ - "${AUTHENTIK_API}/providers/oauth2/${PROVIDER_PK}/set_secret/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{}' | jq -r '.client_secret') - - echo "Created provider pk=${PROVIDER_PK}" - echo "" - echo "┌─────────────────────────────────────────────────────────────────┐" - echo "│ Authentik OIDC app registered. │" - echo "│ Client ID: talos-cli-shell │" - echo "│ Client Secret: ${CLIENT_SECRET} │" - echo "│ Store this secret — it will not be shown again. │" - echo "└─────────────────────────────────────────────────────────────────┘" -fi - -# Idempotent: ensure the application exists and is bound to the provider. -# Runs on both create and patch paths so re-runs always produce a consistent state. -_existing_app=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API}/core/applications/?slug=talos-cli-shell" | jq -r '.results[0].pk // empty') - -if [[ -n "${_existing_app}" ]]; then - echo "Application talos-cli-shell exists (pk=${_existing_app}) — ensuring provider binding..." - curl -sf -X PATCH "${AUTHENTIK_API}/core/applications/${_existing_app}/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "{\"provider\": ${PROVIDER_PK}}" > /dev/null -else - curl -sf -X POST "${AUTHENTIK_API}/core/applications/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "$(jq -n --argjson pk "${PROVIDER_PK}" \ - '{"name":"talos-cli-shell","slug":"talos-cli-shell","provider":$pk}')" > /dev/null - echo "Application talos-cli-shell created and bound to provider pk=${PROVIDER_PK}" -fi - -kill "${AK_PF_PID}" 2>/dev/null || true -unset AK_PF_PID - -# ── 1d. Register vault-browser OIDC app in Authentik (browser / UI login) ──── -# Separate from talos-cli-shell: this uses authorization_code grant so humans -# can log in via the Vault UI or `vault login -method=oidc`. -step authentik_vault_browser "Registering vault-browser OIDC provider in Authentik..." -AUTHENTIK_API_BROWSER="http://127.0.0.1:7000/api/v3" - -kubectl port-forward -n iam svc/authentik-server 7000:80 & -AK_PF2_PID=$! -trap 'kill ${AK_PF2_PID} 2>/dev/null' EXIT -sleep 3 - -_auth_flow_pk2=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API_BROWSER}/flows/instances/?slug=default-provider-authorization-implicit-consent" \ - | jq -r '.results[0].pk // empty') -_inval_flow_pk2=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API_BROWSER}/flows/instances/?slug=default-provider-invalidation-flow" \ - | jq -r '.results[0].pk // empty') -_signing_key_pk2=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API_BROWSER}/crypto/certificatekeypairs/?has_key=true&page_size=1" \ - | jq -r '.results[0].pk // empty') - -_vault_browser_payload=$(jq -n \ - --arg auth_flow "${_auth_flow_pk2}" \ - --arg inval_flow "${_inval_flow_pk2}" \ - --arg signing_key "${_signing_key_pk2}" \ - '{ - name: "vault-browser", - client_type: "confidential", - client_id: "vault-browser", - grant_types: ["authorization_code", "refresh"], - token_validity: "hours=8", - sub_mode: "hashed_user_id", - include_claims_in_id_token: true, - authorization_flow: $auth_flow, - invalidation_flow: $inval_flow, - signing_key: $signing_key, - redirect_uris: [ - {"matching_mode": "strict", "url": "http://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"}, - {"matching_mode": "strict", "url": "http://localhost:8250/oidc/callback"} - ] - }') - -_existing_vault_browser=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API_BROWSER}/providers/oauth2/?name=vault-browser" | jq -r '.results[0].pk // empty') - -if [[ -n "${_existing_vault_browser}" ]]; then - echo "Provider vault-browser exists (pk=${_existing_vault_browser}) — patching..." - VAULT_BROWSER_PK="${_existing_vault_browser}" - curl -sf -X PATCH "${AUTHENTIK_API_BROWSER}/providers/oauth2/${VAULT_BROWSER_PK}/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "${_vault_browser_payload}" > /dev/null - # Re-fetch the client secret from the detail endpoint (list responses mask it) - VAULT_BROWSER_CLIENT_SECRET=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API_BROWSER}/providers/oauth2/${VAULT_BROWSER_PK}/" \ - | jq -r '.client_secret') -else - VAULT_BROWSER_PK=$(curl -sf -X POST "${AUTHENTIK_API_BROWSER}/providers/oauth2/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "${_vault_browser_payload}" | jq -r '.pk') - - VAULT_BROWSER_CLIENT_SECRET=$(curl -sf -X POST \ - "${AUTHENTIK_API_BROWSER}/providers/oauth2/${VAULT_BROWSER_PK}/set_secret/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{}' | jq -r '.client_secret') - - echo "Created vault-browser provider pk=${VAULT_BROWSER_PK}" -fi - -# Bind application -_existing_vault_app=$(curl -sf \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${AUTHENTIK_API_BROWSER}/core/applications/?slug=vault-browser" | jq -r '.results[0].pk // empty') - -if [[ -n "${_existing_vault_app}" ]]; then - curl -sf -X PATCH "${AUTHENTIK_API_BROWSER}/core/applications/${_existing_vault_app}/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "{\"provider\": ${VAULT_BROWSER_PK}}" > /dev/null - echo "Application vault-browser updated." -else - curl -sf -X POST "${AUTHENTIK_API_BROWSER}/core/applications/" \ - -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - -H "Content-Type: application/json" \ - -d "$(jq -n --argjson pk "${VAULT_BROWSER_PK}" \ - '{"name":"vault-browser","slug":"vault-browser","provider":$pk}')" > /dev/null - echo "Application vault-browser created." -fi - -kill "${AK_PF2_PID}" 2>/dev/null || true -unset AK_PF2_PID - -# ── 2. Ensure vault bucket exists in MinIO ──────────────────────────────────── -step minio_bucket "Ensuring 'vault' bucket exists in MinIO..." -kubectl run vault-bucket-init --rm -i --restart=Never \ - --namespace=storage \ - --image=minio/mc:latest \ - --env="MC_HOST_local=http://${MINIO_ROOT_USER}:${MINIO_ROOT_PASSWORD}@minio.storage.svc.cluster.local:9000" \ - --command -- mc mb --ignore-existing local/vault - -# ── 3. Helm install Vault ───────────────────────────────────────────────────── -step helm_install "Installing HashiCorp Vault via Helm..." -helm repo add hashicorp https://helm.releases.hashicorp.com -helm repo update hashicorp - -helm upgrade --install "${RELEASE}" hashicorp/vault \ - --namespace "${NAMESPACE}" \ - --values "${SCRIPT_DIR}/vault-values.yaml" \ - --wait --timeout=120s - -# ── 4. Wait for vault pod ───────────────────────────────────────────────────── -step wait_pod "Waiting for vault-0 pod to be Running..." -kubectl wait pod/vault-0 \ - --namespace="${NAMESPACE}" \ - --for=condition=Ready=false \ - --timeout=60s 2>/dev/null || true # pod starts uninitialized (not Ready) — that's expected - -# Give it a moment to bind the port -sleep 5 - -# ── 5. Port-forward for init/unseal ────────────────────────────────────────── -step port_forward "Starting port-forward to vault-0 on 127.0.0.1:8200..." -kubectl port-forward -n "${NAMESPACE}" pod/vault-0 8200:8200 & -PF_PID=$! -trap 'kill ${PF_PID} 2>/dev/null; on_err' ERR -trap 'kill ${PF_PID} 2>/dev/null' EXIT -sleep 3 - -export VAULT_ADDR="${VAULT_PORT_FWD_ADDR}" - -# ── 6. Init ─────────────────────────────────────────────────────────────────── -step vault_init "Initialising Vault (5 key shares, threshold 3)..." -if vault status 2>/dev/null | grep -q "Initialized.*true"; then - echo "Vault is already initialised — skipping init." -else - vault operator init \ - -key-shares=5 \ - -key-threshold=3 \ - -format=json > "${UNSEAL_KEYS_FILE}" - - chmod 600 "${UNSEAL_KEYS_FILE}" - echo "" - echo "┌─────────────────────────────────────────────────────────────────┐" - echo "│ IMPORTANT: unseal keys + root token saved to: │" - echo "│ ${UNSEAL_KEYS_FILE}" - echo "│ Move this file OFFLINE (USB / password manager) immediately. │" - echo "└─────────────────────────────────────────────────────────────────┘" -fi - -# ── 6b. Store unseal keys in K8s Secret for auto-unseal on restart ─────────── -step vault_unseal_secret "Storing 3 unseal keys in vault-unseal-keys Secret..." -kubectl create secret generic vault-unseal-keys \ - --namespace="${NAMESPACE}" \ - --from-literal=key1="$(jq -r '.unseal_keys_b64[0]' "${UNSEAL_KEYS_FILE}")" \ - --from-literal=key2="$(jq -r '.unseal_keys_b64[1]' "${UNSEAL_KEYS_FILE}")" \ - --from-literal=key3="$(jq -r '.unseal_keys_b64[2]' "${UNSEAL_KEYS_FILE}")" \ - --dry-run=client -o yaml | kubectl apply -f - - -# ── 7. Unseal ───────────────────────────────────────────────────────────────── -step vault_unseal "Unsealing Vault (3 of 5 shares)..." -if vault status 2>/dev/null | grep -q "Sealed.*false"; then - echo "Vault is already unsealed — skipping." -else - for i in 0 1 2; do - KEY=$(jq -r ".unseal_keys_b64[${i}]" "${UNSEAL_KEYS_FILE}") - vault operator unseal "${KEY}" - done -fi - -# ── 8. Login with root token ────────────────────────────────────────────────── -step vault_login "Logging in with root token..." -ROOT_TOKEN=$(jq -r ".root_token" "${UNSEAL_KEYS_FILE}") -vault login "${ROOT_TOKEN}" - -# ── 9. Enable KV v2 ────────────────────────────────────────────────────────── -step kv_enable "Enabling KV v2 at secret/..." -vault secrets enable -path=secret kv-v2 2>/dev/null || echo "KV v2 already enabled." - -# ── 10. Enable JWT auth backend ─────────────────────────────────────────────── -step jwt_enable "Enabling JWT auth backend..." -vault auth enable jwt 2>/dev/null || echo "JWT auth already enabled." - -JWKS_URL="${AUTHENTIK_URL}/application/o/talos-cli-shell/jwks/" -vault write auth/jwt/config \ - jwks_url="${JWKS_URL}" \ - default_role="shell" - -echo "JWT auth configured with JWKS at: ${JWKS_URL}" - -# ── 11. Write policies ──────────────────────────────────────────────────────── -step policies "Writing Vault policies..." - -vault policy write shell-secrets - <<'EOF' -# shell-secrets: read MCP keys and cloud tokens for interactive shell sessions -path "secret/data/mcp/*" { - capabilities = ["read"] -} -path "secret/data/cloud/*" { - capabilities = ["read"] -} -EOF - -vault policy write mcp-readonly - <<'EOF' -# mcp-readonly: each MCP server reads only its own path -path "secret/data/mcp/{{identity.entity.aliases.*.metadata.client_id}}" { - capabilities = ["read"] -} -EOF - -vault policy write cluster-admin - <<'EOF' -# cluster-admin: read cluster service credentials (MinIO, Grafana, etc.) -path "secret/data/cluster/*" { - capabilities = ["read", "update"] -} -path "secret/data/mcp/*" { - capabilities = ["read", "create", "update", "delete"] -} -path "secret/data/cloud/*" { - capabilities = ["read", "create", "update", "delete"] -} -EOF - -# ── 12. Create JWT roles ────────────────────────────────────────────────────── -step jwt_roles "Creating JWT roles..." - -# shell role: interactive shell sessions via talos-cli -vault write auth/jwt/role/shell \ - role_type=jwt \ - bound_audiences="vault" \ - user_claim="sub" \ - policies="shell-secrets" \ - ttl=4h \ - max_ttl=8h - -# mcp role: MCP server processes (short TTL, non-renewable) -vault write auth/jwt/role/mcp \ - role_type=jwt \ - bound_audiences="vault" \ - user_claim="sub" \ - policies="mcp-readonly" \ - ttl=1h \ - max_ttl=1h - -# cluster role: admin operations (bootstrap scripts, rotation jobs) -vault write auth/jwt/role/cluster - <<'EOF' -{ - "role_type": "jwt", - "bound_audiences": ["vault"], - "user_claim": "sub", - "bound_claims": {"groups": ["homelab-admins"]}, - "token_policies": ["cluster-admin"], - "token_ttl": "1h" -} -EOF - -# ── 13. Enable OIDC auth (browser / UI login via Authentik) ────────────────── -step oidc_enable "Enabling OIDC auth method for browser login..." -vault auth enable oidc 2>/dev/null || echo "OIDC auth already enabled." - -vault write auth/oidc/config \ - oidc_discovery_url="http://authentik.riotpiao.com/application/o/vault-browser/" \ - oidc_client_id="vault-browser" \ - oidc_client_secret="${VAULT_BROWSER_CLIENT_SECRET}" \ - default_role="homelab" - -# homelab role: all authenticated Authentik users get shell-secrets + cluster-admin. -# Restrict further by adding bound_claims once group-based mapping is configured. -vault write auth/oidc/role/homelab \ - role_type=oidc \ - bound_audiences="vault-browser" \ - allowed_redirect_uris="http://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \ - user_claim="sub" \ - oidc_scopes="openid,profile,email" \ - token_policies="shell-secrets,cluster-admin" \ - token_ttl=8h \ - token_max_ttl=12h - -echo "OIDC auth configured." -echo " Browser login: http://vault.riotpiao.com → sign in with Authentik" -echo " CLI login: VAULT_ADDR=http://vault.riotpiao.com vault login -method=oidc" - -# ── 14. Seed initial secrets ────────────────────────────────────────────────── -step seed_secrets "Seeding initial secrets from environment (accessible after OIDC login)..." - -# Cluster service credentials -vault kv put secret/cluster/minio \ - user="${MINIO_ROOT_USER}" \ - password="${MINIO_ROOT_PASSWORD}" - -if [[ -n "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then - vault kv put secret/cluster/grafana \ - password="${GRAFANA_ADMIN_PASSWORD}" -fi - -if [[ -n "${DUCKDNS_TOKEN:-}" ]]; then - vault kv put secret/cloud/duckdns \ - token="${DUCKDNS_TOKEN}" -fi - -# MCP / AI tooling keys (optional — add when available) -if [[ -n "${ANTHROPIC_API_KEY:-}" ]]; then - vault kv put secret/mcp/anthropic api_key="${ANTHROPIC_API_KEY}" -fi - -if [[ -n "${GITHUB_TOKEN:-}" ]]; then - vault kv put secret/mcp/github token="${GITHUB_TOKEN}" -fi - -# ── Done ────────────────────────────────────────────────────────────────────── -echo "" -echo "==> Vault bootstrap complete." -echo "" -echo "Next steps:" -echo " 1. Move ${UNSEAL_KEYS_FILE} OFFLINE (USB / password manager)." -echo " 2. Browser login: open http://vault.riotpiao.com → choose OIDC → sign in with Authentik" -echo " 3. CLI login: export VAULT_ADDR=http://vault.riotpiao.com" -echo " vault login -method=oidc" -echo " 4. Read MinIO credentials from Vault after login:" -echo " vault kv get secret/cluster/minio" -echo " 5. talos-cli (JWT): talos secrets login && talos secrets status" diff --git a/k8s/security/iam/verify_existing_oauth_integrations.sh b/k8s/security/iam/verify_existing_oauth_integrations.sh deleted file mode 100755 index e5e0e59..0000000 --- a/k8s/security/iam/verify_existing_oauth_integrations.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env bash -# verify_existing_oauth_integrations.sh -# Verification script that checks existing OIDC integrations -# (Grafana, MinIO, Forgejo, Argo CD) are still working after updates. -# -# Usage: -# bash k8s/talos-iam/verify_existing_oauth_integrations.sh - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}" -export KUBECONFIG - -echo "╔════════════════════════════════════════════════════════════════════╗" -echo "║ OAuth Integrations Verification ║" -echo "╚════════════════════════════════════════════════════════════════════╝" -echo "" - -# Check prerequisites -echo "📋 Checking prerequisites..." -required_cmds=("kubectl" "curl" "jq") -for cmd in "${required_cmds[@]}"; do - if ! command -v "$cmd" &> /dev/null; then - echo "❌ $cmd not found in PATH" - exit 1 - fi -done - -# Check namespaces exist -for ns in iam logging storage; do - if ! kubectl get ns "$ns" &> /dev/null; then - echo "❌ Namespace $ns not found" - exit 1 - fi -done - -echo "✓ All prerequisites met" -echo "" - -# Function to check pod status -check_pod_status() { - local namespace=$1 - local label=$2 - local component=$3 - - echo "Checking $component..." - if kubectl get pods -n "$namespace" -l "$label" -o wide 2>/dev/null | grep -q Running; then - echo " ✓ Running" - return 0 - else - echo " ❌ Not running" - return 1 - fi -} - -# Function to test Authentik API endpoint -test_authentik_api() { - echo "Testing Authentik API..." - if ! kubectl port-forward -n iam svc/authentik-server 7000:80 &> /dev/null & then - sleep 2 - if curl -sf http://localhost:7000/-/health/ready/ &> /dev/null; then - echo " ✓ API healthy (HTTP 204)" - else - echo " ❌ API not responding" - return 1 - fi - fi -} - -# Check core services -echo "🔍 Service Status" -echo "─────────────────────────────────────────────────────────────────────" - -check_pod_status iam "app=authentik,component=server" "Authentik Server" || true -check_pod_status iam "app=authentik,component=worker" "Authentik Worker" || true -check_pod_status iam "app=authentik,component=postgresql" "Authentik PostgreSQL" || true -check_pod_status storage "app.kubernetes.io/name=vault" "Vault" || true -check_pod_status logging "app.kubernetes.io/name=grafana" "Grafana" || true -check_pod_status storage "app.kubernetes.io/name=minio" "MinIO" || true - -echo "" - -# Verify Authentik has expected providers -echo "🔐 Authentik OAuth Providers" -echo "─────────────────────────────────────────────────────────────────────" - -if [[ -z "${AUTHENTIK_BOOTSTRAP_TOKEN:-}" ]]; then - echo "⚠ AUTHENTIK_BOOTSTRAP_TOKEN not set — skipping provider verification" - echo " Set it: export AUTHENTIK_BOOTSTRAP_TOKEN=\"$(talos get cluster/AUTHENTIK_BOOTSTRAP_TOKEN --key AUTHENTIK_BOOTSTRAP_TOKEN 2>/dev/null)\"" - echo "" -else - # Port-forward to Authentik - if ! pgrep -f "kubectl port-forward.*7000:80" > /dev/null; then - kubectl port-forward -n iam svc/authentik-server 7000:80 > /dev/null 2>&1 & - sleep 2 - fi - - api_url="http://localhost:7000/api/v3" - - # Check for expected providers - for provider in grafana minio forgejo argocd talos-cli-shell; do - response=$(curl -sf -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${api_url}/providers/oauth2/?name=${provider}" 2>/dev/null || echo "{}") - - if echo "$response" | jq -e '.results[0]' &> /dev/null; then - pk=$(echo "$response" | jq -r '.results[0].pk') - client_id=$(echo "$response" | jq -r '.results[0].client_id // "N/A"') - echo " ✓ ${provider} (pk=${pk}, client_id=${client_id})" - else - echo " ✗ ${provider} NOT FOUND" - fi - done - - echo "" - - # Check for expected groups - echo "👥 Authentik Groups" - echo "─────────────────────────────────────────────────────────────────────" - - for group in homelab-admins grafana-admins grafana-viewers minio-admins minio-readonly; do - response=$(curl -sf -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \ - "${api_url}/core/groups/?name=${group}" 2>/dev/null || echo "{}") - - if echo "$response" | jq -e '.results[0]' &> /dev/null; then - echo " ✓ ${group}" - else - echo " ✗ ${group} NOT FOUND" - fi - done - - echo "" -fi - -# Verify K8s secrets for apps -echo "🔑 Kubernetes Secrets" -echo "─────────────────────────────────────────────────────────────────────" - -for secret_spec in "grafana-oidc:logging" "minio-oidc:storage" "authentik-oidc-forgejo:cicd" "oidc-secret:cicd"; do - IFS=':' read -r secret_name ns <<< "$secret_spec" - if kubectl get secret "$secret_name" -n "$ns" &> /dev/null 2>&1; then - size=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath='{.data}' 2>/dev/null | wc -c) - echo " ✓ ${secret_name} → ns/${ns} (${size} bytes)" - else - echo " ✗ ${secret_name} → ns/${ns} NOT FOUND" - fi -done - -echo "" - -# Verify Vault JWT auth -echo "🔐 Vault JWT Authentication" -echo "─────────────────────────────────────────────────────────────────────" - -if command -v vault &> /dev/null; then - # Check if Vault auth method is enabled - vault_status=$(kubectl exec -n storage vault-0 -- vault auth list -format=json 2>/dev/null | jq 'keys' || echo "[]") - - if echo "$vault_status" | jq -e '.[] | select(. == "jwt/")' &> /dev/null; then - echo " ✓ JWT auth method enabled" - - # List JWT roles - roles=$(kubectl exec -n storage vault-0 -- vault list auth/jwt/role -format=json 2>/dev/null | jq '.[]' || echo "") - if [[ -n "$roles" ]]; then - echo " ✓ JWT roles found:" - echo "$roles" | while read role; do - echo " - ${role}" - done - else - echo " ✗ No JWT roles found" - fi - else - echo " ⚠ JWT auth method not enabled" - fi -else - echo " ⚠ vault CLI not found — skipping Vault checks" -fi - -echo "" - -# Summary -echo "╔════════════════════════════════════════════════════════════════════╗" -echo "║ Verification Complete ║" -echo "╚════════════════════════════════════════════════════════════════════╝" -echo "" - -echo "Next steps:" -echo "" -echo "1. Verify OIDC login flow (Grafana):" -echo " kubectl port-forward -n logging svc/grafana 3000:80" -echo " # Open http://localhost:3000/login" -echo " # Should show 'Sign in with ...' option" -echo "" - -echo "2. Verify MinIO OIDC (if configured):" -echo " kubectl port-forward -n storage svc/minio 9001:9001" -echo " # Open http://localhost:9001" -echo " # Should show identity provider option" -echo "" - -echo "3. Verify Vault JWT role:" -echo " kubectl port-forward -n storage svc/vault 8200:8200" -echo " export VAULT_ADDR=http://127.0.0.1:8200" -echo " vault read auth/jwt/role/shell" -echo "" - -echo "4. Test JWT authentication to Vault:" -echo " # Get ID token from Authentik (requires app integration)" -echo " # Then authenticate: vault login -method=jwt role=shell jwt=\$ID_TOKEN" -echo "" diff --git a/k8s/security/kustomization.yaml b/k8s/security/kustomization.yaml deleted file mode 100644 index 8b71d08..0000000 --- a/k8s/security/kustomization.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -metadata: - name: security - -# Layer 4: Security — identity, access control, secrets -# Dependencies: bootstrap (networking), platform (logging) -# Order: Applied fourth - -resources: - - iam/ - - ci-cd/ diff --git a/k8s/security/sops-secrets/.gitkeep b/k8s/security/sops-secrets/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/k8s/security/sops-secrets/00-README.md b/k8s/security/sops-secrets/00-README.md deleted file mode 100644 index 942cf5d..0000000 --- a/k8s/security/sops-secrets/00-README.md +++ /dev/null @@ -1,6 +0,0 @@ -# SOPS-Decrypted Secrets - -Placeholder for CMP-generated secrets (pending full CMP plugin setup). - -Currently, manually-created Secrets live in individual namespaces (e.g., grafana-admin in logging/). -Once the CMP plugin is wired, this Application will decode *.enc.yaml and emit all Secrets here. diff --git a/kustomize b/kustomize deleted file mode 100755 index ae9fb32..0000000 Binary files a/kustomize and /dev/null differ diff --git a/migrate-to-gitops.sh b/migrate-to-gitops.sh deleted file mode 100755 index ee46fd4..0000000 --- a/migrate-to-gitops.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env bash -# -# Incremental GitOps Migration - Live Cluster -# Migrates existing cluster to new bootstrap-local + GitOps structure -# -set -euo pipefail - -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log() { echo -e "${GREEN}[$(date +'%H:%M:%S')]${NC} $*"; } -warn() { echo -e "${YELLOW}[$(date +'%H:%M:%S')]${NC} $*"; } -error() { echo -e "${RED}[$(date +'%H:%M:%S')]${NC} $*"; } - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -log "=========================================" -log "GitOps Migration - Live Cluster" -log "=========================================" -echo "" - -# Step 1: Verify prerequisites -log "Step 1: Verifying prerequisites..." -kubectl cluster-info > /dev/null || { error "kubectl not configured"; exit 1; } -command -v argocd > /dev/null || warn "ArgoCD CLI not found - will use kubectl only" - -# Step 2: Update data-schemas path (critical - removes ddb-cluster duplication) -log "Step 2: Updating data-schemas Application path..." -log " Current: k8s/data (includes ddb-cluster - DUPLICATION)" -log " New: k8s/data/schemas (schemas only - no duplication)" - -if kubectl get application data-schemas -n argocd &>/dev/null; then - kubectl patch application data-schemas -n argocd --type=json -p='[ - { - "op": "replace", - "path": "/spec/source/path", - "value": "k8s/data/schemas" - }, - { - "op": "replace", - "path": "/metadata/annotations/argocd.argoproj.io~1sync-wave", - "value": "6" - } - ]' - log " ✓ data-schemas updated" -else - warn " data-schemas Application not found - will be created by homelab-root" -fi - -# Step 3: Apply Longhorn 3-node HA config (already working, but ensure it's in git) -log "Step 3: Verifying Longhorn 3-node HA configuration..." -if [[ -d "k8s/infrastructure/longhorn" ]]; then - kubectl apply -k k8s/infrastructure/longhorn/ || warn "Longhorn config apply failed (may already be applied)" - log " ✓ Longhorn 3-node HA config applied" -else - warn " Longhorn config not found - skipping" -fi - -# Step 4: Update homelab-root if needed -log "Step 4: Ensuring app-of-apps root is up to date..." -kubectl apply -k k8s/argocd/root/ -log " ✓ homelab-root updated" - -# Step 5: Sync all applications in wave order -log "Step 5: Syncing all applications..." -if command -v argocd &>/dev/null; then - log " Using ArgoCD CLI for sync..." - argocd app sync homelab-root --prune || warn "homelab-root sync failed - check manually" - - # Wait a bit for child apps to be created - sleep 5 - - # Sync all apps - argocd app sync --all --timeout 600 || warn "Some apps may need manual intervention" -else - log " ArgoCD CLI not found - triggering sync via kubectl..." - # Refresh all apps - kubectl get applications -n argocd -o name | while read app; do - kubectl patch $app -n argocd --type=merge -p='{"operation":{"initiatedBy":{"username":"kubectl"},"sync":{"revision":"HEAD"}}}' - done -fi - -# Step 6: Wait for critical apps -log "Step 6: Waiting for critical applications..." -log " Waiting for cert-manager..." -kubectl wait --for=condition=available --timeout=300s deployment/cert-manager -n cert-manager 2>/dev/null || warn "cert-manager not ready" - -log " Waiting for ingress-nginx..." -kubectl wait --for=condition=available --timeout=300s deployment/ingress-nginx-controller -n ingress-nginx 2>/dev/null || warn "ingress-nginx not ready" - -log " Waiting for DDB cluster..." -kubectl wait --for=jsonpath='{.status.phase}'='Cluster in healthy state' --timeout=300s cluster/ddb-cluster -n ddb 2>/dev/null || warn "DDB not ready" - -# Step 7: Verify final state -log "Step 7: Verifying final state..." -echo "" - -log "==> ArgoCD Applications:" -kubectl get applications -n argocd --no-headers | head -20 - -echo "" -log "==> Storage verification:" -NODE_COUNT=$(kubectl get nodes.longhorn.io -n longhorn-system --no-headers 2>/dev/null | wc -l | tr -d ' ') -VOLUME_REPLICAS=$(kubectl get volumes.longhorn.io -n longhorn-system -o jsonpath='{.items[0].spec.numberOfReplicas}' 2>/dev/null || echo "0") -log " Longhorn nodes: $NODE_COUNT/3" -log " Volume replicas: $VOLUME_REPLICAS (should be 3)" - -echo "" -log "==> DDB cluster:" -kubectl get cluster -n ddb 2>/dev/null || echo "Not found" - -echo "" -log "==> Forgejo:" -kubectl get deployment,svc -n cicd | grep forgejo || echo "Not found" - -echo "" -log "=========================================" -log "✅ Migration Complete!" -log "=========================================" -echo "" -echo "Next steps:" -echo " 1. Verify all apps are Synced + Healthy:" -echo " kubectl get applications -n argocd" -echo " argocd app list" -echo "" -echo " 2. Access services:" -echo " ArgoCD: https://argocd.riotpiao.com" -echo " Forgejo: https://forgejo.riotpiao.com" -echo " Grafana: https://grafana.riotpiao.com" -echo "" -echo " 3. Update CLAUDE.md to reflect 3-node HA storage" -echo " (all volumes now have 3 replicas across 3 nodes)" -echo ""