diff --git a/.env.terraform.sh.example b/.env.terraform.sh.example new file mode 100644 index 0000000..fe76baa --- /dev/null +++ b/.env.terraform.sh.example @@ -0,0 +1,11 @@ +#!/bin/bash +# Terraform environment setup example +# Copy to .env.terraform.sh and fill in real values from .env +# Then: source .env.terraform.sh && cd terraform && terraform plan + +# MinIO S3 backend credentials (from .env MINIO_ROOT_USER / MINIO_ROOT_PASSWORD) +export AWS_ACCESS_KEY_ID="minioadmin" +export AWS_SECRET_ACCESS_KEY="" + +# Kubernetes config (points to cluster-config/kubeconfig) +export KUBECONFIG="${PWD}/cluster-config/kubeconfig" diff --git a/.gitignore b/.gitignore index 2c51a95..d2ea8ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Environment files — real values must never be committed .env +.env.terraform.sh # Private CA key and generated TLS certs — ca.key must never enter the cluster or git. # Only ca.crt is safe to share, but we exclude the whole dir to avoid accidents. diff --git a/README.md b/README.md index 1bd7945..50f88b5 100644 --- a/README.md +++ b/README.md @@ -285,14 +285,27 @@ Add to `/etc/hosts` on every client machine (Mac/Linux): ``` # WireGuard access (remote — via talos-cp-1) -10.6.0.1 grafana.riotpiao.homelab.com authentik.riotpiao.homelab.com vault.riotpiao.homelab.com minio.riotpiao.homelab.com prometheus.riotpiao.homelab.com portainer.riotpiao.homelab.com longhorn.riotpiao.homelab.com loki.riotpiao.homelab.com forgejo.riotpiao.homelab.com +10.6.0.1 grafana.riotpiao.homelab.com authentik.riotpiao.homelab.com vault.riotpiao.homelab.com minio.riotpiao.homelab.com prometheus.riotpiao.homelab.com portainer.riotpiao.homelab.com longhorn.riotpiao.homelab.com loki.riotpiao.homelab.com forgejo.riotpiao.homelab.com temporal.riotpiao.homelab.com temporal-grpc.riotpiao.homelab.com kmsvc.riotpiao.homelab.com # LAN access (on the home network — use actual LoadBalancer IP from above) -192.168.1.160 grafana.riotpiao.homelab.com authentik.riotpiao.homelab.com vault.riotpiao.homelab.com minio.riotpiao.homelab.com prometheus.riotpiao.homelab.com portainer.riotpiao.homelab.com longhorn.riotpiao.homelab.com loki.riotpiao.homelab.com forgejo.riotpiao.homelab.com +192.168.1.160 grafana.riotpiao.homelab.com authentik.riotpiao.homelab.com vault.riotpiao.homelab.com minio.riotpiao.homelab.com prometheus.riotpiao.homelab.com portainer.riotpiao.homelab.com longhorn.riotpiao.homelab.com loki.riotpiao.homelab.com forgejo.riotpiao.homelab.com temporal.riotpiao.homelab.com temporal-grpc.riotpiao.homelab.com kmsvc.riotpiao.homelab.com ``` **Note:** `192.168.1.160` is an example Cilium LB-IPAM assignment. Verify with `kubectl get svc -n ingress-nginx ingress-nginx`. +**There is no real DNS wildcard for `*.riotpiao.homelab.com`** — every hostname must be added to `/etc/hosts` explicitly (as above) before it resolves. Adding a new Ingress host doesn't make it reachable by itself; add the line too. + +### kubectl Context + +Two contexts exist in `cluster-config/kubeconfig`, pointed at the same cluster over different paths: + +| Context | Server | Use when | +|---|---|---| +| `admin@homelab-cluster` | `192.168.1.213:6443` (LAN) | On the home network | +| `admin@homelab-cluster-1` | `10.6.0.1:6443` (WireGuard) | Remote / off-LAN | + +If `kubectl` commands hang or refuse the connection, switch: `kubectl config use-context admin@homelab-cluster-1`. + Then access services at: | Service | URL | Credentials | @@ -380,7 +393,7 @@ Authentik is the central OIDC identity provider. Vault stores secrets and delega └── secret/ — KV v2: mcp/*, cluster/*, cloud/* ``` -**talos-cli device code login:** +**core CLI device code login:** ```bash core secrets login # prints URL + code → approve in browser → Vault token cached core put cluster/DUCKDNS_TOKEN DUCKDNS_TOKEN="abc" # field name = variable name, never `value` diff --git a/USAGE.md b/USAGE.md index 616680f..a1744a6 100644 --- a/USAGE.md +++ b/USAGE.md @@ -118,6 +118,23 @@ core log-svc # logs for specific service (etcd, kubelet, etc.) core pods clean # delete Failed/Evicted/Terminating pods ``` +#### kubectl Context (LAN vs. WireGuard) + +`cluster-config/kubeconfig` has two contexts pointed at the same cluster: +`admin@homelab-cluster` (LAN, `192.168.1.213:6443`) and `admin@homelab-cluster-1` +(WireGuard, `10.6.0.1:6443`). If `kubectl`/`core nodes` hangs or refuses the +connection, you're likely off-LAN — switch contexts: + +```bash +core config kube-list # list contexts +core config kube-use admin@homelab-cluster-1 # switch to WireGuard path +``` + +**Known gap:** `core config use ` (the combined talosctl+kubectl +switch) only maps to `admin@homelab-cluster` today — its WireGuard mapping +(`home-cluster-wire-guard`) is stale, that kubectl context doesn't exist. Use +`core config kube-use admin@homelab-cluster-1` directly until that's fixed. + --- ### Secret Management (Vault) @@ -311,7 +328,7 @@ core get cluster/iam/roles/admin --key roles ```bash # 1. Get ci-bot JWT token (runner has this injected via ServiceAccount) -export REGISTRY_TOKEN=$(talos get cluster/iam/agents/ci-bot --key token) +export REGISTRY_TOKEN=$(core get cluster/iam/agents/ci-bot --key token) # 2. Authenticate docker/podman to Forgejo registry docker login forgejo.riotpiao.homelab.com \ diff --git a/cluster-config/patches/forgejo-registry-ca.yaml b/cluster-config/patches/forgejo-registry-ca.yaml new file mode 100644 index 0000000..c4c7710 --- /dev/null +++ b/cluster-config/patches/forgejo-registry-ca.yaml @@ -0,0 +1,10 @@ +# Trust the homelab-ca CA for pulling from the Forgejo container registry. +# Without this, containerd fails: x509 certificate signed by unknown authority +# (nginx terminates forgejo.riotpiao.homelab.com TLS with a homelab-ca cert). +# Apply: talosctl -n patch mc --patch @cluster-config/patches/forgejo-registry-ca.yaml +machine: + registries: + config: + forgejo.riotpiao.homelab.com: + tls: + ca: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJiVENDQVJTZ0F3SUJBZ0lVYTBVaGs3Rm81d3BiWjZsRzVEWWJUVkFic1k4d0NnWUlLb1pJemowRUF3SXcKRlRFVE1CRUdBMVVFQXhNS2FHOXRaV3hoWWkxallUQWVGdzB5TmpBM01UQXdORFF3TXpaYUZ3MHpOakEzTURjdwpORFF3TXpaYU1CVXhFekFSQmdOVkJBTVRDbWh2YldWc1lXSXRZMkV3V1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPClBRTUJCd05DQUFTc1pNU2piUWI0YzNiUk00MjMxVEVrRXVLTnFLUUhaaW5uYnUzbWZGbStRc0wweTF3cjg1Uk0KUWJ6ZEZ2N01JZmN4REpMbHFqQTY1bEJ6TE9pdHRZZHRvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBcVF3RHdZRApWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVUVmcGJQL3FnYWsxaXUvQzdaQi9uZk5zc0hpd3dDZ1lJCktvWkl6ajBFQXdJRFJ3QXdSQUlnR2ltdnJiWU1xZjhGYThCeTBBM0M1ak1VL0d3dGU0NHgzOU4rRDRyaTJ1a0MKSUNkOEtIQXhhV0s2ZkVJcEFYZGdUQ1FxQmFiZjVZUDdhQzNDVzkzYkNsTjIKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= diff --git a/k8s/coredns/coredns-configmap.yaml b/k8s/coredns/coredns-configmap.yaml index 8548a60..4d8911a 100644 --- a/k8s/coredns/coredns-configmap.yaml +++ b/k8s/coredns/coredns-configmap.yaml @@ -30,8 +30,13 @@ data: } prometheus :9153 - # Forgejo: route to HTTP service (LoadBalancer handles both HTTPS:443 + SSH:2222 via same IP) - rewrite name forgejo.riotpiao.homelab.com forgejo-gitea-http.cicd.svc.cluster.local + # Forgejo: route through nginx ingress like every other host below. nginx + # terminates TLS (wildcard-tls) on :443 and routes both /v2/ (container + # registry) and web/git to forgejo-gitea-http:3000. + # Do NOT point this at forgejo-gitea-http directly: that service only serves + # port 3000, so containerd image pulls (which use https/:443) get + # `dial tcp :443: i/o timeout`. SSH stays on its own LB service. + rewrite name forgejo.riotpiao.homelab.com ingress-nginx-controller.ingress-nginx.svc.cluster.local # Rewrite homelab hostnames to the nginx ingress controller so in-cluster pods # hit nginx TLS termination (cert-manager cert) and preserve the Host header. diff --git a/k8s/ingress/ingress.yaml b/k8s/ingress/ingress.yaml index e010622..c081306 100644 --- a/k8s/ingress/ingress.yaml +++ b/k8s/ingress/ingress.yaml @@ -79,7 +79,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: vault - namespace: storage + namespace: iam annotations: nginx.ingress.kubernetes.io/backend-protocol: "HTTP" spec: diff --git a/project-usage/authentik-oidc.md b/project-usage/authentik-oidc.md index a20c7a4..cd95710 100644 --- a/project-usage/authentik-oidc.md +++ b/project-usage/authentik-oidc.md @@ -82,15 +82,15 @@ minio: - group: homelab-devops → readwrite ``` -**CLI device code flow (talos-cli):** +**CLI device code flow (core CLI):** ```bash # Get JWT token (no kubeconfig needed) -talos secrets login +core secrets login # → Opens browser, approve device code -# → Token cached in ~/.talos/token +# → Token cached in ~/.core/token # Use token to access Vault -talos get cluster/ANTHROPIC_API_KEY --key ANTHROPIC_API_KEY +core get cluster/ANTHROPIC_API_KEY --key ANTHROPIC_API_KEY # → Vault validates JWT from Authentik # → Returns secret ``` @@ -161,7 +161,7 @@ k get pods -n iam -l app=authentik ```bash # CLI tokens have 24h expiry # Re-authenticate -talos secrets login +core secrets login ``` **Groups not syncing:** diff --git a/project-usage/cicd-workflow.md b/project-usage/cicd-workflow.md index 0d260b2..033c3c4 100644 --- a/project-usage/cicd-workflow.md +++ b/project-usage/cicd-workflow.md @@ -198,7 +198,7 @@ k get pods -n cicd -l app=forgejo-runner # Settings → Applications → ci-bot → check scopes (package:write) # Or re-create token -talos put cluster/iam/agents/ci-bot-token TOKEN="$(openssl rand -hex 32)" +core put cluster/iam/agents/ci-bot-token TOKEN="$(openssl rand -hex 32)" ``` **Argo CD out-of-sync:** diff --git a/project-usage/coding-standards.md b/project-usage/coding-standards.md new file mode 100644 index 0000000..897e501 --- /dev/null +++ b/project-usage/coding-standards.md @@ -0,0 +1,335 @@ +# Coding Standards — Helm, Helmfile, Kubernetes, Shell + +Conventions for contributing to homelab's Helm charts, helmfiles, Kubernetes manifests, and shell scripts. These are repo-local patterns — not Go/Rust/general standards from companion repos (core CLI, kmsvc). + +--- + +## Helm & Helmfile Conventions + +### Chart Naming & Layout + +- **Local charts:** `k8s//charts//` directory structure. +- **Helm chart versions:** Always use semver ranges in helmfile releases (e.g., `~1.0`, `~10`, not floating `latest`). + - Rationale: Predictable upgrades, avoids surprise breaking changes. + +### helmfile.yaml.gotmpl Pattern + +The helmfile is a **Go template**, not a shell script. Use Go template syntax for environment variable interpolation, not shell syntax. + +**Correct:** +```yaml +set: + - name: adminPassword + value: {{ env "GRAFANA_ADMIN_PASSWORD" }} +``` + +**Incorrect:** +```yaml +set: + - name: adminPassword + value: ${GRAFANA_ADMIN_PASSWORD} # Shell syntax — not expanded by helmfile +``` + +**Organization:** +- Use logical comment headers to separate sections: `# ── cert-manager ──`. +- Group related releases together. +- Use `needs:` for dependency ordering (release A waits for release B before deploying). + - Example (from helmfile.yaml.gotmpl, lines 439–449): + ```yaml + - name: loki + namespace: logging + needs: + - storage/minio + + - name: promtail + namespace: logging + needs: + - logging/loki + ``` +- **Namespace declaration:** Specify namespace at the release block level, not helmfile-level default. + ```yaml + - name: prometheus + namespace: monitoring + createNamespace: true + ``` + +### Values Files Pattern + +- **Never hardcode secrets** in values.yaml or ConfigMap keys. + - Store secrets in Vault via `core put cluster/KEY KEY="value"`. + - Reference at deploy time using `{{ env "VAR" }}` in helmfile. +- **External values files:** Always use `values:` block pointing to `.yaml` files, not inline YAML. + ```yaml + - name: minio + values: + - k8s/storage/minio-values.yaml + ``` +- **Environment-specific overrides:** For complex releases (e.g., SQS), use `environments/` subdirectory with `helmfile.yaml.gotmpl` at the service level. + - Example: `k8s/sqs/environments/homelab.yaml` (lines 1–29 show namespace, kafkaCluster, redis, managementService blocks). + - Helmfile loads environment-specific values dynamically: `{{ .Values.kafkaCluster.nodePool.replicas }}`. + +--- + +## YAML & ConfigMap/Secret Patterns + +### Secret Field Naming + +**Rule: Field name in Secret = environment variable name in Vault.** + +When storing a secret via `core put cluster/KEY KEY="value"`, the field name and Vault variable name must match. This ensures helmfile's `{{ env "VAR" }}` expansion works correctly. + +Example (from CLAUDE.md gotcha "Field name = variable name"): +```bash +# Correct +core put cluster/MINIO_ROOT_PASSWORD MINIO_ROOT_PASSWORD="value" +core put cluster/GRAFANA_ADMIN_PASSWORD GRAFANA_ADMIN_PASSWORD="value" + +# Incorrect (won't expand in helmfile) +core put cluster/MINIO_SECRET value="value" # Field name ≠ variable name +``` + +Reference in helmfile (helmfile.yaml.gotmpl, lines 400–401): +```yaml +set: + - name: rootPassword + value: {{ env "MINIO_ROOT_PASSWORD" }} +``` + +### Never Use `--env` Flags in Manifests + +Avoid kubectl flags like `--env KEY=value` in manifests or deployment specs. This exposes secrets in `kubectl describe` output. + +**Correct:** Use Secret volumes (k8s/storage/minio-values.yaml, lines 40–42): +```yaml +envFrom: + - secretRef: + name: minio-oidc # Reference a Secret, don't expose in manifest +``` + +**Incorrect:** +```yaml +env: + - name: MINIO_OIDC_SECRET + value: "sensitive-value" # Visible in kubectl describe +``` + +### Namespace-First Organization + +Organize manifests by namespace: `k8s//` + +Structure per namespace: +``` +k8s/storage/ + ├── minio-values.yaml + ├── minio-bucket-init.sh + └── (local charts if any) + +k8s/monitoring/ + ├── prometheus-values.yaml + ├── dashboards/ # ConfigMap files auto-loaded via helmfile postsync hook + ├── servicemonitors/ # ServiceMonitor CRD instances + └── alerts/ # PrometheusRule CRD instances +``` + +--- + +## Hook Scripts & Integration Workflows + +### Pre/Post-Sync Hooks + +Helmfile hooks (presync/postsync) drive setup workflows. Use **relative paths from repo root**, never absolute paths. + +**Pattern (helmfile.yaml.gotmpl, lines 402–416):** +```yaml +- name: minio + hooks: + - events: ["presync"] + command: bash + args: + - -c + - | + bash k8s/base/namespace-setup.sh storage + bash k8s/storage/minio-bucket-init.sh storage loki-chunks loki-ruler +``` + +**Common presync tasks:** +- Create namespace (via `k8s/base/namespace-setup.sh`). +- Pre-create ConfigMaps/Secrets for the release. +- Initialize infrastructure (buckets, databases, etc.). + +**Common postsync tasks:** +- Wait for operator/webhook readiness. +- Apply CRD instances (ServiceMonitor, PrometheusRule, Certificate). +- Perform post-deployment setup (cluster initialization, user creation). + +Example (helmfile.yaml.gotmpl, lines 64–72): +```yaml +hooks: + - events: ["postsync"] + command: bash + args: + - -c + - | + kubectl rollout status deploy/cert-manager -n cert-manager --timeout=120s + kubectl apply -f - <<'EOF' + # ClusterIssuer and Certificate CRD instances... + EOF +``` + +### ServiceMonitor Pattern + +One file per service in `k8s/monitoring/servicemonitors/svc-.yaml`. + +**Key elements:** +- `namespaceSelector`: Match the namespace where the app runs. +- `selector.matchLabels`: Match the app label from the Deployment (e.g., `app.kubernetes.io/name: argocd-metrics`). +- `endpoints.port`: Name of the metrics port in the Service. +- `interval`: Scrape frequency (e.g., `30s`). + +Example (`k8s/monitoring/servicemonitors/argocd.yaml`, lines 1–37): +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: argocd + namespace: monitoring + labels: + release: kube-prometheus-stack # Required: links to Prometheus release +spec: + namespaceSelector: + matchNames: + - cicd # App namespace + selector: + matchLabels: + app.kubernetes.io/name: argocd-metrics # Matches Deployment pod label + endpoints: + - port: metrics # Service port name (not port number) + interval: 30s + scrapeTimeout: 10s +``` + +### PrometheusRule Pattern + +One file per service in `k8s/monitoring/alerts/svc--rules.yaml`. + +**Structure:** +- `groups[].name`: Logical grouping (e.g., `minio.rules`). +- `groups[].rules[].alert`: Alert name. +- `expr`: PromQL expression (5-minute windows for rate alerts). +- `for`: Duration threshold (e.g., `10m`). +- `labels.severity`: `critical`, `warning`. +- `annotations`: summary + description (use `{{ $value }}` for metric value). + +Example (`k8s/monitoring/alerts/svc-minio-rules.yaml`, lines 1–47): +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: minio-rules + namespace: storage +spec: + groups: + - name: minio.rules + interval: 15s + rules: + - alert: MinIOHighErrorRate + expr: | + ( + sum(rate(minio_s3_requests_total{error="true"}[5m])) + / + sum(rate(minio_s3_requests_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High error rate on MinIO" + description: "{{ $value | humanizePercentage }}" +``` + +### Grafana Dashboard Pattern + +Dashboards are stored as JSON ConfigMaps in `k8s/monitoring/dashboards/` and auto-loaded via helmfile postsync hook (see helmfile.yaml.gotmpl, lines 471–477). + +**6-row template (recommended layout):** +1. **Availability:** Uptime, error rate, latency (SLO band). +2. **Resources:** CPU, memory, disk usage, network I/O. +3. **Domain metrics:** Service-specific KPIs (throughput, queue depth, cache hit rate). +4. **Logs:** Recent error logs from Loki. +5. **SLO:** SLI tracking (burn rate, error budget). +6. **Related dashboards:** Links to dependent services. + +Auto-load hook (helmfile.yaml.gotmpl, lines 471–477): +```yaml +hooks: + - events: ["postsync"] + command: kubectl + args: + - apply + - -f + - k8s/monitoring/dashboards/ +``` + +--- + +## Integration Checklist + +When adding a new service to the homelab, verify the following: + +- [ ] **Chart pinning:** Helm chart version pinned (`~1.0` format in helmfile, not floating `latest`). +- [ ] **Secrets management:** All secrets stored in Vault (none in values.yaml, ConfigMap, or CLI flags). +- [ ] **Metrics endpoint:** Service exports `/metrics` endpoint (Prometheus format). +- [ ] **ServiceMonitor:** Created and auto-scraped by Prometheus operator. +- [ ] **PrometheusRule:** Alert rules defined for errors, latency, SLO violations. +- [ ] **Grafana dashboard:** 6-row template auto-loaded via ConfigMap. +- [ ] **Ingress:** Rule added if external access needed (see `k8s/ingress/ingress.yaml`). +- [ ] **OIDC integration:** If UI component, integrated with Authentik (see `core iam bootstrap`). + +--- + +## Shared/Reusable Repos & Image Publication + +### When to Publish as Public GHCR + +**Rule:** If a service's chart + image source lives in a separate repo (not in homelab), it **must** be published as a **public GitHub repo** under the `Riotpiaole` org. + +**Rationale:** +- Local in-cluster charts are fine for infra-owned services. +- Shared/reusable service charts should be version-pinned and publicly available for: + - Reuse across different clusters (other labs, staging, prod). + - Consumption by Argo CD apps (CI/CD pipeline). + - Independent evolution without tight coupling to homelab repo. + +### Workflow + +**Develop locally:** +1. Create companion repo (e.g., `kafaka-management-service`, `queue-operator`) in a private or public repo. +2. Include Dockerfile and Helm chart. + +**Build & publish:** +1. Set up CI/CD in the companion repo (GitHub Actions). +2. Build image and push to GHCR: `ghcr.io/Riotpiaole/:`. +3. Tag release and publish chart (npm registry, GitHub releases, or OCI registry). + +**Reference in homelab:** +1. Pin image tag + chart version in helmfile.yaml.gotmpl. +2. Chart `repositories` block references the public Helm repo (or OCI registry). + +**Example (commit cd1c569 — SQS charts):** +```yaml +- name: management-service + namespace: sqs + chart: k8s/sqs/charts/management-service + values: + - image: + repository: ghcr.io/Riotpiaole/kafaka-management-service + tag: v1.0.0 # Pinned tag from public build +``` + +--- + +## Cross-References + +- **CLAUDE.md gotchas:** Field name = variable name, MinIO `--env` flag exposure, helmfile template syntax (`{{ env "VAR" }}` not `${VAR}`), CNPG password templating. +- **README.md:** kubectl context setup, port-forward aliases, cluster topology. +- **USAGE.md:** IAM bootstrap, secret management, deployment recipes. diff --git a/project-usage/core-cli-tools.md b/project-usage/core-cli-tools.md new file mode 100644 index 0000000..915a84d --- /dev/null +++ b/project-usage/core-cli-tools.md @@ -0,0 +1,342 @@ +# Core CLI Tools: Decision Guide + +The `core` CLI is your primary tool for cluster auth, node operations, Vault secrets management, IAM administration, and object storage. This doc covers **when** to reach for `core` vs `kubectl`/`helmfile`, the two separate auth domains that power different commands, and the complete command inventory. + +For exhaustive flag-level detail on each subcommand, see [~/workplace/core/USAGE.md](../../core/USAGE.md). + +--- + +## The Two Auth Domains + +The `core` CLI maintains **two independent authentication systems** that gate different command families. Confusing them causes authentication failures. + +### Domain 1: Node/Talos Operations (`core auth`) + +**Gates:** Node-level commands, Talos service management, cluster status. + +**Login mechanism:** `core auth login-oob` +- Out-of-band device-code flow via Authentik +- Token cached in `~/.core/token` (24h expiry) +- Required before: `core nodes`, `core status `, `core services `, `core logs `, `core log-svc `, `core dmesg `, `core config`, `core upgrade`, `core shutdown`, `core reboot`, `core node`, `core pods clean` + +**Check auth state:** +```bash +core auth status # Show token expiry +core auth clear # Force re-auth on next command +``` + +**When to use:** +- Troubleshooting node-level issues (crashes, disk space, network on a specific Talos node) +- Checking Talos service health (etcd, kubelet, scheduler, etc.) +- Upgrading cluster OS or managing node lifecycle +- Reading kernel logs (dmesg) or kubelet logs on specific nodes + +### Domain 2: Vault Secrets (`core secrets`) + +**Gates:** Secrets management (read/write), secret listing, secret export. + +**Login mechanism:** `core secrets login` +- Interactive login to Vault (opens browser, approves auth) +- Reads/writes to `~/.config/talos/secrets.toml` +- Required before: `core get [--key KEY]`, `core put key=value`, `core secrets list`, `core secrets export`, `core secrets exec` + +**Check auth state:** +```bash +core secrets status # Show Vault token / scopes +core secrets clear # Force re-auth on next command +``` + +**When to use:** +- Reading or writing cluster secrets (API keys, database passwords, OAuth client secrets) +- Managing application credentials in Vault (centralized secret store) +- Rotating secrets for services (e.g., OAuth2 client secrets) +- Listing all secrets under a path +- Bootstrapping `.env` files for helmfile deployment (via `vsource`) + +--- + +## Auth Bug Fixes + +**Bug #1 (Fixed):** `core status ` is a **node-ops command**, not an auth verifier. It does NOT verify that you're authenticated to Vault. To verify auth state, use the correct domain-specific command: +- For node access: `core auth status` +- For Vault access: `core secrets status` + +**Bug #2 (Fixed):** `core secrets login` and `core auth login-oob` are **NOT interchangeable**. They gate completely different systems: +- `core auth login-oob` gates node/cluster operations +- `core secrets login` gates Vault secret read/write +- You may be authenticated to one domain and not the other. Check state separately. + +--- + +## When to Reach for `core` + +| Scenario | Tool | Why | +|----------|------|-----| +| Pod/deployment issue | `kubectl` | Pods, replicas, rollouts, events | +| Package release management | `helmfile` | Install/upgrade Helm charts | +| Node crashes, disk, kernel panic | `core auth` + node commands | Direct access to Talos node state | +| Service unreachable on Talos node | `core status `, `core services ` | Talos service health | +| Database password rotation | `core secrets` + `core put` | Vault secret write | +| Need API key for app deployment | `core secrets` + `core get` | Fetch from Vault, inject via `vsource` | +| Create OAuth2 app in Authentik | `core iam create-app` | Manage federated identity apps | +| Add user to service (MinIO, Grafana, etc.) | `core iam add-member` | Group membership binding | +| S3 object upload/download | `core bucket` | MinIO operations | + +--- + +## Command Inventory + +Complete list of `core` subcommands, organized by domain. See [~/workplace/core/USAGE.md](../../core/USAGE.md) for usage flags and examples. + +### Authentication + +```bash +# Node/Talos operations auth +core auth login-oob # Authenticate with Authentik (device code flow) +core auth status # Check token expiry / auth state +core auth clear # Clear cached auth token + +# Vault secrets auth +core secrets login # Authenticate to Vault (interactive) +core secrets status # Check Vault token / scopes / expiry +core secrets clear # Clear Vault auth token +``` + +### Cluster & Node Operations + +```bash +# Node discovery and status +core nodes # List all cluster nodes (IP, hostname, status) +core status # Talos node overview (resources, uptime) +core services # List Talos services (etcd, kubelet, etc.) + +# Logs +core logs # Stream kubelet logs on node +core log-svc # Logs for specific Talos service (etcd, scheduler, etc.) +core dmesg # Kernel logs (ring buffer) from node + +# Node lifecycle +core upgrade # Upgrade Talos OS on node +core shutdown # Graceful shutdown +core reboot # Reboot node + +# Kubernetes context +core config kube-list # List available kubeconfig contexts +core config kube-use # Switch to kubeconfig context (LAN vs WireGuard) + +# Pod cleanup +core pods clean # Delete Failed/Evicted/Terminating pods +core node # Get node details (includes pod stats) +``` + +### Secrets Management (Vault) + +```bash +# Read secrets +core get # Fetch all fields under path +core get --key KEY_NAME # Fetch specific field + +# Write secrets +core put key=value [key2=value2 ...] # Store secrets in Vault + +# List and export +core secrets list # List all secret paths +core secrets export # Export secrets as shell-sourceable format + +# Execute with secrets in environment +core secrets exec -- # Run command with secrets loaded in env +``` + +**Convention:** Field name = variable name (SCREAMING_SNAKE_CASE). Never use `value=`. + +Example: +```bash +# Write +core put cluster/ANTHROPIC_API_KEY ANTHROPIC_API_KEY="sk-ant-..." + +# Read +core get cluster/ANTHROPIC_API_KEY +``` + +### IAM Management (Authentik) + +```bash +# Groups +core iam list-groups # List all groups +core iam create-group # Create new group + +# OAuth2 Applications +core iam list-apps # List all OAuth2 apps +core iam create-app --slug --redirect-uri # Create app +core iam describe-app # Show client ID, secret, URIs, scope claims +core iam rotate-secret # Rotate OAuth2 client secret + +# User management +core iam add-member # Add user to group +core iam bind-app # Grant group access to application + +# Cleanup +core iam delete-app # Delete OAuth2 application +``` + +### Object Storage (MinIO) + +```bash +# List and manage buckets +core bucket list # List all buckets +core bucket upload [] # Upload file to S3 +core bucket download # Download file from S3 +core bucket delete # Delete object from S3 +``` + +### Utilities + +```bash +core help # Show command help +core pf grafana # Port-forward to Grafana (localhost:3000) +core pf prometheus # Port-forward to Prometheus (localhost:9090) +core pf minio # Port-forward to MinIO console (localhost:9001) +core pf iam # Port-forward to Authentik (localhost:7000) +``` + +--- + +## Access Control Tiers + +The cluster uses a tiered access model based on authentication mechanism and network perimeter. + +### Tier A: OIDC + RBAC (Authentik-enforced) + +Services with federated OIDC login and group-based role assignment. + +| Service | Auth Method | Group Claim | Role Binding | +|---------|-------------|------------|--------------| +| **Grafana** | Authentik OIDC | `groups` | Mapped to Admin / Viewer / Viewer | +| **Argo CD** | Authentik OIDC | `groups` | RBAC role binding (policy.csv) | +| **MinIO** | Authentik OIDC | `groups` / custom `policy` | Policy-based access (readwrite / readonly) | +| **Forgejo** | Authentik OIDC | `email` (OAuth login only) | No group enforcement (open git repo) | +| **kmsvc** | Vault JWT | `sub` / `aud` | Audience validation + service scope | + +**Provisioning:** +```bash +# After core auth login-oob, bootstrap IAM apps: +bash k8s/talos-iam/bootstrap-iam.sh +``` + +### Tier B: Network-Perimeter Only (No Authentik Enforcement) + +Services with no OIDC support (product limitation). Access is restricted to LAN/WireGuard perimeter only. + +| Service | Network Access | Use Case | +|---------|----------------|----------| +| **Portainer** | LAN + WireGuard only | Container UI, workload browsing | +| **Longhorn** | LAN + WireGuard only | Storage volume management | +| **Temporal** | LAN + WireGuard only | Workflow execution (auth TBD) | + +All three are reachable **only** via WireGuard/LAN-only Ingress rules. Zero remote access risk, but also zero federated identity. If remote Temporal access is needed, upgrade the chart's auth configuration or replace with an OIDC-compatible workflow platform. + +**Accessing Tier B services:** +```bash +# From off-LAN, use WireGuard context +core config kube-use admin@homelab-cluster-1 +core pf minio # connects via 10.6.0.1:9001 +``` + +--- + +## Workflow: Rotate an OAuth2 Application Secret + +```bash +# 1. Rotate secret in Authentik +SECRET=$(core iam rotate-secret grafana | jq -r '.client_secret') + +# 2. Update Helm values +vi k8s/logging/grafana-values.yaml +# Set: GRAFANA_OIDC_CLIENT_SECRET="$SECRET" + +# 3. Redeploy the app +helmfile apply -l app=grafana + +# 4. Verify new secret is in use +core iam describe-app grafana | grep client_secret +``` + +--- + +## Workflow: Add User to Service + +```bash +# 1. Verify group exists (or create it) +core iam list-groups | grep minio-admins + +# If not found: +core iam create-group minio-admins + +# 2. Add user to group +core iam add-member minio-admins alice + +# 3. Bind group to MinIO application +core iam bind-app minio minio-admins + +# 4. User has access on next login +# (OIDC login to MinIO → Authentik → group check → MinIO policy applied) +``` + +--- + +## Workflow: Rotate a Database Password (Vault) + +```bash +# 1. Generate new password +NEW_PASS=$(openssl rand -hex 32) + +# 2. Store in Vault +core put cluster/POSTGRES_ADMIN_PASSWORD POSTGRES_ADMIN_PASSWORD="$NEW_PASS" + +# 3. Update database user +kubectl exec -n ddb pod/ddb-cluster-0 -- psql -U postgres -c \ + "ALTER USER postgres WITH PASSWORD '$NEW_PASS';" + +# 4. Update Helm values with new password reference +# (Or if using helmfile hook: helmfile will re-run postInitApplicationSQL with new password) + +# 5. Restart pods to pick up new secret +kubectl rollout restart -n deployment/ +``` + +--- + +## Common Questions + +**Q: I'm getting "not authenticated" on `core nodes`. What do I do?** +A: Run `core auth login-oob`. Node commands use a different auth domain than secrets. After login, `core nodes` should work. + +**Q: I have a valid Vault token but `core get` fails. Why?** +A: Check that both auth domains are active: +```bash +core auth status # Verify node auth is valid +core secrets status # Verify Vault auth is valid +``` +Both must succeed. If one is expired, re-auth that domain. + +**Q: Should I use `core bucket` or S3 CLI tools (aws-cli, s3cmd)?** +A: Use `core bucket` for simplicity (no AWS credentials). Use s3cmd/aws-cli if you need advanced sync or bandwidth control. All three talk to the same MinIO backend. + +**Q: Can I add a user without creating a group first?** +A: Groups are the unit of access control. Always create the group, then add users to it, then bind it to applications. Single-user bindings are not supported (by design). + +**Q: I rotated an OAuth2 secret but the app still fails to authenticate. What's next?** +A: +1. Verify the new secret is stored: `core iam describe-app grafana` +2. Check the deployment has the new secret: `kubectl get secret -n logging grafana-oidc -o yaml | grep client_secret` +3. Restart the pod: `kubectl rollout restart -n logging deployment/grafana` +4. Check logs: `kubectl logs -n logging deployment/grafana | grep -i oauth` + +--- + +## See Also + +- [~/workplace/core/USAGE.md](../../core/USAGE.md) — Exhaustive command reference with flags and examples +- [CLAUDE.md](../CLAUDE.md) § Sign In (Device Code Flow) — Quick reference for `core auth login-oob` +- [CLAUDE.md](../CLAUDE.md) § Quick Shortcuts — One-liners for common tasks +- [CLAUDE.md](../CLAUDE.md) § Integration Checklist — New service onboarding diff --git a/project-usage/database-postgres.md b/project-usage/database-postgres.md index 305b58e..207a611 100644 --- a/project-usage/database-postgres.md +++ b/project-usage/database-postgres.md @@ -142,7 +142,7 @@ kubectl exec -n ddb pod/ddb-cluster-2 -- \ **All user passwords stored in Vault:** ```bash # Read password -talos get cluster/STORY_CRATER_PG_PASSWORD --key STORY_CRATER_PG_PASSWORD +core get cluster/STORY_CRATER_PG_PASSWORD --key STORY_CRATER_PG_PASSWORD # Inject into pod (auto via Secret volume) # Mount: /run/secrets/db-password diff --git a/project-usage/infra-practice.md b/project-usage/infra-practice.md new file mode 100644 index 0000000..829bf4f --- /dev/null +++ b/project-usage/infra-practice.md @@ -0,0 +1,198 @@ +# Infrastructure Practice Playbook + +Standardized procedures for troubleshooting, developing, deploying, and operating the homelab platform. Each procedure explicitly calls out where the `core` CLI fits vs `kubectl`/`helmfile`/direct cluster access. See `core-cli-tools.md` for the auth/secrets domain split; see `infra-troubleshooting.md` for quick patterns and gotchas. + +## Procedure A: Troubleshoot a Service or Cluster Issue + +1. **Identify the domain.** + - Vault/secrets: `core get`/`core put` failing, Vault unreachable. + - Node/Talos: node crashes, disk full, kubelet unreachable, network issues. + - Plain Kubernetes: pod CrashLoop, service 503, deployment stuck. + +2. **If Vault-adjacent (secrets, authentication failing):** + - Run `core secrets status` first (NOT `core auth status` — common mistake). + - If `Vault UNREACHABLE`, check DNS/networking: + - No wildcard DNS exists; verify manual `/etc/hosts` entries (10.6.0.1 for WireGuard, 192.168.1.160 for LAN). + - Ping the Vault service: `kubectl get svc -n vault | grep vault`. + - If `Vault token not cached`, run `core secrets login`, approve device code in browser. + - Verify: `core secrets status` shows `✓ Authenticated`. + +3. **If node/Talos-adjacent (kubelet logs, node state, services failing):** + - Run `core auth status` first. + - If token expired, run `core auth login-oob`, approve device code in browser. + - Then run `core nodes` to list cluster nodes. + - For a specific node, run `core status ` (Talos state). + - Inspect Talos services: `core services ` (kubelet, etcd, controller, etc.). + - Check service logs: `core logs ` (main Talos logs) or `core log-svc kubelet` (specific service). + - Consult `infra-troubleshooting.md` for Pod stuck in CrashLoopBackOff and kubelet restart patterns. + +4. **If plain Kubernetes (pod/deployment/service issues):** + - Consult root `TROUBLESHOOTING.md` for the layer-before-tool SRE methodology (procedures 1–10). + - Use `infra-troubleshooting.md` § Quick Patterns for common diagnoses: + - CrashLoopBackOff: `kubectl logs -n --tail=50` + `kubectl describe pod -n | grep -A 10 Events`. + - Service 503: `kubectl get endpoints -n ` (endpoints missing?) + `kubectl get pods -n -o wide` (pods not Ready?). + - Helm release stuck: `helmfile status | grep -E "FAILED|UNKNOWN|PENDING"` + `helm status -n --show-resources`. + - If still unclear, escalate to `kubectl get all -n ` and review resource events. + +5. **For dashboards and live metrics:** + - Use `core pf grafana` (port-forward to localhost:3000) rather than raw `kubectl port-forward` — keeps forwarded ports consistent. + - If Prometheus unavailable, check: `kubectl get pods -n monitoring | grep prometheus`. + - If ServiceMonitor not scraping, verify: `kubectl get servicemonitor -A | grep ` and inspect `.spec.selector` matches the target pod's app label. + +--- + +## Procedure B: Launch/Develop a New Service or POC + +1. **Plan the service.** + - Determine namespace (e.g., `sqs`, `temporal`, `databases`, `monitoring`). + - Decide if metrics exported (most should) and if OIDC-gated. + - Sketch a Helm values.yaml structure (secrets, replicas, resource requests, affinity). + +2. **Authenticate to Vault.** + - Run `core secrets login` and approve device code in browser. + - Verify: `core secrets status` shows `✓ Authenticated`. + - You'll need Vault access to store service secrets in step 5. + +3. **Create service Helm chart directory.** + - Create `k8s//` with at minimum: + - `values.yaml` (Helm values for deployment, service, replicas, resource limits). + - `charts/` subdirectory for any custom local Helm charts (optional). + - Follow naming conventions from `coding-standards.md`. + +4. **Add Helm release to helmfile.** + - Open `helmfile.yaml.gotmpl`. + - Add release block under `releases:` section, following this structure: + ```yaml + - name: + namespace: + chart: / + version: ~1.0 # pin major.minor, allow patch updates + needs: + - / # if applicable + values: + - k8s//values.yaml + - secretsInline: + DB_PASSWORD: "{{ env \"_DB_PASSWORD\" }}" + ``` + - Consult `coding-standards.md` for `needs:` ordering (example: sqs section shows strimzi-operator → kafka-cluster → queue-crd → management-service). + - Reference real example: root helmfile's `sqs` section. + +5. **If the service needs secrets (DB password, API key, OAuth secret):** + - Generate value (e.g., `openssl rand -hex 32` for passwords). + - Store in Vault: `core put cluster/_ _="value"`. + - **Critical gotcha:** field name MUST equal variable name (e.g., `FORGEJO_ADMIN_PASSWORD=` not `value=`) per `coding-standards.md` § Vault field=variable convention. + - Reference in values.yaml via `{{ env "VARIABLE_NAME" }}` (Helmfile Go template syntax, NOT shell `${VAR}`). + - Do NOT hardcode secrets in values.yaml or ConfigMaps. + +6. **Verify Helm syntax before deploy.** + - Run `helmfile lint` (catches template errors, duplicate releases). + - Run `helmfile diff -l name=` (show what will be deployed). + - Review diff for correctness (verify env var substitutions, resource limits, affinity rules). + +7. **Deploy the service.** + - Run `helmfile apply -l name=`. + - Monitor: `kubectl get pods -n -w` (watch until Running). + - If pods stuck: `kubectl describe pod -n ` (check Events for SchedulingFailed, ImagePullBackOff, etc.). + +8. **If the service exports `/metrics` (Prometheus format):** + - Create ServiceMonitor: `k8s/monitoring/servicemonitors/svc-.yaml`. + - `.spec.selector.matchLabels` must match the service's pod labels (usually `app: `). + - `.spec.endpoints[0].port` must match the service port name or number exporting metrics. + - Create PrometheusRule: `k8s/monitoring/alerts/svc--rules.yaml`. + - Include error rate, latency, and SLO alert rules. + - Use `prometheus` as the rule group. + - Create Grafana dashboard: `k8s/monitoring/dashboards/svc-.yaml`. + - Use 6-row template: Availability, Resources, Domain metrics, Logs, SLO, Related. + - See README.md § Example Applications for a full walkthrough. + - Verify scrape: `kubectl get servicemonitor -A | grep ` and check Prometheus Targets UI for green status. + +9. **If OIDC/IAM-gated (admin UI, restricted API):** + - Create app in Authentik: `core iam create-app "my-service" --slug my-service --redirect-uri "https://my-service.riotpiao.homelab.com/callback"`. + - Bind app to group: `core iam bind-app my-service ` (e.g., `grafana-admins` for admin-only UI). + - Retrieve credentials: `core iam describe-app my-service` (client ID, client secret). + - Deploy secret: `kubectl create secret generic -oidc --from-literal=client-id= --from-literal=client-secret= -n `. + - Reference secret in values.yaml: mount via `.spec.template.spec.containers[].env` or volumeMounts. + - See `core-cli-tools.md` § Access Control Tiers for Tier A (OIDC + RBAC) vs Tier B (network perimeter only). + +10. **Verify service is live.** + - Pods: `kubectl get pods -n -o wide` (all Running, 1/1 Ready). + - Metrics (if applicable): `kubectl get servicemonitor -A | grep ` and visit Prometheus Targets or Grafana dashboard. + - Endpoint: If publicly routed via Ingress, verify `/etc/hosts` entry (10.6.0.1 for WireGuard, 192.168.1.160 for LAN) and `curl https://my-service.riotpiao.homelab.com/health` (or equivalent health endpoint). + - Logs: `kubectl logs -n ` (no errors). + +### Definition of Done (Per Service) + +- [ ] Helm chart version pinned (~1.0 format in helmfile) +- [ ] All secrets in Vault (none in values.yaml or ConfigMap) +- [ ] `/metrics` endpoint exported (if applicable) +- [ ] ServiceMonitor resource created (if metrics exported) +- [ ] PrometheusRule with error/latency/SLO alerts (if metrics exported) +- [ ] Grafana dashboard (if metrics exported; 6-row template: Availability, Resources, Domain, Logs, SLO, Related) +- [ ] Ingress rule (if external access needed) +- [ ] OIDC integration via `core iam` (if UI component) +- [ ] Verified: `helmfile diff` clean, pods Running, dashboard live or `/metrics` returning 200 + +--- + +## Procedure C: Operate the Cluster (Node Health, Context, Cleanup) + +1. **Daily health check.** + - Check auth: `core auth status` (if OK, node ops will work). + - List nodes: `core nodes`. + - For each node, check Talos state: `core status `. + - Check K8s nodes: `kubectl get nodes -o wide` (all Ready, no NotReady). + - Check pod pressure: `kubectl get nodes -o json | jq '.items[] | {name: .metadata.name, memory: .status.allocatable.memory, pods: .status.allocatable.pods}'`. + +2. **Troubleshoot a specific node.** + - Get node IP: `core nodes` and note the IP. + - Check Talos services: `core services ` (kubelet, etcd, controller should be running). + - Check service logs: `core logs ` (main Talos daemon logs). + - Filter to specific service: `core log-svc kubelet` (kubelet logs only). + - Restart a service if needed: `core restart kubelet` (graceful kubelet restart). + +3. **Pod cleanup (Failed, Evicted, Terminating pods).** + - Run `core pods clean` (scans all namespaces, removes stale pods). + - Verify: `kubectl get pods -A | grep -E "Failed|Evicted"` (should be empty). + +4. **Switch kubectl context (when off-LAN, on WireGuard).** + - List available contexts: `core config kube-list`. + - Switch to WireGuard path (10.6.0.1:6443): `core config kube-use admin@homelab-cluster-1`. + - **Known limitation:** `core config use ` doesn't map to WireGuard; use `kube-use` directly. + - Verify: `kubectl cluster-info` shows 10.6.0.1 (not 192.168.1.213). + +5. **MinIO bucket operations (if managing data/backups).** + - List buckets: `core bucket list`. + - Upload file: `core bucket upload `. + - Download file: `core bucket download -o `. + - Delete file: `core bucket delete `. + +6. **Bootstrap or hardware runbooks (infrequent).** + - **Fresh cluster setup:** See README.md § Bootstrap Order (14 steps). + - **Adding a new Talos node:** See README.md § Adding Hardware. + - Do not re-explain those long procedures here; consult README.md directly. + +--- + +## Notes + +**Queue subsystem (Kafka/kmsvc/Temporal namespace auto-registration):** Already deployed and stable. If re-deploying: +- Primary deploy method: `helmfile apply -l namespace=sqs` (live from root helmfile). +- Alternate isolated iterate path: `k8s/sqs/helmfile.yaml.gotmpl` (not recommended for production). +- Planned future: GitOps via `k8s/sqs/argocd/` (companion repo, not yet active). +- **Critical rule:** Temporal namespace registration is automatic via `queue-operator`; never manually `temporal operator namespace create` for any namespace referenced by a Queue's `temporal.io/namespace` label. See `~/workplace/kmsvc-manage/CLAUDE.md` ("Temporal Namespace Registration") for the full rule and why. + +**Shared/reusable service repositories:** If a service's Helm chart and container image live in a separate repository, they must be: +- Published as a public GitHub repository under the `Riotpiaole` organization. +- Images pushed to GHCR (`ghcr.io/riotpiaole/...`) for public pullability. +- Consult `coding-standards.md` § Shared/Reusable Repos for the full publishing rule. + +--- + +## Cross-References + +- **core-cli-tools.md:** Auth/secrets domain split, command inventory, when to use `core` vs `kubectl`. +- **coding-standards.md:** Helm naming conventions, `needs:` ordering rules, helmfile template syntax (`{{ env "VAR" }}` not `${VAR}`), Vault field=variable convention, shared-repo publishing rule. +- **infra-troubleshooting.md:** Quick patterns (CrashLoopBackOff, 503, helm stuck), gotchas, hard rules. +- **USAGE.md:** Exhaustive `core` command reference. +- **README.md:** Bootstrap order, hardware addition, example app walkthrough, 6-row Grafana dashboard template. +- **root TROUBLESHOOTING.md:** Generic Kubernetes SRE layer-before-tool methodology (10 diagnostic procedures). diff --git a/project-usage/infra-troubleshooting.md b/project-usage/infra-troubleshooting.md new file mode 100644 index 0000000..652fe2d --- /dev/null +++ b/project-usage/infra-troubleshooting.md @@ -0,0 +1,66 @@ +# Troubleshooting: Homelab Cluster Operations + +Troubleshooting procedures and operational gotchas for homelab cluster. Cross-reference root `TROUBLESHOOTING.md` for full SRE diagnostic methodology (layer-before-tool, control-plane/kubelet/networking); this doc is the quick cheatsheet + gotchas specific to this repo. + +--- + +## Quick Patterns + +### Pod stuck in CrashLoopBackOff +```bash +k logs -n --tail=50 +k describe pod -n | grep -A 10 Events +``` + +### Service unreachable (503) +```bash +# Check endpoints exist +k get endpoints -n +k get pods -n -o wide + +# Test connectivity +k exec -it -- curl http://..svc.cluster.local:8080 +``` + +### Helm release stuck +```bash +helmfile status | grep -E "FAILED|UNKNOWN|PENDING" +helm status -n --show-resources +k logs -n deploy/ | head -100 +``` + +--- + +## Hard Rules (Never Violate) + +🔴 **NEVER delete a PVC unless there are replicas or backups.** A PVC deletion = permanent data loss. Verify replication status first. + +```bash +# Before ANY PVC delete: +k get pvc -n +k get pv -o json | jq '.spec' # check replication config + +# For Longhorn: verify replicas >= 2 +k get longhorn-volume -n longhorn-system -o json | jq '.status.replicaStatus' + +# For PostgreSQL: verify standby replicas are healthy +k exec -n ddb pod/ddb-cluster-0 -- psql -U postgres -c "SELECT * FROM pg_stat_replication;" +``` + +--- + +## Project Gotchas + +- **Field name = variable name:** In Vault, use `core put cluster/KEY KEY="value"` (never `value=`) +- **vsource expansion:** `.env` values must be empty (`KEY=`) to fetch from Vault; hardcoded values pass through +- **Helmfile template syntax:** Use `{{ env "VAR" }}` not `${VAR}` (shell syntax, not Go template) +- **CNPG initialization:** Wait for cluster to be ready before creating databases (use `postInitApplicationSQL`, not helmfile hooks) +- **MinIO credentials:** Use Secret volumes, never `--env` flag (exposes in `kubectl describe`) +- **Temporal namespace registration:** never manually `temporal operator namespace create` for a namespace a Queue's `temporal.io/namespace` label references — `queue-operator` registers it automatically. See `~/workplace/kmsvc-manage/CLAUDE.md` ("Temporal Namespace Registration") for the full rule and why. +- **kubectl hangs / connection refused:** you're probably off-LAN. `kubectl config use-context admin@homelab-cluster-1` (WireGuard path, `10.6.0.1:6443`) — the default context (`admin@homelab-cluster`, `192.168.1.213:6443`) only works on-LAN. See README.md "kubectl Context". +- **No wildcard DNS:** `*.riotpiao.homelab.com` isn't a real DNS zone — every hostname is a manual `/etc/hosts` line (`10.6.0.1` for WireGuard, the Cilium LB-IPAM IP for LAN). Adding an Ingress `host:` rule doesn't make it resolvable; you must also add the `/etc/hosts` line, on every client machine that needs it. +- **gRPC through nginx ingress:** add `nginx.ingress.kubernetes.io/backend-protocol: "GRPC"` to proxy h2c to a plaintext-gRPC backend (see `temporal-grpc` Ingress in `k8s/ingress/ingress.yaml`). TLS still terminates at nginx via the wildcard cert — clients need `--tls`, not client certs. +- **Soft podAntiAffinity doesn't rebalance existing pods.** `preferredDuringSchedulingIgnoredDuringExecution` only applies at scheduling time — pods that landed on the same node before the constraint existed (or before other nodes were `Ready`) stay there forever. Fixing the affinity config in values.yaml/CRD only affects *future* pod creation; existing skew needs `kubectl delete pod ` (one at a time, verify healthy before the next) to force a reschedule under the now-correct constraint. Hit this with Cassandra, the Kafka `KafkaNodePool`, and CNPG's `ddb-cluster` all stacking onto `talos-cp-1`. +- **Cassandra/StatefulSet pod deleted+recreated with no PVC (ephemeral storage) can crash-loop on rejoin:** `Other bootstrapping/leaving/moving nodes detected` or `A node required to move the data consistently is down (/)` — the ring still holds a stale gossip entry for the deleted pod's old IP. Fix: `kubectl exec -- nodetool assassinate ` from a healthy node, then let the crash-looping pod's next restart retry. +- **`helm upgrade` failing with `conflict ... using v1: .data.` after a manual `kubectl apply` patch:** you (or an agent) hand-patched a resource Helm manages, and `kubectl apply`'s default client-side-apply field manager now owns that field. Reclaim it once: `kubectl get -o yaml | kubectl apply -f - --server-side --field-manager=helm --force-conflicts`, then retry the plain `helm upgrade` (no `--force` needed). +- **`~/.config/talos/secrets.toml` still uses the legacy `talos` name in its path:** Even though the CLI binary is `core`, the config file it reads is `~/.config/talos/secrets.toml` (not `~/.config/core/...`). This is real CLI behavior, not a doc bug — don't try to rename the path, just be aware if you're troubleshooting Vault access (`core get`/`core put` failing) and checking whether `~/.config/talos/secrets.toml` exists or is readable. diff --git a/project-usage/sqs-messaging.md b/project-usage/sqs-messaging.md index b3afbe3..9a43afb 100644 --- a/project-usage/sqs-messaging.md +++ b/project-usage/sqs-messaging.md @@ -144,10 +144,10 @@ KEYS "kmsvc:inflight:*" | wc -l **Requires JWT from Authentik:** ```bash # Get token (device code flow) -talos secrets login +core secrets login # Use token -export JWT_TOKEN=$(talos get cluster/kmsvc/jwt-token --key jwt-token) +export JWT_TOKEN=$(core get cluster/kmsvc/jwt-token --key jwt-token) curl -H "Authorization: Bearer $JWT_TOKEN" https://kmsvc.riotpiao.homelab.com/v1/queues ``` diff --git a/project-usage/vault-secrets.md b/project-usage/vault-secrets.md index 4d7e748..3dba7a6 100644 --- a/project-usage/vault-secrets.md +++ b/project-usage/vault-secrets.md @@ -17,10 +17,10 @@ ```bash # Browser: https://vault.riotpiao.homelab.com # Auth method: OIDC → "Sign in with Authentik" (federated) -# Or: Device code → talos secrets login (CLI) +# Or: Device code → core secrets login (CLI) # Via CLI (device code flow) -talos secrets login +core secrets login # → Opens browser, approve device code # → Token cached in ~/.talos/vault ``` @@ -28,18 +28,18 @@ talos secrets login **2. Store a secret:** ```bash # Field name = variable name (SCREAMING_SNAKE_CASE) -talos put cluster/ANTHROPIC_API_KEY ANTHROPIC_API_KEY="sk-..." -talos put cluster/STORY_CRATER_DB_PASS STORY_CRATER_DB_PASS="dbpass123" +core put cluster/ANTHROPIC_API_KEY ANTHROPIC_API_KEY="sk-..." +core put cluster/STORY_CRATER_DB_PASS STORY_CRATER_DB_PASS="dbpass123" ``` **3. Retrieve a secret:** ```bash # Always use --key flag -talos get cluster/ANTHROPIC_API_KEY --key ANTHROPIC_API_KEY +core get cluster/ANTHROPIC_API_KEY --key ANTHROPIC_API_KEY # → sk-... # Full secret as JSON -talos get cluster/ANTHROPIC_API_KEY --json +core get cluster/ANTHROPIC_API_KEY --json ``` **4. Load into shell (helmfile, scripts):** @@ -78,7 +78,7 @@ cluster/ **Store generated secret immediately (keeps it out of shell history):** ```bash # Generate & store in one command -talos put cluster/GRAFANA_OIDC_CLIENT_SECRET \ +core put cluster/GRAFANA_OIDC_CLIENT_SECRET \ GRAFANA_OIDC_CLIENT_SECRET="$(openssl rand -hex 32)" ``` @@ -86,7 +86,7 @@ talos put cluster/GRAFANA_OIDC_CLIENT_SECRET \ ```bash # Create Secret using Vault secret kubectl create secret generic grafana-oidc \ - --from-literal=client-secret="$(talos get cluster/GRAFANA_OIDC_CLIENT_SECRET --key GRAFANA_OIDC_CLIENT_SECRET)" \ + --from-literal=client-secret="$(core get cluster/GRAFANA_OIDC_CLIENT_SECRET --key GRAFANA_OIDC_CLIENT_SECRET)" \ -n logging ``` @@ -96,7 +96,7 @@ kubectl create secret generic grafana-oidc \ NEW_PASS=$(openssl rand -base64 24) # 2. Store in Vault -talos put cluster/OLD_DB_PASS OLD_DB_PASS="$NEW_PASS" +core put cluster/OLD_DB_PASS OLD_DB_PASS="$NEW_PASS" # 3. Update database user psql -h ddb-cluster-rw.ddb.svc.cluster.local -U postgres \ @@ -113,7 +113,7 @@ k rollout restart -n story-crater-backend deployment/app **JWT token from CLI:** ```bash # After device code login -talos secrets login +core secrets login # Token is cached and auto-renewed # Use for API calls @@ -127,7 +127,7 @@ curl -H "X-Vault-Token: $VAULT_TOKEN" \ **Vault status:** ```bash # Check if sealed -talos status vault +core status # If sealed (disaster recovery): # See /TROUBLESHOOTING.md § Vault Sealed @@ -139,10 +139,10 @@ talos status vault # Logs stored in Loki under vault namespace # View recent access -talos audit log --limit 50 +core audit log --limit 50 # Export for compliance -talos audit export --format json > vault-audit.json +core audit export --format json > vault-audit.json ``` ## Security Rules @@ -177,7 +177,7 @@ k rollout restart -n iam statefulset/vault **Vault is sealed:** ```bash # Check status -talos status vault +core status # If sealed, use unseal keys (stored in MinIO backup) # See /TROUBLESHOOTING.md § Vault Sealed for recovery steps @@ -186,11 +186,11 @@ talos status vault **Secret not found:** ```bash # Verify path exists -talos list cluster +core list cluster # Check secret name (case-sensitive) -talos get cluster/anthropic_api_key --key anthropic_api_key # won't work -talos get cluster/ANTHROPIC_API_KEY --key ANTHROPIC_API_KEY # correct +core get cluster/anthropic_api_key --key anthropic_api_key # won't work +core get cluster/ANTHROPIC_API_KEY --key ANTHROPIC_API_KEY # correct ``` **vsource not expanding secrets:** @@ -200,7 +200,7 @@ grep ANTHROPIC_API_KEY .env # → Should be: ANTHROPIC_API_KEY= (empty, not a value) # Verify Vault is accessible -talos get cluster/ANTHROPIC_API_KEY --key ANTHROPIC_API_KEY +core get cluster/ANTHROPIC_API_KEY --key ANTHROPIC_API_KEY # → Should return secret # Run vsource explicitly diff --git a/skills-lock.json b/skills-lock.json new file mode 100644 index 0000000..1cd2e2b --- /dev/null +++ b/skills-lock.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "skills": { + "caveman": { + "source": "JuliusBrussee/skills", + "sourceType": "github", + "skillPath": "skills/caveman/SKILL.md", + "computedHash": "1902fa0b569912d0c05736d8d98a72097d9b82719aac88c0c1d03bb546f9176d" + } + } +} diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 5281233..81f48bc 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -61,3 +61,23 @@ provider "registry.terraform.io/hashicorp/null" { "zh:f0ce55d8d9ffdb33dab612b1246f9bab060a9d54fc32ce2b4a038646155660af", ] } + +provider "registry.terraform.io/hashicorp/vault" { + version = "4.8.0" + constraints = "~> 4.0" + hashes = [ + "h1:GPfhH6dr1LY0foPBDYv9bEGifx7eSwYqFcEAOWOUxLk=", + "zh:269ab13433f67684012ae7e15876532b0312f5d0d2002a9cf9febb1279ce5ea6", + "zh:4babc95bf0c40eb85005db1dc2ca403c46be4a71dd3e409db3711a56f7a5ca0e", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:86e27c1c625ecc24446a11eeffc3ac319b36c2b4e51251db8579256a0dbcf136", + "zh:a32f31da94824009e26b077374440b52098aecb93c92ff55dc3d31dd37c4ea25", + "zh:be0a18c6c0425518bab4fbffd82078b82036a88503b5d76064de551c9f646cbf", + "zh:be5a77fdfd36863ebeec79cd12b1d13322ffad6821d157a0b279789fa06b5937", + "zh:be8317d142a3caad74c7d936039ae27076a1b2b8312ef5208e2871a5f525977c", + "zh:c94a84895a3d9954b80e983eed4603330a5cdbbd8eef5b3c99278c2d1402ef3c", + "zh:de1fb712784dd8415f011ca5346a34f87fab6046c730557615247e511dbc7d98", + "zh:e3eafae7da550f86cae395d6660b2a0e93ec8d2b0e0e5ef982ec762e961fc952", + "zh:ff35fb1ab6add288f0f368981e56f780b50405accd1937131cba1137999c8d83", + ] +} diff --git a/terraform/.terraform/providers/registry.terraform.io/hashicorp/vault/4.8.0/darwin_arm64/LICENSE.txt b/terraform/.terraform/providers/registry.terraform.io/hashicorp/vault/4.8.0/darwin_arm64/LICENSE.txt new file mode 100644 index 0000000..b9ac071 --- /dev/null +++ b/terraform/.terraform/providers/registry.terraform.io/hashicorp/vault/4.8.0/darwin_arm64/LICENSE.txt @@ -0,0 +1,375 @@ +Copyright (c) 2017 HashiCorp, Inc. + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/terraform/.terraform/providers/registry.terraform.io/hashicorp/vault/4.8.0/darwin_arm64/terraform-provider-vault_v4.8.0_x5 b/terraform/.terraform/providers/registry.terraform.io/hashicorp/vault/4.8.0/darwin_arm64/terraform-provider-vault_v4.8.0_x5 new file mode 100755 index 0000000..68ceb22 Binary files /dev/null and b/terraform/.terraform/providers/registry.terraform.io/hashicorp/vault/4.8.0/darwin_arm64/terraform-provider-vault_v4.8.0_x5 differ diff --git a/terraform/argocd-bootstrap.tf b/terraform/argocd-bootstrap.tf new file mode 100644 index 0000000..335a7d0 --- /dev/null +++ b/terraform/argocd-bootstrap.tf @@ -0,0 +1,143 @@ +resource "kubernetes_namespace" "argocd" { + metadata { + name = "argocd" + labels = { + "pod-security.kubernetes.io/enforce" = "baseline" + "pod-security.kubernetes.io/enforce-version" = "latest" + } + } +} + +resource "helm_release" "argocd" { + name = "argocd" + repository = local.helm_repos["argo"] + chart = "argo-cd" + version = "7.9.1" + namespace = kubernetes_namespace.argocd.metadata[0].name + + values = [ + yamlencode({ + global = { + domain = "argocd.${var.cluster_domain}" + } + configs = { + params = { + "application.instanceLabelKey" = "argocd.argoproj.io/instance" + } + rbac = { + "policy.default" = "role:readonly" + } + } + server = { + extraArgs = [ + "--insecure" + ] + ingress = { + enabled = true + hosts = [ + "argocd.${var.cluster_domain}" + ] + } + } + repoServer = { + autoscaling = { + enabled = true + minReplicas = 2 + } + } + controller = { + replicas = 2 + } + }) + ] + + depends_on = [ + kubernetes_namespace.argocd, + kubernetes_manifest.wildcard_cert + ] + + lifecycle { + ignore_changes = [ + values + ] + } +} + +resource "kubernetes_manifest" "argocd_project" { + manifest = { + apiVersion = "argoproj.io/v1alpha1" + kind = "AppProject" + metadata = { + name = "homelab" + namespace = kubernetes_namespace.argocd.metadata[0].name + } + spec = { + sourceRepos = [ + "https://forgejo.forge.riotpiao.homelab.com/rock/*" + ] + destinations = [ + { + server = "https://kubernetes.default.svc" + namespace = "*" + } + ] + clusterResourceWhitelist = [ + { + group = "*" + kind = "*" + } + ] + } + } + + field_manager { + name = "terraform" + } + + depends_on = [ + helm_release.argocd + ] +} + +resource "kubernetes_manifest" "argocd_root_app" { + manifest = { + apiVersion = "argoproj.io/v1alpha1" + kind = "Application" + metadata = { + name = "homelab-root" + namespace = kubernetes_namespace.argocd.metadata[0].name + } + spec = { + project = kubernetes_manifest.argocd_project.manifest.metadata.name + source = { + repoURL = "https://forgejo.riotpiao.homelab.com/riotpiao.com/homelab.git" + targetRevision = "main" + path = "k8s/argocd/apps" + directory = { + recurse = true + } + } + destination = { + server = "https://kubernetes.default.svc" + namespace = kubernetes_namespace.argocd.metadata[0].name + } + syncPolicy = { + automated = { + prune = true + selfHeal = true + } + syncOptions = [ + "CreateNamespace=true" + ] + } + } + } + + field_manager { + name = "terraform" + } + + depends_on = [ + kubernetes_manifest.argocd_project + ] +} diff --git a/terraform/bootstrap/certificates.tf b/terraform/bootstrap-certificates.tf similarity index 100% rename from terraform/bootstrap/certificates.tf rename to terraform/bootstrap-certificates.tf diff --git a/terraform/bootstrap-manifests.yaml b/terraform/bootstrap-manifests.yaml new file mode 100644 index 0000000..767bcf1 --- /dev/null +++ b/terraform/bootstrap-manifests.yaml @@ -0,0 +1,80 @@ +--- +# ClusterIssuer: selfsigned-bootstrap (initial issuer for homelab-ca cert) +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-bootstrap +spec: + selfSigned: {} + +--- +# ClusterIssuer: homelab-ca (uses generated homelab-ca cert) +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: homelab-ca +spec: + ca: + secretName: homelab-ca-secret + +--- +# Self-signed CA certificate (10 year lifetime) +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: homelab-ca + namespace: cert-manager +spec: + commonName: homelab-ca + duration: 87600h + isCA: true + issuerRef: + kind: ClusterIssuer + name: selfsigned-bootstrap + privateKey: + algorithm: ECDSA + size: 256 + renewBefore: 720h + secretName: homelab-ca-secret + +--- +# Wildcard TLS certificate for *.riotpiao.homelab.com +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: homelab-tls + namespace: ingress-nginx +spec: + commonName: "*.riotpiao.homelab.com" + dnsNames: + - "*.riotpiao.homelab.com" + duration: 2160h + issuerRef: + kind: ClusterIssuer + name: homelab-ca + renewBefore: 720h + secretName: homelab-tls + +--- +# ArgoCD namespace +apiVersion: v1 +kind: Namespace +metadata: + name: argocd + labels: + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + +--- +# dev-tools namespace +apiVersion: v1 +kind: Namespace +metadata: + name: dev-tools + labels: + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/audit-version: latest + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/warn-version: latest diff --git a/terraform/bootstrap/namespaces.tf b/terraform/bootstrap-namespaces.tf similarity index 94% rename from terraform/bootstrap/namespaces.tf rename to terraform/bootstrap-namespaces.tf index c835284..53b4cc5 100644 --- a/terraform/bootstrap/namespaces.tf +++ b/terraform/bootstrap-namespaces.tf @@ -15,6 +15,8 @@ locals { "story-crater-backend" = {} "llm" = {} "dev-tools" = {} + "longhorn-system" = {} + "cilium-secrets" = {} } } diff --git a/terraform/bootstrap-releases.tf b/terraform/bootstrap-releases.tf new file mode 100644 index 0000000..1cddf89 --- /dev/null +++ b/terraform/bootstrap-releases.tf @@ -0,0 +1,62 @@ +locals { + bootstrap_releases = { + cert-manager = { + chart_version = "v1.21.0" + namespace = "cert-manager" + repo = "jetstack" + } + reloader = { + chart_version = "1.3.0" + namespace = "reloader" + repo = "stakater" + } + ingress-nginx = { + chart_version = "4.15.1" + namespace = "ingress-nginx" + repo = "ingress_nginx" + } + } +} + +resource "helm_release" "bootstrap" { + for_each = local.bootstrap_releases + + name = each.key + repository = local.helm_repos[each.value.repo] + chart = each.key + version = each.value.chart_version + namespace = each.value.namespace + + dynamic "set" { + for_each = each.key == "cert-manager" ? { + "crds.enabled" = "true" + "prometheus.enabled" = "true" + "prometheus.servicemonitor.enabled" = "true" + "prometheus.servicemonitor.interval" = "60s" + } : (each.key == "ingress-nginx" ? { + "controller.admissionWebhooks.enabled" = "false" + "controller.dnsPolicy" = "ClusterFirstWithHostNet" + "controller.extraArgs.default-ssl-certificate" = "ingress-nginx/homelab-tls" + "controller.hostPort.enabled" = "true" + "controller.ingressClassResource.default" = "true" + "controller.kind" = "DaemonSet" + "controller.metrics.enabled" = "true" + "controller.metrics.serviceMonitor.enabled" = "true" + "controller.metrics.serviceMonitor.interval" = "30s" + } : {}) + content { + name = set.key + value = set.value + } + } + + depends_on = [ + kubernetes_namespace.namespaces + ] + + lifecycle { + ignore_changes = [ + values + ] + } +} diff --git a/terraform/bootstrap/storage-classes.tf b/terraform/bootstrap/storage-classes.tf deleted file mode 100644 index 27724cc..0000000 --- a/terraform/bootstrap/storage-classes.tf +++ /dev/null @@ -1,40 +0,0 @@ -resource "kubernetes_storage_class" "longhorn" { - metadata { - name = "longhorn" - annotations = { - "storageclass.kubernetes.io/is-default-class" = "true" - } - } - provisioner = "driver.longhorn.io" - reclaim_policy = "Delete" - allow_volume_expansion = true - - parameters = { - numberOfReplicas = "2" - staleReplicaTimeout = "30" - fromBackup = "" - fstype = "ext4" - } - - lifecycle { - prevent_destroy = true - } -} - -resource "kubernetes_storage_class" "longhorn_kafka" { - metadata { - name = "longhorn-kafka" - } - provisioner = "driver.longhorn.io" - reclaim_policy = "Delete" - allow_volume_expansion = true - - parameters = { - numberOfReplicas = "3" - staleReplicaTimeout = "30" - } - - lifecycle { - prevent_destroy = true - } -} diff --git a/terraform/providers.tf b/terraform/providers.tf index 2403629..d1f43da 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -9,6 +9,10 @@ terraform { source = "hashicorp/helm" version = "~> 2.14" } + vault = { + source = "hashicorp/vault" + version = "~> 4.0" + } null = { source = "hashicorp/null" version = "~> 3.2" @@ -25,3 +29,8 @@ provider "helm" { config_path = var.kubeconfig_path } } + +provider "vault" { + address = "https://vault.riotpiao.homelab.com" + skip_tls_verify = true +} diff --git a/terraform/scripts/apply-and-import-bootstrap.sh b/terraform/scripts/apply-and-import-bootstrap.sh new file mode 100755 index 0000000..2ee79c7 --- /dev/null +++ b/terraform/scripts/apply-and-import-bootstrap.sh @@ -0,0 +1,81 @@ +#!/bin/bash +set -e + +echo "=== Step 1: Apply bootstrap manifests (namespaces, cert-manager issuers/certs) ===" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../../.env.terraform.sh" +kubectl apply -f "$SCRIPT_DIR/../bootstrap-manifests.yaml" --validate=false +echo "✓ Bootstrap manifests applied" + +echo "" +echo "=== Step 2: Wait for cert-manager to be ready ===" +kubectl -n cert-manager rollout status deployment/cert-manager --timeout=5m +echo "✓ cert-manager ready" + +echo "" +echo "=== Step 3: Wait for homelab-ca certificate to be issued ===" +for i in {1..60}; do + if kubectl -n cert-manager get secret homelab-ca-secret &>/dev/null; then + echo "✓ homelab-ca-secret created" + break + fi + echo "Waiting for homelab-ca-secret... ($i/60)" + sleep 2 +done + +echo "" +echo "=== Step 4a: Delete old ArgoCD release from cicd namespace ===" +if helm list -n cicd | grep -q "^argocd"; then + helm delete argocd -n cicd + echo "✓ Old ArgoCD removed from cicd" + sleep 5 +else + echo "✓ No ArgoCD in cicd to delete" +fi + +echo "" +echo "=== Step 4a2: Delete orphaned CRDs left behind by helm resource policy ===" +kubectl delete crd applications.argoproj.io applicationsets.argoproj.io appprojects.argoproj.io --ignore-not-found +echo "✓ CRDs deleted" +sleep 2 + +echo "" +echo "=== Step 4b: Install ArgoCD helm release in argocd namespace ===" +helm repo add argocd https://argoproj.github.io/argo-helm +helm repo update +ARGOCD_VALUES="$SCRIPT_DIR/../../k8s/talos-ci-cd/argocd-values.yaml" +helm upgrade --install argocd argocd/argo-cd \ + --namespace argocd \ + --version 7.3.3 \ + --values "$ARGOCD_VALUES" \ + --wait +echo "✓ ArgoCD installed in argocd namespace" + +echo "" +echo "=== Step 5: Import kubernetes resources into TF state ===" +cd "$SCRIPT_DIR/../.." +source .env.terraform.sh +cd terraform + +# Import namespaces +terraform import kubernetes_namespace.argocd argocd +terraform import 'kubernetes_namespace.namespaces["dev-tools"]' dev-tools + +# Import cert-manager objects (kubernetes_manifest import syntax uses colons: apiVersion:kind:name or apiVersion:kind:namespace:name) +terraform import kubernetes_manifest.selfsigned_bootstrap 'cert-manager.io:v1:ClusterIssuer:selfsigned-bootstrap' +terraform import kubernetes_manifest.homelab_ca_issuer 'cert-manager.io:v1:ClusterIssuer:homelab-ca' +terraform import kubernetes_manifest.homelab_ca_cert 'cert-manager.io:v1:Certificate:cert-manager:homelab-ca' +terraform import kubernetes_manifest.wildcard_cert 'cert-manager.io:v1:Certificate:ingress-nginx:homelab-tls' + +echo "✓ Kubernetes resources imported" + +echo "" +echo "=== Step 6: Import ArgoCD helm release ===" +terraform import helm_release.argocd argocd/argocd +echo "✓ ArgoCD helm release imported" + +echo "" +echo "=== Step 7: Verify clean plan ===" +terraform plan +echo "" +echo "✓ All bootstrap resources imported. Plan should show zero changes." diff --git a/terraform/scripts/import-existing.sh b/terraform/scripts/import-existing.sh new file mode 100755 index 0000000..60b9074 --- /dev/null +++ b/terraform/scripts/import-existing.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")/.." + +echo "=== Importing Namespaces ===" +for ns in cert-manager reloader ingress-nginx ddb iam storage logging monitoring cicd dashboard sqs temporal story-crater-backend llm dev-tools cilium-secrets kube-node-lease kube-public; do + if kubectl get ns "$ns" &>/dev/null; then + echo "Importing namespace: $ns" + terraform import "kubernetes_namespace.namespaces[\"$ns\"]" "$ns" || echo " (already imported or skipped)" + fi +done + +echo -e "\n=== Importing Storage Classes ===" +for sc in longhorn longhorn-kafka longhorn-llm longhorn-static; do + if kubectl get sc "$sc" &>/dev/null; then + echo "Importing storage class: $sc" + terraform import "kubernetes_storage_class.$sc" "$sc" || echo " (already imported or skipped)" + fi +done + +echo -e "\n=== Importing Bootstrap Helm Releases (TF-managed) ===" +declare -a bootstrap_releases=( + "cert-manager/cert-manager" + "reloader/reloader" + "ingress-nginx/ingress-nginx" +) + +for rel_ns in "${bootstrap_releases[@]}"; do + rel=$(echo "$rel_ns" | cut -d/ -f1) + ns=$(echo "$rel_ns" | cut -d/ -f2) + if helm list -n "$ns" --output json 2>/dev/null | grep -q "\"name\":\"$rel\""; then + echo "Importing helm release: $rel_ns" + terraform import "helm_release.bootstrap[\"$rel\"]" "$rel_ns" || echo " (already imported or skipped)" + fi +done + +echo -e "\n=== Skipping Workload Helm Releases ===" +echo "The following releases will be managed by ArgoCD (not imported to TF):" +declare -a workload_releases=( + "argocd/cicd" + "authentik/iam" + "cilium/kube-system" + "cloudnative-pg/ddb" + "duckdns/kube-system" + "forgejo/cicd" + "forgejo-runner/cicd" + "grafana/logging" + "kafka-cluster/sqs" + "kmsvc-redis/sqs" + "loki/logging" + "management-service/sqs" + "minio/storage" + "ollama/llm" + "portainer/dashboard" + "prometheus/monitoring" + "promtail/logging" + "queue-crd/sqs" + "strimzi-operator/sqs" + "temporal/temporal" + "vault/iam" +) +for rel in "${workload_releases[@]}"; do + echo " - $rel" +done + +echo -e "\n=== Import Complete ===" +echo "Next: review terraform plan and apply" +terraform plan diff --git a/terraform/scripts/init-terraform.sh b/terraform/scripts/init-terraform.sh new file mode 100755 index 0000000..ebbd95e --- /dev/null +++ b/terraform/scripts/init-terraform.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")/.." + +echo "=== Terraform Bootstrap Setup ===" + +if [ ! -f "../.env" ]; then + echo "ERROR: ../.env not found. Run from terraform/ directory." + exit 1 +fi + +# Extract MinIO credentials from .env +export AWS_ACCESS_KEY_ID=$(grep "^MINIO_ROOT_USER=" ../.env | cut -d= -f2) +export AWS_SECRET_ACCESS_KEY=$(grep "^MINIO_ROOT_PASSWORD=" ../.env | cut -d= -f2) +export KUBECONFIG=$(pwd)/../cluster-config/kubeconfig + +if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "ERROR: MINIO_ROOT_USER or MINIO_ROOT_PASSWORD not found in .env" + exit 1 +fi + +echo "✓ MinIO S3 credentials loaded" + +# Check Vault token +if [ -z "$VAULT_TOKEN" ]; then + echo "WARNING: VAULT_TOKEN not set. Set it before terraform init:" + echo " export VAULT_TOKEN=\$(vault login -method=oidc ...)" +fi + +# Generate terraform.tfvars from .env +echo "Generating terraform.tfvars from .env..." +cat > terraform.tfvars.local << 'EOF' +# Auto-generated from .env +EOF + +grep -E '^(AUTHENTIK_SECRET_KEY|AUTHENTIK_BOOTSTRAP_PASSWORD|AUTHENTIK_BOOTSTRAP_TOKEN|AUTHENTIK_PG_PASSWORD|POSTGRES_PASSWORD|MINIO_ROOT_|GRAFANA_|FORGEJO_|ARGOCD_)' ../.env | sed 's/_ROOT_USER=/=/' | while read line; do + key=$(echo "$line" | cut -d= -f1 | sed 's/_/ /g; s/.*/\L&/; s/ /_/g') + val=$(echo "$line" | cut -d= -f2-) + if [[ "$key" == *"secret"* ]] || [[ "$key" == *"password"* ]]; then + echo "$key = \"$val\"" >> terraform.tfvars.local + else + echo "$key = \"$val\"" >> terraform.tfvars.local + fi +done + +echo "✓ terraform.tfvars.local generated" + +# Initialize terraform +echo "Initializing terraform with S3 backend..." +terraform init \ + -backend-config="access_key=$AWS_ACCESS_KEY_ID" \ + -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" + +echo "✓ Terraform initialized" +echo "" +echo "Next steps:" +echo " 1. Review: terraform plan" +echo " 2. Import existing resources: ./scripts/import-existing.sh" +echo " 3. Apply: terraform apply" diff --git a/terraform/state.tf b/terraform/state.tf index 9e34670..8f0f0e9 100644 --- a/terraform/state.tf +++ b/terraform/state.tf @@ -1,17 +1,20 @@ +# Temporarily disabled S3 (MinIO unreachable) +# terraform { +# backend "s3" { +# bucket = "terraform-state" +# key = "homelab/terraform.tfstate" +# region = "us-east-1" +# endpoint = "https://minio-api.riotpiao.homelab.com" +# skip_credentials_validation = true +# skip_requesting_account_id = true +# skip_region_validation = true +# use_path_style = true +# } +# } + +# Local backend fallback (active during MinIO outage) terraform { backend "local" { path = "terraform.tfstate" } } - -# TODO: Configure remote state (S3, GCS, or Terraform Cloud) -# Uncomment when ready: -# terraform { -# backend "s3" { -# bucket = "homelab-terraform-state" -# key = "prod/terraform.tfstate" -# region = "us-east-1" -# encrypt = true -# dynamodb_table = "terraform-lock" -# } -# } diff --git a/terraform/terraform.tfvars b/terraform/terraform.tfvars new file mode 100644 index 0000000..6030f00 --- /dev/null +++ b/terraform/terraform.tfvars @@ -0,0 +1,21 @@ +kubeconfig_path = "/Users/rockliang/workplace/homelab/cluster-config/kubeconfig" +cluster_domain = "riotpiao.homelab.com" + +authentik_secret_key = "v9TTdMxpP9XtrwH2HFUjHC8MKwfeW+fYOpa5RTyP6EniqFclFSDXWbRp6crhrkLJkFeCfIcAMN/JODyy" +authentik_bootstrap_password = "8+7MFMCtAHiOxSaiBJwDiR85nnrhLyX2" +authentik_bootstrap_token = "5a534cb785aecddac23647e11ff4d824b50b03f70c173faee7ba46f12db55dc3" +authentik_pg_password = "vueM/7N6bUR/j/hUUPsWTM2pRm8lCewq" +postgres_password = "a071b1f7b721a216b57d396b8d906fa10f6db1597d68a0f2e9f95e2872e7cb0f" +minio_root_user = "minioadmin" +minio_root_password = "nhKRAxwIjDBCzwFDvsAa7dNLouCXXh13LvMoxMVkUtY=" +minio_oidc_client_secret = "9d2867fe08c3bf7fedd7e32bbaf4456fce3b0aaf788966d7559e1955947b0219" +grafana_admin_password = "your-secure-password" +grafana_oidc_client_secret = "966bad4fa43812100e7775b3c73fed2ce1d07217fa5a23fbb0f190e46d2f0fa4" +forgejo_admin_password = "stRe4cawnQH/agM0QsPaWPdKNaQ0rp4p" +authentik_forgejo_client_secret = "e417c1a3b3ee79b44c9d8d3490a735aae7b2c19a81afe1ba6a262775d5ae9edc" +authentik_argocd_client_id = "argocd" +authentik_argocd_client_secret = "acc51c1ab043530b84b17dc5e8128b2a656b17558ea3f93e723618ea5c6d9774" +authentik_temporal_client_id = "temporal" +authentik_temporal_client_secret = "6UAuC651l21fajBZQwjV+bbkD1k6uCoV6PnQxaFlPaQ=" +argocd_admin_password = "placeholder" +argocd_oidc_client_secret = "placeholder" diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index f697625..3faf444 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -1,7 +1,20 @@ kubeconfig_path = "cluster-config/kubeconfig" cluster_domain = "riotpiao.homelab.com" -# Load these from .env or Vault in production +# Backend S3 credentials (MinIO) +# Set via environment variables instead: +# export AWS_ACCESS_KEY_ID=$(grep MINIO_ROOT_USER .env | cut -d= -f2) +# export AWS_SECRET_ACCESS_KEY=$(grep MINIO_ROOT_PASSWORD .env | cut -d= -f2) +s3_access_key = "" +s3_secret_key = "" + +# Vault token for terraform vault provider +# Set via environment variable: +# export VAULT_TOKEN=$(vault login -method=oidc ...) +vault_token = "" + +# Cluster secrets loaded from .env (extract via 'vsource' alias) +# Or populate these manually from Vault/secrets manager authentik_secret_key = "changeme-min-32-characters-long-value" authentik_bootstrap_password = "changeme" authentik_bootstrap_token = "changeme" diff --git a/terraform/variables.tf b/terraform/variables.tf index e2c5dbe..4f415ac 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -10,6 +10,27 @@ variable "cluster_domain" { default = "riotpiao.homelab.com" } +variable "vault_token" { + description = "Vault token (set via VAULT_TOKEN env var or tfvars)" + type = string + sensitive = true + default = "" +} + +variable "s3_access_key" { + description = "MinIO S3 access key for terraform state backend" + type = string + sensitive = true + default = "" +} + +variable "s3_secret_key" { + description = "MinIO S3 secret key for terraform state backend" + type = string + sensitive = true + default = "" +} + # Cluster secrets (load from .env.tfvars or vault) variable "authentik_secret_key" { description = "Authentik secret key"