From 6312674058704b1b0279f5cc8567de8f27b9f339 Mon Sep 17 00:00:00 2001 From: rock Date: Sat, 24 Oct 2026 20:05:19 -0700 Subject: [PATCH] chore(sms): retire the stuck macOS namespace The macOS workload is scaled to 0 and its PVC macos-disk has been Pending for 72 days against storage class longhorn-imessage-local. No node carries the workload=imessage label it pins to. The namespace only holds that dead PVC plus two unused Services. Remove the Application, the manifests and the namespace so ArgoCD prunes them. --- k8s/apps/sms/deployment-macos.yaml | 120 --------------------------- k8s/apps/sms/kustomization.yaml | 10 --- k8s/apps/sms/namespace.yaml | 19 ----- k8s/apps/sms/networkpolicy.yaml | 22 ----- k8s/apps/sms/pvc-macos.yaml | 25 ------ k8s/apps/sms/service.yaml | 31 ------- k8s/apps/sms/storageclass.yaml | 32 ------- k8s/argocd/apps/60-applications.yaml | 31 ------- 8 files changed, 290 deletions(-) delete mode 100644 k8s/apps/sms/deployment-macos.yaml delete mode 100644 k8s/apps/sms/kustomization.yaml delete mode 100644 k8s/apps/sms/namespace.yaml delete mode 100644 k8s/apps/sms/networkpolicy.yaml delete mode 100644 k8s/apps/sms/pvc-macos.yaml delete mode 100644 k8s/apps/sms/service.yaml delete mode 100644 k8s/apps/sms/storageclass.yaml diff --git a/k8s/apps/sms/deployment-macos.yaml b/k8s/apps/sms/deployment-macos.yaml deleted file mode 100644 index cda9cd1..0000000 --- a/k8s/apps/sms/deployment-macos.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# macOS VM (Docker-OSX) hosting the BlueBubbles server. -# -# ── Why a VM and not a container ──────────────────────────────────────────── -# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus -# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists -# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a -# container — a VM in a box, not a macOS container. -# -# ── Why this works on worker-2 ────────────────────────────────────────────── -# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and -# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a -# module). Bare metal, so no nested virtualisation needed. -# -# ── Read this before relying on it ────────────────────────────────────────── -# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over -# VNC (:5999), erase the disk in Disk Utility, install, create a user, sign -# into iMessage, THEN install BlueBubbles inside the guest. This manifest -# only provides the machine; it does not provision macOS. -# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles' -# own guidance: "test sending an iMessage to yourself. If it does not -# succeed, it's likely best to restart from the beginning." -# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This -# is a Hackintosh. Use a throwaway Apple ID, not a primary one. -# 4. BlueBubbles labels this path "not for beginners", "no guarantees or -# warranty". -# -# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled -# inside the guest and is NOT required for plain send/receive. Skip it. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: macos-bluebubbles - labels: - app.kubernetes.io/name: macos-bluebubbles - app.kubernetes.io/part-of: sms -spec: - # Scaled to 0 on 2026-10-24. The pod pinned itself to a node labelled - # workload=imessage, and no such node exists any more. The PVC macos-disk also - # uses StorageClass longhorn-imessage-local, which pins the same dead node and - # stayed unbound for 71 days. The macOS guest therefore cannot start. Raise - # this to 1 again only after a KVM-capable node carries that label and taint. - replicas: 0 - # Recreate: the qcow2 disk is RWO and a second pod must never attach it - # concurrently — two QEMU processes on one image corrupts it. - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: macos-bluebubbles - template: - metadata: - labels: - app.kubernetes.io/name: macos-bluebubbles - app.kubernetes.io/part-of: sms - spec: - # Dedicated node. The taint keeps everything else off worker-2; this - # toleration is what lets the VM on. Both halves are required. - nodeSelector: - workload: imessage - tolerations: - - key: workload - operator: Equal - value: imessage - effect: NoSchedule - containers: - - name: macos - image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c - # QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this - # only schedules because the sms namespace is labelled privileged. - securityContext: - privileged: true - env: - # Generates a unique serial / board-serial / UUID / MAC and persists - # them to bootdisk.qcow2. This synthetic identity is what iMessage - # activates against — it must stay stable across restarts, which is - # why the PVC matters. - - name: GENERATE_UNIQUE - value: "true" - # Identity is only plausible if it matches a real product line. - - name: DEVICE_MODEL - value: "iMacPro1,1" - - name: RAM - value: "12" - - name: CORES - value: "6" - - name: EXTRA - # Expose the BlueBubbles server port from the guest to the pod. - # Guest :1234 (BlueBubbles default) -> pod :1234. - value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234" - ports: - - name: vnc - containerPort: 5999 - protocol: TCP - - name: bluebubbles - containerPort: 1234 - protocol: TCP - resources: - requests: - cpu: "6" - memory: 14Gi - limits: - cpu: "12" - memory: 20Gi - volumeMounts: - - name: macos-disk - mountPath: /home/arch/OSX-KVM/disk - - name: kvm - mountPath: /dev/kvm - # No readiness probe on purpose. The guest takes many minutes to boot, - # and until macOS + BlueBubbles are installed BY HAND there is nothing - # listening on 1234. A probe here would crash-loop the pod through the - # entire interactive install. - volumes: - - name: macos-disk - persistentVolumeClaim: - claimName: macos-disk - - name: kvm - hostPath: - path: /dev/kvm - type: CharDevice diff --git a/k8s/apps/sms/kustomization.yaml b/k8s/apps/sms/kustomization.yaml deleted file mode 100644 index 13c3377..0000000 --- a/k8s/apps/sms/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: sms -resources: - - namespace.yaml - - storageclass.yaml - - pvc-macos.yaml - - deployment-macos.yaml - - service.yaml - - networkpolicy.yaml diff --git a/k8s/apps/sms/namespace.yaml b/k8s/apps/sms/namespace.yaml deleted file mode 100644 index 73d4f05..0000000 --- a/k8s/apps/sms/namespace.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# iMessage delivery for the cluster. -# -# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper -# that hooks Messages.app private APIs — it cannot be containerised on Linux, -# because containers share the host kernel and macOS needs XNU + Cocoa. The only -# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which -# needs a privileged pod with /dev/kvm. -# -# Hence privileged PodSecurity: the cluster default from the Talos controlplane -# is `enforce: baseline`, which forbids privileged containers and host devices. -# Scope is limited to this namespace. -apiVersion: v1 -kind: Namespace -metadata: - name: sms - labels: - pod-security.kubernetes.io/enforce: privileged - pod-security.kubernetes.io/audit: privileged - pod-security.kubernetes.io/warn: privileged diff --git a/k8s/apps/sms/networkpolicy.yaml b/k8s/apps/sms/networkpolicy.yaml deleted file mode 100644 index a2aada0..0000000 --- a/k8s/apps/sms/networkpolicy.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple -# ID and exposes an unauthenticated VNC console; nothing should reach it except -# opted-in clients. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: sms-default-deny -spec: - podSelector: - matchLabels: - app.kubernetes.io/part-of: sms - policyTypes: - - Ingress - ingress: - - from: - - namespaceSelector: {} - podSelector: - matchLabels: - sms-client: "true" - ports: - - protocol: TCP - port: 1234 diff --git a/k8s/apps/sms/pvc-macos.yaml b/k8s/apps/sms/pvc-macos.yaml deleted file mode 100644 index 32118c3..0000000 --- a/k8s/apps/sms/pvc-macos.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2). -# -# This volume is NOT disposable: it holds the VM's serial number, board serial, -# UUID and MAC, which together form the identity iMessage was activated against. -# Losing it means re-running activation, which is the least reliable step of the -# whole setup. -# -# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for -# the installer, the base system, and qcow2 growth. -# -# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3). -# Losing worker-2's disk therefore means losing the activated identity and -# redoing iMessage activation. Once the guest is installed and activated, take -# a Longhorn snapshot/backup of this volume; that is the only redundancy here. -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: macos-disk -spec: - accessModes: - - ReadWriteOnce - storageClassName: longhorn-imessage-local - resources: - requests: - storage: 200Gi diff --git a/k8s/apps/sms/service.yaml b/k8s/apps/sms/service.yaml deleted file mode 100644 index decfa71..0000000 --- a/k8s/apps/sms/service.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# VNC is how you drive the interactive macOS install. Deliberately ClusterIP — -# it is an unauthenticated console onto a machine holding a live Apple ID -# session. Reach it with `kubectl port-forward`, never an Ingress. -apiVersion: v1 -kind: Service -metadata: - name: macos-vnc -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: macos-bluebubbles - ports: - - name: vnc - port: 5999 - targetPort: vnc ---- -# The BlueBubbles REST API, once installed inside the guest. This is the stable -# name cluster services use, so callers never depend on the pod IP or on whether -# the backend is this VM or a real Mac mini later. -apiVersion: v1 -kind: Service -metadata: - name: bluebubbles -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: macos-bluebubbles - ports: - - name: http - port: 1234 - targetPort: bluebubbles diff --git a/k8s/apps/sms/storageclass.yaml b/k8s/apps/sms/storageclass.yaml deleted file mode 100644 index 2a96c2c..0000000 --- a/k8s/apps/sms/storageclass.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Dedicated StorageClass for the macOS VM disk. -# -# The default `longhorn` class does not work here, for two independent reasons: -# -# 1. Replica count. Default is 3, and Longhorn schedules against -# storageMaximum - storageScheduled with over-provisioning at 100%. Free -# space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi -# volume has only one node that can hold even a single replica — a 3-replica -# volume fails outright with ReplicaSchedulingFailure. -# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is -# created, before any pod is scheduled. Combined with strict-local that -# pins the data to an arbitrary node, not the one the VM runs on. -# -# So: one replica, kept local to the VM, bound only once the pod has a node. -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: longhorn-imessage-local -provisioner: driver.longhorn.io -allowVolumeExpansion: true -reclaimPolicy: Delete -# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so -# the replica is placed on that node and not somewhere else. -volumeBindingMode: WaitForFirstConsumer -parameters: - # A qcow2 backing a live VM is latency-sensitive and rewritten constantly. - # Serving it over the network from another node's disk would be the single - # worst thing for guest responsiveness, so force it local. - numberOfReplicas: "1" - dataLocality: "strict-local" - staleReplicaTimeout: "30" - fsType: "ext4" diff --git a/k8s/argocd/apps/60-applications.yaml b/k8s/argocd/apps/60-applications.yaml index f63f0af..e7a1534 100644 --- a/k8s/argocd/apps/60-applications.yaml +++ b/k8s/argocd/apps/60-applications.yaml @@ -110,37 +110,6 @@ spec: syncOptions: - CreateNamespace=true --- -# iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX) -# running the BlueBubbles server, plus its dedicated local StorageClass. -# -# Pinned to worker-2 via nodeSelector `workload: imessage` + a matching -# toleration for that node's taint. Until worker-2 is provisioned this app -# syncs everything except the pod, which stays Pending — that is expected. -# -# No CreateNamespace: namespace.yaml carries `pod-security: privileged`, which -# the VM needs (/dev/kvm, privileged), and an ArgoCD-created namespace would -# not have those labels. -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: sms - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "8" -spec: - project: homelab - source: - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git - targetRevision: main - path: k8s/apps/sms - destination: - server: https://kubernetes.default.svc - namespace: sms - syncPolicy: - automated: - prune: true - selfHeal: true ---- # Document management. Raw manifests (no Helm): postgres is the dedicated # paperless-db CNPG cluster in k8s/infra/databases (wave 2), redis is # in-cluster only (no PVC), media lives on the cp-3 USB HDD (see