From a07af6bf07a3d5c50ccdbf8b817f99cf47b3d4c2 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:15:02 -0700 Subject: [PATCH] =?UTF-8?q?feat(sms):=20add=20BlueBubbles=20iMessage=20del?= =?UTF-8?q?ivery=20(Docker-OSX=20macOS=20VM=20pinned=20to=20worker-2)=20+?= =?UTF-8?q?=20ArgoCD=20app=20+=20dedicated=20longhorn-imessage-local=20SC?= =?UTF-8?q?=20=E2=80=94=20default=20longhorn=20SC=20can't=20schedule=20a?= =?UTF-8?q?=203-replica=20200Gi=20volume=20(only=20worker-1=20has=20200Gi?= =?UTF-8?q?=20free=20at=20100%=20over-provisioning)=20and=20Immediate=20bi?= =?UTF-8?q?nding=20would=20pin=20the=20qcow2=20to=20the=20wrong=20node?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - namespace: PodSecurity privileged, needed for /dev/kvm + privileged QEMU - storageclass: 1 replica, strict-local, WaitForFirstConsumer - deployment: nodeSelector workload=imessage + matching NoSchedule toleration, Recreate strategy (two QEMU procs on one qcow2 corrupts it), no readiness probe (guest install is interactive and takes many minutes) - services: ClusterIP only; VNC is an unauthenticated console, reach it with port-forward, never an Ingress - networkpolicy: default-deny, opt-in via sms-client=true on port 1234 --- k8s/apps/sms/deployment-macos.yaml | 115 +++++++++++++++++++++++++++ k8s/apps/sms/kustomization.yaml | 10 +++ k8s/apps/sms/namespace.yaml | 19 +++++ k8s/apps/sms/networkpolicy.yaml | 22 +++++ k8s/apps/sms/pvc-macos.yaml | 25 ++++++ k8s/apps/sms/service.yaml | 31 ++++++++ k8s/apps/sms/storageclass.yaml | 32 ++++++++ k8s/argocd/apps/60-applications.yaml | 31 ++++++++ 8 files changed, 285 insertions(+) create mode 100644 k8s/apps/sms/deployment-macos.yaml create mode 100644 k8s/apps/sms/kustomization.yaml create mode 100644 k8s/apps/sms/namespace.yaml create mode 100644 k8s/apps/sms/networkpolicy.yaml create mode 100644 k8s/apps/sms/pvc-macos.yaml create mode 100644 k8s/apps/sms/service.yaml create mode 100644 k8s/apps/sms/storageclass.yaml diff --git a/k8s/apps/sms/deployment-macos.yaml b/k8s/apps/sms/deployment-macos.yaml new file mode 100644 index 0000000..667ac29 --- /dev/null +++ b/k8s/apps/sms/deployment-macos.yaml @@ -0,0 +1,115 @@ +# macOS VM (Docker-OSX) hosting the BlueBubbles server. +# +# ── Why a VM and not a container ──────────────────────────────────────────── +# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus +# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists +# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a +# container — a VM in a box, not a macOS container. +# +# ── Why this works on worker-2 ────────────────────────────────────────────── +# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and +# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a +# module). Bare metal, so no nested virtualisation needed. +# +# ── Read this before relying on it ────────────────────────────────────────── +# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over +# VNC (:5999), erase the disk in Disk Utility, install, create a user, sign +# into iMessage, THEN install BlueBubbles inside the guest. This manifest +# only provides the machine; it does not provision macOS. +# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles' +# own guidance: "test sending an iMessage to yourself. If it does not +# succeed, it's likely best to restart from the beginning." +# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This +# is a Hackintosh. Use a throwaway Apple ID, not a primary one. +# 4. BlueBubbles labels this path "not for beginners", "no guarantees or +# warranty". +# +# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled +# inside the guest and is NOT required for plain send/receive. Skip it. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: macos-bluebubbles + labels: + app.kubernetes.io/name: macos-bluebubbles + app.kubernetes.io/part-of: sms +spec: + replicas: 1 + # Recreate: the qcow2 disk is RWO and a second pod must never attach it + # concurrently — two QEMU processes on one image corrupts it. + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: macos-bluebubbles + template: + metadata: + labels: + app.kubernetes.io/name: macos-bluebubbles + app.kubernetes.io/part-of: sms + spec: + # Dedicated node. The taint keeps everything else off worker-2; this + # toleration is what lets the VM on. Both halves are required. + nodeSelector: + workload: imessage + tolerations: + - key: workload + operator: Equal + value: imessage + effect: NoSchedule + containers: + - name: macos + image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c + # QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this + # only schedules because the sms namespace is labelled privileged. + securityContext: + privileged: true + env: + # Generates a unique serial / board-serial / UUID / MAC and persists + # them to bootdisk.qcow2. This synthetic identity is what iMessage + # activates against — it must stay stable across restarts, which is + # why the PVC matters. + - name: GENERATE_UNIQUE + value: "true" + # Identity is only plausible if it matches a real product line. + - name: DEVICE_MODEL + value: "iMacPro1,1" + - name: RAM + value: "12" + - name: CORES + value: "6" + - name: EXTRA + # Expose the BlueBubbles server port from the guest to the pod. + # Guest :1234 (BlueBubbles default) -> pod :1234. + value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234" + ports: + - name: vnc + containerPort: 5999 + protocol: TCP + - name: bluebubbles + containerPort: 1234 + protocol: TCP + resources: + requests: + cpu: "6" + memory: 14Gi + limits: + cpu: "12" + memory: 20Gi + volumeMounts: + - name: macos-disk + mountPath: /home/arch/OSX-KVM/disk + - name: kvm + mountPath: /dev/kvm + # No readiness probe on purpose. The guest takes many minutes to boot, + # and until macOS + BlueBubbles are installed BY HAND there is nothing + # listening on 1234. A probe here would crash-loop the pod through the + # entire interactive install. + volumes: + - name: macos-disk + persistentVolumeClaim: + claimName: macos-disk + - name: kvm + hostPath: + path: /dev/kvm + type: CharDevice diff --git a/k8s/apps/sms/kustomization.yaml b/k8s/apps/sms/kustomization.yaml new file mode 100644 index 0000000..13c3377 --- /dev/null +++ b/k8s/apps/sms/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: sms +resources: + - namespace.yaml + - storageclass.yaml + - pvc-macos.yaml + - deployment-macos.yaml + - service.yaml + - networkpolicy.yaml diff --git a/k8s/apps/sms/namespace.yaml b/k8s/apps/sms/namespace.yaml new file mode 100644 index 0000000..73d4f05 --- /dev/null +++ b/k8s/apps/sms/namespace.yaml @@ -0,0 +1,19 @@ +# iMessage delivery for the cluster. +# +# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper +# that hooks Messages.app private APIs — it cannot be containerised on Linux, +# because containers share the host kernel and macOS needs XNU + Cocoa. The only +# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which +# needs a privileged pod with /dev/kvm. +# +# Hence privileged PodSecurity: the cluster default from the Talos controlplane +# is `enforce: baseline`, which forbids privileged containers and host devices. +# Scope is limited to this namespace. +apiVersion: v1 +kind: Namespace +metadata: + name: sms + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/k8s/apps/sms/networkpolicy.yaml b/k8s/apps/sms/networkpolicy.yaml new file mode 100644 index 0000000..a2aada0 --- /dev/null +++ b/k8s/apps/sms/networkpolicy.yaml @@ -0,0 +1,22 @@ +# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple +# ID and exposes an unauthenticated VNC console; nothing should reach it except +# opted-in clients. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: sms-default-deny +spec: + podSelector: + matchLabels: + app.kubernetes.io/part-of: sms + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: {} + podSelector: + matchLabels: + sms-client: "true" + ports: + - protocol: TCP + port: 1234 diff --git a/k8s/apps/sms/pvc-macos.yaml b/k8s/apps/sms/pvc-macos.yaml new file mode 100644 index 0000000..32118c3 --- /dev/null +++ b/k8s/apps/sms/pvc-macos.yaml @@ -0,0 +1,25 @@ +# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2). +# +# This volume is NOT disposable: it holds the VM's serial number, board serial, +# UUID and MAC, which together form the identity iMessage was activated against. +# Losing it means re-running activation, which is the least reliable step of the +# whole setup. +# +# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for +# the installer, the base system, and qcow2 growth. +# +# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3). +# Losing worker-2's disk therefore means losing the activated identity and +# redoing iMessage activation. Once the guest is installed and activated, take +# a Longhorn snapshot/backup of this volume; that is the only redundancy here. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: macos-disk +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn-imessage-local + resources: + requests: + storage: 200Gi diff --git a/k8s/apps/sms/service.yaml b/k8s/apps/sms/service.yaml new file mode 100644 index 0000000..decfa71 --- /dev/null +++ b/k8s/apps/sms/service.yaml @@ -0,0 +1,31 @@ +# VNC is how you drive the interactive macOS install. Deliberately ClusterIP — +# it is an unauthenticated console onto a machine holding a live Apple ID +# session. Reach it with `kubectl port-forward`, never an Ingress. +apiVersion: v1 +kind: Service +metadata: + name: macos-vnc +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: macos-bluebubbles + ports: + - name: vnc + port: 5999 + targetPort: vnc +--- +# The BlueBubbles REST API, once installed inside the guest. This is the stable +# name cluster services use, so callers never depend on the pod IP or on whether +# the backend is this VM or a real Mac mini later. +apiVersion: v1 +kind: Service +metadata: + name: bluebubbles +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: macos-bluebubbles + ports: + - name: http + port: 1234 + targetPort: bluebubbles diff --git a/k8s/apps/sms/storageclass.yaml b/k8s/apps/sms/storageclass.yaml new file mode 100644 index 0000000..2a96c2c --- /dev/null +++ b/k8s/apps/sms/storageclass.yaml @@ -0,0 +1,32 @@ +# Dedicated StorageClass for the macOS VM disk. +# +# The default `longhorn` class does not work here, for two independent reasons: +# +# 1. Replica count. Default is 3, and Longhorn schedules against +# storageMaximum - storageScheduled with over-provisioning at 100%. Free +# space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi +# volume has only one node that can hold even a single replica — a 3-replica +# volume fails outright with ReplicaSchedulingFailure. +# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is +# created, before any pod is scheduled. Combined with strict-local that +# pins the data to an arbitrary node, not the one the VM runs on. +# +# So: one replica, kept local to the VM, bound only once the pod has a node. +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-imessage-local +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Delete +# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so +# the replica is placed on that node and not somewhere else. +volumeBindingMode: WaitForFirstConsumer +parameters: + # A qcow2 backing a live VM is latency-sensitive and rewritten constantly. + # Serving it over the network from another node's disk would be the single + # worst thing for guest responsiveness, so force it local. + numberOfReplicas: "1" + dataLocality: "strict-local" + staleReplicaTimeout: "30" + fsType: "ext4" diff --git a/k8s/argocd/apps/60-applications.yaml b/k8s/argocd/apps/60-applications.yaml index 3d9926f..c9a61ed 100644 --- a/k8s/argocd/apps/60-applications.yaml +++ b/k8s/argocd/apps/60-applications.yaml @@ -84,6 +84,37 @@ spec: syncOptions: - CreateNamespace=true --- +# iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX) +# running the BlueBubbles server, plus its dedicated local StorageClass. +# +# Pinned to worker-2 via nodeSelector `workload: imessage` + a matching +# toleration for that node's taint. Until worker-2 is provisioned this app +# syncs everything except the pod, which stays Pending — that is expected. +# +# No CreateNamespace: namespace.yaml carries `pod-security: privileged`, which +# the VM needs (/dev/kvm, privileged), and an ArgoCD-created namespace would +# not have those labels. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: sms + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "8" +spec: + project: homelab + source: + repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git + targetRevision: main + path: k8s/apps/sms + destination: + server: https://kubernetes.default.svc + namespace: sms + syncPolicy: + automated: + prune: true + selfHeal: true +--- # Consolidated: homarr + homarr-patches → homarr # Helm chart + values + PostSync hook patch (fix-probes-job.yaml) apiVersion: argoproj.io/v1alpha1