feat(sms): add BlueBubbles iMessage delivery (Docker-OSX macOS VM pinned to worker-2) + ArgoCD app + dedicated longhorn-imessage-local SC — default longhorn SC can't schedule a 3-replica 200Gi volume (only worker-1 has 200Gi free at 100% over-provisioning) and Immediate binding would pin the qcow2 to the wrong node
- namespace: PodSecurity privileged, needed for /dev/kvm + privileged QEMU - storageclass: 1 replica, strict-local, WaitForFirstConsumer - deployment: nodeSelector workload=imessage + matching NoSchedule toleration, Recreate strategy (two QEMU procs on one qcow2 corrupts it), no readiness probe (guest install is interactive and takes many minutes) - services: ClusterIP only; VNC is an unauthenticated console, reach it with port-forward, never an Ingress - networkpolicy: default-deny, opt-in via sms-client=true on port 1234
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,115 @@
|
||||
# macOS VM (Docker-OSX) hosting the BlueBubbles server.
|
||||
#
|
||||
# ── Why a VM and not a container ────────────────────────────────────────────
|
||||
# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus
|
||||
# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists
|
||||
# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a
|
||||
# container — a VM in a box, not a macOS container.
|
||||
#
|
||||
# ── Why this works on worker-2 ──────────────────────────────────────────────
|
||||
# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and
|
||||
# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a
|
||||
# module). Bare metal, so no nested virtualisation needed.
|
||||
#
|
||||
# ── Read this before relying on it ──────────────────────────────────────────
|
||||
# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over
|
||||
# VNC (:5999), erase the disk in Disk Utility, install, create a user, sign
|
||||
# into iMessage, THEN install BlueBubbles inside the guest. This manifest
|
||||
# only provides the machine; it does not provision macOS.
|
||||
# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles'
|
||||
# own guidance: "test sending an iMessage to yourself. If it does not
|
||||
# succeed, it's likely best to restart from the beginning."
|
||||
# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This
|
||||
# is a Hackintosh. Use a throwaway Apple ID, not a primary one.
|
||||
# 4. BlueBubbles labels this path "not for beginners", "no guarantees or
|
||||
# warranty".
|
||||
#
|
||||
# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled
|
||||
# inside the guest and is NOT required for plain send/receive. Skip it.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: macos-bluebubbles
|
||||
labels:
|
||||
app.kubernetes.io/name: macos-bluebubbles
|
||||
app.kubernetes.io/part-of: sms
|
||||
spec:
|
||||
replicas: 1
|
||||
# Recreate: the qcow2 disk is RWO and a second pod must never attach it
|
||||
# concurrently — two QEMU processes on one image corrupts it.
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: macos-bluebubbles
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: macos-bluebubbles
|
||||
app.kubernetes.io/part-of: sms
|
||||
spec:
|
||||
# Dedicated node. The taint keeps everything else off worker-2; this
|
||||
# toleration is what lets the VM on. Both halves are required.
|
||||
nodeSelector:
|
||||
workload: imessage
|
||||
tolerations:
|
||||
- key: workload
|
||||
operator: Equal
|
||||
value: imessage
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: macos
|
||||
image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c
|
||||
# QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this
|
||||
# only schedules because the sms namespace is labelled privileged.
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
# Generates a unique serial / board-serial / UUID / MAC and persists
|
||||
# them to bootdisk.qcow2. This synthetic identity is what iMessage
|
||||
# activates against — it must stay stable across restarts, which is
|
||||
# why the PVC matters.
|
||||
- name: GENERATE_UNIQUE
|
||||
value: "true"
|
||||
# Identity is only plausible if it matches a real product line.
|
||||
- name: DEVICE_MODEL
|
||||
value: "iMacPro1,1"
|
||||
- name: RAM
|
||||
value: "12"
|
||||
- name: CORES
|
||||
value: "6"
|
||||
- name: EXTRA
|
||||
# Expose the BlueBubbles server port from the guest to the pod.
|
||||
# Guest :1234 (BlueBubbles default) -> pod :1234.
|
||||
value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234"
|
||||
ports:
|
||||
- name: vnc
|
||||
containerPort: 5999
|
||||
protocol: TCP
|
||||
- name: bluebubbles
|
||||
containerPort: 1234
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: "6"
|
||||
memory: 14Gi
|
||||
limits:
|
||||
cpu: "12"
|
||||
memory: 20Gi
|
||||
volumeMounts:
|
||||
- name: macos-disk
|
||||
mountPath: /home/arch/OSX-KVM/disk
|
||||
- name: kvm
|
||||
mountPath: /dev/kvm
|
||||
# No readiness probe on purpose. The guest takes many minutes to boot,
|
||||
# and until macOS + BlueBubbles are installed BY HAND there is nothing
|
||||
# listening on 1234. A probe here would crash-loop the pod through the
|
||||
# entire interactive install.
|
||||
volumes:
|
||||
- name: macos-disk
|
||||
persistentVolumeClaim:
|
||||
claimName: macos-disk
|
||||
- name: kvm
|
||||
hostPath:
|
||||
path: /dev/kvm
|
||||
type: CharDevice
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: sms
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- storageclass.yaml
|
||||
- pvc-macos.yaml
|
||||
- deployment-macos.yaml
|
||||
- service.yaml
|
||||
- networkpolicy.yaml
|
||||
@@ -0,0 +1,19 @@
|
||||
# iMessage delivery for the cluster.
|
||||
#
|
||||
# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper
|
||||
# that hooks Messages.app private APIs — it cannot be containerised on Linux,
|
||||
# because containers share the host kernel and macOS needs XNU + Cocoa. The only
|
||||
# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which
|
||||
# needs a privileged pod with /dev/kvm.
|
||||
#
|
||||
# Hence privileged PodSecurity: the cluster default from the Talos controlplane
|
||||
# is `enforce: baseline`, which forbids privileged containers and host devices.
|
||||
# Scope is limited to this namespace.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: sms
|
||||
labels:
|
||||
pod-security.kubernetes.io/enforce: privileged
|
||||
pod-security.kubernetes.io/audit: privileged
|
||||
pod-security.kubernetes.io/warn: privileged
|
||||
@@ -0,0 +1,22 @@
|
||||
# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple
|
||||
# ID and exposes an unauthenticated VNC console; nothing should reach it except
|
||||
# opted-in clients.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: sms-default-deny
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/part-of: sms
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector: {}
|
||||
podSelector:
|
||||
matchLabels:
|
||||
sms-client: "true"
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 1234
|
||||
@@ -0,0 +1,25 @@
|
||||
# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2).
|
||||
#
|
||||
# This volume is NOT disposable: it holds the VM's serial number, board serial,
|
||||
# UUID and MAC, which together form the identity iMessage was activated against.
|
||||
# Losing it means re-running activation, which is the least reliable step of the
|
||||
# whole setup.
|
||||
#
|
||||
# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for
|
||||
# the installer, the base system, and qcow2 growth.
|
||||
#
|
||||
# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3).
|
||||
# Losing worker-2's disk therefore means losing the activated identity and
|
||||
# redoing iMessage activation. Once the guest is installed and activated, take
|
||||
# a Longhorn snapshot/backup of this volume; that is the only redundancy here.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: macos-disk
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn-imessage-local
|
||||
resources:
|
||||
requests:
|
||||
storage: 200Gi
|
||||
@@ -0,0 +1,31 @@
|
||||
# VNC is how you drive the interactive macOS install. Deliberately ClusterIP —
|
||||
# it is an unauthenticated console onto a machine holding a live Apple ID
|
||||
# session. Reach it with `kubectl port-forward`, never an Ingress.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: macos-vnc
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: macos-bluebubbles
|
||||
ports:
|
||||
- name: vnc
|
||||
port: 5999
|
||||
targetPort: vnc
|
||||
---
|
||||
# The BlueBubbles REST API, once installed inside the guest. This is the stable
|
||||
# name cluster services use, so callers never depend on the pod IP or on whether
|
||||
# the backend is this VM or a real Mac mini later.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: bluebubbles
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: macos-bluebubbles
|
||||
ports:
|
||||
- name: http
|
||||
port: 1234
|
||||
targetPort: bluebubbles
|
||||
@@ -0,0 +1,32 @@
|
||||
# Dedicated StorageClass for the macOS VM disk.
|
||||
#
|
||||
# The default `longhorn` class does not work here, for two independent reasons:
|
||||
#
|
||||
# 1. Replica count. Default is 3, and Longhorn schedules against
|
||||
# storageMaximum - storageScheduled with over-provisioning at 100%. Free
|
||||
# space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi
|
||||
# volume has only one node that can hold even a single replica — a 3-replica
|
||||
# volume fails outright with ReplicaSchedulingFailure.
|
||||
# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is
|
||||
# created, before any pod is scheduled. Combined with strict-local that
|
||||
# pins the data to an arbitrary node, not the one the VM runs on.
|
||||
#
|
||||
# So: one replica, kept local to the VM, bound only once the pod has a node.
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: longhorn-imessage-local
|
||||
provisioner: driver.longhorn.io
|
||||
allowVolumeExpansion: true
|
||||
reclaimPolicy: Delete
|
||||
# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so
|
||||
# the replica is placed on that node and not somewhere else.
|
||||
volumeBindingMode: WaitForFirstConsumer
|
||||
parameters:
|
||||
# A qcow2 backing a live VM is latency-sensitive and rewritten constantly.
|
||||
# Serving it over the network from another node's disk would be the single
|
||||
# worst thing for guest responsiveness, so force it local.
|
||||
numberOfReplicas: "1"
|
||||
dataLocality: "strict-local"
|
||||
staleReplicaTimeout: "30"
|
||||
fsType: "ext4"
|
||||
Reference in New Issue
Block a user