feat(sms): add BlueBubbles iMessage delivery (Docker-OSX macOS VM pinned to worker-2) + ArgoCD app + dedicated longhorn-imessage-local SC — default longhorn SC can't schedule a 3-replica 200Gi volume (only worker-1 has 200Gi free at 100% over-provisioning) and Immediate binding would pin the qcow2 to the wrong node
- namespace: PodSecurity privileged, needed for /dev/kvm + privileged QEMU - storageclass: 1 replica, strict-local, WaitForFirstConsumer - deployment: nodeSelector workload=imessage + matching NoSchedule toleration, Recreate strategy (two QEMU procs on one qcow2 corrupts it), no readiness probe (guest install is interactive and takes many minutes) - services: ClusterIP only; VNC is an unauthenticated console, reach it with port-forward, never an Ingress - networkpolicy: default-deny, opt-in via sms-client=true on port 1234
This commit is contained in:
@@ -0,0 +1,115 @@
|
|||||||
|
# macOS VM (Docker-OSX) hosting the BlueBubbles server.
|
||||||
|
#
|
||||||
|
# ── Why a VM and not a container ────────────────────────────────────────────
|
||||||
|
# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus
|
||||||
|
# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists
|
||||||
|
# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a
|
||||||
|
# container — a VM in a box, not a macOS container.
|
||||||
|
#
|
||||||
|
# ── Why this works on worker-2 ──────────────────────────────────────────────
|
||||||
|
# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and
|
||||||
|
# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a
|
||||||
|
# module). Bare metal, so no nested virtualisation needed.
|
||||||
|
#
|
||||||
|
# ── Read this before relying on it ──────────────────────────────────────────
|
||||||
|
# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over
|
||||||
|
# VNC (:5999), erase the disk in Disk Utility, install, create a user, sign
|
||||||
|
# into iMessage, THEN install BlueBubbles inside the guest. This manifest
|
||||||
|
# only provides the machine; it does not provision macOS.
|
||||||
|
# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles'
|
||||||
|
# own guidance: "test sending an iMessage to yourself. If it does not
|
||||||
|
# succeed, it's likely best to restart from the beginning."
|
||||||
|
# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This
|
||||||
|
# is a Hackintosh. Use a throwaway Apple ID, not a primary one.
|
||||||
|
# 4. BlueBubbles labels this path "not for beginners", "no guarantees or
|
||||||
|
# warranty".
|
||||||
|
#
|
||||||
|
# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled
|
||||||
|
# inside the guest and is NOT required for plain send/receive. Skip it.
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: macos-bluebubbles
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: macos-bluebubbles
|
||||||
|
app.kubernetes.io/part-of: sms
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
# Recreate: the qcow2 disk is RWO and a second pod must never attach it
|
||||||
|
# concurrently — two QEMU processes on one image corrupts it.
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: macos-bluebubbles
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: macos-bluebubbles
|
||||||
|
app.kubernetes.io/part-of: sms
|
||||||
|
spec:
|
||||||
|
# Dedicated node. The taint keeps everything else off worker-2; this
|
||||||
|
# toleration is what lets the VM on. Both halves are required.
|
||||||
|
nodeSelector:
|
||||||
|
workload: imessage
|
||||||
|
tolerations:
|
||||||
|
- key: workload
|
||||||
|
operator: Equal
|
||||||
|
value: imessage
|
||||||
|
effect: NoSchedule
|
||||||
|
containers:
|
||||||
|
- name: macos
|
||||||
|
image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c
|
||||||
|
# QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this
|
||||||
|
# only schedules because the sms namespace is labelled privileged.
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
env:
|
||||||
|
# Generates a unique serial / board-serial / UUID / MAC and persists
|
||||||
|
# them to bootdisk.qcow2. This synthetic identity is what iMessage
|
||||||
|
# activates against — it must stay stable across restarts, which is
|
||||||
|
# why the PVC matters.
|
||||||
|
- name: GENERATE_UNIQUE
|
||||||
|
value: "true"
|
||||||
|
# Identity is only plausible if it matches a real product line.
|
||||||
|
- name: DEVICE_MODEL
|
||||||
|
value: "iMacPro1,1"
|
||||||
|
- name: RAM
|
||||||
|
value: "12"
|
||||||
|
- name: CORES
|
||||||
|
value: "6"
|
||||||
|
- name: EXTRA
|
||||||
|
# Expose the BlueBubbles server port from the guest to the pod.
|
||||||
|
# Guest :1234 (BlueBubbles default) -> pod :1234.
|
||||||
|
value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234"
|
||||||
|
ports:
|
||||||
|
- name: vnc
|
||||||
|
containerPort: 5999
|
||||||
|
protocol: TCP
|
||||||
|
- name: bluebubbles
|
||||||
|
containerPort: 1234
|
||||||
|
protocol: TCP
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: "6"
|
||||||
|
memory: 14Gi
|
||||||
|
limits:
|
||||||
|
cpu: "12"
|
||||||
|
memory: 20Gi
|
||||||
|
volumeMounts:
|
||||||
|
- name: macos-disk
|
||||||
|
mountPath: /home/arch/OSX-KVM/disk
|
||||||
|
- name: kvm
|
||||||
|
mountPath: /dev/kvm
|
||||||
|
# No readiness probe on purpose. The guest takes many minutes to boot,
|
||||||
|
# and until macOS + BlueBubbles are installed BY HAND there is nothing
|
||||||
|
# listening on 1234. A probe here would crash-loop the pod through the
|
||||||
|
# entire interactive install.
|
||||||
|
volumes:
|
||||||
|
- name: macos-disk
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: macos-disk
|
||||||
|
- name: kvm
|
||||||
|
hostPath:
|
||||||
|
path: /dev/kvm
|
||||||
|
type: CharDevice
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: sms
|
||||||
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
|
- storageclass.yaml
|
||||||
|
- pvc-macos.yaml
|
||||||
|
- deployment-macos.yaml
|
||||||
|
- service.yaml
|
||||||
|
- networkpolicy.yaml
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# iMessage delivery for the cluster.
|
||||||
|
#
|
||||||
|
# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper
|
||||||
|
# that hooks Messages.app private APIs — it cannot be containerised on Linux,
|
||||||
|
# because containers share the host kernel and macOS needs XNU + Cocoa. The only
|
||||||
|
# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which
|
||||||
|
# needs a privileged pod with /dev/kvm.
|
||||||
|
#
|
||||||
|
# Hence privileged PodSecurity: the cluster default from the Talos controlplane
|
||||||
|
# is `enforce: baseline`, which forbids privileged containers and host devices.
|
||||||
|
# Scope is limited to this namespace.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: sms
|
||||||
|
labels:
|
||||||
|
pod-security.kubernetes.io/enforce: privileged
|
||||||
|
pod-security.kubernetes.io/audit: privileged
|
||||||
|
pod-security.kubernetes.io/warn: privileged
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple
|
||||||
|
# ID and exposes an unauthenticated VNC console; nothing should reach it except
|
||||||
|
# opted-in clients.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: NetworkPolicy
|
||||||
|
metadata:
|
||||||
|
name: sms-default-deny
|
||||||
|
spec:
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/part-of: sms
|
||||||
|
policyTypes:
|
||||||
|
- Ingress
|
||||||
|
ingress:
|
||||||
|
- from:
|
||||||
|
- namespaceSelector: {}
|
||||||
|
podSelector:
|
||||||
|
matchLabels:
|
||||||
|
sms-client: "true"
|
||||||
|
ports:
|
||||||
|
- protocol: TCP
|
||||||
|
port: 1234
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2).
|
||||||
|
#
|
||||||
|
# This volume is NOT disposable: it holds the VM's serial number, board serial,
|
||||||
|
# UUID and MAC, which together form the identity iMessage was activated against.
|
||||||
|
# Losing it means re-running activation, which is the least reliable step of the
|
||||||
|
# whole setup.
|
||||||
|
#
|
||||||
|
# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for
|
||||||
|
# the installer, the base system, and qcow2 growth.
|
||||||
|
#
|
||||||
|
# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3).
|
||||||
|
# Losing worker-2's disk therefore means losing the activated identity and
|
||||||
|
# redoing iMessage activation. Once the guest is installed and activated, take
|
||||||
|
# a Longhorn snapshot/backup of this volume; that is the only redundancy here.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: macos-disk
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: longhorn-imessage-local
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 200Gi
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# VNC is how you drive the interactive macOS install. Deliberately ClusterIP —
|
||||||
|
# it is an unauthenticated console onto a machine holding a live Apple ID
|
||||||
|
# session. Reach it with `kubectl port-forward`, never an Ingress.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: macos-vnc
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: macos-bluebubbles
|
||||||
|
ports:
|
||||||
|
- name: vnc
|
||||||
|
port: 5999
|
||||||
|
targetPort: vnc
|
||||||
|
---
|
||||||
|
# The BlueBubbles REST API, once installed inside the guest. This is the stable
|
||||||
|
# name cluster services use, so callers never depend on the pod IP or on whether
|
||||||
|
# the backend is this VM or a real Mac mini later.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: bluebubbles
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: macos-bluebubbles
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 1234
|
||||||
|
targetPort: bluebubbles
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# Dedicated StorageClass for the macOS VM disk.
|
||||||
|
#
|
||||||
|
# The default `longhorn` class does not work here, for two independent reasons:
|
||||||
|
#
|
||||||
|
# 1. Replica count. Default is 3, and Longhorn schedules against
|
||||||
|
# storageMaximum - storageScheduled with over-provisioning at 100%. Free
|
||||||
|
# space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi
|
||||||
|
# volume has only one node that can hold even a single replica — a 3-replica
|
||||||
|
# volume fails outright with ReplicaSchedulingFailure.
|
||||||
|
# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is
|
||||||
|
# created, before any pod is scheduled. Combined with strict-local that
|
||||||
|
# pins the data to an arbitrary node, not the one the VM runs on.
|
||||||
|
#
|
||||||
|
# So: one replica, kept local to the VM, bound only once the pod has a node.
|
||||||
|
apiVersion: storage.k8s.io/v1
|
||||||
|
kind: StorageClass
|
||||||
|
metadata:
|
||||||
|
name: longhorn-imessage-local
|
||||||
|
provisioner: driver.longhorn.io
|
||||||
|
allowVolumeExpansion: true
|
||||||
|
reclaimPolicy: Delete
|
||||||
|
# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so
|
||||||
|
# the replica is placed on that node and not somewhere else.
|
||||||
|
volumeBindingMode: WaitForFirstConsumer
|
||||||
|
parameters:
|
||||||
|
# A qcow2 backing a live VM is latency-sensitive and rewritten constantly.
|
||||||
|
# Serving it over the network from another node's disk would be the single
|
||||||
|
# worst thing for guest responsiveness, so force it local.
|
||||||
|
numberOfReplicas: "1"
|
||||||
|
dataLocality: "strict-local"
|
||||||
|
staleReplicaTimeout: "30"
|
||||||
|
fsType: "ext4"
|
||||||
@@ -84,6 +84,37 @@ spec:
|
|||||||
syncOptions:
|
syncOptions:
|
||||||
- CreateNamespace=true
|
- CreateNamespace=true
|
||||||
---
|
---
|
||||||
|
# iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX)
|
||||||
|
# running the BlueBubbles server, plus its dedicated local StorageClass.
|
||||||
|
#
|
||||||
|
# Pinned to worker-2 via nodeSelector `workload: imessage` + a matching
|
||||||
|
# toleration for that node's taint. Until worker-2 is provisioned this app
|
||||||
|
# syncs everything except the pod, which stays Pending — that is expected.
|
||||||
|
#
|
||||||
|
# No CreateNamespace: namespace.yaml carries `pod-security: privileged`, which
|
||||||
|
# the VM needs (/dev/kvm, privileged), and an ArgoCD-created namespace would
|
||||||
|
# not have those labels.
|
||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: sms
|
||||||
|
namespace: argocd
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "8"
|
||||||
|
spec:
|
||||||
|
project: homelab
|
||||||
|
source:
|
||||||
|
repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
|
||||||
|
targetRevision: main
|
||||||
|
path: k8s/apps/sms
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: sms
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
---
|
||||||
# Consolidated: homarr + homarr-patches → homarr
|
# Consolidated: homarr + homarr-patches → homarr
|
||||||
# Helm chart + values + PostSync hook patch (fix-probes-job.yaml)
|
# Helm chart + values + PostSync hook patch (fix-probes-job.yaml)
|
||||||
apiVersion: argoproj.io/v1alpha1
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
|||||||
Reference in New Issue
Block a user