// Package sessions — SandboxManager provisions per-session k8s resources: // // PVC workspace-session- Longhorn 1Gi // Pod pi-session- single container, /workspace <- PVC // Svc pi-session- ClusterIP :8787 // NetPol pi-session-sandbox shared for all pi-session pods (created once) // // All k8s API calls use raw HTTP + in-cluster SA token (no client-go). // Same pattern as internal/webhook/promoter.go. package sessions import ( "bytes" "context" "crypto/tls" "crypto/x509" "encoding/json" "fmt" "io" "log" "net/http" "os" "strings" "time" ) const ( sandboxNamespace = "poimen" sandboxImage = "forgejo.riotpiao.com/riotpiao-poimen/poimen-pi-session-runtime:latest" sandboxPort = 8787 defaultPVCSize = "1Gi" defaultStorageClass = "longhorn" defaultReadyTimeout = 30 * time.Second defaultReaperInterval = 5 * time.Minute sandboxSATokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" sandboxCACertPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" ) // SandboxManager manages the k8s lifecycle of pi-session Pods, PVCs, Services. type SandboxManager struct { k8sHost string k8sClient *http.Client pvcSize string readyTimeout time.Duration } // SandboxResult is returned by CreateOrResume. type SandboxResult struct { SessionID string `json:"session_id"` PodName string `json:"pod_name"` PVCName string `json:"pvc_name"` SvcDNS string `json:"svc_dns"` Resumed bool `json:"resumed"` // true = pod already existed } // NewSandboxManager creates a SandboxManager from env + in-cluster SA files. func NewSandboxManager() *SandboxManager { host := os.Getenv("K8S_API_HOST") if host == "" { host = "https://kubernetes.default.svc" } pvcSize := os.Getenv("SANDBOX_PVC_SIZE") if pvcSize == "" { pvcSize = defaultPVCSize } caCert, _ := os.ReadFile(sandboxCACertPath) return &SandboxManager{ k8sHost: host, k8sClient: buildSandboxK8sClient(caCert), pvcSize: pvcSize, readyTimeout: defaultReadyTimeout, } } func buildSandboxK8sClient(caCert []byte) *http.Client { transport := &http.Transport{} if len(caCert) > 0 { pool := x509.NewCertPool() pool.AppendCertsFromPEM(caCert) transport.TLSClientConfig = &tls.Config{RootCAs: pool} } else { log.Printf("sandbox: WARNING no SA CA cert at %s — using InsecureSkipVerify", sandboxCACertPath) transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec } return &http.Client{Timeout: 30 * time.Second, Transport: transport} } func sandboxSAToken() string { b, err := os.ReadFile(sandboxSATokenPath) if err != nil { log.Printf("sandbox: WARNING failed to read SA token %s: %v", sandboxSATokenPath, err) return "" } return strings.TrimSpace(string(b)) } func pvcName(sessionID string) string { return "workspace-session-" + sessionID } func podName(sessionID string) string { return "pi-session-" + sessionID } func svcName(sessionID string) string { return "pi-session-" + sessionID } func svcDNS(sessionID string) string { return fmt.Sprintf("%s.%s.svc.cluster.local:%d", svcName(sessionID), sandboxNamespace, sandboxPort) } // ── public API ──────────────────────────────────────────────────────────── // CreateOrResume provisions or resumes a sandbox for sessionID. // Idempotent: if the pod is already Running, returns its DNS immediately. func (s *SandboxManager) CreateOrResume(ctx context.Context, sessionID string) (*SandboxResult, error) { res := &SandboxResult{ SessionID: sessionID, PodName: podName(sessionID), PVCName: pvcName(sessionID), SvcDNS: svcDNS(sessionID), } // 1. Ensure PVC exists if err := s.ensurePVC(ctx, sessionID); err != nil { return nil, fmt.Errorf("ensure PVC: %w", err) } // 2. Check if pod already running (resume case) phase, err := s.getPodPhase(ctx, sessionID) if err != nil { return nil, fmt.Errorf("check pod: %w", err) } if phase == "Running" { res.Resumed = true return res, nil } // 3. Ensure NetworkPolicy for all pi-session pods exists if err := s.ensureNetworkPolicy(ctx); err != nil { log.Printf("sandbox: WARNING failed to ensure network policy: %v", err) } // 4. Create Pod + Service if err := s.createPod(ctx, sessionID); err != nil { return nil, fmt.Errorf("create pod: %w", err) } if err := s.ensureService(ctx, sessionID); err != nil { return nil, fmt.Errorf("ensure service: %w", err) } // 5. Wait for Ready if err := s.waitPodReady(ctx, sessionID); err != nil { return nil, fmt.Errorf("wait ready: %w", err) } return res, nil } // Delete removes the Pod, Service, and PVC for sessionID. func (s *SandboxManager) Delete(ctx context.Context, sessionID string) error { var errs []string for _, fn := range []func(context.Context, string) error{ s.deletePod, s.deleteService, s.deletePVC, } { if err := fn(ctx, sessionID); err != nil && !isNotFound(err) { errs = append(errs, err.Error()) } } if len(errs) > 0 { return fmt.Errorf("delete sandbox: %s", strings.Join(errs, "; ")) } return nil } // ReapIdlePods lists all pi-session-* pods and deletes those that have exited // (Succeeded/Failed) — bridge.js self-exits after idle timeout. // The PVC is preserved so the next CreateOrResume reattaches the same workspace. func (s *SandboxManager) ReapIdlePods(ctx context.Context) { pods, err := s.listSandboxPods(ctx) if err != nil { log.Printf("sandbox reaper: list pods: %v", err) return } for _, podAny := range pods { pod, _ := podAny.(map[string]any) phase, _ := pod["status"].(map[string]any)["phase"].(string) name, _ := pod["metadata"].(map[string]any)["name"].(string) if phase == "Succeeded" || phase == "Failed" { log.Printf("sandbox reaper: deleting exited pod %s (phase=%s)", name, phase) if err := s.k8sDelete(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/pods/"+name); err != nil && !isNotFound(err) { log.Printf("sandbox reaper: delete pod %s: %v", name, err) } } } } // ── k8s resource helpers ───────────────────────────────────────────────── func (s *SandboxManager) ensurePVC(ctx context.Context, sessionID string) error { name := pvcName(sessionID) _, err := s.k8sGet(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/persistentvolumeclaims/"+name) if err == nil { return nil // already exists } if !isNotFound(err) { return err } // Create body := map[string]any{ "apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": map[string]any{ "name": name, "namespace": sandboxNamespace, "labels": map[string]string{"app": "pi-session", "session-id": sessionID}, }, "spec": map[string]any{ "accessModes": []string{"ReadWriteOnce"}, "storageClassName": defaultStorageClass, "resources": map[string]any{ "requests": map[string]string{"storage": s.pvcSize}, }, }, } return s.k8sCreate(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/persistentvolumeclaims", body) } func (s *SandboxManager) getPodPhase(ctx context.Context, sessionID string) (string, error) { pod, err := s.k8sGet(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/pods/"+podName(sessionID)) if isNotFound(err) { return "", nil } if err != nil { return "", err } phase, _ := pod["status"].(map[string]any)["phase"].(string) return phase, nil } func (s *SandboxManager) createPod(ctx context.Context, sessionID string) error { body := map[string]any{ "apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{ "name": podName(sessionID), "namespace": sandboxNamespace, "labels": map[string]string{"app": "pi-session", "session-id": sessionID}, }, "spec": map[string]any{ "restartPolicy": "Never", "automountServiceAccountToken": false, "securityContext": map[string]any{ "runAsNonRoot": true, "runAsUser": int64(1000), "seccompProfile": map[string]string{"type": "RuntimeDefault"}, }, "containers": []map[string]any{{ "name": "pi-session", "image": sandboxImage, "ports": []map[string]any{{"containerPort": sandboxPort}}, "securityContext": map[string]any{ "readOnlyRootFilesystem": true, "allowPrivilegeEscalation": false, "capabilities": map[string]any{"drop": []string{"ALL"}}, }, "resources": map[string]any{ "requests": map[string]string{"cpu": "500m", "memory": "512Mi"}, "limits": map[string]string{"cpu": "1", "memory": "1Gi"}, }, "readinessProbe": map[string]any{ "tcpSocket": map[string]any{"port": sandboxPort}, "initialDelaySeconds": 3, "periodSeconds": 2, "failureThreshold": 15, }, "volumeMounts": []map[string]any{ {"name": "workspace", "mountPath": "/workspace"}, {"name": "tmp", "mountPath": "/tmp"}, }, }}, "volumes": []map[string]any{ {"name": "workspace", "persistentVolumeClaim": map[string]string{"claimName": pvcName(sessionID)}}, {"name": "tmp", "emptyDir": map[string]any{}}, }, }, } return s.k8sCreate(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/pods", body) } func (s *SandboxManager) ensureService(ctx context.Context, sessionID string) error { name := svcName(sessionID) _, err := s.k8sGet(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/services/"+name) if err == nil { return nil } if !isNotFound(err) { return err } body := map[string]any{ "apiVersion": "v1", "kind": "Service", "metadata": map[string]any{ "name": name, "namespace": sandboxNamespace, "labels": map[string]string{"app": "pi-session", "session-id": sessionID}, }, "spec": map[string]any{ "type": "ClusterIP", "selector": map[string]string{"app": "pi-session", "session-id": sessionID}, "ports": []map[string]any{{ "port": sandboxPort, "targetPort": sandboxPort, "protocol": "TCP", }}, }, } return s.k8sCreate(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/services", body) } func (s *SandboxManager) ensureNetworkPolicy(ctx context.Context) error { const npName = "pi-session-sandbox" _, err := s.k8sGet(ctx, "/apis/networking.k8s.io/v1/namespaces/"+sandboxNamespace+"/networkpolicies/"+npName) if err == nil { return nil // already exists } if !isNotFound(err) { return err } body := map[string]any{ "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": map[string]any{ "name": npName, "namespace": sandboxNamespace, "labels": map[string]string{"app": "pi-session"}, }, "spec": map[string]any{ "podSelector": map[string]any{"matchLabels": map[string]string{"app": "pi-session"}}, "policyTypes": []string{"Ingress", "Egress"}, "ingress": []map[string]any{{ "from": []map[string]any{{ "podSelector": map[string]any{"matchLabels": map[string]string{"app": "poimen-frontend"}}, }}, "ports": []map[string]any{{"protocol": "TCP", "port": sandboxPort}}, }}, "egress": []map[string]any{ // DNS { "to": []map[string]any{{"namespaceSelector": map[string]any{"matchLabels": map[string]string{"kubernetes.io/metadata.name": "kube-system"}}}}, "ports": []map[string]any{{"protocol": "UDP", "port": 53}, {"protocol": "TCP", "port": 53}}, }, // api-gateway (LLM inference) { "to": []map[string]any{{"namespaceSelector": map[string]any{"matchLabels": map[string]string{"kubernetes.io/metadata.name": "api"}}}}, "ports": []map[string]any{{"protocol": "TCP", "port": 8080}}, }, }, }, } return s.k8sCreate(ctx, "/apis/networking.k8s.io/v1/namespaces/"+sandboxNamespace+"/networkpolicies", body) } func (s *SandboxManager) waitPodReady(ctx context.Context, sessionID string) error { deadline := time.Now().Add(s.readyTimeout) for time.Now().Before(deadline) { pod, err := s.k8sGet(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/pods/"+podName(sessionID)) if err != nil { return fmt.Errorf("poll pod: %w", err) } status, _ := pod["status"].(map[string]any) phase, _ := status["phase"].(string) if phase == "Failed" || phase == "Succeeded" { return fmt.Errorf("pod entered terminal phase %s before ready", phase) } if phase == "Running" { for _, cond := range toSlice(status["conditions"]) { if c, ok := cond.(map[string]any); ok { if c["type"] == "Ready" && c["status"] == "True" { return nil } } } } select { case <-time.After(2 * time.Second): case <-ctx.Done(): return ctx.Err() } } return fmt.Errorf("pod not ready after %v", s.readyTimeout) } func (s *SandboxManager) deletePod(ctx context.Context, sessionID string) error { return s.k8sDelete(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/pods/"+podName(sessionID)) } func (s *SandboxManager) deleteService(ctx context.Context, sessionID string) error { return s.k8sDelete(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/services/"+svcName(sessionID)) } func (s *SandboxManager) deletePVC(ctx context.Context, sessionID string) error { return s.k8sDelete(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/persistentvolumeclaims/"+pvcName(sessionID)) } func (s *SandboxManager) listSandboxPods(ctx context.Context) ([]any, error) { body, err := s.k8sGet(ctx, "/api/v1/namespaces/"+sandboxNamespace+"/pods?labelSelector=app%3Dpi-session") if err != nil { return nil, err } items, _ := body["items"].([]any) return items, nil } // ── raw k8s HTTP ───────────────────────────────────────────────────────── func (s *SandboxManager) k8sGet(ctx context.Context, path string) (map[string]any, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.k8sHost+path, nil) if err != nil { return nil, err } req.Header.Set("Authorization", "Bearer "+sandboxSAToken()) resp, err := s.k8sClient.Do(req) if err != nil { return nil, fmt.Errorf("k8s GET %s: %w", path, err) } defer resp.Body.Close() if resp.StatusCode == 404 { return nil, &k8sNotFound{path: path} } if resp.StatusCode >= 400 { b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return nil, fmt.Errorf("k8s GET %s returned %d: %s", path, resp.StatusCode, b) } var result map[string]any if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return nil, fmt.Errorf("decode: %w", err) } return result, nil } func (s *SandboxManager) k8sCreate(ctx context.Context, path string, body any) error { payload, err := json.Marshal(body) if err != nil { return fmt.Errorf("marshal: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.k8sHost+path, bytes.NewReader(payload)) if err != nil { return err } req.Header.Set("Authorization", "Bearer "+sandboxSAToken()) req.Header.Set("Content-Type", "application/json") resp, err := s.k8sClient.Do(req) if err != nil { return fmt.Errorf("k8s POST %s: %w", path, err) } defer resp.Body.Close() if resp.StatusCode == 409 { // AlreadyExists — idempotent return nil } if resp.StatusCode < 200 || resp.StatusCode >= 300 { b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return fmt.Errorf("k8s POST %s returned %d: %s", path, resp.StatusCode, b) } return nil } func (s *SandboxManager) k8sDelete(ctx context.Context, path string) error { req, err := http.NewRequestWithContext(ctx, http.MethodDelete, s.k8sHost+path, nil) if err != nil { return err } req.Header.Set("Authorization", "Bearer "+sandboxSAToken()) resp, err := s.k8sClient.Do(req) if err != nil { return fmt.Errorf("k8s DELETE %s: %w", path, err) } defer resp.Body.Close() if resp.StatusCode == 404 { return &k8sNotFound{path: path} } if resp.StatusCode >= 400 { b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) return fmt.Errorf("k8s DELETE %s returned %d: %s", path, resp.StatusCode, b) } return nil } // ── error helpers ───────────────────────────────────────────────────────── type k8sNotFound struct{ path string } func (e *k8sNotFound) Error() string { return "not found: " + e.path } func isNotFound(err error) bool { if err == nil { return false } _, ok := err.(*k8sNotFound) return ok } func toSlice(v any) []any { if s, ok := v.([]any); ok { return s } return nil }