feat(T1.8): implement health checks for Kubernetes deployment
- Add internal/health package with health checker - Implement three endpoints: /health, /health/live, /health/ready - /health returns full JSON report with component status, latency, timestamp - /health/live for K8s liveness probe (service running) - /health/ready for K8s readiness probe (ready to accept traffic) - Temporal connectivity check via GetWorkflow call with timeout - Health check caching (30s interval) to prevent excessive checks - Graceful shutdown: health server stops on SIGINT/SIGTERM - Add --health flag to starter command to run health check - Worker runs health server on port 8081 alongside task queue worker - 10/10 unit tests passing - All verification criteria met Closes T1.8
This commit is contained in:
@@ -0,0 +1,167 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.temporal.io/sdk/client"
|
||||
)
|
||||
|
||||
// Status represents the health status of a component
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusHealthy Status = "healthy"
|
||||
StatusUnhealthy Status = "unhealthy"
|
||||
StatusUnknown Status = "unknown"
|
||||
)
|
||||
|
||||
// ComponentHealth represents the health of a single system component
|
||||
type ComponentHealth struct {
|
||||
Name string `json:"name"`
|
||||
Status Status `json:"status"`
|
||||
Latency int64 `json:"latency_ms"`
|
||||
LastCheck time.Time `json:"last_check"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// HealthReport is the overall health status of the system
|
||||
type HealthReport struct {
|
||||
Status Status `json:"status"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Components map[string]ComponentHealth `json:"components"`
|
||||
Latency int64 `json:"latency_ms"`
|
||||
}
|
||||
|
||||
// Checker provides health check functionality
|
||||
type Checker struct {
|
||||
temporalClient client.Client
|
||||
mu sync.RWMutex
|
||||
lastReport *HealthReport
|
||||
lastCheckTime time.Time
|
||||
checkInterval time.Duration
|
||||
}
|
||||
|
||||
// NewChecker creates a new health checker
|
||||
func NewChecker(temporalClient client.Client) *Checker {
|
||||
return &Checker{
|
||||
temporalClient: temporalClient,
|
||||
checkInterval: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Check performs a comprehensive health check
|
||||
func (h *Checker) Check(ctx context.Context) *HealthReport {
|
||||
startTime := time.Now()
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
// Skip check if recently done
|
||||
if time.Since(h.lastCheckTime) < h.checkInterval && h.lastReport != nil {
|
||||
return h.lastReport
|
||||
}
|
||||
|
||||
components := make(map[string]ComponentHealth)
|
||||
|
||||
// Check Temporal connectivity
|
||||
temporalHealth := h.checkTemporal(ctx)
|
||||
components["temporal"] = temporalHealth
|
||||
|
||||
// Determine overall status
|
||||
overallStatus := StatusHealthy
|
||||
for _, comp := range components {
|
||||
if comp.Status == StatusUnhealthy {
|
||||
overallStatus = StatusUnhealthy
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
latency := time.Since(startTime).Milliseconds()
|
||||
report := &HealthReport{
|
||||
Status: overallStatus,
|
||||
Timestamp: time.Now(),
|
||||
Components: components,
|
||||
Latency: latency,
|
||||
}
|
||||
|
||||
h.lastReport = report
|
||||
h.lastCheckTime = time.Now()
|
||||
|
||||
return report
|
||||
}
|
||||
|
||||
// checkTemporal verifies Temporal cluster connectivity
|
||||
func (h *Checker) checkTemporal(ctx context.Context) ComponentHealth {
|
||||
startTime := time.Now()
|
||||
comp := ComponentHealth{
|
||||
Name: "temporal",
|
||||
Status: StatusHealthy,
|
||||
LastCheck: time.Now(),
|
||||
}
|
||||
|
||||
if h.temporalClient == nil {
|
||||
comp.Status = StatusUnhealthy
|
||||
comp.Error = "Temporal client not initialized"
|
||||
comp.Latency = time.Since(startTime).Milliseconds()
|
||||
return comp
|
||||
}
|
||||
|
||||
// Create a short timeout context for the health check
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Try to get a workflow to verify connectivity
|
||||
// Use a dummy workflow ID that likely doesn't exist - we're just testing connectivity
|
||||
// If Temporal is unreachable, this will fail; if it's reachable, it will return NotFound error (which is fine)
|
||||
wfRun := h.temporalClient.GetWorkflow(checkCtx, "health-check-dummy-id-"+time.Now().Format("20060102150405"), "")
|
||||
comp.Latency = time.Since(startTime).Milliseconds()
|
||||
|
||||
// We're only checking connectivity, so we try to peek at the result
|
||||
// This will fail if Temporal is unreachable, but return nil error if it just doesn't exist
|
||||
var result interface{}
|
||||
err := wfRun.Get(checkCtx, &result)
|
||||
|
||||
if err != nil {
|
||||
errMsg := err.Error()
|
||||
// NotFound errors mean Temporal responded but workflow doesn't exist - this is healthy
|
||||
if !contains(errMsg, "not found") && !contains(errMsg, "NotFound") {
|
||||
comp.Status = StatusUnhealthy
|
||||
comp.Error = err.Error()
|
||||
}
|
||||
}
|
||||
|
||||
return comp
|
||||
}
|
||||
|
||||
// contains checks if a string contains a substring (case-insensitive)
|
||||
func contains(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsHealthy returns true if the system is healthy
|
||||
func (h *Checker) IsHealthy(ctx context.Context) bool {
|
||||
report := h.Check(ctx)
|
||||
return report.Status == StatusHealthy
|
||||
}
|
||||
|
||||
// GetReport returns the last health report
|
||||
func (h *Checker) GetReport(ctx context.Context) *HealthReport {
|
||||
return h.Check(ctx)
|
||||
}
|
||||
|
||||
// ToJSON converts the health report to JSON
|
||||
func (report *HealthReport) ToJSON() ([]byte, error) {
|
||||
return json.MarshalIndent(report, "", " ")
|
||||
}
|
||||
|
||||
// String returns the health status as a string
|
||||
func (s Status) String() string {
|
||||
return string(s)
|
||||
}
|
||||
Reference in New Issue
Block a user