- Add internal/health package with health checker - Implement three endpoints: /health, /health/live, /health/ready - /health returns full JSON report with component status, latency, timestamp - /health/live for K8s liveness probe (service running) - /health/ready for K8s readiness probe (ready to accept traffic) - Temporal connectivity check via GetWorkflow call with timeout - Health check caching (30s interval) to prevent excessive checks - Graceful shutdown: health server stops on SIGINT/SIGTERM - Add --health flag to starter command to run health check - Worker runs health server on port 8081 alongside task queue worker - 10/10 unit tests passing - All verification criteria met Closes T1.8
168 lines
4.4 KiB
Go
168 lines
4.4 KiB
Go
package health
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sync"
|
|
"time"
|
|
|
|
"go.temporal.io/sdk/client"
|
|
)
|
|
|
|
// Status represents the health status of a component
|
|
type Status string
|
|
|
|
const (
|
|
StatusHealthy Status = "healthy"
|
|
StatusUnhealthy Status = "unhealthy"
|
|
StatusUnknown Status = "unknown"
|
|
)
|
|
|
|
// ComponentHealth represents the health of a single system component
|
|
type ComponentHealth struct {
|
|
Name string `json:"name"`
|
|
Status Status `json:"status"`
|
|
Latency int64 `json:"latency_ms"`
|
|
LastCheck time.Time `json:"last_check"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// HealthReport is the overall health status of the system
|
|
type HealthReport struct {
|
|
Status Status `json:"status"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
Components map[string]ComponentHealth `json:"components"`
|
|
Latency int64 `json:"latency_ms"`
|
|
}
|
|
|
|
// Checker provides health check functionality
|
|
type Checker struct {
|
|
temporalClient client.Client
|
|
mu sync.RWMutex
|
|
lastReport *HealthReport
|
|
lastCheckTime time.Time
|
|
checkInterval time.Duration
|
|
}
|
|
|
|
// NewChecker creates a new health checker
|
|
func NewChecker(temporalClient client.Client) *Checker {
|
|
return &Checker{
|
|
temporalClient: temporalClient,
|
|
checkInterval: 30 * time.Second,
|
|
}
|
|
}
|
|
|
|
// Check performs a comprehensive health check
|
|
func (h *Checker) Check(ctx context.Context) *HealthReport {
|
|
startTime := time.Now()
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
|
|
// Skip check if recently done
|
|
if time.Since(h.lastCheckTime) < h.checkInterval && h.lastReport != nil {
|
|
return h.lastReport
|
|
}
|
|
|
|
components := make(map[string]ComponentHealth)
|
|
|
|
// Check Temporal connectivity
|
|
temporalHealth := h.checkTemporal(ctx)
|
|
components["temporal"] = temporalHealth
|
|
|
|
// Determine overall status
|
|
overallStatus := StatusHealthy
|
|
for _, comp := range components {
|
|
if comp.Status == StatusUnhealthy {
|
|
overallStatus = StatusUnhealthy
|
|
break
|
|
}
|
|
}
|
|
|
|
latency := time.Since(startTime).Milliseconds()
|
|
report := &HealthReport{
|
|
Status: overallStatus,
|
|
Timestamp: time.Now(),
|
|
Components: components,
|
|
Latency: latency,
|
|
}
|
|
|
|
h.lastReport = report
|
|
h.lastCheckTime = time.Now()
|
|
|
|
return report
|
|
}
|
|
|
|
// checkTemporal verifies Temporal cluster connectivity
|
|
func (h *Checker) checkTemporal(ctx context.Context) ComponentHealth {
|
|
startTime := time.Now()
|
|
comp := ComponentHealth{
|
|
Name: "temporal",
|
|
Status: StatusHealthy,
|
|
LastCheck: time.Now(),
|
|
}
|
|
|
|
if h.temporalClient == nil {
|
|
comp.Status = StatusUnhealthy
|
|
comp.Error = "Temporal client not initialized"
|
|
comp.Latency = time.Since(startTime).Milliseconds()
|
|
return comp
|
|
}
|
|
|
|
// Create a short timeout context for the health check
|
|
checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
|
|
// Try to get a workflow to verify connectivity
|
|
// Use a dummy workflow ID that likely doesn't exist - we're just testing connectivity
|
|
// If Temporal is unreachable, this will fail; if it's reachable, it will return NotFound error (which is fine)
|
|
wfRun := h.temporalClient.GetWorkflow(checkCtx, "health-check-dummy-id-"+time.Now().Format("20060102150405"), "")
|
|
comp.Latency = time.Since(startTime).Milliseconds()
|
|
|
|
// We're only checking connectivity, so we try to peek at the result
|
|
// This will fail if Temporal is unreachable, but return nil error if it just doesn't exist
|
|
var result interface{}
|
|
err := wfRun.Get(checkCtx, &result)
|
|
|
|
if err != nil {
|
|
errMsg := err.Error()
|
|
// NotFound errors mean Temporal responded but workflow doesn't exist - this is healthy
|
|
if !contains(errMsg, "not found") && !contains(errMsg, "NotFound") {
|
|
comp.Status = StatusUnhealthy
|
|
comp.Error = err.Error()
|
|
}
|
|
}
|
|
|
|
return comp
|
|
}
|
|
|
|
// contains checks if a string contains a substring (case-insensitive)
|
|
func contains(s, substr string) bool {
|
|
for i := 0; i <= len(s)-len(substr); i++ {
|
|
if s[i:i+len(substr)] == substr {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// IsHealthy returns true if the system is healthy
|
|
func (h *Checker) IsHealthy(ctx context.Context) bool {
|
|
report := h.Check(ctx)
|
|
return report.Status == StatusHealthy
|
|
}
|
|
|
|
// GetReport returns the last health report
|
|
func (h *Checker) GetReport(ctx context.Context) *HealthReport {
|
|
return h.Check(ctx)
|
|
}
|
|
|
|
// ToJSON converts the health report to JSON
|
|
func (report *HealthReport) ToJSON() ([]byte, error) {
|
|
return json.MarshalIndent(report, "", " ")
|
|
}
|
|
|
|
// String returns the health status as a string
|
|
func (s Status) String() string {
|
|
return string(s)
|
|
}
|