Add TTFT & ITL Metrics for LLM Inference (#28)

Time-to-First-Token (TTFT) and Inter-Token Latency (ITL) metrics for LLM inference observability Metrics: llm_ttft_seconds, llm_itl_seconds, llm_tokens_total Closes #31 #32 #33 --------- Co-authored-by: poimen <[email protected]> Reviewed-on: #28
2026-09-15 08:53:58 +00:00
parent 30e0a83a50
commit d439536ca9
22 changed files with 2224 additions and 199 deletions
@@ -0,0 +1,260 @@
+package notification
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// GotifyClient is a CRUD client for the Gotify API.
+type GotifyClient struct {
+	baseURL    string
+	appToken   string // token for sending messages (application token)
+	clientToken string // token for reading/managing (client token)
+	httpClient *http.Client
+}
+
+// NewGotifyClient creates a Gotify API client.
+// appToken is used for sending messages.
+// clientToken is used for listing/deleting messages and managing applications.
+func NewGotifyClient(baseURL, appToken, clientToken string) *GotifyClient {
+	return &GotifyClient{
+		baseURL:     baseURL,
+		appToken:    appToken,
+		clientToken: clientToken,
+		httpClient: &http.Client{
+			Timeout: 10 * time.Second,
+		},
+	}
+}
+
+// --- Message Types ---
+
+// GotifyMessage represents a Gotify message.
+type GotifyMessage struct {
+	ID       int                    `json:"id,omitempty"`
+	AppID    int                    `json:"appid,omitempty"`
+	Title    string                 `json:"title"`
+	Message  string                 `json:"message"`
+	Priority int                    `json:"priority,omitempty"`
+	Date     string                 `json:"date,omitempty"`
+	Extras   map[string]interface{} `json:"extras,omitempty"`
+}
+
+// GotifyMessageList is a paginated list of messages.
+type GotifyMessageList struct {
+	Messages []GotifyMessage `json:"messages"`
+	Paging   GotifyPaging    `json:"paging"`
+}
+
+// GotifyPaging represents pagination info.
+type GotifyPaging struct {
+	Size  int    `json:"size"`
+	Since int    `json:"since"`
+	Limit int    `json:"limit"`
+	Next  string `json:"next,omitempty"`
+}
+
+// --- Application Types ---
+
+// GotifyApplication represents a Gotify application.
+type GotifyApplication struct {
+	ID          int    `json:"id,omitempty"`
+	Token       string `json:"token,omitempty"`
+	Name        string `json:"name"`
+	Description string `json:"description,omitempty"`
+	Image       string `json:"image,omitempty"`
+	Internal    bool   `json:"internal,omitempty"`
+}
+
+// --- Message CRUD ---
+
+// SendMessage sends a message via Gotify (uses app token).
+func (c *GotifyClient) SendMessage(msg GotifyMessage) (*GotifyMessage, error) {
+	body, err := json.Marshal(msg)
+	if err != nil {
+		return nil, fmt.Errorf("marshal message: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPost, c.baseURL+"/message", bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-Gotify-Key", c.appToken)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("send message: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
+		return nil, c.readError(resp)
+	}
+
+	var result GotifyMessage
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("decode response: %w", err)
+	}
+	return &result, nil
+}
+
+// ListMessages lists messages (uses client token).
+func (c *GotifyClient) ListMessages(limit int) (*GotifyMessageList, error) {
+	url := fmt.Sprintf("%s/message?limit=%d", c.baseURL, limit)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("X-Gotify-Key", c.clientToken)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("list messages: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, c.readError(resp)
+	}
+
+	var result GotifyMessageList
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("decode response: %w", err)
+	}
+	return &result, nil
+}
+
+// DeleteMessage deletes a message by ID (uses client token).
+func (c *GotifyClient) DeleteMessage(id int) error {
+	url := fmt.Sprintf("%s/message/%d", c.baseURL, id)
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	if err != nil {
+		return fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("X-Gotify-Key", c.clientToken)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("delete message: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
+		return c.readError(resp)
+	}
+	return nil
+}
+
+// DeleteAllMessages deletes all messages (uses client token).
+func (c *GotifyClient) DeleteAllMessages() error {
+	req, err := http.NewRequest(http.MethodDelete, c.baseURL+"/message", nil)
+	if err != nil {
+		return fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("X-Gotify-Key", c.clientToken)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("delete all messages: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
+		return c.readError(resp)
+	}
+	return nil
+}
+
+// --- Application CRUD ---
+
+// ListApplications lists all applications (uses client token).
+func (c *GotifyClient) ListApplications() ([]GotifyApplication, error) {
+	req, err := http.NewRequest(http.MethodGet, c.baseURL+"/application", nil)
+	if err != nil {
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("X-Gotify-Key", c.clientToken)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("list applications: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, c.readError(resp)
+	}
+
+	var result []GotifyApplication
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("decode response: %w", err)
+	}
+	return result, nil
+}
+
+// CreateApplication creates a new application (uses client token).
+func (c *GotifyClient) CreateApplication(app GotifyApplication) (*GotifyApplication, error) {
+	body, err := json.Marshal(app)
+	if err != nil {
+		return nil, fmt.Errorf("marshal application: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPost, c.baseURL+"/application", bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-Gotify-Key", c.clientToken)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("create application: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
+		return nil, c.readError(resp)
+	}
+
+	var result GotifyApplication
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("decode response: %w", err)
+	}
+	return &result, nil
+}
+
+// DeleteApplication deletes an application by ID (uses client token).
+func (c *GotifyClient) DeleteApplication(id int) error {
+	url := fmt.Sprintf("%s/application/%d", c.baseURL, id)
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	if err != nil {
+		return fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("X-Gotify-Key", c.clientToken)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("delete application: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
+		return c.readError(resp)
+	}
+	return nil
+}
+
+// --- Helpers ---
+
+func (c *GotifyClient) readError(resp *http.Response) error {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("gotify API error (HTTP %d): failed to read body: %w", resp.StatusCode, err)
+	}
+	return fmt.Errorf("gotify API error (HTTP %d): %s", resp.StatusCode, string(body))
+}
@@ -7,154 +7,285 @@ import (
 	"net/http"
 	"net/smtp"
 	"os"
+	"strconv"
 )

-// SendMsgRequest represents a sendMsg API request.
-type SendMsgRequest struct {
-	Format  string            `json:"format"`  // "smtp" or "sms"
-	Title   string            `json:"title"`
-	Message string            `json:"message"`
-	Priority int             `json:"priority,omitempty"`
-	Extras  map[string]string `json:"extras,omitempty"` // e.g., {"to_email": "[email protected]", "phone": "+1234567890"}
-}
-
-// SendMsgResponse represents a sendMsg API response.
-type SendMsgResponse struct {
-	Status  string `json:"status"`
-	MessageID string `json:"messageId,omitempty"`
-	Error  string `json:"error,omitempty"`
-}
-
-// Handler handles sendMsg requests and forwards to appropriate channel (email, SMS, or Gotify push).
+// Handler handles notification requests routed via X-Resource header.
+// Supports: send-email, send-gotify, list-messages, delete-message,
+// delete-all-messages, list-applications, create-application, delete-application.
 type Handler struct {
-	smtpHost     string
-	smtpPort     string
-	smtpFrom     string
-	smtpUser     string
-	smtpPass     string
-	smsAPIURL    string
-	smsAPIKey    string
-	gotifyURL    string
-	gotifyToken  string
+	smtpHost    string
+	smtpPort    string
+	smtpFrom    string
+	smtpUser    string
+	smtpPass    string
+	gotify      *GotifyClient
 }

-// NewHandler creates a new notification handler from environment variables.
+// NewHandler creates a notification handler from environment variables.
 func NewHandler() *Handler {
+	var gotify *GotifyClient
+	gotifyURL := os.Getenv("GOTIFY_URL")
+	if gotifyURL != "" {
+		gotify = NewGotifyClient(
+			gotifyURL,
+			os.Getenv("GOTIFY_APP_TOKEN"),
+			os.Getenv("GOTIFY_CLIENT_TOKEN"),
+		)
+	}
+
 	return &Handler{
-		smtpHost:    os.Getenv("SMTP_HOST"),
-		smtpPort:    os.Getenv("SMTP_PORT"),
-		smtpFrom:    os.Getenv("SMTP_FROM"),
-		smtpUser:    os.Getenv("SMTP_USER"),
-		smtpPass:    os.Getenv("SMTP_PASS"),
-		smsAPIURL:   os.Getenv("SMS_API_URL"),
-		smsAPIKey:   os.Getenv("SMS_API_KEY"),
-		gotifyURL:   os.Getenv("GOTIFY_URL"),
-		gotifyToken: os.Getenv("GOTIFY_TOKEN"),
+		smtpHost: os.Getenv("SMTP_HOST"),
+		smtpPort: os.Getenv("SMTP_PORT"),
+		smtpFrom: os.Getenv("SMTP_FROM"),
+		smtpUser: os.Getenv("SMTP_USER"),
+		smtpPass: os.Getenv("SMTP_PASS"),
+		gotify:   gotify,
 	}
 }

-// ServeHTTP handles sendMsg requests.
+// ServeHTTP routes requests by X-Upstream-Path (set by dispatcher after resource matching).
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodPost {
-		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
-		return
-	}
+	resource := r.Header.Get("X-Resource")

-	var req SendMsgRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(http.StatusBadRequest)
-		json.NewEncoder(w).Encode(SendMsgResponse{
-			Status: "error",
-			Error:  "invalid request: " + err.Error(),
-		})
-		return
-	}
+	switch resource {
+	// --- Email ---
+	case "send-email":
+		h.handleSendEmail(w, r)
+
+	// --- Gotify Messages ---
+	case "send-message":
+		h.handleSendGotify(w, r)
+	case "list-messages":
+		h.handleListMessages(w, r)
+	case "delete-message":
+		h.handleDeleteMessage(w, r)
+	case "delete-all-messages":
+		h.handleDeleteAllMessages(w, r)
+
+	// --- Gotify Applications ---
+	case "list-applications":
+		h.handleListApplications(w, r)
+	case "create-application":
+		h.handleCreateApplication(w, r)
+	case "delete-application":
+		h.handleDeleteApplication(w, r)

-	// Route based on format
-	var resp SendMsgResponse
-	switch req.Format {
-	case "smtp":
-		resp = h.sendEmail(req)
-	case "sms":
-		resp = h.sendSMS(req)
 	default:
-		resp = SendMsgResponse{
-			Status: "error",
-			Error:  "unsupported format: " + req.Format,
-		}
+		h.writeJSON(w, http.StatusNotFound, map[string]string{
+			"error": fmt.Sprintf("unknown resource: %s", resource),
+		})
 	}
-
-	w.Header().Set("Content-Type", "application/json")
-	if resp.Error != "" {
-		w.WriteHeader(http.StatusInternalServerError)
-	} else {
-		w.WriteHeader(http.StatusOK)
-	}
-	json.NewEncoder(w).Encode(resp)
 }

-// sendEmail sends an email via SMTP.
-func (h *Handler) sendEmail(req SendMsgRequest) SendMsgResponse {
-	toEmail := req.Extras["to_email"]
-	if toEmail == "" {
-		return SendMsgResponse{
-			Status: "error",
-			Error:  "missing to_email in extras",
-		}
+// --- Email ---
+
+type SendEmailRequest struct {
+	To      string `json:"to"`
+	CC      string `json:"cc,omitempty"`
+	Subject string `json:"subject"`
+	Body    string `json:"body"`
+}
+
+func (h *Handler) handleSendEmail(w http.ResponseWriter, r *http.Request) {
+	var req SendEmailRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		h.writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request: " + err.Error()})
+		return
 	}

-	subject := req.Title
+	if req.To == "" {
+		h.writeJSON(w, http.StatusBadRequest, map[string]string{"error": "missing 'to' field"})
+		return
+	}
+
+	subject := req.Subject
 	if subject == "" {
 		subject = "Notification"
 	}

-	// Construct email body
-	body := req.Message
-	if req.Extras != nil {
-		if cc := req.Extras["cc"]; cc != "" {
-			body = fmt.Sprintf("CC: %s\n\n%s", cc, body)
-		}
-	}
-
 	msg := fmt.Sprintf(
 		"From: %s\r\nTo: %s\r\nSubject: %s\r\nContent-Type: text/plain; charset=UTF-8\r\n\r\n%s",
-		h.smtpFrom, toEmail, subject, body,
+		h.smtpFrom, req.To, subject, req.Body,
 	)

-	// Send via SMTP
 	smtpAddr := fmt.Sprintf("%s:%s", h.smtpHost, h.smtpPort)
 	auth := smtp.PlainAuth("", h.smtpUser, h.smtpPass, h.smtpHost)

-	if err := smtp.SendMail(smtpAddr, auth, h.smtpFrom, []string{toEmail}, []byte(msg)); err != nil {
-		log.Printf("error sending email to %s: %v", toEmail, err)
-		return SendMsgResponse{
-			Status: "error",
-			Error:  "failed to send email: " + err.Error(),
+	if err := smtp.SendMail(smtpAddr, auth, h.smtpFrom, []string{req.To}, []byte(msg)); err != nil {
+		log.Printf("error sending email to %s: %v", req.To, err)
+		h.writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to send email: " + err.Error()})
+		return
+	}
+
+	h.writeJSON(w, http.StatusOK, map[string]string{
+		"status":    "success",
+		"messageId": fmt.Sprintf("email-%s", req.To),
+	})
+}
+
+// --- Gotify Messages ---
+
+func (h *Handler) handleSendGotify(w http.ResponseWriter, r *http.Request) {
+	if h.gotify == nil {
+		h.writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Gotify not configured"})
+		return
+	}
+
+	var msg GotifyMessage
+	if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
+		h.writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request: " + err.Error()})
+		return
+	}
+
+	result, err := h.gotify.SendMessage(msg)
+	if err != nil {
+		log.Printf("error sending gotify message: %v", err)
+		h.writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
+		return
+	}
+
+	h.writeJSON(w, http.StatusOK, result)
+}
+
+func (h *Handler) handleListMessages(w http.ResponseWriter, r *http.Request) {
+	if h.gotify == nil {
+		h.writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Gotify not configured"})
+		return
+	}
+
+	limit := 50
+	if l := r.URL.Query().Get("limit"); l != "" {
+		if parsed, err := strconv.Atoi(l); err == nil && parsed > 0 {
+			limit = parsed
 		}
 	}

-	return SendMsgResponse{
-		Status:    "success",
-		MessageID: fmt.Sprintf("email-%s", toEmail),
-	}
-}
-
-// sendSMS sends an SMS via configured provider.
-// Placeholder: integrate with Twilio, AWS SNS, or similar.
-func (h *Handler) sendSMS(req SendMsgRequest) SendMsgResponse {
-	phone := req.Extras["phone"]
-	if phone == "" {
-		return SendMsgResponse{
-			Status: "error",
-			Error:  "missing phone in extras",
-		}
+	result, err := h.gotify.ListMessages(limit)
+	if err != nil {
+		log.Printf("error listing gotify messages: %v", err)
+		h.writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
+		return
 	}

-	// TODO: Implement SMS provider integration (Twilio, AWS SNS, etc.)
-	// For now, return error
-	return SendMsgResponse{
-		Status: "error",
-		Error:  "SMS not implemented yet",
+	h.writeJSON(w, http.StatusOK, result)
+}
+
+func (h *Handler) handleDeleteMessage(w http.ResponseWriter, r *http.Request) {
+	if h.gotify == nil {
+		h.writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Gotify not configured"})
+		return
 	}
+
+	var req struct {
+		ID int `json:"id"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		h.writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request: " + err.Error()})
+		return
+	}
+
+	if req.ID == 0 {
+		h.writeJSON(w, http.StatusBadRequest, map[string]string{"error": "missing 'id' field"})
+		return
+	}
+
+	if err := h.gotify.DeleteMessage(req.ID); err != nil {
+		log.Printf("error deleting gotify message %d: %v", req.ID, err)
+		h.writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
+		return
+	}
+
+	h.writeJSON(w, http.StatusOK, map[string]string{"status": "deleted"})
+}
+
+func (h *Handler) handleDeleteAllMessages(w http.ResponseWriter, r *http.Request) {
+	if h.gotify == nil {
+		h.writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Gotify not configured"})
+		return
+	}
+
+	if err := h.gotify.DeleteAllMessages(); err != nil {
+		log.Printf("error deleting all gotify messages: %v", err)
+		h.writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
+		return
+	}
+
+	h.writeJSON(w, http.StatusOK, map[string]string{"status": "all messages deleted"})
+}
+
+// --- Gotify Applications ---
+
+func (h *Handler) handleListApplications(w http.ResponseWriter, r *http.Request) {
+	if h.gotify == nil {
+		h.writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Gotify not configured"})
+		return
+	}
+
+	result, err := h.gotify.ListApplications()
+	if err != nil {
+		log.Printf("error listing gotify applications: %v", err)
+		h.writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
+		return
+	}
+
+	h.writeJSON(w, http.StatusOK, result)
+}
+
+func (h *Handler) handleCreateApplication(w http.ResponseWriter, r *http.Request) {
+	if h.gotify == nil {
+		h.writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Gotify not configured"})
+		return
+	}
+
+	var app GotifyApplication
+	if err := json.NewDecoder(r.Body).Decode(&app); err != nil {
+		h.writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request: " + err.Error()})
+		return
+	}
+
+	result, err := h.gotify.CreateApplication(app)
+	if err != nil {
+		log.Printf("error creating gotify application: %v", err)
+		h.writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
+		return
+	}
+
+	h.writeJSON(w, http.StatusCreated, result)
+}
+
+func (h *Handler) handleDeleteApplication(w http.ResponseWriter, r *http.Request) {
+	if h.gotify == nil {
+		h.writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "Gotify not configured"})
+		return
+	}
+
+	var req struct {
+		ID int `json:"id"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		h.writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request: " + err.Error()})
+		return
+	}
+
+	if req.ID == 0 {
+		h.writeJSON(w, http.StatusBadRequest, map[string]string{"error": "missing 'id' field"})
+		return
+	}
+
+	if err := h.gotify.DeleteApplication(req.ID); err != nil {
+		log.Printf("error deleting gotify application %d: %v", req.ID, err)
+		h.writeJSON(w, http.StatusBadGateway, map[string]string{"error": err.Error()})
+		return
+	}
+
+	h.writeJSON(w, http.StatusOK, map[string]string{"status": "deleted"})
+}
+
+// --- Helpers ---
+
+func (h *Handler) writeJSON(w http.ResponseWriter, status int, data interface{}) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(data)
 }
@@ -0,0 +1,513 @@
+package notification
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// mockGotifyServer creates a test server that simulates the Gotify API.
+func mockGotifyServer() *httptest.Server {
+	mux := http.NewServeMux()
+
+	// POST /message — send message
+	mux.HandleFunc("/message", func(w http.ResponseWriter, r *http.Request) {
+		switch r.Method {
+		case http.MethodPost:
+			var msg GotifyMessage
+			if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
+				http.Error(w, "bad request", http.StatusBadRequest)
+				return
+			}
+			msg.ID = 42
+			msg.AppID = 1
+			w.Header().Set("Content-Type", "application/json")
+			json.NewEncoder(w).Encode(msg)
+
+		case http.MethodGet:
+			// list messages
+			w.Header().Set("Content-Type", "application/json")
+			json.NewEncoder(w).Encode(GotifyMessageList{
+				Messages: []GotifyMessage{
+					{ID: 1, Title: "Test", Message: "hello", Priority: 3},
+					{ID: 2, Title: "Alert", Message: "world", Priority: 7},
+				},
+				Paging: GotifyPaging{Size: 2, Limit: 50},
+			})
+
+		case http.MethodDelete:
+			// delete all messages
+			w.WriteHeader(http.StatusOK)
+			json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
+
+		default:
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		}
+	})
+
+	// DELETE /message/{id}
+	mux.HandleFunc("/message/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodDelete {
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
+	})
+
+	// GET/POST/DELETE /application
+	mux.HandleFunc("/application", func(w http.ResponseWriter, r *http.Request) {
+		switch r.Method {
+		case http.MethodGet:
+			w.Header().Set("Content-Type", "application/json")
+			json.NewEncoder(w).Encode([]GotifyApplication{
+				{ID: 1, Name: "app1", Token: "tok1"},
+				{ID: 2, Name: "app2", Token: "tok2"},
+			})
+
+		case http.MethodPost:
+			var app GotifyApplication
+			if err := json.NewDecoder(r.Body).Decode(&app); err != nil {
+				http.Error(w, "bad request", http.StatusBadRequest)
+				return
+			}
+			app.ID = 10
+			app.Token = "new-token"
+			w.WriteHeader(http.StatusCreated)
+			w.Header().Set("Content-Type", "application/json")
+			json.NewEncoder(w).Encode(app)
+
+		default:
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		}
+	})
+
+	// DELETE /application/{id}
+	mux.HandleFunc("/application/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodDelete {
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
+	})
+
+	return httptest.NewServer(mux)
+}
+
+func newTestHandler(gotifyURL string) *Handler {
+	h := &Handler{}
+	if gotifyURL != "" {
+		h.gotify = NewGotifyClient(gotifyURL, "test-app-token", "test-client-token")
+	}
+	return h
+}
+
+func doRequest(h *Handler, method, resource string, body interface{}) *httptest.ResponseRecorder {
+	var reqBody io.Reader
+	if body != nil {
+		b, _ := json.Marshal(body)
+		reqBody = bytes.NewReader(b)
+	}
+
+	req := httptest.NewRequest(method, "/", reqBody)
+	req.Header.Set("X-Resource", resource)
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	w := httptest.NewRecorder()
+	h.ServeHTTP(w, req)
+	return w
+}
+
+func decodeResponse(t *testing.T, w *httptest.ResponseRecorder) map[string]interface{} {
+	t.Helper()
+	var result map[string]interface{}
+	if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil {
+		t.Fatalf("decode response: %v (body: %s)", err, w.Body.String())
+	}
+	return result
+}
+
+// ============================================================
+// Handler routing tests
+// ============================================================
+
+func TestHandler_UnknownResource(t *testing.T) {
+	h := newTestHandler("")
+	w := doRequest(h, "GET", "unknown-resource", nil)
+	if w.Code != http.StatusNotFound {
+		t.Errorf("expected 404, got %d", w.Code)
+	}
+	data := decodeResponse(t, w)
+	if _, ok := data["error"]; !ok {
+		t.Error("expected error in response")
+	}
+}
+
+func TestHandler_GotifyNotConfigured(t *testing.T) {
+	h := newTestHandler("") // no gotify
+
+	resources := []struct {
+		method   string
+		resource string
+		body     interface{}
+	}{
+		{"POST", "send-message", map[string]string{"title": "t", "message": "m"}},
+		{"GET", "list-messages", nil},
+		{"DELETE", "delete-message", map[string]int{"id": 1}},
+		{"DELETE", "delete-all-messages", nil},
+		{"GET", "list-applications", nil},
+		{"POST", "create-application", map[string]string{"name": "app"}},
+		{"DELETE", "delete-application", map[string]int{"id": 1}},
+	}
+
+	for _, tc := range resources {
+		t.Run(tc.resource, func(t *testing.T) {
+			w := doRequest(h, tc.method, tc.resource, tc.body)
+			if w.Code != http.StatusServiceUnavailable {
+				t.Errorf("expected 503, got %d", w.Code)
+			}
+		})
+	}
+}
+
+// ============================================================
+// Gotify message tests (via handler)
+// ============================================================
+
+func TestHandler_SendMessage(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "POST", "send-message", map[string]interface{}{
+		"title":    "Test Alert",
+		"message":  "Something happened",
+		"priority": 5,
+	})
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+
+	data := decodeResponse(t, w)
+	if data["title"] != "Test Alert" {
+		t.Errorf("expected title 'Test Alert', got %v", data["title"])
+	}
+	if int(data["id"].(float64)) != 42 {
+		t.Errorf("expected id 42, got %v", data["id"])
+	}
+}
+
+func TestHandler_SendMessage_InvalidJSON(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	req := httptest.NewRequest("POST", "/", bytes.NewReader([]byte("not json")))
+	req.Header.Set("X-Resource", "send-message")
+	w := httptest.NewRecorder()
+	h.ServeHTTP(w, req)
+
+	if w.Code != http.StatusBadRequest {
+		t.Errorf("expected 400, got %d", w.Code)
+	}
+}
+
+func TestHandler_ListMessages(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "GET", "list-messages", nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+
+	var result GotifyMessageList
+	if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if len(result.Messages) != 2 {
+		t.Errorf("expected 2 messages, got %d", len(result.Messages))
+	}
+}
+
+func TestHandler_DeleteMessage(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "DELETE", "delete-message", map[string]int{"id": 1})
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+}
+
+func TestHandler_DeleteMessage_MissingID(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "DELETE", "delete-message", map[string]int{"id": 0})
+	if w.Code != http.StatusBadRequest {
+		t.Errorf("expected 400, got %d", w.Code)
+	}
+}
+
+func TestHandler_DeleteAllMessages(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "DELETE", "delete-all-messages", nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+}
+
+// ============================================================
+// Gotify application tests (via handler)
+// ============================================================
+
+func TestHandler_ListApplications(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "GET", "list-applications", nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+
+	var apps []GotifyApplication
+	if err := json.Unmarshal(w.Body.Bytes(), &apps); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if len(apps) != 2 {
+		t.Errorf("expected 2 apps, got %d", len(apps))
+	}
+}
+
+func TestHandler_CreateApplication(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "POST", "create-application", map[string]string{
+		"name":        "my-app",
+		"description": "test app",
+	})
+	if w.Code != http.StatusCreated {
+		t.Fatalf("expected 201, got %d: %s", w.Code, w.Body.String())
+	}
+
+	data := decodeResponse(t, w)
+	if data["name"] != "my-app" {
+		t.Errorf("expected name 'my-app', got %v", data["name"])
+	}
+}
+
+func TestHandler_DeleteApplication(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "DELETE", "delete-application", map[string]int{"id": 1})
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+	}
+}
+
+func TestHandler_DeleteApplication_MissingID(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	h := newTestHandler(srv.URL)
+
+	w := doRequest(h, "DELETE", "delete-application", map[string]int{"id": 0})
+	if w.Code != http.StatusBadRequest {
+		t.Errorf("expected 400, got %d", w.Code)
+	}
+}
+
+// ============================================================
+// Email tests (routing only, no SMTP)
+// ============================================================
+
+func TestHandler_SendEmail_MissingTo(t *testing.T) {
+	h := newTestHandler("")
+
+	w := doRequest(h, "POST", "send-email", map[string]string{
+		"subject": "Test",
+		"body":    "Hello",
+	})
+	if w.Code != http.StatusBadRequest {
+		t.Errorf("expected 400, got %d", w.Code)
+	}
+}
+
+func TestHandler_SendEmail_InvalidJSON(t *testing.T) {
+	h := newTestHandler("")
+
+	req := httptest.NewRequest("POST", "/", bytes.NewReader([]byte("{bad")))
+	req.Header.Set("X-Resource", "send-email")
+	w := httptest.NewRecorder()
+	h.ServeHTTP(w, req)
+
+	if w.Code != http.StatusBadRequest {
+		t.Errorf("expected 400, got %d", w.Code)
+	}
+}
+
+// ============================================================
+// GotifyClient direct tests
+// ============================================================
+
+func TestGotifyClient_SendMessage(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	msg, err := client.SendMessage(GotifyMessage{
+		Title:    "Direct Test",
+		Message:  "Hello",
+		Priority: 3,
+	})
+	if err != nil {
+		t.Fatalf("send: %v", err)
+	}
+	if msg.ID != 42 {
+		t.Errorf("expected id 42, got %d", msg.ID)
+	}
+	if msg.Title != "Direct Test" {
+		t.Errorf("expected title 'Direct Test', got %s", msg.Title)
+	}
+}
+
+func TestGotifyClient_ListMessages(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	list, err := client.ListMessages(50)
+	if err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if len(list.Messages) != 2 {
+		t.Errorf("expected 2 messages, got %d", len(list.Messages))
+	}
+}
+
+func TestGotifyClient_DeleteMessage(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	if err := client.DeleteMessage(1); err != nil {
+		t.Fatalf("delete: %v", err)
+	}
+}
+
+func TestGotifyClient_DeleteAllMessages(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	if err := client.DeleteAllMessages(); err != nil {
+		t.Fatalf("delete all: %v", err)
+	}
+}
+
+func TestGotifyClient_ListApplications(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	apps, err := client.ListApplications()
+	if err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if len(apps) != 2 {
+		t.Errorf("expected 2 apps, got %d", len(apps))
+	}
+}
+
+func TestGotifyClient_CreateApplication(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	app, err := client.CreateApplication(GotifyApplication{
+		Name:        "new-app",
+		Description: "test",
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	if app.ID != 10 {
+		t.Errorf("expected id 10, got %d", app.ID)
+	}
+}
+
+func TestGotifyClient_DeleteApplication(t *testing.T) {
+	srv := mockGotifyServer()
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	if err := client.DeleteApplication(1); err != nil {
+		t.Fatalf("delete: %v", err)
+	}
+}
+
+func TestGotifyClient_ErrorResponse(t *testing.T) {
+	// Server that returns 500 for everything
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, "internal error")
+	}))
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	_, err := client.SendMessage(GotifyMessage{Title: "test"})
+	if err == nil {
+		t.Fatal("expected error")
+	}
+
+	_, err = client.ListMessages(10)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+
+	_, err = client.ListApplications()
+	if err == nil {
+		t.Fatal("expected error")
+	}
+}
+
+func TestGotifyClient_RateLimited(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusTooManyRequests)
+		fmt.Fprint(w, "rate limited")
+	}))
+	defer srv.Close()
+	client := NewGotifyClient(srv.URL, "app-token", "client-token")
+
+	_, err := client.SendMessage(GotifyMessage{Title: "test"})
+	if err == nil {
+		t.Fatal("expected error on 429")
+	}
+}
+
+func TestGotifyClient_InvalidURL(t *testing.T) {
+	client := NewGotifyClient("http://localhost:1", "app-token", "client-token")
+
+	_, err := client.SendMessage(GotifyMessage{Title: "test"})
+	if err == nil {
+		t.Fatal("expected connection error")
+	}
+}