Day 46 — Backend Health Checking

Learning Objectives

• Build an active health checker with configurable failure thresholds
• Emit state change events — hooks into your alert system
• Understand the difference between active and passive health checking

Key Concepts

Active health check:
  Every N seconds: TCP connect or HTTP GET /health
  Track consecutive successes/failures
  Transition: healthy → unhealthy after K failures
  Transition: unhealthy → healthy after M successes

Passive health check:
  Observe in-flight traffic
  Track error rate over a time window
  Transition based on observed error rate

When to use which:
  Active: more reliable, detects failures before real traffic hits backend
  Passive: lower overhead, but detects failures only during real traffic
  Production: use both

Exercise: Active Health Checker

package main

import (
    "fmt"
    "net"
    "sync"
    "time"
)

type HealthState int

const (
    Healthy   HealthState = iota
    Unhealthy
    Unknown
)

func (s HealthState) String() string {
    switch s {
    case Healthy:   return "HEALTHY"
    case Unhealthy: return "UNHEALTHY"
    default:        return "UNKNOWN"
    }
}

type BackendHealth struct {
    addr             string
    state            HealthState
    consecutiveFails int
    consecutiveOK    int
    lastCheck        time.Time
    mu               sync.RWMutex
}

type HealthChecker struct {
    backends    []*BackendHealth
    interval    time.Duration
    failThresh  int // consecutive failures to mark unhealthy
    okThresh    int // consecutive successes to mark healthy
    onStateChange func(addr string, old, new HealthState)
}

func NewHealthChecker(addrs []string, interval time.Duration) *HealthChecker {
    hc := &HealthChecker{
        interval:   interval,
        failThresh: 3,
        okThresh:   2,
        onStateChange: func(addr string, old, new HealthState) {
            fmt.Printf("[HEALTH] %s: %s → %s\n", addr, old, new)
        },
    }
    for _, addr := range addrs {
        hc.backends = append(hc.backends, &BackendHealth{addr: addr, state: Unknown})
    }
    return hc
}

func (hc *HealthChecker) checkOne(b *BackendHealth) {
    // TCP health check: can we connect?
    conn, err := net.DialTimeout("tcp", b.addr, 2*time.Second)
    if err == nil {
        conn.Close()
    }

    b.mu.Lock()
    defer b.mu.Unlock()
    b.lastCheck = time.Now()

    oldState := b.state
    if err != nil {
        b.consecutiveFails++
        b.consecutiveOK = 0
        if b.consecutiveFails >= hc.failThresh {
            b.state = Unhealthy
        }
    } else {
        b.consecutiveOK++
        b.consecutiveFails = 0
        if b.consecutiveOK >= hc.okThresh {
            b.state = Healthy
        }
    }

    if b.state != oldState && hc.onStateChange != nil {
        go hc.onStateChange(b.addr, oldState, b.state)
    }
}

func (hc *HealthChecker) Run(stop <-chan struct{}) {
    ticker := time.NewTicker(hc.interval)
    defer ticker.Stop()
    for {
        select {
        case <-stop:
            return
        case <-ticker.C:
            for _, b := range hc.backends {
                go hc.checkOne(b)
            }
        }
    }
}

func (hc *HealthChecker) HealthyBackends() []string {
    var healthy []string
    for _, b := range hc.backends {
        b.mu.RLock()
        if b.state == Healthy {
            healthy = append(healthy, b.addr)
        }
        b.mu.RUnlock()
    }
    return healthy
}

func main() {
    checker := NewHealthChecker(
        []string{"127.0.0.1:8081", "127.0.0.1:8082", "127.0.0.1:8083"},
        5*time.Second,
    )

    stop := make(chan struct{})
    go checker.Run(stop)

    // Run for 30s, check healthy backends every 10s
    for i := 0; i < 3; i++ {
        time.Sleep(10 * time.Second)
        fmt.Printf("Healthy backends: %v\n", checker.HealthyBackends())
    }
    close(stop)
}

Real-World Context

Health checking is where most load balancer misconfigurations live. Typical mistakes: health check interval too long (60s → 60s of traffic to dead backend), failure threshold too high (3 failures × 5s interval = 15s of bad traffic), health endpoint too expensive (full DB check on each probe overwhelms the DB during recovery).