Day 50 — Observability at L4

Learning Objectives

• Add metrics to every component of your load balancer
• Build a /stats endpoint with human-readable backend status
• Know what to look at during an L4 incident

Exercise: Load Balancer Observability

package main

import (
    "encoding/json"
    "expvar"
    "fmt"
    "net/http"
    "time"
)

// L4Stats exposes all LB metrics
type L4Stats struct {
    ConnAccepted  *expvar.Int
    ConnActive    *expvar.Int
    ConnRejected  *expvar.Int
    BytesIn       *expvar.Int
    BytesOut      *expvar.Int
    ByBackend     *expvar.Map  // per-backend: connections, bytes
    HealthStates  *expvar.Map  // per-backend: health state
}

func NewL4Stats() *L4Stats {
    return &L4Stats{
        ConnAccepted: expvar.NewInt("lb_connections_accepted"),
        ConnActive:   expvar.NewInt("lb_connections_active"),
        ConnRejected: expvar.NewInt("lb_connections_rejected"),
        BytesIn:      expvar.NewInt("lb_bytes_in"),
        BytesOut:     expvar.NewInt("lb_bytes_out"),
        ByBackend:    expvar.NewMap("lb_by_backend"),
        HealthStates: expvar.NewMap("lb_backend_health"),
    }
}

func (s *L4Stats) RecordBackendHealth(addr string, state HealthState) {
    s.HealthStates.Set(addr, expvar.Func(func() any { return state.String() }))
}

// /stats — human-readable status page
func statsHandler(stats *L4Stats, checker *HealthChecker) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        type backendStatus struct {
            Addr         string `json:"addr"`
            Health       string `json:"health"`
            ActiveConns  int64  `json:"active_connections"`
        }

        type response struct {
            Timestamp    string          `json:"timestamp"`
            Backends     []backendStatus `json:"backends"`
            GlobalActive int64           `json:"global_active_connections"`
            TotalAccepted int64          `json:"total_accepted"`
            TotalRejected int64          `json:"total_rejected"`
        }

        var backends []backendStatus
        for _, b := range checker.backends {
            b.mu.RLock()
            state := b.state.String()
            b.mu.RUnlock()
            backends = append(backends, backendStatus{
                Addr:   b.addr,
                Health: state,
            })
        }

        w.Header().Set("Content-Type", "application/json")
        json.NewEncoder(w).Encode(response{
            Timestamp:    time.Now().UTC().Format(time.RFC3339),
            Backends:     backends,
            GlobalActive: stats.ConnActive.Value(),
            TotalAccepted: stats.ConnAccepted.Value(),
            TotalRejected: stats.ConnRejected.Value(),
        })
    }
}

func main() {
    stats := NewL4Stats()
    checker := NewHealthChecker(
        []string{"127.0.0.1:8081", "127.0.0.1:8082"},
        5*time.Second,
    )

    // Serve metrics and debug endpoints
    mux := http.NewServeMux()
    mux.HandleFunc("/stats", statsHandler(stats, checker))
    // /debug/vars served automatically by expvar
    // /debug/pprof served automatically by net/http/pprof
    go http.ListenAndServe(":9090", mux)

    fmt.Println("LB observability on :9090/stats and :9090/debug/vars")
    select {} // block
}

Real-World Context

During a DDoS incident, the first thing you check is connection rates per backend. Is one backend getting all the traffic? Are connections being accepted and immediately closed (rejected)? Is lb_connections_active growing without bound? These metrics answer those questions in real-time.