Day 88 — Load Testing the Security Stack

Learning Objectives

• Measure the latency overhead of each security middleware
• Identify the bottleneck (usually WAF regex or rate limiter mutex)
• Generate the numbers needed to set SLOs

Exercise: Security Middleware Overhead Measurement

package main

import (
    "fmt"
    "net/http"
    "net/http/httptest"
    "sort"
    "sync"
    "time"
)

type OverheadMeasurement struct {
    Name      string
    Samples   []time.Duration
}

func (m *OverheadMeasurement) P99() time.Duration {
    if len(m.Samples) == 0 { return 0 }
    sorted := make([]time.Duration, len(m.Samples))
    copy(sorted, m.Samples)
    sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
    return sorted[int(float64(len(sorted))*0.99)]
}

func measureOverhead(name string, handler http.Handler, requests int) *OverheadMeasurement {
    m := &OverheadMeasurement{Name: name}
    var mu sync.Mutex

    var wg sync.WaitGroup
    concurrency := 50
    perWorker := requests / concurrency

    for i := 0; i < concurrency; i++ {
        wg.Add(1)
        go func() {
            defer wg.Done()
            for j := 0; j < perWorker; j++ {
                w := httptest.NewRecorder()
                r := httptest.NewRequest("GET", "/api/data?q=legitimate+search+term", nil)
                r.Header.Set("Authorization", "Bearer valid-token")

                start := time.Now()
                handler.ServeHTTP(w, r)
                elapsed := time.Since(start)

                mu.Lock()
                m.Samples = append(m.Samples, elapsed)
                mu.Unlock()
            }
        }()
    }
    wg.Wait()
    return m
}

func main() {
    baseHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        w.WriteHeader(200)
    })

    configs := []struct{ name string; handler http.Handler }{
        {"baseline (no middleware)", baseHandler},
        {"+ RequestID", Chain(baseHandler, RequestID)},
        {"+ SecurityHeaders", Chain(baseHandler, RequestID, SecurityHeaders)},
        {"+ Auth", Chain(baseHandler, RequestID, SecurityHeaders, ecdsaAuth)},
        {"+ Rate Limit", Chain(baseHandler, RequestID, SecurityHeaders, ecdsaAuth, rateLimiter)},
        {"+ WAF", Chain(baseHandler, RequestID, SecurityHeaders, ecdsaAuth, rateLimiter, waf)},
        {"full stack", fullStack},
    }

    fmt.Println("=== Security Middleware Overhead ===")
    fmt.Printf("%-35s %8s %8s\n", "Stack", "P50 (µs)", "P99 (µs)")
    fmt.Println(strings.Repeat("-", 55))

    for _, cfg := range configs {
        m := measureOverhead(cfg.name, cfg.handler, 10000)
        samples := m.Samples
        sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] })
        p50 := samples[len(samples)/2]
        p99 := m.P99()
        fmt.Printf("%-35s %8.0f %8.0f\n",
            cfg.name,
            float64(p50.Microseconds()),
            float64(p99.Microseconds()),
        )
    }
}

Expected output on developer machine:

Stack                               P50 (µs)  P99 (µs)
-------------------------------------------------------
baseline (no middleware)                   8        45
+ RequestID                               12        55
+ SecurityHeaders                         15        60
+ Auth                                    28        85
+ Rate Limit                              35       100
+ WAF                                     95       280
full stack                               110       320

WAF adds ~85µs p50, ~220µs p99. At 1000 rps: 85ms CPU/second for WAF scanning. At 10000 rps: 850ms CPU/second. This is the data you need to justify WAF hardware or horizontal scaling.