Day 110 — Chaos Engineering

Exercise — Chaos Middleware + Circuit Breaker

package chaos

import (
    "math/rand"
    "net/http"
    "sync/atomic"
    "time"
)

// ChaosConfig controls fault injection rates (all zero by default — safe to deploy)
type ChaosConfig struct {
    ErrorRate   float64       // 0.0–1.0: probability of returning 500
    DelayRate   float64       // 0.0–1.0: probability of injecting delay
    MinDelay    time.Duration
    MaxDelay    time.Duration
    CloseRate   float64       // 0.0–1.0: probability of hijacking and closing connection
    Enabled     atomic.Bool   // master kill switch
}

func NewChaosConfig() *ChaosConfig {
    cfg := &ChaosConfig{
        ErrorRate: 0.05, MaxDelay: 2 * time.Second, MinDelay: 100 * time.Millisecond,
        DelayRate: 0.02, CloseRate: 0.01,
    }
    // Disabled by default — enable via CHAOS_ENABLED env var or config
    return cfg
}

// ChaosMiddleware injects random faults per ChaosConfig
func ChaosMiddleware(cfg *ChaosConfig, next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        if !cfg.Enabled.Load() {
            next.ServeHTTP(w, r)
            return
        }

        roll := rand.Float64()

        // Error injection
        if roll < cfg.ErrorRate {
            http.Error(w, `{"error":"chaos_injected"}`, http.StatusInternalServerError)
            return
        }

        // Latency injection
        if roll < cfg.ErrorRate+cfg.DelayRate {
            delay := cfg.MinDelay + time.Duration(rand.Int63n(int64(cfg.MaxDelay-cfg.MinDelay)))
            time.Sleep(delay)
        }

        // Connection close injection — requires Hijacker
        if roll < cfg.ErrorRate+cfg.DelayRate+cfg.CloseRate {
            if h, ok := w.(http.Hijacker); ok {
                conn, _, err := h.Hijack()
                if err == nil {
                    conn.Close()
                    return
                }
            }
        }

        next.ServeHTTP(w, r)
    })
}

// CircuitBreaker — 3 states: closed (normal), open (blocking), half-open (testing)
type CircuitBreaker struct {
    failures    atomic.Int64
    successes   atomic.Int64
    state       atomic.Int32 // 0=closed, 1=open, 2=half-open
    lastFailure atomic.Int64 // unix nano

    failureThreshold int64
    successThreshold int64
    openDuration     time.Duration
    halfOpenAllowed  atomic.Int32 // permit count in half-open state
}

const (
    stateClosed   int32 = 0
    stateOpen     int32 = 1
    stateHalfOpen int32 = 2
)

func NewCircuitBreaker(failureThreshold, successThreshold int, openDuration time.Duration) *CircuitBreaker {
    return &CircuitBreaker{
        failureThreshold: int64(failureThreshold),
        successThreshold: int64(successThreshold),
        openDuration:     openDuration,
    }
}

// Allow returns true if the request should proceed
func (cb *CircuitBreaker) Allow() bool {
    switch cb.state.Load() {
    case stateClosed:
        return true

    case stateOpen:
        // Check if enough time has passed to try half-open
        lastFail := time.Unix(0, cb.lastFailure.Load())
        if time.Since(lastFail) > cb.openDuration {
            if cb.state.CompareAndSwap(stateOpen, stateHalfOpen) {
                cb.halfOpenAllowed.Store(1) // allow one request to test
            }
            return cb.state.Load() == stateHalfOpen && cb.halfOpenAllowed.Add(-1) >= 0
        }
        return false

    case stateHalfOpen:
        return cb.halfOpenAllowed.Add(-1) >= 0
    }
    return true
}

// RecordSuccess records a successful request
func (cb *CircuitBreaker) RecordSuccess() {
    if cb.state.Load() == stateHalfOpen {
        if cb.successes.Add(1) >= cb.successThreshold {
            cb.successes.Store(0)
            cb.failures.Store(0)
            cb.state.Store(stateClosed)
        }
    }
}

// RecordFailure records a failed request
func (cb *CircuitBreaker) RecordFailure() {
    cb.lastFailure.Store(time.Now().UnixNano())
    if cb.failures.Add(1) >= cb.failureThreshold {
        cb.state.Store(stateOpen)
        cb.failures.Store(0)
        cb.successes.Store(0)
    }
}

func (cb *CircuitBreaker) State() string {
    switch cb.state.Load() {
    case stateClosed:
        return "closed"
    case stateOpen:
        return "open"
    case stateHalfOpen:
        return "half_open"
    }
    return "unknown"
}

// Middleware wraps an HTTP handler with circuit breaker protection
func (cb *CircuitBreaker) Middleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        if !cb.Allow() {
            http.Error(w, `{"error":"circuit_open"}`, http.StatusServiceUnavailable)
            return
        }

        // Use statusCapture to detect failures
        sc := &statusCapture{ResponseWriter: w}
        next.ServeHTTP(sc, r)

        if sc.status >= 500 {
            cb.RecordFailure()
        } else {
            cb.RecordSuccess()
        }
    })
}

type statusCapture struct {
    http.ResponseWriter
    status int
}

func (sc *statusCapture) WriteHeader(code int) {
    sc.status = code
    sc.ResponseWriter.WriteHeader(code)
}