Day 51 — Resilience Patterns

Learning Objectives

• Implement a circuit breaker that protects backends from cascading failures
• Build exponential backoff with jitter for retry logic
• Understand when to retry vs when to fail fast

Key Concepts

Circuit Breaker States:
  Closed (normal):  requests pass through, track failures
  Open (tripped):   reject all requests immediately (fast fail)
                    protect backend from being overwhelmed during recovery
  Half-Open (probe): after timeout, let ONE request through
                     if success → close; if fail → open again

Retry with exponential backoff + jitter:
  Attempt 1: wait 100ms ± 10ms
  Attempt 2: wait 200ms ± 20ms
  Attempt 3: wait 400ms ± 40ms
  Max 3 retries

Jitter prevents "thundering herd": without jitter, all retrying clients
retry at the same time (100ms, 200ms...) and overload the recovering backend.
With jitter: retries are spread out over the window.

Exercise: Circuit Breaker + Retry

package main

import (
    "errors"
    "fmt"
    "math/rand"
    "sync"
    "sync/atomic"
    "time"
)

var ErrCircuitOpen = errors.New("circuit breaker open")

type CBState int32

const (
    CBClosed   CBState = iota
    CBOpen
    CBHalfOpen
)

type CircuitBreaker struct {
    state          atomic.Int32
    failures       atomic.Int64
    successes      atomic.Int64
    failThresh     int64
    successThresh  int64
    openDuration   time.Duration
    openedAt       time.Time
    mu             sync.Mutex
}

func NewCircuitBreaker(failThresh, successThresh int64, openDuration time.Duration) *CircuitBreaker {
    return &CircuitBreaker{
        failThresh:    failThresh,
        successThresh: successThresh,
        openDuration:  openDuration,
    }
}

func (cb *CircuitBreaker) Allow() error {
    switch CBState(cb.state.Load()) {
    case CBClosed:
        return nil
    case CBOpen:
        cb.mu.Lock()
        defer cb.mu.Unlock()
        if time.Since(cb.openedAt) > cb.openDuration {
            cb.state.Store(int32(CBHalfOpen))
            cb.successes.Store(0)
            fmt.Println("[CB] OPEN → HALF-OPEN (probing)")
            return nil
        }
        return ErrCircuitOpen
    case CBHalfOpen:
        return nil // let one request through to probe
    }
    return nil
}

func (cb *CircuitBreaker) RecordSuccess() {
    cb.failures.Store(0)
    if CBState(cb.state.Load()) == CBHalfOpen {
        if cb.successes.Add(1) >= cb.successThresh {
            cb.state.Store(int32(CBClosed))
            fmt.Println("[CB] HALF-OPEN → CLOSED (recovered)")
        }
    }
}

func (cb *CircuitBreaker) RecordFailure() {
    if CBState(cb.state.Load()) == CBHalfOpen {
        cb.mu.Lock()
        cb.state.Store(int32(CBOpen))
        cb.openedAt = time.Now()
        cb.mu.Unlock()
        fmt.Println("[CB] HALF-OPEN → OPEN (probe failed)")
        return
    }

    failures := cb.failures.Add(1)
    if failures >= cb.failThresh {
        cb.mu.Lock()
        if CBState(cb.state.Load()) == CBClosed {
            cb.state.Store(int32(CBOpen))
            cb.openedAt = time.Now()
            fmt.Printf("[CB] CLOSED → OPEN (failed %d times)\n", failures)
        }
        cb.mu.Unlock()
    }
}

// RetryWithBackoff executes fn with exponential backoff + jitter
func RetryWithBackoff(maxAttempts int, base time.Duration, fn func() error) error {
    var lastErr error
    for attempt := 0; attempt < maxAttempts; attempt++ {
        if attempt > 0 {
            // Exponential backoff with ±20% jitter
            delay := base * (1 << uint(attempt-1))
            jitter := time.Duration(float64(delay) * 0.2 * (rand.Float64()*2 - 1))
            time.Sleep(delay + jitter)
        }

        lastErr = fn()
        if lastErr == nil {
            return nil
        }

        // Don't retry on circuit breaker open — it's already protecting the backend
        if errors.Is(lastErr, ErrCircuitOpen) {
            return lastErr
        }
        fmt.Printf("[RETRY] attempt %d failed: %v\n", attempt+1, lastErr)
    }
    return fmt.Errorf("all %d attempts failed: %w", maxAttempts, lastErr)
}

func main() {
    cb := NewCircuitBreaker(5, 2, 10*time.Second)
    failures := 0

    makeRequest := func() error {
        if err := cb.Allow(); err != nil {
            return err
        }
        failures++
        if failures <= 7 { // first 7 fail, then recover
            cb.RecordFailure()
            return fmt.Errorf("backend error")
        }
        cb.RecordSuccess()
        return nil
    }

    // Send 20 requests
    for i := 1; i <= 20; i++ {
        err := RetryWithBackoff(3, 100*time.Millisecond, makeRequest)
        state := CBState(cb.state.Load())
        stateStr := []string{"CLOSED", "OPEN", "HALF-OPEN"}[state]
        fmt.Printf("Request %d: err=%v state=%s\n", i, err, stateStr)
        time.Sleep(500 * time.Millisecond)
    }
}

Real-World Context

Netflix's Hystrix (now Resilience4j) popularized circuit breakers. Without them, a slow database causes HTTP handlers to block, threads fill up, and your service crashes — this is cascading failure. With a circuit breaker, slow database → circuit opens → requests fail fast → service stays responsive. This is OWASP A05 mitigation.