Chapter 21 — Resilience between services: rate limits, retries, idempotency, and circuit breakers

Build it in Go (60 min) — all four, measured

resilience runs a flaky "create order" server and a well-behaved client against it. 30% of first attempts fail after the order has been committed, the hardest case for retries, because the client can't tell the order exists:

// resilience: the four mechanisms that keep retries from becoming outages.
//
//	server: per-client token-bucket rate limiting (429 + Retry-After)
//	        Idempotency-Key handling (a retried POST never creates two orders)
//	client: retries with exponential backoff + full jitter, a retry budget,
//	        and a circuit breaker
//
//	go run ./resilience
package main

import (
	"crypto/rand"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"math"
	mrand "math/rand/v2"
	"net/http"
	"net/http/httptest"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

// ------------------------------------------------------------ server side --

// bucket is a token bucket: capacity `burst`, refilled at `rate` per second.
type bucket struct {
	tokens float64
	last   time.Time
}

type limiter struct {
	mu      sync.Mutex
	rate    float64
	burst   float64
	buckets map[string]*bucket
}

// allow reports whether the client may proceed, and if not, how long to wait.
func (l *limiter) allow(key string) (bool, time.Duration) {
	l.mu.Lock()
	defer l.mu.Unlock()
	now := time.Now()
	b := l.buckets[key]
	if b == nil {
		b = &bucket{tokens: l.burst, last: now}
		l.buckets[key] = b
	}
	b.tokens = math.Min(l.burst, b.tokens+now.Sub(b.last).Seconds()*l.rate)
	b.last = now
	if b.tokens >= 1 {
		b.tokens--
		return true, 0
	}
	return false, time.Duration((1 - b.tokens) / l.rate * float64(time.Second))
}

func rateLimit(l *limiter, next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		key := r.Header.Get("X-API-Key") // identity, not IP: NAT puts many users behind one IP
		if ok, wait := l.allow(key); !ok {
			w.Header().Set("Retry-After", strconv.Itoa(int(math.Ceil(wait.Seconds()))))
			http.Error(w, "slow down", http.StatusTooManyRequests)
			return
		}
		next.ServeHTTP(w, r)
	})
}

// orders is the "database". The idempotency key is recorded IN THE SAME
// TRANSACTION as the side effect, so a retry -- even one whose first attempt
// committed and then lost its response -- finds the existing order instead
// of creating a second one. (Caching HTTP responses is NOT enough: a request
// that committed and then failed has no successful response to cache.)
type orders struct {
	mu    sync.Mutex
	byKey map[string]int
	next  int
}

// create returns the order for this key, creating it only the first time.
func (o *orders) create(key string) (id int, replay bool) {
	o.mu.Lock() // production: INSERT ... ON CONFLICT (idempotency_key) DO NOTHING
	defer o.mu.Unlock()
	if id, ok := o.byKey[key]; ok {
		return id, true
	}
	o.next++
	o.byKey[key] = o.next
	return o.next, false
}

// ------------------------------------------------------------ client side --

type breaker struct {
	mu       sync.Mutex
	failures int
	openTill time.Time
}

var errOpen = errors.New("circuit open: failing fast without calling the server")

func (b *breaker) before() error {
	b.mu.Lock()
	defer b.mu.Unlock()
	if time.Now().Before(b.openTill) {
		return errOpen // OPEN: don't even try
	}
	return nil // CLOSED, or HALF-OPEN after the cool-down (one trial gets through)
}

func (b *breaker) after(ok bool) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if ok {
		b.failures = 0
		return
	}
	b.failures++
	if b.failures >= 5 {
		b.openTill = time.Now().Add(2 * time.Second)
		b.failures = 0
	}
}

type client struct {
	http     *http.Client
	brk      breaker
	requests atomic.Int64
	retries  atomic.Int64
}

// budgetOK caps retries at 20% of first attempts: under a real outage,
// retries can't multiply the load by more than 1.2x.
func (c *client) budgetOK() bool {
	return float64(c.retries.Load()) < 0.2*float64(c.requests.Load())+5
}

func (c *client) post(url, key string) (string, error) {
	c.requests.Add(1)
	for attempt := 0; ; attempt++ {
		if err := c.brk.before(); err != nil {
			return "", err
		}
		req, _ := http.NewRequest("POST", url, nil)
		req.Header.Set("Idempotency-Key", key) // SAME key on every retry: that's the point
		req.Header.Set("X-API-Key", "team-a")
		resp, err := c.http.Do(req)
		var body []byte
		status := 0
		if err == nil {
			body, _ = io.ReadAll(resp.Body)
			resp.Body.Close()
			status = resp.StatusCode
		}
		c.brk.after(err == nil && status < 500)
		if err == nil && status < 300 {
			return string(body), nil
		}
		retryable := err != nil || status == 429 || status >= 500
		if !retryable || attempt == 3 || !c.budgetOK() {
			return "", fmt.Errorf("giving up after %d attempts (last status %d, err %v)", attempt+1, status, err)
		}
		c.retries.Add(1)
		// Exponential backoff with FULL jitter: random in [0, base*2^attempt].
		// Jitter spreads retries out so clients don't retry in lockstep.
		backoff := time.Duration(mrand.Int64N(int64(50*time.Millisecond) << attempt))
		if status == 429 && resp.Header.Get("Retry-After") != "" {
			s, _ := strconv.Atoi(resp.Header.Get("Retry-After"))
			backoff = max(backoff, time.Duration(s)*time.Second)
		}
		time.Sleep(backoff)
	}
}

func newKey() string {
	b := make([]byte, 8)
	rand.Read(b)
	return hex.EncodeToString(b)
}

// ------------------------------------------------------------------- demo --

func main() {
	db := &orders{byKey: map[string]int{}}
	var down atomic.Bool
	flaky := func(w http.ResponseWriter, r *http.Request) {
		if down.Load() {
			http.Error(w, "down", http.StatusServiceUnavailable)
			return
		}
		key := r.Header.Get("Idempotency-Key")
		if key == "" {
			http.Error(w, "Idempotency-Key required", http.StatusBadRequest)
			return
		}
		id, replay := db.create(key) // the side effect, deduplicated by key
		if !replay && mrand.Float64() < 0.3 {
			// The order WAS created, but the response is lost -- the worst case for retries.
			http.Error(w, "oops after commit", http.StatusInternalServerError)
			return
		}
		w.WriteHeader(http.StatusCreated)
		fmt.Fprintf(w, "order %d", id)
	}
	lim := &limiter{rate: 50, burst: 20, buckets: map[string]*bucket{}}
	mux := http.NewServeMux()
	mux.HandleFunc("POST /orders", flaky)
	srv := httptest.NewServer(rateLimit(lim, mux))
	defer srv.Close()

	c := &client{http: &http.Client{Timeout: 2 * time.Second}}
	ok, failed := 0, 0
	for i := 0; i < 40; i++ {
		if _, err := c.post(srv.URL+"/orders", newKey()); err != nil {
			failed++
		} else {
			ok++
		}
	}
	db.mu.Lock()
	n := db.next
	db.mu.Unlock()
	fmt.Printf("phase 1, flaky server (30%% of first attempts fail AFTER committing):\n")
	fmt.Printf("  40 distinct orders sent, %d confirmed, %d gave up, %d retries\n", ok, failed, c.retries.Load())
	fmt.Printf("  orders in the database: %d  <- never more than 40: no duplicates\n", n)

	// Same idempotency key twice: the server replays the first answer.
	key := newKey()
	for {
		if a, err := c.post(srv.URL+"/orders", key); err == nil {
			b, _ := c.post(srv.URL+"/orders", key)
			fmt.Printf("\nsame key sent twice: %q then %q -> one order, two identical answers\n", a, b)
			break
		}
	}

	down.Store(true)
	start := time.Now()
	fast := 0
	for i := 0; i < 20; i++ {
		if _, err := c.post(srv.URL+"/orders", newKey()); errors.Is(err, errOpen) {
			fast++
		}
	}
	fmt.Printf("\nphase 2, server down: %d of 20 calls failed fast with the breaker open (%v total)\n",
		fast, time.Since(start).Round(time.Millisecond))
}

$ go run ./resilience
phase 1, flaky server (30% of first attempts fail AFTER committing):
  40 distinct orders sent, 38 confirmed, 2 gave up, 12 retries
  orders in the database: 40  <- never more than 40: no duplicates

same key sent twice: "order 41" then "order 41" -> one order, two identical answers

phase 2, server down: 17 of 20 calls failed fast with the breaker open (1.141s total)

What to notice:

• No duplicates. 12 retries, and the database still has exactly one order per key. The "2 gave up" orders exist in the database even though the client saw an error. That's why the client must also be able to look an operation up by its key later.
• An earlier version of this lab was wrong in an instructive way. It implemented idempotency as middleware that cached successful responses. Because 30% of first attempts committed the order and then failed, there was no successful response to cache, and the retry created a second order: 40 distinct keys produced 52 orders. The key must be recorded with the side effect itself (in SQL, a unique constraint on idempotency_key with INSERT ... ON CONFLICT DO NOTHING), not in a response cache.
• The breaker turned 20 doomed calls into 3 real attempts and 17 instant failures. The server got a rest, and callers got an immediate answer instead of waiting for timeouts.
• Rate limits are keyed by API key, not IP (NAT puts many users behind one address; see the OSI guide, Layer 3). The response carries Retry-After, and the client honours it.

Exercises:

1. Lower the limiter to rate: 5, burst: 5, rerun, and count the 429s. Does the client's backoff respect Retry-After?
2. Remove the jitter (sleep exactly 50ms << attempt), run 50 clients concurrently against a server that fails for 2 seconds, and plot when requests arrive. You'll see the synchronised waves.
3. Add deadline propagation: give each call a context.WithTimeout and have the server stop work when r.Context() is cancelled.
4. Replace the in-memory orders map with SQLite or Postgres and a unique index on the key, then run two clients that use the same key concurrently.