Chapter 57 — Production load balancing: L4 vs L7, health checks, draining, retries

L4 vs L7

The per-connection vs per-request difference is critical for long-lived connections. HTTP/2 and gRPC clients send all their requests over one connection. Behind an L4 balancer, that client is pinned to one backend forever: new pods get no traffic and hot pods melt. gRPC needs an L7 balancer, a service mesh, or client-side balancing (OSI guide, Part 15).

Health checks must test what matters

Bad:

GET / -> 200

Better:

GET /readyz -> 200 only if this instance can serve real traffic right now

• Readiness should fail when the instance should be removed from rotation: starting up, draining, or unable to reach a dependency it can't work without.
• Liveness should fail only when the process should be restarted: deadlocked or corrupted. Never make liveness depend on a database. Otherwise a database outage restarts every pod at once, and they all stampede the database when it comes back.
• Hysteresis: eject after N consecutive failures and restore after M successes, so one slow probe doesn't flap a backend in and out.
• Passive health checks (outlier detection): also eject backends based on real traffic, such as 5 consecutive 5xx errors. Active probes run every few seconds, while real traffic notices failures in milliseconds.

Draining

Before shutting down a backend:

1. mark it not-ready (fail /readyz);
2. wait for the load balancer to stop sending new traffic (at least failure threshold × check interval);
3. let in-flight requests finish, up to a deadline;
4. exit.

Skipping draining creates random user-visible errors on every deploy.

// lbdrain: Chapter 57 in one program -- backends with /readyz and graceful
// draining, and a round-robin load balancer with active health checks.
// Kill a backend gracefully mid-traffic and count the errors (there should be 0).
//
//	go build -o lbdrain ./lbdrain
//	./lbdrain backend 127.0.0.1:9081 &
//	./lbdrain backend 127.0.0.1:9082 &
//	./lbdrain lb 127.0.0.1:9080 http://127.0.0.1:9081 http://127.0.0.1:9082 &
//	for i in $(seq 1 2000); do curl -s -o /dev/null -w '%{http_code}\n' localhost:9080/; done | sort | uniq -c &
//	kill -TERM %1      # graceful: readiness fails first, then in-flight requests finish
package main

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"
)

func main() {
	if len(os.Args) < 3 {
		fmt.Println("usage: lbdrain backend ADDR | lbdrain lb ADDR BACKEND_URL...")
		os.Exit(2)
	}
	switch os.Args[1] {
	case "backend":
		backend(os.Args[2])
	case "lb":
		loadBalancer(os.Args[2], os.Args[3:])
	}
}

// ---------------------------------------------------------------- backend --

func backend(addr string) {
	var draining atomic.Bool
	mux := http.NewServeMux()
	mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
		if draining.Load() {
			http.Error(w, "draining", http.StatusServiceUnavailable) // "stop sending me new work"
			return
		}
		w.Write([]byte("ok"))
	})
	mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
		time.Sleep(50 * time.Millisecond) // pretend to work
		fmt.Fprintf(w, "hello from %s\n", addr)
	})
	srv := &http.Server{Addr: addr, Handler: mux,
		ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 60 * time.Second}

	go func() {
		if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
			log.Fatal(err)
		}
	}()
	log.Printf("backend %s up (pid %d)", addr, os.Getpid())

	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
	defer stop()
	<-ctx.Done()

	// The drain sequence from Chapter 57:
	log.Printf("backend %s: SIGTERM -> failing readiness", addr)
	draining.Store(true)        // 1. mark not-ready
	time.Sleep(3 * time.Second) // 2. wait for the LB's health checks to notice
	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	if err := srv.Shutdown(shutdownCtx); err != nil { // 3. stop accepting, finish in-flight
		log.Printf("forced close: %v", err)
	}
	log.Printf("backend %s: drained, exiting", addr) // 4. exit
}

// ----------------------------------------------------------- load balancer --

type upstream struct {
	url     *url.URL
	proxy   *httputil.ReverseProxy
	healthy atomic.Bool
}

func loadBalancer(addr string, targets []string) {
	var ups []*upstream
	for _, t := range targets {
		u, err := url.Parse(t)
		if err != nil {
			log.Fatal(err)
		}
		up := &upstream{url: u, proxy: httputil.NewSingleHostReverseProxy(u)}
		up.healthy.Store(true)
		ups = append(ups, up)
		go healthCheck(up)
	}

	var next atomic.Uint64
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Round-robin over HEALTHY upstreams only.
		for range ups {
			up := ups[next.Add(1)%uint64(len(ups))]
			if up.healthy.Load() {
				up.proxy.ServeHTTP(w, r)
				return
			}
		}
		http.Error(w, "no healthy upstream", http.StatusServiceUnavailable)
	})
	log.Printf("lb on %s -> %v", addr, targets)
	log.Fatal(http.ListenAndServe(addr, handler))
}

// healthCheck polls /readyz. Two failures to eject, two successes to restore:
// hysteresis stops one slow probe from flapping a backend in and out.
func healthCheck(up *upstream) {
	client := &http.Client{Timeout: time.Second}
	fails, oks := 0, 0
	for range time.Tick(time.Second) {
		resp, err := client.Get(up.url.String() + "/readyz")
		ok := err == nil && resp.StatusCode == http.StatusOK
		if resp != nil {
			resp.Body.Close()
		}
		if ok {
			oks, fails = oks+1, 0
		} else {
			fails, oks = fails+1, 0
		}
		switch {
		case fails == 2 && up.healthy.Load():
			up.healthy.Store(false)
			log.Printf("lb: EJECT %s", up.url)
		case oks == 2 && !up.healthy.Load():
			up.healthy.Store(true)
			log.Printf("lb: RESTORE %s", up.url)
		}
	}
}

$ go build -o lbdrain ./lbdrain
$ ./lbdrain backend 127.0.0.1:9081 &        # job %1
$ ./lbdrain backend 127.0.0.1:9082 &
$ ./lbdrain lb 127.0.0.1:9080 http://127.0.0.1:9081 http://127.0.0.1:9082 &
$ (for i in $(seq 1 300); do curl -s -o /dev/null -w '%{http_code}\n' localhost:9080/; done | sort | uniq -c) &
$ kill -TERM %1
backend 127.0.0.1:9081: SIGTERM -> failing readiness
lb: EJECT http://127.0.0.1:9081
backend 127.0.0.1:9081: drained, exiting
 300 200                                        <- zero errors

 276 200
  24 502                                        <- every in-flight or not-yet-ejected request fails

Who is the client? X-Forwarded-For and the PROXY protocol

Behind any proxy, the backend's TCP peer is the proxy. An L7 proxy can add a header (X-Forwarded-For: 203.0.113.7). An L4 proxy can't change HTTP, because it may not even be HTTP or may be encrypted. Instead it uses the PROXY protocol: one line of text, sent before the client's bytes.

// proxyproto: recover the real client address behind an L4 load balancer.
//
// An L4 proxy (HAProxy, AWS NLB with proxy protocol enabled, Envoy) opens its
// OWN connection to your backend, so RemoteAddr() is the proxy, not the
// client. With the PROXY protocol, the proxy writes one header line first:
//
//	PROXY TCP4 203.0.113.7 10.0.1.15 61001 443\r\n
//
//	go run ./proxyproto            # runs a server and a fake proxy against it
package main

import (
	"bufio"
	"errors"
	"fmt"
	"net"
	"net/netip"
	"strconv"
	"strings"
)

// readProxyV1 parses a PROXY protocol v1 header. Only call it on listeners
// that ONLY your load balancer can reach: otherwise any client can claim to
// be any IP address.
func readProxyV1(r *bufio.Reader) (src, dst netip.AddrPort, err error) {
	line, err := r.ReadString('\n')
	if err != nil {
		return src, dst, err
	}
	if len(line) > 107 || !strings.HasSuffix(line, "\r\n") { // 107 = spec maximum
		return src, dst, errors.New("malformed PROXY header")
	}
	f := strings.Fields(strings.TrimSuffix(line, "\r\n"))
	if len(f) < 2 || f[0] != "PROXY" {
		return src, dst, errors.New("missing PROXY header")
	}
	if f[1] == "UNKNOWN" { // e.g. the LB's own health checks
		return src, dst, nil
	}
	if len(f) != 6 || (f[1] != "TCP4" && f[1] != "TCP6") {
		return src, dst, fmt.Errorf("unsupported PROXY header %q", line)
	}
	sip, err1 := netip.ParseAddr(f[2])
	dip, err2 := netip.ParseAddr(f[3])
	sp, err3 := strconv.ParseUint(f[4], 10, 16)
	dp, err4 := strconv.ParseUint(f[5], 10, 16)
	if err := errors.Join(err1, err2, err3, err4); err != nil {
		return src, dst, fmt.Errorf("bad PROXY header: %w", err)
	}
	return netip.AddrPortFrom(sip, uint16(sp)), netip.AddrPortFrom(dip, uint16(dp)), nil
}

func main() {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	done := make(chan struct{})
	go func() {
		defer close(done)
		for i := 0; i < 2; i++ {
			c, err := ln.Accept()
			if err != nil {
				return
			}
			r := bufio.NewReader(c) // keep using r: it may have buffered request bytes
			src, _, err := readProxyV1(r)
			if err != nil {
				fmt.Println("rejected:", err)
				c.Close()
				continue
			}
			req, _ := r.ReadString('\n')
			fmt.Printf("TCP peer (the proxy): %s\nreal client:          %s\nrequest:              %q\n",
				c.RemoteAddr(), src, req)
			c.Close()
		}
	}()

	// Pretend to be the load balancer: header first, then the client's bytes.
	c, _ := net.Dial("tcp", ln.Addr().String())
	fmt.Fprint(c, "PROXY TCP4 203.0.113.7 10.0.1.15 61001 443\r\nGET / HTTP/1.1\r\n")
	c.Close()
	// And a client that skipped the proxy and talks to the backend directly.
	c, _ = net.Dial("tcp", ln.Addr().String())
	fmt.Fprint(c, "GET / HTTP/1.1\r\n")
	c.Close()
	<-done
}

$ go run ./proxyproto
TCP peer (the proxy): 127.0.0.1:52937
real client:          203.0.113.7:61001
request:              "GET / HTTP/1.1\r\n"
rejected: missing PROXY header

Both mechanisms are trivially forgeable by anyone who can reach the backend directly. Accept PROXY headers or X-Forwarded-For only from your load balancer's addresses, and for X-Forwarded-For take the address added by your proxy (the right-most trusted entry), never the left-most, which the client wrote.

Retry trap

Retries are useful only when:

• the operation is idempotent or has an idempotency key;
• the retry count is bounded, and only one layer retries (client, LB, and service mesh each retrying 3 times means 27 attempts);
• backoff with jitter exists;
• a retry budget caps retries at a fraction of normal traffic (say 10%), so retries can't multiply load during an outage;
• the total deadline still matters: a retry that can't finish in time is pure waste.

Blind retries turn "one slow backend" into "three times the traffic during an outage."

Timeouts must nest

Every hop needs a timeout, and they must fit inside each other:

 client 10s  >  CDN 9s  >  LB idle 60s / request 8s  >  app handler 7s  >  DB query 5s

And for idle keep-alive connections, the rule is reversed: the backend must keep idle connections open longer than the load balancer (e.g. app 65 s > LB 60 s), so the LB always closes first. Otherwise the LB occasionally sends a request on a connection the backend is closing, and the user gets a 502 that appears in no application log. The OSI guide's Layer 5 war story walks through this exact incident.