Chapter 20 — Reverse proxies in Go: the client's identity, framing, and request smuggling

Build it in Go (40 min) — a reverse proxy that gets the details right

// revproxy: a path-routing reverse proxy, built to show the details that
// matter in production -- who the client is (X-Forwarded-*), hop-by-hop
// headers, upstream timeouts, and 502 vs 504.
//
//	go run ./revproxy -demo
package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"net/url"
	"strings"
	"time"
)

type route struct {
	prefix string
	target *url.URL
}

// newProxy routes by path prefix, first match wins, so list longer prefixes
// first. Rewrite (Go 1.20+) is the safe API: the outgoing request starts
// WITHOUT the client's X-Forwarded-* headers, and SetXForwarded adds fresh
// ones from the real TCP peer.
func newProxy(routes []route) http.Handler {
	transport := &http.Transport{
		DialContext:           (&net.Dialer{Timeout: 2 * time.Second}).DialContext,
		ResponseHeaderTimeout: 2 * time.Second, // upstream must START answering in 2s
		IdleConnTimeout:       60 * time.Second,
		MaxIdleConnsPerHost:   64, // the default of 2 causes connection churn under load
	}
	return &httputil.ReverseProxy{
		Rewrite: func(pr *httputil.ProxyRequest) {
			for _, rt := range routes {
				if strings.HasPrefix(pr.In.URL.Path, rt.prefix) {
					pr.SetURL(rt.target)
					break
				}
			}
			pr.SetXForwarded() // X-Forwarded-For/Host/Proto from the TCP connection
		},
		Transport: transport,
		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
			status := http.StatusBadGateway // 502: couldn't get a valid response
			if errors.Is(err, context.DeadlineExceeded) || isTimeout(err) {
				status = http.StatusGatewayTimeout // 504: upstream too slow
			}
			log.Printf("proxy %s %s -> %d: %v", r.Method, r.URL.Path, status, err)
			http.Error(w, http.StatusText(status), status)
		},
	}
}

func isTimeout(err error) bool {
	var ne net.Error
	return errors.As(err, &ne) && ne.Timeout()
}

// legacyProxy is the pre-Go-1.20 Director style, shown for contrast: it
// APPENDS to whatever X-Forwarded-For the client sent.
func legacyProxy(target *url.URL) http.Handler {
	return httputil.NewSingleHostReverseProxy(target)
}

func main() {
	demo := flag.Bool("demo", false, "run a scripted demo")
	flag.Parse()
	if !*demo {
		fmt.Println("run with -demo")
		return
	}
	log.SetFlags(0)

	echo := func(name string) *httptest.Server {
		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if r.URL.Path == "/api/slow" {
				time.Sleep(3 * time.Second)
			}
			fmt.Fprintf(w, "%s saw path=%s XFF=%q Proto=%q Connection=%q",
				name, r.URL.Path, r.Header.Get("X-Forwarded-For"), r.Header.Get("X-Forwarded-Proto"), r.Header.Get("Connection"))
		}))
	}
	api, web := echo("api"), echo("web")
	defer api.Close()
	defer web.Close()
	apiURL, _ := url.Parse(api.URL)
	webURL, _ := url.Parse(web.URL)
	dead, _ := url.Parse("http://127.0.0.1:1") // nothing listens on port 1

	proxy := httptest.NewServer(newProxy([]route{{"/api/", apiURL}, {"/down/", dead}, {"/", webURL}}))
	defer proxy.Close()
	legacy := httptest.NewServer(legacyProxy(webURL))
	defer legacy.Close()

	get := func(label, base, path string, hdr map[string]string) {
		req, _ := http.NewRequest("GET", base+path, nil)
		for k, v := range hdr {
			req.Header.Set(k, v)
		}
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			fmt.Println(label, err)
			return
		}
		b, _ := io.ReadAll(resp.Body)
		resp.Body.Close()
		fmt.Printf("%-22s %d  %s\n", label, resp.StatusCode, strings.TrimSpace(string(b)))
	}
	spoof := map[string]string{"X-Forwarded-For": "6.6.6.6", "Connection": "X-Secret", "X-Secret": "hop"}

	get("route /api/users", proxy.URL, "/api/users", nil)
	get("route /index.html", proxy.URL, "/index.html", nil)
	get("spoofed XFF (Rewrite)", proxy.URL, "/", spoof)
	get("spoofed XFF (Director)", legacy.URL, "/", spoof)
	get("dead upstream", proxy.URL, "/down/x", nil)
	get("slow upstream", proxy.URL, "/api/slow", nil)
}

$ go run ./revproxy -demo
route /api/users       200  api saw path=/api/users XFF="127.0.0.1" Proto="http" Connection=""
route /index.html      200  web saw path=/index.html XFF="127.0.0.1" Proto="http" Connection=""
spoofed XFF (Rewrite)  200  web saw path=/ XFF="127.0.0.1" Proto="http" Connection=""
spoofed XFF (Director) 200  web saw path=/ XFF="6.6.6.6, 127.0.0.1" Proto="" Connection=""
proxy GET /down/x -> 502: dial tcp 127.0.0.1:1: connect: connection refused
dead upstream          502  Bad Gateway
proxy GET /api/slow -> 504: net/http: timeout awaiting response headers
slow upstream          504  Gateway Timeout

What to notice:

• Rewrite vs Director: the client sent X-Forwarded-For: 6.6.6.6. With Go 1.20's Rewrite API, the backend saw only the real peer address. With the older Director API (NewSingleHostReverseProxy), the client's fake value reached the backend first in the list, and an application that reads the first entry would rate-limit, geo-locate, or audit-log 6.6.6.6.
• Connection: X-Secret was stripped by both: hop-by-hop handling is built in.
• 502 vs 504 is a diagnosis. 502: the proxy couldn't get a valid response (here, connection refused). 504: the upstream accepted the connection but didn't answer within ResponseHeaderTimeout.
• MaxIdleConnsPerHost: 64: Go's default is 2, so a busy proxy would open and close upstream connections constantly, wasting handshakes and filling TIME_WAIT (TCP/IP guide, Chapter 23).
• Route order matters. An earlier draft of this lab stored routes in a Go map, and map iteration order is random, so /api/users sometimes went to the catch-all / backend. Routing tables must be ordered (longest prefix first) or use a router that guarantees it.

Build it in Go (15 min) — request smuggling's root cause

desync sends one message carrying both framing headers to a Go server it starts itself, and shows how the server splits it:

// desync: see HTTP request smuggling's root cause -- two parsers disagreeing
// about where a request ends. Self-contained: it only talks to a test server
// it starts itself.
//
//	go run ./desync
package main

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"strings"
	"time"
)

func main() {
	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "backend handled: %s %s\n", r.Method, r.URL.Path)
	}))
	defer backend.Close()

	// ONE message with BOTH framing headers. A front end that trusts
	// Content-Length (10 bytes: "0\r\n\r\nGET /") sees one POST whose body
	// ends mid-line. The backend follows RFC 9112 -- Transfer-Encoding wins --
	// so the body ends at the "0" chunk and everything after it is parsed as a
	// SECOND request the front end never inspected.
	msg := "POST /search HTTP/1.1\r\nHost: shop\r\n" +
		"Content-Length: 10\r\nTransfer-Encoding: chunked\r\n\r\n" +
		"0\r\n\r\n" +
		"GET /admin/delete-user HTTP/1.1\r\nHost: shop\r\n\r\n"

	c, err := net.Dial("tcp", strings.TrimPrefix(backend.URL, "http://"))
	if err != nil {
		panic(err)
	}
	defer c.Close()
	c.SetDeadline(time.Now().Add(2 * time.Second))
	io.WriteString(c, msg)
	out, _ := io.ReadAll(c)
	for _, line := range strings.Split(string(out), "\n") {
		if strings.HasPrefix(line, "HTTP/") || strings.HasPrefix(line, "backend") {
			fmt.Println(strings.TrimSpace(line))
		}
	}
	fmt.Println("\nOne message in, TWO requests handled. If a front end had checked only")
	fmt.Println("POST /search, then GET /admin/delete-user was smuggled past it.")
}

$ go run ./desync
HTTP/1.1 200 OK
backend handled: POST /search
HTTP/1.1 200 OK
backend handled: GET /admin/delete-user

One message in, TWO requests handled. If a front end had checked only
POST /search, then GET /admin/delete-user was smuggled past it.

What's happening: Go follows RFC 9112, where Transfer-Encoding overrides Content-Length. The chunked body ends at 0\r\n\r\n, and the rest of the bytes are parsed as a new request on the same connection. A front end that had framed the message by Content-Length would have forwarded all of it as one request after checking only POST /search.

Two more facts from testing this with Go 1.26:

• Go's handler never sees the conflict: the Content-Length header is removed before your code runs. You can't detect smuggling inside a Go handler. It has to be stopped by the parser in front.
• RFC 9112 says a server must close the connection after responding to a request with both headers. In this test the Go server answered the second request on the same connection. Don't rely on any single component's parser. Reject ambiguous requests at the edge, and prefer HTTP/2 between proxies and backends.