Day 100 — HTTP Header Fingerprinting

Key Concepts

Chrome 120 always sends these headers (in this order):

:method, :authority, :scheme, :path  (HTTP/2 pseudo-headers)
sec-ch-ua, sec-ch-ua-mobile, sec-ch-ua-platform
Upgrade-Insecure-Requests
User-Agent
Accept
Sec-Fetch-Site, Sec-Fetch-Mode, Sec-Fetch-Dest
Referer (when navigating)
Accept-Encoding: gzip, deflate, br
Accept-Language

curl sends:

User-Agent, Accept, Host (only)

Exercise — Header Consistency Detector

package bot

import (
    "net/http"
    "strings"
)

// browserProfile defines expected headers for a claimed browser
type browserProfile struct {
    requiredHeaders  []string // must be present
    forbiddenValues  map[string]string // header → value that browser never sends
    minHeaderCount   int
}

var profiles = map[string]browserProfile{
    "Chrome": {
        requiredHeaders: []string{"Accept-Language", "Accept-Encoding"},
        forbiddenValues: map[string]string{
            "Accept-Encoding": "compress", // Chrome never sends compress
        },
        minHeaderCount: 6,
    },
    "Firefox": {
        requiredHeaders: []string{"Accept-Language", "Accept-Encoding"},
        forbiddenValues: map[string]string{},
        minHeaderCount: 5,
    },
    "Safari": {
        requiredHeaders: []string{"Accept-Language"},
        forbiddenValues: map[string]string{},
        minHeaderCount: 5,
    },
}

// claimedBrowser extracts the browser name from User-Agent
func claimedBrowser(ua string) string {
    ua = strings.ToLower(ua)
    switch {
    case strings.Contains(ua, "chrome") && !strings.Contains(ua, "edg"):
        return "Chrome"
    case strings.Contains(ua, "firefox"):
        return "Firefox"
    case strings.Contains(ua, "safari") && !strings.Contains(ua, "chrome"):
        return "Safari"
    case strings.Contains(ua, "curl"):
        return "curl"
    case strings.Contains(ua, "python"):
        return "python"
    case strings.Contains(ua, "go-http-client"):
        return "go-http"
    }
    return ""
}

// HeaderConsistencyDetector implements Detector
type HeaderConsistencyDetector struct{}

func (d *HeaderConsistencyDetector) Detect(r *http.Request, session *SessionScore) {
    ua := r.Header.Get("User-Agent")
    if ua == "" {
        session.AddSignal(Signal{"missing_ua", 20, CategoryHTTP, "No User-Agent header"})
        return
    }

    browser := claimedBrowser(ua)

    // Known non-browser clients — flag immediately
    switch browser {
    case "curl":
        session.AddSignal(Signal{"curl_ua", 25, CategoryHTTP, "curl User-Agent detected"})
        return
    case "python":
        session.AddSignal(Signal{"python_ua", 25, CategoryHTTP, "Python http client detected"})
        return
    case "go-http":
        session.AddSignal(Signal{"go_http_ua", 20, CategoryHTTP, "Go HTTP client detected"})
        return
    case "":
        session.AddSignal(Signal{"unknown_ua", 10, CategoryHTTP, "Unrecognized User-Agent"})
    }

    profile, ok := profiles[browser]
    if !ok {
        return
    }

    // Check required headers
    for _, h := range profile.requiredHeaders {
        if r.Header.Get(h) == "" {
            session.AddSignal(SigMissingAcceptLang)
        }
    }

    // Check forbidden values
    for header, forbidden := range profile.forbiddenValues {
        if strings.Contains(r.Header.Get(header), forbidden) {
            session.AddSignal(SigBadAcceptEncoding)
        }
    }

    // Check minimum header count
    if len(r.Header) < profile.minHeaderCount {
        session.AddSignal(Signal{
            "low_header_count", 15, CategoryHTTP,
            fmt.Sprintf("%s normally sends ≥%d headers, got %d", browser, profile.minHeaderCount, len(r.Header)),
        })
    }

    // Check Accept header consistency
    accept := r.Header.Get("Accept")
    if browser == "Chrome" && accept == "*/*" {
        // Chrome never sends Accept: */* for page navigation
        session.AddSignal(Signal{"bad_accept", 10, CategoryHTTP, "Chrome never sends Accept: */*"})
    }

    // User-Agent vs Sec-CH-UA consistency (HTTP/2 only)
    secCHUA := r.Header.Get("Sec-CH-UA")
    if browser == "Chrome" && r.ProtoMajor >= 2 && secCHUA == "" {
        session.AddSignal(SigUserAgentMismatch)
    }
}