Day 99 — Bot Taxonomy & Signal Framework

Key Concepts

Bot taxonomy:

Signal categories:

• Network signals: IP ASN, datacenter vs residential, TLS fingerprint (JA3)
• HTTP signals: Header order/values, User-Agent consistency, Accept-Encoding
• Behavioral signals: Request velocity, path diversity, referrer chain, session depth

Exercise — BotScorer Signal Framework

package bot

import (
    "net/http"
    "sync"
    "time"
)

// SignalCategory groups related signals for reporting
type SignalCategory string

const (
    CategoryNetwork   SignalCategory = "network"
    CategoryHTTP      SignalCategory = "http"
    CategoryBehavior  SignalCategory = "behavioral"
    CategoryReputation SignalCategory = "reputation"
)

// Signal represents a single detection signal with its weight
type Signal struct {
    Name     string
    Weight   float64
    Category SignalCategory
    Reason   string // human-readable explanation
}

// Common signals
var (
    SigMissingAcceptLang   = Signal{"missing_accept_language", 10, CategoryHTTP, "Real browsers always send Accept-Language"}
    SigBadAcceptEncoding   = Signal{"bad_accept_encoding", 15, CategoryHTTP, "Accept-Encoding inconsistent with claimed browser"}
    SigDatacenterIP        = Signal{"datacenter_ip", 15, CategoryNetwork, "Source IP belongs to known hosting/datacenter ASN"}
    SigKnownBadJA3         = Signal{"known_bad_ja3", 30, CategoryNetwork, "TLS fingerprint matches curl/requests/scrapy"}
    SigHighVelocity        = Signal{"high_velocity", 20, CategoryBehavior, "Request rate > 10 req/s"}
    SigLowPathDiversity    = Signal{"low_path_diversity", 10, CategoryBehavior, "< 2 unique paths in session"}
    SigMissingReferrer     = Signal{"missing_referrer", 10, CategoryBehavior, "> 80% of requests have no Referer"}
    SigHoneypotHit         = Signal{"honeypot_hit", 100, CategoryBehavior, "Request to known honeypot endpoint"}
    SigUserAgentMismatch   = Signal{"user_agent_mismatch", 25, CategoryHTTP, "UA claims browser but headers don't match"}
)

// SessionScore tracks the aggregated score for a session/IP
type SessionScore struct {
    mu      sync.Mutex
    signals []Signal
    score   float64
    created time.Time
    lastSee time.Time
}

func (ss *SessionScore) AddSignal(s Signal) {
    ss.mu.Lock()
    defer ss.mu.Unlock()
    ss.signals = append(ss.signals, s)
    ss.score += s.Weight
    ss.lastSee = time.Now()
}

func (ss *SessionScore) Score() float64 {
    ss.mu.Lock()
    defer ss.mu.Unlock()
    return ss.score
}

func (ss *SessionScore) Signals() []Signal {
    ss.mu.Lock()
    defer ss.mu.Unlock()
    out := make([]Signal, len(ss.signals))
    copy(out, ss.signals)
    return out
}

// BotScorer maintains per-IP scores and dispatches detection signals
type BotScorer struct {
    mu       sync.RWMutex
    sessions map[string]*SessionScore
    detectors []Detector
    done     chan struct{}

    // Thresholds
    ChallengeScore float64 // score >= this → PoW challenge
    BlockScore     float64 // score >= this → block
}

// Detector is the interface for individual signal detectors
type Detector interface {
    Detect(r *http.Request, session *SessionScore)
}

func NewBotScorer(detectors ...Detector) *BotScorer {
    bs := &BotScorer{
        sessions:       make(map[string]*SessionScore),
        detectors:      detectors,
        ChallengeScore: 40,
        BlockScore:     70,
        done:           make(chan struct{}),
    }
    go bs.cleanup()
    return bs
}

func (bs *BotScorer) sessionFor(ip string) *SessionScore {
    bs.mu.RLock()
    s, ok := bs.sessions[ip]
    bs.mu.RUnlock()
    if ok {
        return s
    }
    bs.mu.Lock()
    defer bs.mu.Unlock()
    s, ok = bs.sessions[ip]
    if !ok {
        s = &SessionScore{created: time.Now(), lastSee: time.Now()}
        bs.sessions[ip] = s
    }
    return s
}

func (bs *BotScorer) Score(r *http.Request) (float64, []Signal) {
    ip := clientIP(r)
    session := bs.sessionFor(ip)

    for _, d := range bs.detectors {
        d.Detect(r, session)
    }

    return session.Score(), session.Signals()
}

// Action returns the mitigation action based on score
func (bs *BotScorer) Action(score float64) string {
    switch {
    case score >= bs.BlockScore:
        return "block"
    case score >= bs.ChallengeScore:
        return "challenge"
    default:
        return "allow"
    }
}

func (bs *BotScorer) Middleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        score, _ := bs.Score(r)
        switch bs.Action(score) {
        case "block":
            http.Error(w, `{"error":"forbidden"}`, http.StatusForbidden)
        case "challenge":
            // In production: redirect to PoW challenge page
            w.Header().Set("X-Bot-Score", fmt.Sprintf("%.0f", score))
            http.Error(w, `{"error":"challenge_required"}`, http.StatusTooManyRequests)
        default:
            next.ServeHTTP(w, r)
        }
    })
}

func (bs *BotScorer) cleanup() {
    ticker := time.NewTicker(10 * time.Minute)
    defer ticker.Stop()
    for {
        select {
        case <-bs.done:
            return
        case <-ticker.C:
            cutoff := time.Now().Add(-30 * time.Minute)
            bs.mu.Lock()
            for ip, s := range bs.sessions {
                s.mu.Lock()
                stale := s.lastSee.Before(cutoff)
                s.mu.Unlock()
                if stale {
                    delete(bs.sessions, ip)
                }
            }
            bs.mu.Unlock()
        }
    }
}

func clientIP(r *http.Request) string {
    host, _, _ := net.SplitHostPort(r.RemoteAddr)
    return host
}