Day 73 — WAF Normalization Pipeline

Learning Objectives

• Build the multi-layer normalization pipeline that separates basic WAFs from production ones
• Test 10+ encoding bypass attempts against your normalized matcher
• Understand why ModSecurity has 8 normalization passes

Exercise: Normalization Pipeline

package waf

import (
    "html"
    "net/url"
    "regexp"
    "strings"
    "unicode"
)

var (
    sqlCommentRe    = regexp.MustCompile(`\/\*.*?\*\/`)
    multiSpaceRe    = regexp.MustCompile(`\s+`)
    nullByteRe      = regexp.MustCompile(`\x00`)
    htmlEntityRe    = regexp.MustCompile(`&#x?[0-9a-fA-F]+;?|&\w+;`)
)

// NormalizationPipeline applies layers of decoding before rule matching.
// Each layer decodes a different encoding that attackers might use.
func NormalizationPipeline(input string) string {
    result := input

    // Pass 1: Remove null bytes (used to truncate strings in C-based systems)
    result = nullByteRe.ReplaceAllString(result, "")

    // Pass 2: URL decode (handles %XX encoding)
    if decoded, err := url.QueryUnescape(result); err == nil {
        result = decoded
    }

    // Pass 3: HTML entity decode (handles < > < <)
    result = html.UnescapeString(result)

    // Pass 4: Second URL decode (catches double-encoded: %2520 → %20 → space)
    if decoded, err := url.QueryUnescape(result); err == nil {
        result = decoded
    }

    // Pass 5: Strip SQL comments (/**/ used to split keywords: UN/**/ION)
    result = sqlCommentRe.ReplaceAllString(result, " ")

    // Pass 6: Normalize whitespace (tab, newline used to split keywords)
    result = strings.Map(func(r rune) rune {
        if unicode.IsSpace(r) { return ' ' }
        return r
    }, result)

    // Pass 7: Collapse multiple spaces
    result = multiSpaceRe.ReplaceAllString(result, " ")

    // Pass 8: Lowercase (for case-insensitive matching)
    result = strings.ToLower(strings.TrimSpace(result))

    return result
}

// Test bypass attempts — all should be normalized to detectable form:
var bypassAttempts = []struct {
    input, expectedNorm, attack string
}{
    {
        input:       "UN/**/ION SE/**/LECT",
        expectedNorm: "union select",
        attack:      "SQL comment evasion",
    },
    {
        input:       "%55NION%20%53ELECT",
        expectedNorm: "union select",
        attack:      "URL encoding",
    },
    {
        input:       "%2555NION",
        expectedNorm: "union",
        attack:      "Double URL encoding (%25 → % → %55 → U)",
    },
    {
        input:       "UNION",
        expectedNorm: "union",
        attack:      "HTML entity encoding (U = U)",
    },
    {
        input:       "sElEcT\t\n*\tFrOm\tusers",
        expectedNorm: "select * from users",
        attack:      "Mixed case + whitespace",
    },
    {
        input:       "\x00UNION SELECT",
        expectedNorm: "union select",
        attack:      "Null byte prefix",
    },
}

func TestNormalizationBypass() {
    for _, tc := range bypassAttempts {
        norm := NormalizationPipeline(tc.input)
        detected := strings.Contains(norm, "union") || strings.Contains(norm, "select")
        fmt.Printf("[%s] input=%q norm=%q detected=%v\n", tc.attack, tc.input, norm, detected)
    }
}

Real-World Context

OWASP ModSecurity Core Rule Set (CRS) — the reference implementation used by nginx, Apache, and cloud WAFs — has 8 specific normalization transformation steps. Without normalization, a WAF that detects UNION SELECT is trivially bypassed by %55NION SELECT. This is exactly why attackers test their payloads against WAFs first.