Day 26 — expvar & Runtime Metrics

Learning Objectives

• Expose runtime metrics via /debug/vars — zero-dependency observability
• Build the metrics foundation for your proxy's /health endpoint
• Understand why self-instrumentation matters before adding Prometheus

Key Concepts

// expvar — built-in, zero-dependency metrics
import "expvar"

var (
    reqTotal  = expvar.NewInt("requests_total")
    reqErrors = expvar.NewInt("requests_errors")
    reqLatency = expvar.NewFloat("requests_latency_p99_ms")
    connActive = expvar.NewInt("connections_active")
)

// Increment
reqTotal.Add(1)
connActive.Add(1)
// Decrement on close
defer connActive.Add(-1)

// Map for per-endpoint stats
var endpointStats = expvar.NewMap("endpoints")
func recordEndpoint(path string) {
    endpointStats.Add(path, 1)
}

// Accessed at: GET /debug/vars
// Returns JSON: {"requests_total": 1234, "connections_active": 42, ...}

Exercise: Instrumented Proxy Metrics

package main

import (
    "expvar"
    "net/http"
    _ "net/http/pprof"   // registers /debug/pprof automatically
    "time"
)

// ProxyMetrics collects metrics for the security proxy
type ProxyMetrics struct {
    ReqTotal    *expvar.Int
    ReqErrors   *expvar.Int
    ReqBlocked  *expvar.Int    // WAF blocks
    ConnActive  *expvar.Int
    ConnTotal   *expvar.Int
    BytesIn     *expvar.Int
    BytesOut    *expvar.Int
    ByEndpoint  *expvar.Map    // requests per path
    ByStatus    *expvar.Map    // requests per status code

    // latency ring buffer (not expvar — compute percentiles ourselves)
    latencies   *RingBuffer[int64]
    latMu       sync.Mutex
}

func NewProxyMetrics() *ProxyMetrics {
    m := &ProxyMetrics{
        ReqTotal:   expvar.NewInt("proxy_requests_total"),
        ReqErrors:  expvar.NewInt("proxy_requests_errors"),
        ReqBlocked: expvar.NewInt("proxy_requests_blocked"),
        ConnActive: expvar.NewInt("proxy_connections_active"),
        ConnTotal:  expvar.NewInt("proxy_connections_total"),
        BytesIn:    expvar.NewInt("proxy_bytes_in"),
        BytesOut:   expvar.NewInt("proxy_bytes_out"),
        ByEndpoint: expvar.NewMap("proxy_by_endpoint"),
        ByStatus:   expvar.NewMap("proxy_by_status"),
        latencies:  NewRingBuffer[int64](10000),
    }

    // Register computed P99 as expvar.Func (re-computed on each scrape)
    expvar.Publish("proxy_latency_p99_ms", expvar.Func(func() any {
        return m.percentile(99)
    }))
    expvar.Publish("proxy_latency_p95_ms", expvar.Func(func() any {
        return m.percentile(95)
    }))

    return m
}

func (m *ProxyMetrics) RecordRequest(path, status string, latencyMS int64, blocked bool) {
    m.ReqTotal.Add(1)
    m.ByEndpoint.Add(path, 1)
    m.ByStatus.Add(status, 1)
    if blocked { m.ReqBlocked.Add(1) }

    m.latMu.Lock()
    m.latencies.Add(latencyMS)
    m.latMu.Unlock()
}

func (m *ProxyMetrics) percentile(p float64) float64 {
    m.latMu.Lock()
    samples := m.latencies.All()
    m.latMu.Unlock()

    if len(samples) == 0 { return 0 }
    sorted := make([]int64, len(samples))
    copy(sorted, samples)
    sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
    idx := int(float64(len(sorted)) * p / 100)
    if idx >= len(sorted) { idx = len(sorted) - 1 }
    return float64(sorted[idx])
}

func main() {
    metrics := NewProxyMetrics()

    // Simulate requests
    for i := 0; i < 100; i++ {
        latency := int64(rand.Intn(100) + 1)
        metrics.RecordRequest("/api/data", "200", latency, false)
    }
    metrics.RecordRequest("/api/admin", "403", 5, true)

    http.ListenAndServe(":8080", nil) // /debug/vars and /debug/pprof served automatically
}

Real-World Context

/debug/vars is the first observability tool you add to any Go service. When an incident happens and your Datadog is down, curl http://service:8080/debug/vars still gives you request rates, error counts, and latency percentiles. Every internal service should expose this endpoint — not to the internet, but behind your VPN or internal load balancer.