Chapter 46 — Where the numbers come from: counters, flows, and probes

Practice (25 min)

A. SNMP against your own router (if it supports it — most home routers do, sometimes needing enabling in the admin page):

sudo apt install snmp   # or: brew install net-snmp
snmpwalk -v2c -c public 192.168.1.1 1.3.6.1.2.1.1        # system info
snmpwalk -v2c -c public 192.168.1.1 IF-MIB::ifDescr      # interface names
snmpget  -v2c -c public 192.168.1.1 IF-MIB::ifHCInOctets.2

To turn a counter into a rate: take two readings, subtract, divide by the seconds between them, multiply by 8 for bits/second.

B. Generate and read your own flow records:

sudo apt install softflowd nfdump
sudo softflowd -i eth0 -n 127.0.0.1:9995 -v 9
nfcapd -w -D -l /tmp/flows -p 9995 &
# ... browse the web for two minutes ...
nfdump -R /tmp/flows -s srcip/bytes -n 10       # your top talkers
nfdump -R /tmp/flows -s dstport/bytes -n 10     # top ports

You just built the thing that answers "what's using my bandwidth?"

C. Write a layered probe script:

#!/bin/bash
H=${1:-example.com}
echo -n "DNS:  "; dig +short +time=2 $H | head -1 || echo FAIL
echo -n "PING: "; ping -c2 -W2 $H >/dev/null 2>&1 && echo ok || echo FAIL
echo -n "TCP:  "; nc -zv -w3 $H 443 >/dev/null 2>&1 && echo ok || echo FAIL
echo -n "TLS:  "; echo | openssl s_client -connect $H:443 -servername $H 2>/dev/null \
                   | openssl x509 -noout -enddate || echo FAIL
echo -n "HTTP: "; curl -s -o /dev/null -w '%{http_code} in %{time_total}s\n' https://$H

Run it against a few sites. This is a monitoring system in twelve lines — the real ones just do it continuously and store the results.

Build it in Go (45 min) — a blackbox prober with a latency histogram

"Active probes" from this chapter are what Prometheus's blackbox_exporter does. Writing a small one shows you why monitoring systems store histograms, not averages (Chapter 45): a histogram is a set of cumulative counters, cheap to store and aggregate, from which percentiles can be estimated later.

// prober: a tiny blackbox exporter (Chapters 45-47). It probes TCP targets on
// a schedule and exposes Prometheus-format metrics -- a latency histogram,
// success/failure counters -- on /metrics.
//
//	go run ./prober -listen :9115 example.com:443 1.1.1.1:53 10.255.255.1:80
//	curl -s localhost:9115/metrics
package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"sort"
	"sync"
	"time"
)

// Bucket upper bounds in seconds. Choose them around your SLO, not at random.
var buckets = []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5}

type series struct {
	counts   []uint64 // cumulative-later; raw per-bucket here
	sum      float64
	total    uint64
	failures uint64
}

type registry struct {
	mu sync.Mutex
	m  map[string]*series
}

func (r *registry) observe(target string, d time.Duration, ok bool) {
	r.mu.Lock()
	defer r.mu.Unlock()
	s := r.m[target]
	if s == nil {
		s = &series{counts: make([]uint64, len(buckets)+1)}
		r.m[target] = s
	}
	if !ok {
		s.failures++
		return
	}
	sec := d.Seconds()
	i := sort.SearchFloat64s(buckets, sec) // first bucket >= sec; len(buckets) = +Inf
	s.counts[i]++
	s.sum += sec
	s.total++
}

// ServeHTTP writes the Prometheus text exposition format by hand -- it is
// just lines of "name{labels} value".
func (r *registry) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
	r.mu.Lock()
	defer r.mu.Unlock()
	w.Header().Set("Content-Type", "text/plain; version=0.0.4")
	fmt.Fprintln(w, "# HELP probe_tcp_connect_seconds Time to complete the TCP handshake.")
	fmt.Fprintln(w, "# TYPE probe_tcp_connect_seconds histogram")
	targets := make([]string, 0, len(r.m))
	for t := range r.m {
		targets = append(targets, t)
	}
	sort.Strings(targets)
	for _, t := range targets {
		s := r.m[t]
		var cum uint64
		for i, le := range buckets {
			cum += s.counts[i]
			fmt.Fprintf(w, "probe_tcp_connect_seconds_bucket{target=%q,le=\"%g\"} %d\n", t, le, cum)
		}
		cum += s.counts[len(buckets)]
		fmt.Fprintf(w, "probe_tcp_connect_seconds_bucket{target=%q,le=\"+Inf\"} %d\n", t, cum)
		fmt.Fprintf(w, "probe_tcp_connect_seconds_sum{target=%q} %g\n", t, s.sum)
		fmt.Fprintf(w, "probe_tcp_connect_seconds_count{target=%q} %d\n", t, s.total)
	}
	fmt.Fprintln(w, "# TYPE probe_failures_total counter")
	for _, t := range targets {
		fmt.Fprintf(w, "probe_failures_total{target=%q} %d\n", t, r.m[t].failures)
	}
}

func main() {
	listen := flag.String("listen", ":9115", "address for /metrics")
	every := flag.Duration("every", 5*time.Second, "probe interval")
	timeout := flag.Duration("timeout", 2*time.Second, "per-probe timeout")
	flag.Parse()

	reg := &registry{m: map[string]*series{}}
	for _, target := range flag.Args() {
		go func() {
			for {
				start := time.Now()
				c, err := net.DialTimeout("tcp", target, *timeout)
				if err == nil {
					c.Close()
				}
				reg.observe(target, time.Since(start), err == nil)
				time.Sleep(*every)
			}
		}()
	}
	http.Handle("/metrics", reg)
	log.Printf("probing %v every %v; metrics on http://localhost%s/metrics", flag.Args(), *every, *listen)
	log.Fatal(http.ListenAndServe(*listen, nil))
}

$ go run ./prober -every 1s example.com:443 1.1.1.1:53 10.255.255.1:80 &
$ curl -s localhost:9115/metrics | grep -E 'example.com|failures'
probe_tcp_connect_seconds_bucket{target="example.com:443",le="0.05"} 5
probe_tcp_connect_seconds_bucket{target="example.com:443",le="+Inf"} 6
probe_tcp_connect_seconds_count{target="example.com:443"} 6
probe_failures_total{target="1.1.1.1:53"} 0
probe_failures_total{target="10.255.255.1:80"} 2
probe_failures_total{target="example.com:443"} 0

What to notice:

• Buckets are cumulative. le="0.05"} 5 means 5 of the 6 probes finished in 50 ms or less. To find the p99 you look for the first bucket that holds 99% of _count, which is exactly what PromQL's histogram_quantile() does.
• Choose bucket edges around your objective. If the SLO is "connect under 100 ms", you need edges just below and above 0.1. Otherwise the histogram can't tell you whether you're meeting it.
• Failures are a separate counter. Mixing timeouts into the latency histogram would hide them in the +Inf bucket. Alert on the failure rate (Chapter 47) and track latency separately.
• The exposition format is plain text. No client library is needed to expose metrics, though in production you'd use github.com/prometheus/client_golang (see the Go guide's observability chapter).

Exercises:

1. Point a local Prometheus at it (scrape_configs: [{job_name: probe, static_configs: [{targets: ['localhost:9115']}]}]) and graph histogram_quantile(0.99, rate(probe_tcp_connect_seconds_bucket[5m])).
2. Add a TLS probe type that records handshake time separately from TCP connect time, and an HTTP probe that checks the status code. Now you can tell which layer got slow.
3. Write the Chapter 47 alert for it: "more than 10% of probes to a target failed for 5 minutes", plus a runbook link.