Chapter 23 — SLIs, SLOs, and error budgets for HTTPS services

Build it in Go (45 min) — burn-rate alerting, simulated

slo counts good and total events in one-minute buckets, computes SLIs over any window, and evaluates the alert table above. Its Middleware shows how real HTTP traffic feeds it. The demo replays 30 days of synthetic traffic with a simulated clock: a 30-minute outage on day 20, and a slow leak from day 25.

// slo: SLIs, an SLO, an error budget, and multi-window burn-rate alerts --
// the way Google's SRE Workbook recommends alerting on them.
//
// The tracker counts good/total events in one-minute buckets. A real service
// feeds it from HTTP middleware (see Middleware); the demo replays 30 days of
// synthetic traffic with a simulated clock so you can watch alerts fire.
//
//	go run ./slo
package main

import (
	"fmt"
	"net/http"
	"time"
)

const (
	objective = 0.999 // 99.9% of requests good over 30 days
	window    = 30 * 24 * time.Hour
)

type bucket struct{ good, total int64 }

type tracker struct {
	buckets map[int64]*bucket // unix minute -> counts
	now     func() time.Time
}

func newTracker(now func() time.Time) *tracker {
	return &tracker{buckets: map[int64]*bucket{}, now: now}
}

// Record is one event: good means "fast enough AND not a server error".
func (t *tracker) Record(good bool) {
	m := t.now().Unix() / 60
	b := t.buckets[m]
	if b == nil {
		b = &bucket{}
		t.buckets[m] = b
	}
	b.total++
	if good {
		b.good++
	}
}

// sli = good / total over the trailing duration d.
func (t *tracker) sli(d time.Duration) (sli float64, total int64) {
	end := t.now().Unix() / 60
	var good int64
	for m := end - int64(d/time.Minute) + 1; m <= end; m++ {
		if b := t.buckets[m]; b != nil {
			good += b.good
			total += b.total
		}
	}
	if total == 0 {
		return 1, 0
	}
	return float64(good) / float64(total), total
}

// burnRate: how fast we are spending the error budget. 1.0 = exactly on pace
// to use 100% of the budget in 30 days; 14.4 = the whole budget in ~2 days.
func (t *tracker) burnRate(d time.Duration) float64 {
	s, _ := t.sli(d)
	return (1 - s) / (1 - objective)
}

func (t *tracker) budgetRemaining() float64 {
	s, total := t.sli(window)
	allowed := (1 - objective) * float64(total)
	bad := (1 - s) * float64(total)
	if allowed == 0 {
		return 1
	}
	return 1 - bad/allowed
}

// alert implements the multi-window, multi-burn-rate policy: a long window
// proves the problem is significant, a short window proves it is STILL
// happening (so alerts stop soon after recovery).
func (t *tracker) alert() string {
	switch {
	case t.burnRate(time.Hour) > 14.4 && t.burnRate(5*time.Minute) > 14.4:
		return "PAGE   (2% of the monthly budget burned in 1h)"
	case t.burnRate(6*time.Hour) > 6 && t.burnRate(30*time.Minute) > 6:
		return "PAGE   (5% of the budget burned in 6h)"
	case t.burnRate(3*24*time.Hour) > 1 && t.burnRate(6*time.Hour) > 1:
		return "TICKET (on pace to miss the SLO this month)"
	}
	return "ok"
}

// Middleware turns real traffic into SLI events: 5xx = bad, slower than
// `slow` = bad (a latency SLI), everything else = good. 4xx are the
// client's fault and count as good.
func (t *tracker) Middleware(slow time.Duration, next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		rec := &status{ResponseWriter: w, code: 200}
		next.ServeHTTP(rec, r)
		t.Record(rec.code < 500 && time.Since(start) < slow)
	})
}

type status struct {
	http.ResponseWriter
	code int
}

func (s *status) WriteHeader(c int) { s.code = c; s.ResponseWriter.WriteHeader(c) }

// ------------------------------------------------------------------ demo --

func main() {
	clock := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
	t := newTracker(func() time.Time { return clock })

	// 600 requests/minute. Baseline: 0.02% bad. Two incidents:
	//   day 20, 14:00-14:30  hard outage, 20% of requests fail
	//   day 25 onwards       slow leak, 0.25% bad (2.5x the allowed rate)
	errRate := func(c time.Time) float64 {
		day := c.Sub(time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)).Hours() / 24
		switch {
		case day >= 19 && day < 20 && c.Hour() == 14 && c.Minute() < 30:
			return 0.20
		case day >= 24:
			return 0.0025
		}
		return 0.0002
	}
	report := func(label string) {
		s, _ := t.sli(window)
		fmt.Printf("%-34s 30d SLI=%.4f%%  budget left=%6.1f%%  burn 1h=%5.1f 6h=%5.1f  -> %s\n",
			label, s*100, t.budgetRemaining()*100, t.burnRate(time.Hour), t.burnRate(6*time.Hour), t.alert())
	}
	var acc float64
	step := func(minutes int) {
		for i := 0; i < minutes; i++ {
			clock = clock.Add(time.Minute)
			for r := 0; r < 600; r++ {
				acc += errRate(clock) // deterministic: exactly errRate of requests fail
				bad := acc >= 1
				if bad {
					acc--
				}
				t.Record(!bad)
			}
		}
	}

	step(19*24*60 + 14*60 + 5) // up to day 20, 14:05
	report("day 20 14:05 (outage, 5 min in)")
	step(10)
	report("day 20 14:15 (outage ongoing)")
	step(45)
	report("day 20 15:00 (30 min after fix)")
	step(4*24*60 + 9*60)
	report("day 25 00:00 (slow leak begins)")
	step(3 * 24 * 60)
	report("day 28 00:00 (leak for 3 days)")
	step(2*24*60 + 23*60)
	report("day 30 23:00 (month end)")
}

$ go run ./slo
day 20 14:05 (outage, 5 min in)    30d SLI=99.9758%  budget left=  75.8%  burn 1h= 20.2 6h=  3.5  -> PAGE   (2% of the monthly budget burned in 1h)
day 20 14:15 (outage ongoing)      30d SLI=99.9687%  budget left=  68.7%  burn 1h= 53.5 6h=  9.1  -> PAGE   (2% of the monthly budget burned in 1h)
day 20 15:00 (30 min after fix)    30d SLI=99.9588%  budget left=  58.8%  burn 1h= 96.8 6h= 16.9  -> TICKET (on pace to miss the SLO this month)
day 25 00:00 (slow leak begins)    30d SLI=99.9627%  budget left=  62.7%  burn 1h=  0.2 6h=  0.2  -> ok
day 28 00:00 (leak for 3 days)     30d SLI=99.9390%  budget left=  39.0%  burn 1h=  2.5 6h=  2.5  -> TICKET (on pace to miss the SLO this month)
day 30 23:00 (month end)           30d SLI=99.9204%  budget left=  20.4%  burn 1h=  2.5 6h=  2.5  -> TICKET (on pace to miss the SLO this month)

Read it as an on-call engineer would:

• The outage paged within 5 minutes (1 h burn 20.2 > 14.4, and the 5-minute window agreed).
• 30 minutes after the fix it no longer pages, even though the 1-hour burn rate is at its highest (96.8), because the 5-minute window is clean. That's the short window doing its job: no pages for problems that are already over.
• The slow leak never paged (burn 2.5), but it opened a ticket after 3 days. A 0.25% error rate doesn't need anyone out of bed. It does need fixing this week, because it consumed 40% of the month's budget.
• The month ended inside the SLO (99.92% > 99.9%), with 20% of the budget left. That number is what a team uses to decide whether this sprint ships features or reliability work.

Exercises:

1. Change the objective to 99.99%. How long can the outage last before the monthly SLO is lost? (About 4.3 minutes of total outage per 30 days.)
2. Add a latency SLI: record good only if the request also finished under a threshold, and simulate a latency regression that causes no errors at all.
3. Wire Middleware into Chapter 19's server and expose the burn rates on /metrics in Prometheus format (the TCP/IP guide's prober lab shows the text format).