Chapter 34 — A troubleshooting method that works

Practice (30 min) — break things on purpose

The single best way to learn troubleshooting is to cause the failures yourself, so you recognise them later. Do these on a spare machine or VM, and undo each one.

# 1. Break DNS
sudo cp /etc/resolv.conf /tmp/resolv.backup
echo "nameserver 192.0.2.1" | sudo tee /etc/resolv.conf   # a black hole
# symptom: ping by IP works, by name doesn't
sudo cp /tmp/resolv.backup /etc/resolv.conf                # UNDO

# 2. Break the default route
ip route | grep default                                    # note it first!
sudo ip route del default
# symptom: local works, internet doesn't, "Network is unreachable"
sudo ip route add default via <your gateway>               # UNDO

# 3. Break the MTU
sudo ip link set eth0 mtu 900
# symptom: small things work, large downloads stall
sudo ip link set eth0 mtu 1500                             # UNDO

# 4. Simulate loss and latency
sudo tc qdisc add dev eth0 root netem loss 10% delay 200ms
sudo tc qdisc del dev eth0 root                            # UNDO

# 5. Block a port
sudo iptables -A OUTPUT -p tcp --dport 443 -j DROP
# symptom: TIMEOUT (not refused) -- feel the difference
sudo iptables -D OUTPUT -p tcp --dport 443 -j DROP         # UNDO

For each one: before undoing it, run the diagnostic script above and note which step first shows the failure. Write it in notes.md. That table becomes your personal troubleshooting reference.

Build it in Go (35 min) — the troubleshooting ladder as one command

The method in this chapter is "climb the layers and stop at the first one that fails". That's an algorithm, so here it is as a program. Each rung only runs if the one below it passed, and the program tells you which layer to investigate.

// netcheck: Chapter 34's troubleshooting ladder as one program. It climbs the
// stack -- route/interface, DNS, TCP, TLS, HTTP -- stops at the first rung
// that fails, and tells you which layer to look at.
//
//	go run ./netcheck https://example.com/
//	go run ./netcheck https://example.com:81/      # watch it stop at TCP
package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"time"
)

type step struct {
	layer string
	run   func() (string, error)
}

func main() {
	if len(os.Args) != 2 {
		fmt.Println("usage: netcheck URL")
		os.Exit(2)
	}
	u, err := url.Parse(os.Args[1])
	if err != nil || u.Hostname() == "" {
		fmt.Println("bad URL:", os.Args[1])
		os.Exit(2)
	}
	host, port := u.Hostname(), u.Port()
	if port == "" {
		port = map[string]string{"http": "80", "https": "443"}[u.Scheme]
	}
	var ips []net.IP
	var addr string

	steps := []step{
		{"L3 route", func() (string, error) {
			c, err := net.Dial("udp", "1.1.1.1:53") // sends nothing; just asks the routing table
			if err != nil {
				return "", fmt.Errorf("no route to the internet: %w", err)
			}
			defer c.Close()
			src := c.LocalAddr().(*net.UDPAddr).IP
			// Which interface owns that source address? That's the one that matters.
			ifs, _ := net.Interfaces()
			for _, i := range ifs {
				as, _ := i.Addrs()
				for _, a := range as {
					if n, ok := a.(*net.IPNet); ok && n.IP.Equal(src) {
						return fmt.Sprintf("default route via %s (up=%v), source %s", i.Name, i.Flags&net.FlagUp != 0, src), nil
					}
				}
			}
			return "default route exists, source " + src.String(), nil
		}},
		{"L7 DNS", func() (string, error) {
			ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
			defer cancel()
			start := time.Now()
			addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
			if err != nil {
				return "", err
			}
			for _, a := range addrs {
				ips = append(ips, a.IP)
			}
			return fmt.Sprintf("%s -> %v in %v", host, ips, time.Since(start).Round(time.Millisecond)), nil
		}},
		{"L4 TCP", func() (string, error) {
			addr = net.JoinHostPort(ips[0].String(), port)
			start := time.Now()
			c, err := net.DialTimeout("tcp", addr, 3*time.Second)
			if err != nil {
				return "", err
			}
			c.Close()
			return fmt.Sprintf("connected to %s in %v (~1 RTT)", addr, time.Since(start).Round(time.Millisecond)), nil
		}},
		{"L5/6 TLS", func() (string, error) {
			if u.Scheme != "https" {
				return "skipped (plain http)", nil
			}
			start := time.Now()
			c, err := tls.DialWithDialer(&net.Dialer{Timeout: 3 * time.Second}, "tcp", addr,
				&tls.Config{ServerName: host})
			if err != nil {
				return "", err
			}
			defer c.Close()
			st := c.ConnectionState()
			days := int(time.Until(st.PeerCertificates[0].NotAfter).Hours() / 24)
			return fmt.Sprintf("%s, cert valid %d more days, TCP+TLS %v",
				tls.VersionName(st.Version), days, time.Since(start).Round(time.Millisecond)), nil
		}},
		{"L7 HTTP", func() (string, error) {
			client := &http.Client{Timeout: 5 * time.Second,
				CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
			start := time.Now()
			resp, err := client.Get(u.String())
			if err != nil {
				return "", err
			}
			resp.Body.Close()
			if resp.StatusCode >= 500 {
				return "", fmt.Errorf("server error %s", resp.Status)
			}
			return fmt.Sprintf("%s in %v (server: %q)", resp.Status, time.Since(start).Round(time.Millisecond), resp.Header.Get("Server")), nil
		}},
	}

	for _, s := range steps {
		detail, err := s.run()
		if err != nil {
			fmt.Printf("FAIL  %-11s %v\n\n=> Everything below %s works. Investigate %s first.\n", s.layer, err, s.layer, s.layer)
			os.Exit(1)
		}
		fmt.Printf("ok    %-11s %s\n", s.layer, detail)
	}
	fmt.Println("\n=> every layer answered. If users still complain, look at latency, not reachability.")
}

$ go run ./netcheck https://example.com/
ok    L3 route    default route via en0 (up=true), source 192.168.1.2
ok    L7 DNS      example.com -> [104.20.23.154 172.66.147.243 2606:4700:...] in 2ms
ok    L4 TCP      connected to 104.20.23.154:443 in 38ms (~1 RTT)
ok    L5/6 TLS    TLS 1.3, cert valid 83 more days, TCP+TLS 96ms
ok    L7 HTTP     200 OK in 514ms (server: "cloudflare")

$ go run ./netcheck https://example.com:81/
ok    L3 route    default route via en0 (up=true), source 192.168.1.2
ok    L7 DNS      example.com -> [...] in 3ms
FAIL  L4 TCP      dial tcp 172.66.147.243:81: i/o timeout

=> Everything below L4 TCP works. Investigate L4 TCP first.

What to notice:

• The order isn't the OSI order. DNS (an L7 protocol) runs before TCP, because you need the address before you can connect. The OSI guide's "layers are not steps in time" section explains why this is correct.
• Each rung is the smallest test that can prove its layer. TCP is checked with a bare connect, not an HTTP request. A failing HTTP request alone can't tell you whether DNS, TCP, TLS, or the app is at fault.
• Timing on every rung. "Everything works, but TCP took 900 ms" is also a finding: SYN loss and a retry (Chapter 21).

Exercises:

1. Break each layer in turn, using the "break things on purpose" practice above (a bad resolver, no default route, a blocked port), and check that the tool names the right layer each time.
2. Add an MTU rung: download something larger than 1,500 bytes and fail if the small request works but the large one stalls (Chapter 15's fingerprint).
3. Add -json output and run it from cron on three networks (home, office, cloud VM). You now have a minimal synthetic monitor. Chapter 46's prober lab turns that into proper metrics.

Across the series: the same ladder appears as curl -w timings in the OSI guide's debugging playbook and in the HTTPS guide's debugging chapter. The Linux guide's Appendix C is the host-side equivalent for "the server is slow".