Build it in Go (20 min) -- count the threads

//go:build linux

// threads: how many OS threads does a Go program really use? It depends on
// HOW goroutines wait. Run it and compare the three cases.
//
//	go run ./threads
package main

import (
	"fmt"
	"net"
	"os"
	"runtime"
	"strings"
	"syscall"
	"time"
)

func threads() string {
	b, _ := os.ReadFile("/proc/self/status")
	for _, l := range strings.Split(string(b), "\n") {
		if strings.HasPrefix(l, "Threads:") {
			return strings.TrimSpace(strings.TrimPrefix(l, "Threads:"))
		}
	}
	return "?"
}

func report(label string) {
	time.Sleep(500 * time.Millisecond) // let the runtime settle
	fmt.Printf("%-52s goroutines=%-6d OS threads=%s\n", label, runtime.NumGoroutine(), threads())
}

func main() {
	report("start")

	// 1. 10,000 goroutines in time.Sleep: parked by the Go scheduler, no thread each.
	for i := 0; i < 10000; i++ {
		go time.Sleep(time.Hour)
	}
	report("+10,000 goroutines sleeping")

	// 2. 10,000 goroutines blocked reading network sockets: parked in the
	//    netpoller (one epoll instance), still no thread each.
	ln, _ := net.Listen("tcp", "127.0.0.1:0")
	go func() {
		for {
			c, err := ln.Accept()
			if err != nil {
				return
			}
			go func() { buf := make([]byte, 1); c.Read(buf) }() // waits forever
		}
	}()
	var conns []net.Conn
	for i := 0; i < 5000; i++ { // 5,000 client + 5,000 server goroutines
		c, err := net.Dial("tcp", ln.Addr().String())
		if err != nil {
			fmt.Println("dial:", err)
			break
		}
		conns = append(conns, c)
		go func() { buf := make([]byte, 1); c.Read(buf) }()
	}
	report("+10,000 goroutines blocked on socket reads")

	// 3. 100 goroutines blocked in a RAW blocking system call: each one pins
	//    an OS thread, and the runtime starts new threads to keep running Go code.
	for i := 0; i < 100; i++ {
		var p [2]int
		syscall.Pipe(p[:])
		go func() { buf := make([]byte, 1); syscall.Read(p[0], buf) }() // blocks the thread itself
	}
	report("+100 goroutines blocked in raw read(2) syscalls")
	runtime.KeepAlive(conns)
}

Real output (Linux, 14 CPUs):

start                                                goroutines=1      OS threads=5
+10,000 goroutines sleeping                          goroutines=10001  OS threads=16
+10,000 goroutines blocked on socket reads           goroutines=20002  OS threads=16
+100 goroutines blocked in raw read(2) syscalls      goroutines=20102  OS threads=108

What to notice:

• 20,000 goroutines, 16 threads. Sleeping and network-blocked goroutines cost no threads. Ten thousand idle TCP connections are cheap in Go, which is why it's popular for proxies and gateways.
• 100 blocking system calls, +92 threads. Each raw read(2) on a pipe that has no data pins its thread inside the kernel. (Go's os.File puts pipes in the netpoller too. We used syscall.Read to bypass it.)
• Regular files can't use epoll on Linux, so os.ReadFile on a slow disk or network filesystem blocks a thread for real. A thousand goroutines reading from a stalled NFS mount means about a thousand threads. The runtime crashes the program at 10,000 (runtime/debug.SetMaxThreads).

Exercises:

1. Run any lab under strace -f -c -o trace.txt and read the summary. Which system call is called most?
2. Run GODEBUG=schedtrace=1000 ./threads and read the scheduler's once-a-second report: gomaxprocs, threads, idleprocs, runqueue.
3. Lower the limit with debug.SetMaxThreads(50) and rerun. What happens at the syscall step?