Chapter 80 -- A container runtime in 150 lines of Go

Build it in Go (45 min) -- minicontainer

//go:build linux

// minicontainer: a container runtime in ~150 lines. Namespaces give the
// process its own view of the system; chroot gives it its own filesystem;
// a cgroup limits what it can consume. That's all a container is.
//
// Run as root on a Linux machine/VM (cgroup v2), with a root filesystem:
//
//	mkdir -p /tmp/rootfs && docker export $(docker create busybox) | tar -x -C /tmp/rootfs
//	go build -o minicontainer ./minicontainer
//	sudo ./minicontainer run /tmp/rootfs /bin/sh
package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
)

func main() {
	if len(os.Args) < 4 {
		fmt.Println("usage: minicontainer run <rootfs> <command> [args...]")
		os.Exit(2)
	}
	switch os.Args[1] {
	case "run":
		run(os.Args[2], os.Args[3:])
	case "child":
		child(os.Args[2], os.Args[3:])
	}
}

// run is the "runtime": it re-executes this same binary as `child` inside new
// namespaces and a new cgroup, and waits for it.
func run(rootfs string, argv []string) {
	cg, err := makeCgroup()
	must(err)
	defer os.Remove(cg) // rmdir: only works once every process in it has exited
	cgfd, err := syscall.Open(cg, syscall.O_DIRECTORY|syscall.O_RDONLY, 0)
	must(err)
	defer syscall.Close(cgfd)

	cmd := exec.Command("/proc/self/exe", append([]string{"child", rootfs}, argv...)...)
	cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: syscall.CLONE_NEWUTS | // own hostname
			syscall.CLONE_NEWPID | // own PID numbering: the child is PID 1
			syscall.CLONE_NEWNS | // own mount table
			syscall.CLONE_NEWIPC | // own System V IPC / message queues
			syscall.CLONE_NEWNET, // own network stack: only a loopback, and it's down
		Unshareflags: syscall.CLONE_NEWNS, // don't propagate our mounts back to the host
		UseCgroupFD:  true,                // Go 1.20+: start the child directly inside the cgroup
		CgroupFD:     cgfd,
	}
	if err := cmd.Run(); err != nil {
		fmt.Println("container exited:", err)
	}
	if b, err := os.ReadFile(filepath.Join(cg, "pids.events")); err == nil {
		fmt.Printf("cgroup pids.events: %s", b) // "max N" = N forks refused by the limit
	}
}

// child runs INSIDE the new namespaces, as PID 1 of its own PID namespace.
func child(rootfs string, argv []string) {
	must(syscall.Sethostname([]byte("minicontainer")))
	must(syscall.Chroot(rootfs)) // production runtimes use pivot_root, which can't be escaped like chroot
	must(os.Chdir("/"))
	must(syscall.Mount("proc", "/proc", "proc", 0, "")) // a /proc that only shows OUR PID namespace
	defer syscall.Unmount("/proc", 0)

	cmd := exec.Command(argv[0], argv[1:]...)
	cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr
	cmd.Env = []string{"PATH=/bin:/usr/bin:/sbin:/usr/sbin", "PS1=container# "}
	if err := cmd.Run(); err != nil {
		os.Exit(1)
	}
}

// makeCgroup creates a cgroup v2 group with a process limit (a fork-bomb
// guard) and a memory limit. Requires root and the pids + memory controllers
// enabled in the parent's cgroup.subtree_control.
func makeCgroup() (string, error) {
	cg := filepath.Join("/sys/fs/cgroup", "minicontainer-"+strconv.Itoa(os.Getpid()))
	if err := os.Mkdir(cg, 0o755); err != nil {
		return "", fmt.Errorf("create cgroup (are you root, on cgroup v2?): %w", err)
	}
	for file, value := range map[string]string{
		"pids.max":   "20",       // at most 20 processes
		"memory.max": "64000000", // 64 MB
	} {
		if err := os.WriteFile(filepath.Join(cg, file), []byte(value), 0o644); err != nil {
			os.Remove(cg)
			return "", fmt.Errorf("%s: %w (enable the controller in the parent's cgroup.subtree_control)", file, err)
		}
	}
	return cg, nil
}

func must(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

Run it as root on a Linux VM, or inside a privileged container as below:

mkdir -p rootfs && docker export $(docker create busybox) | tar -x -C rootfs
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o bin/minicontainer ./minicontainer
docker run --rm -it --privileged --cgroupns=private -v "$PWD":/w debian:bookworm sh -c '
  mkdir /sys/fs/cgroup/init && echo $$ > /sys/fs/cgroup/init/cgroup.procs   # cgroup v2: move ourselves
  echo "+pids +memory" > /sys/fs/cgroup/cgroup.subtree_control                # into a leaf, enable controllers
  /w/bin/minicontainer run /w/rootfs /bin/sh'

(The two cgroup lines are needed only inside a container. cgroup v2 doesn't let a group both contain processes and delegate controllers, so the shell moves itself into a leaf group first. On a systemd host, run as root and skip them.)

Real output from a scripted run:

== host view: hostname=3b127053433e pid=1
inside: hostname=minicontainer my-pid=7
PID   USER     TIME  COMMAND
    1 root      0:00 /proc/self/exe child /b/rootfs /bin/sh -c ...
    7 root      0:00 /bin/sh -c ...
    9 root      0:00 ps

1: lo: <LOOPBACK> mtu 65536 qdisc noop qlen 1000
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop qlen 1000
...                                                    (every interface DOWN: "noop")

fork test:
/bin/sh: line 0: can't fork: Resource temporarily unavailable
cgroup pids.events: max 1

What to notice:

• Our child is PID 1, and ps sees only three processes, because the freshly mounted /proc belongs to the new PID namespace.
• The new network namespace has only a loopback, and it's down. (The tunl0, gre0... devices appear automatically in every network namespace when those tunnel modules are loaded on the host, also down.) The container has no network until something creates a veth pair for it: Docker's job, built by hand in the TCP/IP guide.
• The fork bomb stopped at 20 processes, and pids.events recorded it. That one line is why a container can't take down its host by forking.
• UseCgroupFD/CgroupFD start the child directly inside the cgroup, with no window in which it runs unlimited.

Exercises:

1. Inside the container run hostname foo, then check the host's hostname. The UTS namespace kept them apart.
2. Add CLONE_NEWUSER with a UID mapping so it runs without root: a rootless container.
3. Add networking: create a veth pair, move one end into the child's network namespace (ip link set veth1 netns <pid>), and give both ends addresses.
4. Copy Chapter 76's memlimit binary into the rootfs and run it with -live 100. The 64 MB memory.max kills it. Then read memory.events in the cgroup directory to see the oom_kill count.