Chapter 77 -- Files that survive crashes: page cache, fsync, and atomic replacement

Build it in Go (25 min) -- WriteFileAtomic, and what safety costs

// atomicwrite: replace a file so that, after a crash or power cut, readers
// see EITHER the complete old version OR the complete new version -- never a
// half-written mix. Also measures what durability costs.
//
//	go run ./atomicwrite
package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"
)

// WriteFileAtomic is the standard recipe used by databases, package managers,
// and config tools:
//  1. write to a temporary file IN THE SAME DIRECTORY (same filesystem)
//  2. fsync it                  -> the new bytes are on stable storage
//  3. rename over the target    -> atomic: the name points at old OR new
//  4. fsync the directory       -> the rename itself is on stable storage
func WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
	dir := filepath.Dir(path)
	tmp, err := os.CreateTemp(dir, ".tmp-"+filepath.Base(path)+"-*")
	if err != nil {
		return err
	}
	defer os.Remove(tmp.Name()) // no-op after a successful rename
	if _, err := tmp.Write(data); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Sync(); err != nil { // fsync(2)
		tmp.Close()
		return err
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	if err := os.Chmod(tmp.Name(), perm); err != nil {
		return err
	}
	if err := os.Rename(tmp.Name(), path); err != nil { // rename(2): atomic within a filesystem
		return err
	}
	d, err := os.Open(dir)
	if err != nil {
		return err
	}
	defer d.Close()
	return d.Sync() // fsync the directory entry
}

func main() {
	dir, _ := os.MkdirTemp("", "atomic")
	defer os.RemoveAll(dir)
	path := filepath.Join(dir, "config.json")
	data := []byte(`{"version": 2, "replicas": 3}` + "\n")

	// What does each level of safety cost? 200 small writes each.
	measure := func(label string, write func(i int) error) {
		start := time.Now()
		for i := 0; i < 200; i++ {
			if err := write(i); err != nil {
				fmt.Println(label, err)
				return
			}
		}
		fmt.Printf("%-44s %8.3f ms per write\n", label, float64(time.Since(start).Microseconds())/200/1000)
	}
	measure("os.WriteFile (page cache only, NOT durable)", func(int) error {
		return os.WriteFile(path, data, 0o644)
	})
	measure("write + fsync (durable, NOT atomic)", func(int) error {
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
		if err != nil {
			return err
		}
		defer f.Close()
		if _, err := f.Write(data); err != nil {
			return err
		}
		return f.Sync()
	})
	measure("WriteFileAtomic (durable AND atomic)", func(int) error {
		return WriteFileAtomic(path, data, 0o644)
	})
	b, _ := os.ReadFile(path)
	fmt.Printf("final content: %s", b)
}

Real output:

== Linux (container, overlay filesystem)
os.WriteFile (page cache only, NOT durable)      0.070 ms per write
write + fsync (durable, NOT atomic)              0.699 ms per write
WriteFileAtomic (durable AND atomic)             1.047 ms per write

== macOS (APFS, Apple SSD)
os.WriteFile (page cache only, NOT durable)      0.040 ms per write
write + fsync (durable, NOT atomic)              4.624 ms per write
WriteFileAtomic (durable AND atomic)             9.540 ms per write

What to notice:

• Durability costs 10× to 100×. That's why databases batch many commits into one fsync ("group commit"), and why fsync-per-request designs don't scale.
• macOS is slower for a good reason: Go's File.Sync on macOS uses F_FULLFSYNC, which forces the drive's own cache to flush. Plain fsync there doesn't guarantee that.
• Cloud and container numbers vary widely. Network block storage often takes milliseconds per fsync, so measure on the volume you actually deploy to.
• The defer os.Remove(tmp) cleans up a failed attempt. After a successful rename it's a harmless no-op.

Exercises:

1. Use WriteFileAtomic in a program that saves its state on SIGTERM (Chapter 74), and kill it repeatedly in a loop. The state file must always parse.
2. Write 1,000 records with an fsync after each, then with one fsync per 100. That's the group-commit trade-off between throughput and how much can be lost.
3. Recreate Chapter 15's "deleted but still open" mystery from Go: open a file, os.Remove it, keep writing, and watch df and ls -l /proc/<pid>/fd.