11. Terminal project: a word-frequency counter

Goal: a CLI that prints the most common words in one or more files, or in standard input. It uses argparse, generators, Counter, pathlib, and correct exit codes.

# labs/wordfreq.py
"""Print the most common words in files or stdin."""
import argparse
import re
import sys
from collections import Counter
from collections.abc import Iterable, Iterator
from pathlib import Path

WORD = re.compile(r"[a-z']+")

def words(lines: Iterable[str]) -> Iterator[str]:
    for line in lines:
        yield from WORD.findall(line.lower())

def main(argv: list[str] | None = None) -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("files", nargs="*", type=Path, help="files to read (default: stdin)")
    p.add_argument("-n", "--top", type=int, default=10, help="how many words to show")
    p.add_argument("--min-len", type=int, default=1)
    args = p.parse_args(argv)

    counts: Counter[str] = Counter()
    if not args.files:
        counts.update(words(sys.stdin))
    for path in args.files:
        try:
            with path.open(encoding="utf-8", errors="replace") as f:
                counts.update(words(f))
        except OSError as e:
            print(f"wordfreq: {path}: {e.strerror}", file=sys.stderr)
            return 1

    for word, n in counts.most_common():
        if len(word) < args.min_len:
            continue
        print(f"{n:>7}  {word}")
        args.top -= 1
        if args.top == 0:
            break
    return 0

if __name__ == "__main__":
    raise SystemExit(main())

$ curl -s https://www.gutenberg.org/files/11/11-0.txt -o alice.txt
$ uv run labs/wordfreq.py alice.txt -n 5 --min-len 4
    ... the five most common words of four or more letters
$ cat alice.txt | uv run labs/wordfreq.py -n 3
$ uv run labs/wordfreq.py missing.txt; echo "exit=$?"
wordfreq: missing.txt: No such file or directory
exit=1

What to notice:

• words() is a generator (Ch 13): the file is never loaded into memory, so a 10 GB file uses the same memory as a 10 KB one.
• Errors go to stderr and set a non-zero exit code, so the tool composes in shell pipelines (OS Ch 28).
• main(argv) takes an optional argument list, so a test can call main(["file.txt", "-n", "3"]) without spawning a process.

Extend it: add --stopwords FILE; add --json output; compare its speed with the Rust version on a 500 MB file and explain the difference using Chapter 25.