Chapter 16 — Certificate and domain operations: ACME, rotation, CT monitoring, and emergency revocation

Build it in Go (40 min) — a certificate inventory scanner

Monitoring the certificate in the directory isn't enough. You need to check the certificate each endpoint actually serves. certcheck scans many endpoints concurrently, grades each one, exits non-zero for cron/CI, and can ask Certificate Transparency what has been issued for a domain:

// certcheck: a certificate inventory scanner. For every endpoint, check the
// certificate actually SERVED: days to expiry, name coverage, chain sent,
// issuer, key type. Optionally list what Certificate Transparency logs know.
//
//	go run ./certcheck -warn 30 example.com github.com expired.badssl.com
//	go run ./certcheck -ct example.com        # also query crt.sh (CT logs)
package main

import (
	"context"
	"crypto/ecdsa"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"
	"sync"
	"time"
)

type result struct {
	target string
	lines  []string
	level  int // 0 ok, 1 warn, 2 critical
}

func main() {
	warn := flag.Int("warn", 30, "warn when a certificate expires within this many days")
	crit := flag.Int("crit", 7, "critical within this many days")
	ct := flag.Bool("ct", false, "also list recent certificates from Certificate Transparency (crt.sh)")
	flag.Parse()

	results := make([]result, flag.NArg())
	var wg sync.WaitGroup
	for i, t := range flag.Args() {
		wg.Add(1)
		go func() { defer wg.Done(); results[i] = check(t, *warn, *crit) }()
	}
	wg.Wait()

	worst := 0
	for _, r := range results {
		tag := []string{"OK  ", "WARN", "CRIT"}[r.level]
		fmt.Printf("[%s] %s\n", tag, r.target)
		for _, l := range r.lines {
			fmt.Println("       " + l)
		}
		worst = max(worst, r.level)
	}
	if *ct {
		for _, t := range flag.Args() {
			host, _, _ := strings.Cut(t, ":")
			ctLookup(host)
		}
	}
	os.Exit(worst) // 0/1/2: plugs straight into cron, CI, or Nagios-style monitoring
}

func check(target string, warnDays, critDays int) result {
	r := result{target: target}
	host, port, err := net.SplitHostPort(target)
	if err != nil {
		host, port = target, "443"
	}
	// Verify ourselves AFTER the handshake so we can still report on bad certs.
	conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp",
		net.JoinHostPort(host, port), &tls.Config{ServerName: host, InsecureSkipVerify: true})
	if err != nil {
		r.level, r.lines = 2, []string{"handshake failed: " + err.Error()}
		return r
	}
	defer conn.Close()
	certs := conn.ConnectionState().PeerCertificates
	leaf := certs[0]
	days := int(time.Until(leaf.NotAfter).Hours() / 24)

	r.lines = append(r.lines, fmt.Sprintf("subject %s, issuer %q", leaf.Subject.CommonName, leaf.Issuer.CommonName))
	r.lines = append(r.lines, fmt.Sprintf("expires %s (%d days), key %s, lifetime %d days",
		leaf.NotAfter.Format("2006-01-02"), days, keyType(leaf),
		int(leaf.NotAfter.Sub(leaf.NotBefore).Hours()/24)))
	switch {
	case days < 0:
		r.level = 2
		r.lines = append(r.lines, "EXPIRED")
	case days < critDays:
		r.level = 2
	case days < warnDays:
		r.level = 1
	}
	if err := leaf.VerifyHostname(host); err != nil {
		r.level = 2
		r.lines = append(r.lines, "NAME NOT COVERED: SANs are "+strings.Join(leaf.DNSNames, ", "))
	}

	// Chain check using only what the server sent. Run this on Linux: on macOS
	// (and Windows) Go asks the OS to verify, and the OS quietly downloads
	// missing intermediates -- hiding exactly the bug we are looking for.
	inter := x509.NewCertPool()
	for _, c := range certs[1:] {
		inter.AddCert(c)
	}
	if _, err := leaf.Verify(x509.VerifyOptions{DNSName: host, Intermediates: inter}); err != nil {
		r.level = max(r.level, 2)
		r.lines = append(r.lines, fmt.Sprintf("chain does not verify with the %d cert(s) sent: %v", len(certs), err))
	}
	return r
}

func keyType(c *x509.Certificate) string {
	switch k := c.PublicKey.(type) {
	case *rsa.PublicKey:
		return fmt.Sprintf("RSA-%d", k.N.BitLen())
	case *ecdsa.PublicKey:
		return "ECDSA-" + k.Curve.Params().Name
	}
	return c.PublicKeyAlgorithm.String()
}

// ctLookup lists certificates that public CT logs recorded for a domain in
// the last 90 days -- including ones YOU didn't request. crt.sh is a free
// community service: be gentle, and expect it to be slow at times.
func ctLookup(domain string) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	u := "https://crt.sh/?output=json&exclude=expired&q=" + url.QueryEscape(domain)
	req, _ := http.NewRequestWithContext(ctx, "GET", u, nil)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Printf("\nCT %s: %v\n", domain, err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK || !strings.Contains(resp.Header.Get("Content-Type"), "json") {
		fmt.Printf("\nCT %s: crt.sh answered %s (it is often overloaded; try again later)\n", domain, resp.Status)
		return
	}
	var rows []struct {
		IssuerName string `json:"issuer_name"`
		NameValue  string `json:"name_value"`
		NotBefore  string `json:"not_before"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&rows); err != nil {
		fmt.Printf("\nCT %s: %v\n", domain, err)
		return
	}
	issuers := map[string]int{}
	cutoff := time.Now().AddDate(0, 0, -90).Format("2006-01-02")
	recent := 0
	for _, r := range rows {
		if r.NotBefore >= cutoff {
			recent++
			issuers[r.IssuerName]++
		}
	}
	fmt.Printf("\nCT %s: %d unexpired certificates logged, %d issued in the last 90 days, by:\n", domain, len(rows), recent)
	names := make([]string, 0, len(issuers))
	for n := range issuers {
		names = append(names, n)
	}
	sort.Slice(names, func(i, j int) bool { return issuers[names[i]] > issuers[names[j]] })
	for _, n := range names {
		fmt.Printf("  %4d  %s\n", issuers[n], n)
	}
	fmt.Println("  -> any issuer here that you don't use is worth investigating (and a CAA record would block it)")
}

Real output, run on Linux:

$ go run ./certcheck -warn 30 example.com github.com expired.badssl.com wrong.host.badssl.com incomplete-chain.badssl.com
[OK  ] example.com
       subject example.com, issuer "Cloudflare TLS Issuing ECC CA 3"
       expires 2026-12-25 (83 days), key ECDSA-P-256, lifetime 90 days
[OK  ] github.com
       subject github.com, issuer "Sectigo Public Server Authentication CA DV E36"
       expires 2026-11-29 (57 days), key ECDSA-P-256, lifetime 89 days
[CRIT] expired.badssl.com
       subject *.badssl.com, issuer "COMODO RSA Domain Validation Secure Server CA"
       expires 2015-04-12 (-4191 days), key RSA-2048, lifetime 3 days
       EXPIRED
       chain does not verify with the 3 cert(s) sent: x509: certificate has expired or is not yet valid: ...
[CRIT] wrong.host.badssl.com
       subject *.badssl.com, issuer "YR1"
       expires 2026-12-28 (86 days), key RSA-2048, lifetime 89 days
       NAME NOT COVERED: SANs are *.badssl.com, badssl.com
       chain does not verify with the 3 cert(s) sent: x509: certificate is valid for *.badssl.com, badssl.com, not wrong.host.badssl.com
[CRIT] incomplete-chain.badssl.com
       subject *.badssl.com, issuer "YR1"
       expires 2026-12-28 (86 days), key RSA-2048, lifetime 89 days
       chain does not verify with the 1 cert(s) sent: x509: certificate signed by unknown authority
exit status 2

What to notice:

• InsecureSkipVerify: true is deliberate here, and only here. The scanner must complete the handshake to report on a bad certificate, and then it verifies explicitly with VerifyHostname and Verify. Never copy that flag into a client that sends data.
• The incomplete-chain check passed on macOS and failed on Linux. On macOS (and Windows), Go asks the operating system to verify certificates, and the OS silently downloads the missing intermediate. Linux clients, many containers, Java, and Python don't. Run chain checks from the kind of client your users have, or at least from Linux.
• Look at the lifetimes: 89–90 days for both real sites. The industry is already most of the way to short-lived certificates.
• The exit code (0 OK, 1 warn, 2 critical) makes it a monitoring plugin with no extra code.
• -ct queries crt.sh, a free community CT search engine. It's frequently overloaded (it returned 502 during testing), which is why the code checks the status and content type before parsing. For production monitoring, use a CT monitoring service or run your own log watcher.

Exercises:

1. Put every hostname your organisation serves into a file and run certcheck daily from cron, alerting on exit code ≥ 1. You now have the inventory this chapter asks for.
2. Add a check that the key type and size meet your policy (e.g. reject RSA-1024) and that the certificate isn't valid for more than 398 days.
3. Add a CAA check by extending the TCP/IP guide's dnsquery lab to query type 257, and warn if the issuing CA isn't listed.

Operational checklist:

[ ] Automated renewal tested, not merely configured.
[ ] Expiry alert fires well before the shortest certificate lifetime.
[ ] Private keys are generated and stored in approved systems.
[ ] Intermediate chain is served correctly from every edge.
[ ] CT logs are monitored for unexpected certificates.
[ ] CAA records match the CAs you actually use.
[ ] Emergency revocation and reissue runbook exists.