Day 27 — Graceful Shutdown

Learning Objectives

• Implement 3-stage graceful shutdown: stop accepting → drain in-flight → force close
• Handle SIGTERM and SIGINT correctly
• Prevent connection drops during deployment

Key Concepts

// Signal handling
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()

// http.Server graceful shutdown
srv := &http.Server{Addr: ":8080", Handler: handler}
go srv.ListenAndServe()

<-ctx.Done()  // wait for signal
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
srv.Shutdown(shutdownCtx)  // stops accepting, waits for in-flight to complete

Exercise: 3-Stage Graceful Shutdown

package main

import (
    "context"
    "fmt"
    "net/http"
    "os/signal"
    "sync"
    "sync/atomic"
    "syscall"
    "time"
)

type GracefulServer struct {
    srv        *http.Server
    inFlight   sync.WaitGroup
    reqCount   atomic.Int64
    shutting   atomic.Bool
}

func (g *GracefulServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    if g.shutting.Load() {
        // Stage 1: stop accepting new requests (return 503 with Retry-After)
        w.Header().Set("Retry-After", "10")
        http.Error(w, "service shutting down", http.StatusServiceUnavailable)
        return
    }

    g.inFlight.Add(1)
    defer g.inFlight.Done()
    g.reqCount.Add(1)

    // Simulate work — in a real proxy, this forwards the request
    time.Sleep(100 * time.Millisecond)
    fmt.Fprintf(w, `{"request":%d}`, g.reqCount.Load())
}

func (g *GracefulServer) Shutdown() {
    fmt.Println("shutdown: stage 1 — stopping new requests")
    g.shutting.Store(true)

    // Stage 2: wait for in-flight with 30s budget
    done := make(chan struct{})
    go func() {
        g.inFlight.Wait()
        close(done)
    }()

    fmt.Println("shutdown: stage 2 — draining in-flight requests")
    select {
    case <-done:
        fmt.Println("shutdown: all requests drained cleanly")
    case <-time.After(30 * time.Second):
        fmt.Println("shutdown: drain timeout, force-closing remaining")
    }

    // Stage 3: shut down the HTTP server
    fmt.Println("shutdown: stage 3 — closing server")
    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    defer cancel()
    g.srv.Shutdown(ctx)
}

func main() {
    gs := &GracefulServer{}
    gs.srv = &http.Server{Addr: ":8080", Handler: gs}

    ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
    defer stop()

    go func() {
        fmt.Println("server listening on :8080")
        if err := gs.srv.ListenAndServe(); err != http.ErrServerClosed {
            fmt.Printf("server error: %v\n", err)
        }
    }()

    <-ctx.Done()
    fmt.Println("signal received")
    gs.Shutdown()
    fmt.Println("shutdown complete")
}

Test it:

# Start server
go run .

# Send some requests (in another terminal)
for i in $(seq 1 20); do curl -s http://localhost:8080/ & done

# Send SIGINT while requests are in-flight
kill -SIGINT $(pgrep -f 'go run')

Real-World Context

Kubernetes sends SIGTERM when scaling down or deploying. Your pod has a terminationGracePeriodSeconds budget (default 30s). If your server doesn't drain in-flight requests, users see connection resets. Every production Go service must implement graceful shutdown.