init

internal/deploy/assemble.go

@@ -0,0 +1,147 @@
+// Package deploy owns the deployment lifecycle: creating one, negotiating its
+// manifest, accepting blob uploads, assembling the directory tree, and (from M3)
+// switching a project over to it.
+//
+// It is the only package that writes to $DATA_DIR/deployments. Everything it
+// writes lands in a staging directory first and becomes visible with a single
+// rename, which is the same discipline the CAS uses for blobs and the registry
+// uses for the active pointer.
+package deploy
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strconv"
+
+	"github.com/iceBear67/simplepages/internal/cas"
+	"github.com/iceBear67/simplepages/internal/pathutil"
+	"github.com/iceBear67/simplepages/internal/store"
+)
+
+const (
+	dirMode = 0o755
+	// stagingSuffix marks a tree that is still being built. Recovery deletes
+	// every directory carrying it, since by definition nothing references one.
+	stagingSuffix = ".staging"
+)
+
+// DeploymentDir is where a deployment's assembled tree lives.
+//
+// The project id rather than its name: a project that is renamed keeps its
+// deployments where they are, and no user-chosen string is ever a path segment
+// under $DATA_DIR.
+func DeploymentDir(root string, projectID int64, publicID string) string {
+	return filepath.Join(root, strconv.FormatInt(projectID, 10), publicID)
+}
+
+// Assemble builds destDir from the CAS.
+//
+// The tree is built under destDir+".staging" and moved into place with one
+// rename, so destDir either does not exist or is the complete deployment —
+// there is no state in which a reader could walk a half-built tree.
+//
+// It is idempotent in the way finalize needs: an existing destDir is a finished
+// tree (rename is atomic, so a partial one cannot survive a crash) and is left
+// alone.
+func Assemble(ctx context.Context, cs *cas.Store, files []store.FileRow, destDir string) error {
+	staging := destDir + stagingSuffix
+	if fi, err := os.Stat(destDir); err == nil {
+		if !fi.IsDir() {
+			return fmt.Errorf("deploy: %s exists and is not a directory", destDir)
+		}
+		return os.RemoveAll(staging)
+	} else if !os.IsNotExist(err) {
+		return err
+	}
+
+	// Whatever an earlier attempt left behind is unreferenced by construction.
+	if err := os.RemoveAll(staging); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(staging, dirMode); err != nil {
+		return err
+	}
+	// One cleanup for every failure path: a staging tree that outlives its
+	// attempt is wasted disk that only recovery would find.
+	ok := false
+	defer func() {
+		if !ok {
+			os.RemoveAll(staging)
+		}
+	}()
+
+	// The set of directories that had to be created, so each can be fsynced
+	// once. Recorded per ancestor because MkdirAll creates parents silently.
+	dirs := map[string]bool{".": true}
+	for _, f := range files {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		// The manifest was validated when it was accepted. Checking again here
+		// costs a few hundred nanoseconds per file and means the one place that
+		// turns stored strings into filesystem paths does not depend on a
+		// promise made by a different package at a different time.
+		if err := pathutil.Validate(f.Path); err != nil {
+			return fmt.Errorf("deploy: manifest path %q: %w", f.Path, err)
+		}
+		if dir := path.Dir(f.Path); !dirs[dir] {
+			if err := os.MkdirAll(filepath.Join(staging, filepath.FromSlash(dir)), dirMode); err != nil {
+				return err
+			}
+			for d := dir; !dirs[d]; d = path.Dir(d) {
+				dirs[d] = true
+			}
+		}
+		if err := cs.LinkInto(f.Digest, staging, f.Path); err != nil {
+			return fmt.Errorf("deploy: %s: %w", f.Path, err)
+		}
+	}
+
+	// Blob content is already durable — Put fsynced it, and a hardlink shares
+	// that inode — but the directory entries pointing at it are not. Without
+	// this, a crash could leave a renamed-into-place tree with missing files,
+	// which is exactly the half-updated site the whole design exists to avoid.
+	if err := syncDirs(staging, dirs); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(destDir), dirMode); err != nil {
+		return err
+	}
+	if err := os.Rename(staging, destDir); err != nil {
+		return err
+	}
+	ok = true
+	return syncDir(filepath.Dir(destDir))
+}
+
+// syncDirs fsyncs every directory of the staged tree, deepest first, so a
+// parent is only made durable once the entries it names are.
+func syncDirs(staging string, dirs map[string]bool) error {
+	rel := make([]string, 0, len(dirs))
+	for d := range dirs {
+		rel = append(rel, d)
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(rel)))
+	for _, d := range rel {
+		if err := syncDir(filepath.Join(staging, filepath.FromSlash(d))); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func syncDir(dir string) error {
+	f, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	return f.Close()
+}

internal/deploy/assemble_test.go

@@ -0,0 +1,228 @@
+package deploy
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/iceBear67/simplepages/internal/cas"
+	"github.com/iceBear67/simplepages/internal/store"
+)
+
+// fixture is a CAS holding some content plus the manifest that names it.
+type fixture struct {
+	cas   *cas.Store
+	dir   string // deployments root
+	files []store.FileRow
+}
+
+func newFixture(t *testing.T, contents map[string]string) *fixture {
+	t.Helper()
+	base := t.TempDir()
+	deployDir := filepath.Join(base, "deployments")
+	cs, err := cas.Open(filepath.Join(base, "cas"), cas.Options{ProbeDir: deployDir})
+	if err != nil {
+		t.Fatalf("cas.Open: %v", err)
+	}
+	t.Cleanup(func() { cs.Close() })
+
+	f := &fixture{cas: cs, dir: deployDir}
+	for path, content := range contents {
+		d := cas.Sum([]byte(content))
+		if _, err := cs.Put(t.Context(), d, int64(len(content)), 1<<20, strings.NewReader(content)); err != nil {
+			t.Fatalf("put %s: %v", path, err)
+		}
+		f.files = append(f.files, store.FileRow{Path: path, Digest: d, Size: int64(len(content))})
+	}
+	return f
+}
+
+// walk reads back an assembled tree as path -> content.
+func walk(t *testing.T, dir string) map[string]string {
+	t.Helper()
+	out := map[string]string{}
+	err := filepath.WalkDir(dir, func(p string, e fs.DirEntry, err error) error {
+		if err != nil || e.IsDir() {
+			return err
+		}
+		b, err := os.ReadFile(p)
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(dir, p)
+		if err != nil {
+			return err
+		}
+		out[filepath.ToSlash(rel)] = string(b)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("walk %s: %v", dir, err)
+	}
+	return out
+}
+
+func TestAssembleBuildsTheTree(t *testing.T) {
+	want := map[string]string{
+		"index.html":         "<h1>hi</h1>",
+		"assets/app.js":      "console.log(1)",
+		"assets/css/app.css": "body{}",
+		"a/b/c/d/deep.txt":   "deep",
+		"copy.html":          "<h1>hi</h1>", // shares a blob with index.html
+	}
+	f := newFixture(t, want)
+	dest := DeploymentDir(f.dir, 7, "dpl_0123456789abcdef")
+
+	if err := Assemble(t.Context(), f.cas, f.files, dest); err != nil {
+		t.Fatalf("Assemble: %v", err)
+	}
+	got := walk(t, dest)
+	if len(got) != len(want) {
+		t.Fatalf("assembled %d files, want %d: %v", len(got), len(want), got)
+	}
+	for p, content := range want {
+		if got[p] != content {
+			t.Errorf("%s = %q, want %q", p, got[p], content)
+		}
+	}
+	// The staging directory is gone: it became the tree by rename.
+	if _, err := os.Stat(dest + stagingSuffix); !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("staging directory survived: %v", err)
+	}
+	// The project id is a path segment, so the tree is where the registry will
+	// later expect to find it.
+	if !strings.HasSuffix(filepath.Dir(dest), string(filepath.Separator)+"7") {
+		t.Errorf("deployment dir %q is not under its project id", dest)
+	}
+
+	// Where the filesystem allows it, an assembled file is the blob rather than a
+	// copy of it. This is what keeps a hundred deployments of one site costing
+	// one site's worth of disk, so it is worth asserting rather than assuming.
+	if f.cas.LinkMode() != cas.LinkHard {
+		t.Skipf("link mode is %s on this filesystem; skipping the hardlink assertion", f.cas.LinkMode())
+	}
+	for _, e := range f.files {
+		blob, err := os.Stat(f.cas.Path(e.Digest))
+		if err != nil {
+			t.Fatal(err)
+		}
+		placed, err := os.Stat(filepath.Join(dest, e.Path))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !os.SameFile(blob, placed) {
+			t.Errorf("%s is a copy of its blob, not a link to it", e.Path)
+		}
+	}
+}
+
+// Finalize can be retried, so assembling onto a finished tree must be a no-op
+// rather than a rebuild — a rebuild would briefly unlink files that an
+// external consumer of $WEBROOT is reading.
+func TestAssembleIsIdempotent(t *testing.T) {
+	f := newFixture(t, map[string]string{"index.html": "one"})
+	dest := DeploymentDir(f.dir, 1, "dpl_a")
+	if err := Assemble(t.Context(), f.cas, f.files, dest); err != nil {
+		t.Fatal(err)
+	}
+	before, err := os.Stat(filepath.Join(dest, "index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := Assemble(t.Context(), f.cas, f.files, dest); err != nil {
+		t.Fatalf("second Assemble: %v", err)
+	}
+	after, err := os.Stat(filepath.Join(dest, "index.html"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !os.SameFile(before, after) {
+		t.Error("the second Assemble replaced a file that was already in place")
+	}
+}
+
+// A crash leaves a staging tree behind. The next attempt must clear it rather
+// than build on top of files it did not put there.
+func TestAssembleDiscardsALeftoverStagingTree(t *testing.T) {
+	f := newFixture(t, map[string]string{"index.html": "real"})
+	dest := DeploymentDir(f.dir, 1, "dpl_a")
+	staging := dest + stagingSuffix
+	if err := os.MkdirAll(staging, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	// Same path as a manifest entry, so a build that did not clear this would
+	// fail on the exclusive create rather than silently serve the wrong bytes.
+	if err := os.WriteFile(filepath.Join(staging, "index.html"), []byte("stale"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(staging, "orphan.txt"), []byte("stale"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := Assemble(t.Context(), f.cas, f.files, dest); err != nil {
+		t.Fatalf("Assemble: %v", err)
+	}
+	got := walk(t, dest)
+	if len(got) != 1 || got["index.html"] != "real" {
+		t.Errorf("tree = %v, want just the real index.html", got)
+	}
+}
+
+// A failure must leave nothing behind: no partial tree at the destination, and
+// no staging directory quietly consuming disk until recovery notices it.
+func TestAssembleLeavesNothingBehindOnFailure(t *testing.T) {
+	f := newFixture(t, map[string]string{"index.html": "real"})
+	missing := store.FileRow{Path: "gone.txt", Digest: cas.Sum([]byte("never stored")), Size: 12}
+	dest := DeploymentDir(f.dir, 1, "dpl_a")
+
+	err := Assemble(t.Context(), f.cas, append(f.files, missing), dest)
+	if !errors.Is(err, cas.ErrNotFound) {
+		t.Fatalf("err = %v, want cas.ErrNotFound", err)
+	}
+	for _, p := range []string{dest, dest + stagingSuffix} {
+		if _, err := os.Stat(p); !errors.Is(err, fs.ErrNotExist) {
+			t.Errorf("%s still exists: %v", p, err)
+		}
+	}
+}
+
+// Assembly is the one place that turns stored strings into filesystem paths, so
+// it re-checks them even though the API validated the manifest on the way in.
+// A row that got past the API — or into the table by some other route — must not
+// be able to place a file outside the tree being built.
+func TestAssembleRejectsAnEscapingPath(t *testing.T) {
+	f := newFixture(t, map[string]string{"index.html": "real"})
+	dest := DeploymentDir(f.dir, 1, "dpl_a")
+
+	for _, bad := range []string{"../../etc/passwd", "/etc/passwd", "a/../../b", "a//b", `a\..\b`, "a/./b", ""} {
+		row := store.FileRow{Path: bad, Digest: f.files[0].Digest, Size: f.files[0].Size}
+		if err := Assemble(t.Context(), f.cas, []store.FileRow{row}, dest); err == nil {
+			t.Errorf("Assemble accepted %q", bad)
+		}
+		// Nothing is left behind for the next attempt to inherit, and in
+		// particular no directory was created on the way to the rejection.
+		for _, p := range []string{dest, dest + stagingSuffix} {
+			if _, err := os.Stat(p); !errors.Is(err, fs.ErrNotExist) {
+				t.Fatalf("%q left %s behind: %v", bad, p, err)
+			}
+		}
+	}
+}
+
+func TestAssembleHonoursCancellation(t *testing.T) {
+	f := newFixture(t, map[string]string{"index.html": "real"})
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+
+	dest := DeploymentDir(f.dir, 1, "dpl_a")
+	if err := Assemble(ctx, f.cas, f.files, dest); !errors.Is(err, context.Canceled) {
+		t.Fatalf("err = %v, want context.Canceled", err)
+	}
+	if _, err := os.Stat(dest + stagingSuffix); !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("staging directory survived cancellation: %v", err)
+	}
+}

internal/deploy/atomicity_test.go

@@ -0,0 +1,454 @@
+package deploy
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/iceBear67/simplepages/internal/cas"
+	"github.com/iceBear67/simplepages/internal/site"
+	"github.com/iceBear67/simplepages/internal/store"
+	"github.com/iceBear67/simplepages/internal/webroot"
+)
+
+// These are the tests the whole program exists for.
+//
+// The failure they are written against is the one rsync has: during a
+// deployment a visitor sees the new HTML with the old JavaScript, or an asset
+// that is not there yet. Everything else in this repository — the immutable
+// snapshot, the single pointer store, the database-before-memory ordering — is
+// a means to making that state unobservable, and unobservable is a claim about
+// concurrent behaviour that only a concurrent test can support.
+//
+// What is actually asserted, and why it is the strongest true statement:
+//
+//   - Every single response is internally consistent. big.bin is half a
+//     megabyte of one repeated byte, so it spans many writes and a switch
+//     landing mid-body would show up as a seam. Every byte of it equal to the
+//     same version is the claim "no request ever saw a half-updated site".
+//   - A reader's successive responses never go backwards. Requests within one
+//     reader are strictly ordered — the next is not sent until the previous has
+//     been read to completion — so the version it observes may only rise.
+//
+// It would be tempting to also demand that three separate GETs issued around
+// the same time report the same version. That is not a property this or any
+// design has: they are three requests, an activation may legitimately land
+// between any two of them, and asserting otherwise would be asserting that the
+// switch never happens. The per-response guarantee above is what "atomic
+// deployment" means.
+
+const (
+	// Large enough that a response spans many socket writes, so a switch has
+	// somewhere to land mid-body.
+	stormFileSize = 512 << 10
+	// The in-flight tests need the server to still be blocked writing when the
+	// test does something underneath it, which means comfortably more than a
+	// loopback socket will buffer for a client that has stopped reading.
+	inflightFileSize = 8 << 20
+)
+
+// switchEnv is an env with the serving layer attached: a registry, a webroot
+// and an HTTP server, which is the only configuration in which the switch is
+// observable from the outside.
+type switchEnv struct {
+	*env
+	reg   *site.Registry
+	wrDir string
+	srv   *httptest.Server
+	cl    *http.Client
+}
+
+func newSwitchEnv(t *testing.T) *switchEnv {
+	t.Helper()
+	e := newEnv(t)
+	log := slog.New(slog.DiscardHandler)
+
+	reg := site.NewRegistry()
+	wrDir := t.TempDir()
+	wr, err := webroot.Open(wrDir, e.dir)
+	if err != nil {
+		t.Fatalf("webroot.Open: %v", err)
+	}
+	e.svc.Sites = reg
+	e.svc.Webroot = wr
+	reg.Put(e.p)
+
+	srv := httptest.NewServer(&site.Handler{Registry: reg, CAS: e.cas, Log: log})
+	t.Cleanup(srv.Close)
+
+	// The default transport keeps two idle connections per host, which would
+	// turn 64 readers into a connection churn benchmark instead of a switching
+	// one.
+	cl := &http.Client{Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512}}
+	t.Cleanup(cl.CloseIdleConnections)
+
+	return &switchEnv{env: e, reg: reg, wrDir: wrDir, srv: srv, cl: cl}
+}
+
+// versionFiles is deployment n: three small files that name their version, and
+// one large one filled with the single byte n so that any part of it identifies
+// the whole.
+func versionFiles(n, size int) map[string]string {
+	v := strconv.Itoa(n)
+	return map[string]string{
+		"marker.txt": v,
+		"a.txt":      v,
+		"b.txt":      v,
+		"big.bin":    string(bytes.Repeat([]byte{byte(n)}, size)),
+	}
+}
+
+// publish takes version n all the way to ready without activating it.
+func (e *switchEnv) publish(t *testing.T, n, size int) *store.Deployment {
+	t.Helper()
+	contents := versionFiles(n, size)
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
+		t.Fatalf("SetManifest v%d: %v", n, err)
+	}
+	names := make([]string, 0, len(contents))
+	for p := range contents {
+		names = append(names, p)
+	}
+	e.upload(t, contents, names...)
+	dep, err := e.svc.Finalize(t.Context(), e.p, dep)
+	if err != nil {
+		t.Fatalf("Finalize v%d: %v", n, err)
+	}
+	return dep
+}
+
+func (e *switchEnv) activate(t *testing.T, dep *store.Deployment) *store.Deployment {
+	t.Helper()
+	out, err := e.svc.Activate(t.Context(), e.p, dep)
+	if err != nil {
+		t.Fatalf("Activate %s: %v", dep.PublicID, err)
+	}
+	return out
+}
+
+// get fetches one file from the served site. It returns errors rather than
+// failing the test, because most of its callers are goroutines.
+func (e *switchEnv) get(name string) ([]byte, error) {
+	resp, err := e.cl.Get(e.srv.URL + "/~demo/" + name)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		io.Copy(io.Discard, resp.Body)
+		return nil, fmt.Errorf("GET %s: status %d", name, resp.StatusCode)
+	}
+	return io.ReadAll(resp.Body)
+}
+
+// bigVersion is the assertion that matters: it reports which version a big.bin
+// body came from, and fails if the body is not entirely from one version.
+func bigVersion(b []byte, size int) (int, error) {
+	if len(b) != size {
+		return 0, fmt.Errorf("big.bin is %d bytes, want %d", len(b), size)
+	}
+	first := b[0]
+	for i, c := range b {
+		if c != first {
+			return 0, fmt.Errorf(
+				"big.bin mixes two deployments: byte 0 is from version %d but byte %d is from version %d",
+				first, i, c)
+		}
+	}
+	return int(first), nil
+}
+
+func smallVersion(b []byte) (int, error) {
+	n, err := strconv.Atoi(string(b))
+	if err != nil {
+		return 0, fmt.Errorf("file body %q is not a version number: %v", b, err)
+	}
+	return n, nil
+}
+
+func TestActivationAtomicity(t *testing.T) {
+	const versions = 50
+	const readers = 64
+
+	e := newSwitchEnv(t)
+	deps := make([]*store.Deployment, versions)
+	valid := make(map[string]bool, versions)
+	for i := range deps {
+		deps[i] = e.publish(t, i+1, stormFileSize)
+		valid[deps[i].PublicID] = true
+	}
+	e.activate(t, deps[0])
+
+	ctx, stop := context.WithCancel(t.Context())
+	var wg sync.WaitGroup
+
+	// Anti-vacuity: if every reader only ever saw the last version the
+	// invariants above hold trivially and prove nothing.
+	var mu sync.Mutex
+	lowest, highest := versions+1, 0
+	record := func(n int) {
+		mu.Lock()
+		defer mu.Unlock()
+		lowest = min(lowest, n)
+		highest = max(highest, n)
+	}
+
+	for range readers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			seen := 0
+			for ctx.Err() == nil {
+				for _, name := range []string{"a.txt", "b.txt", "big.bin"} {
+					body, err := e.get(name)
+					if err != nil {
+						if ctx.Err() == nil {
+							t.Errorf("%v", err)
+						}
+						return
+					}
+					var n int
+					if name == "big.bin" {
+						n, err = bigVersion(body, stormFileSize)
+					} else {
+						n, err = smallVersion(body)
+					}
+					if err != nil {
+						t.Errorf("%s: %v", name, err)
+						return
+					}
+					if n < seen {
+						t.Errorf("%s reported version %d after version %d had already been "+
+							"served to this reader: the switch was observed running backwards",
+							name, n, seen)
+						return
+					}
+					seen = n
+					record(n)
+				}
+			}
+		}()
+	}
+
+	// The symlink is not what serves the site, but an external reader — a
+	// reverse proxy, a backup job — follows it, and it must never be missing or
+	// dangling while the switch runs.
+	link := filepath.Join(e.wrDir, "~demo")
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for ctx.Err() == nil {
+			target, err := os.Readlink(link)
+			if err != nil {
+				t.Errorf("$WEBROOT/~demo: %v", err)
+				return
+			}
+			// Stat follows the link, so a dangling one fails here.
+			fi, err := os.Stat(target)
+			if err != nil {
+				t.Errorf("$WEBROOT/~demo -> %s: %v", target, err)
+				return
+			}
+			if !fi.IsDir() {
+				t.Errorf("$WEBROOT/~demo -> %s is not a directory", target)
+				return
+			}
+			if !valid[filepath.Base(target)] {
+				t.Errorf("$WEBROOT/~demo -> %s, which is not a deployment of this project", target)
+				return
+			}
+		}
+	}()
+
+	for _, dep := range deps[1:] {
+		if _, err := e.svc.Activate(t.Context(), e.p, dep); err != nil {
+			t.Errorf("Activate %s: %v", dep.PublicID, err)
+			break
+		}
+	}
+	stop()
+	wg.Wait()
+
+	if t.Failed() {
+		return
+	}
+	if lowest >= versions {
+		t.Fatalf("every observation was of version %d or later: the readers never "+
+			"overlapped the switching, so this test proved nothing", lowest)
+	}
+	if lowest == highest {
+		t.Fatalf("every observation was of version %d: no switch was observed", lowest)
+	}
+	for _, name := range []string{"marker.txt", "a.txt", "b.txt"} {
+		body, err := e.get(name)
+		if err != nil {
+			t.Fatalf("%v", err)
+		}
+		if got, err := smallVersion(body); err != nil || got != versions {
+			t.Errorf("after the storm %s = %q (%v), want version %d", name, body, err, versions)
+		}
+	}
+}
+
+// openBig starts a request for big.bin and reads only its first kilobyte, so
+// the response is still open and the server is still blocked writing it.
+func (e *switchEnv) openBig(t *testing.T) (*http.Response, []byte) {
+	t.Helper()
+	resp, err := e.cl.Get(e.srv.URL + "/~demo/big.bin")
+	if err != nil {
+		t.Fatalf("GET big.bin: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		t.Fatalf("GET big.bin: status %d", resp.StatusCode)
+	}
+	head := make([]byte, 1024)
+	if _, err := io.ReadFull(resp.Body, head); err != nil {
+		resp.Body.Close()
+		t.Fatalf("reading the start of big.bin: %v", err)
+	}
+	return resp, head
+}
+
+// drain finishes a response opened by openBig and reports which version the
+// whole body came from.
+func drain(t *testing.T, resp *http.Response, head []byte) int {
+	t.Helper()
+	defer resp.Body.Close()
+	rest, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("reading the rest of big.bin: %v", err)
+	}
+	n, err := bigVersion(append(head, rest...), inflightFileSize)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	return n
+}
+
+func TestAnInFlightRequestKeepsReadingTheDeploymentItStartedOn(t *testing.T) {
+	e := newSwitchEnv(t)
+	v1 := e.publish(t, 1, inflightFileSize)
+	v2 := e.publish(t, 2, inflightFileSize)
+	e.activate(t, v1)
+
+	resp, head := e.openBig(t)
+	e.activate(t, v2)
+
+	// This is the property a symlink rename cannot give you: the switch has
+	// already happened, and this response is still the one it started as.
+	if n := drain(t, resp, head); n != 1 {
+		t.Errorf("a request that started before the switch finished on version %d, want 1", n)
+	}
+	body, err := e.get("marker.txt")
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	if n, _ := smallVersion(body); n != 2 {
+		t.Errorf("a request that started after the switch got version %d, want 2", n)
+	}
+}
+
+func TestAnInFlightRequestSurvivesTheContentBeingCollected(t *testing.T) {
+	e := newSwitchEnv(t)
+	v1 := e.publish(t, 1, inflightFileSize)
+	v2 := e.publish(t, 2, inflightFileSize)
+	e.activate(t, v1)
+
+	resp, head := e.openBig(t)
+	e.activate(t, v2)
+
+	// Everything a collector could possibly remove, with no grace period at
+	// all: the assembled tree and the content it was hardlinked from. Both,
+	// because removing only one of them leaves the inode alive through the
+	// other and the test would prove nothing.
+	if err := os.RemoveAll(DeploymentDir(e.dir, e.p.ID, v1.PublicID)); err != nil {
+		t.Fatalf("removing the old tree: %v", err)
+	}
+	for _, c := range versionFiles(1, inflightFileSize) {
+		if err := e.cas.Remove(cas.Sum([]byte(c))); err != nil {
+			t.Fatalf("removing old content: %v", err)
+		}
+	}
+
+	// The handler is holding an open descriptor, and POSIX keeps the inode
+	// alive until it closes. The grace period in the collector exists so this
+	// never has to be relied on, but relying on it has to work.
+	if n := drain(t, resp, head); n != 1 {
+		t.Errorf("a request whose content was deleted under it finished on version %d, want 1", n)
+	}
+	if _, err := e.get("marker.txt"); err != nil {
+		t.Errorf("the live deployment stopped serving after the old one was collected: %v", err)
+	}
+}
+
+func TestAFailedActivationChangesNothing(t *testing.T) {
+	e := newSwitchEnv(t)
+	v1 := e.publish(t, 1, 64)
+	v2 := e.publish(t, 2, 64)
+	e.activate(t, v1)
+
+	link := filepath.Join(e.wrDir, "~demo")
+	before, err := os.Readlink(link)
+	if err != nil {
+		t.Fatalf("$WEBROOT/~demo: %v", err)
+	}
+
+	// Make v2's manifest unreadable, which fails Activate inside index() —
+	// before the transaction, before the pointer store, before the symlink. The
+	// bogus blobs row exists only to satisfy the foreign key; the point is the
+	// one-byte digest, which cas.FromBytes refuses.
+	err = e.db.Tx(t.Context(), func(tx *sql.Tx) error {
+		if _, err := tx.ExecContext(t.Context(), `
+			INSERT INTO blobs (digest, size, present, created_at, last_ref_at)
+			VALUES (x'00', 1, 1, 0, 0) ON CONFLICT(digest) DO NOTHING`); err != nil {
+			return err
+		}
+		_, err := tx.ExecContext(t.Context(),
+			`UPDATE deployment_files SET digest = x'00' WHERE deployment_id = ?`, v2.ID)
+		return err
+	})
+	if err != nil {
+		t.Fatalf("corrupting the manifest: %v", err)
+	}
+
+	if _, err := e.svc.Activate(t.Context(), e.p, v2); err == nil {
+		t.Fatal("Activate succeeded on a deployment whose manifest cannot be read")
+	}
+
+	// In memory.
+	body, err := e.get("marker.txt")
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	if n, _ := smallVersion(body); n != 1 {
+		t.Errorf("after the failed activation the site serves version %d, want 1", n)
+	}
+	// In the database, which is what a restart would come back to.
+	active, err := e.db.ActiveDeployment(t.Context(), e.p.ID)
+	if err != nil {
+		t.Fatalf("ActiveDeployment: %v", err)
+	}
+	if active.PublicID != v1.PublicID {
+		t.Errorf("the database says %s is active, want %s", active.PublicID, v1.PublicID)
+	}
+	// And on disk.
+	after, err := os.Readlink(link)
+	if err != nil {
+		t.Fatalf("$WEBROOT/~demo: %v", err)
+	}
+	if after != before {
+		t.Errorf("$WEBROOT/~demo moved to %s, want it left at %s", after, before)
+	}
+}

internal/deploy/gc.go

@@ -0,0 +1,259 @@
+package deploy
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/iceBear67/simplepages/api"
+	"github.com/iceBear67/simplepages/internal/store"
+)
+
+const (
+	// defaultBlobGrace is how long a blob must have been unreferenced before
+	// its content is removed.
+	//
+	// It is what makes the read path safe without reference counting requests.
+	// A handler resolves a digest from its snapshot and then opens it; between
+	// those two instants the collector could in principle delete the file. An
+	// hour is an enormous margin for a gap that is measured in microseconds,
+	// and it costs only some disk that was going to be reclaimed anyway.
+	defaultBlobGrace = time.Hour
+
+	// failedRetention is how long a failed deployment's row is kept. Its
+	// manifest is already gone, so this is purely so that an operator
+	// investigating a broken CI job can still see that it failed and why.
+	failedRetention = 24 * time.Hour
+)
+
+// Collect runs one garbage collection pass: retention first, then the content
+// nothing references any more.
+//
+// Two passes in that order, and not one, because the first is what makes work
+// for the second. Deleting a deployment drops its manifest rows, the delete
+// trigger takes each blob's refcount down, and only then can a blob be seen to
+// be unreferenced. The second pass will not act on those blobs in this same
+// run — the trigger sets last_ref_at to now and the grace period has not
+// elapsed — which is deliberate: content that just became unreachable is
+// exactly the content some in-flight request is most likely to still be
+// reading.
+//
+// A dry run reports the deployments that would be deleted and the blobs that
+// are collectable right now. It cannot report the blobs the deletions would
+// free, because nothing has been deleted; the number is a floor, not an
+// estimate, and it is honest about being one.
+func (s *Service) Collect(ctx context.Context, dryRun bool) (api.GCStats, error) {
+	stats := api.GCStats{DryRun: dryRun}
+
+	if !dryRun {
+		// Abandoned uploads first, so that whatever only they referenced is
+		// already unreferenced by the time the blob pass looks.
+		n, err := s.DB.ExpireStaleDeployments(ctx, time.Now().Add(-staleUploadAge),
+			"abandoned: no activity for "+staleUploadAge.String())
+		if err != nil {
+			return stats, err
+		}
+		if n > 0 {
+			s.Log.InfoContext(ctx, "expired unfinished deployments", "count", n)
+		}
+	}
+
+	projects, err := s.DB.AllProjects(ctx)
+	if err != nil {
+		return stats, err
+	}
+	for _, p := range projects {
+		n, err := s.collectProject(ctx, p, dryRun)
+		stats.DeploymentsDeleted += n
+		if err != nil {
+			return stats, err
+		}
+	}
+
+	blobs, err := s.DB.UnreferencedBlobs(ctx, time.Now().Add(-s.blobGrace()), 0)
+	if err != nil {
+		return stats, err
+	}
+	for _, b := range blobs {
+		if dryRun {
+			stats.BlobsDeleted++
+			stats.BytesFreed += b.Size
+			continue
+		}
+		deleted, err := s.DB.DeleteBlob(ctx, b.Digest, func() error { return s.CAS.Remove(b.Digest) })
+		if err != nil {
+			return stats, err
+		}
+		// Not deleted means the blob was referenced again between the listing
+		// and the delete — a new deployment naming content that was about to be
+		// collected. Leaving it alone is the whole point of rechecking the
+		// refcount inside the transaction.
+		if deleted {
+			stats.BlobsDeleted++
+			stats.BytesFreed += b.Size
+		}
+	}
+	return stats, nil
+}
+
+// collectProject applies one project's retention policy.
+//
+// The active deployment is not considered at all: it is excluded by the query,
+// so no counting mistake here can reach it.
+func (s *Service) collectProject(ctx context.Context, p *store.Project, dryRun bool) (int, error) {
+	deps, err := s.DB.InactiveDeployments(ctx, p.ID)
+	if err != nil {
+		return 0, err
+	}
+	grace := time.Duration(p.RetentionGraceS) * time.Second
+	now := time.Now()
+
+	var deleted, kept int
+	for _, dep := range deps {
+		switch dep.State {
+		case store.StateReady:
+			// Newest first, so the first RetentionCount of them are the ones
+			// worth keeping for a rollback.
+			if kept < p.RetentionCount {
+				kept++
+				continue
+			}
+			if now.Sub(retiredAt(dep)) < grace {
+				continue
+			}
+		case store.StateFailed:
+			if now.Sub(dep.CreatedAt) < failedRetention {
+				continue
+			}
+		case store.StateDeleting:
+			// Already claimed by a sweep that did not finish. No grace applies:
+			// nothing may serve a tree that is half removed.
+		default:
+			// pending or uploading. Someone may still be uploading to it, and
+			// ExpireStaleDeployments is what decides when they are not.
+			continue
+		}
+		if dryRun {
+			deleted++
+			continue
+		}
+		if err := s.claim(ctx, p, dep); err != nil {
+			switch {
+			case errors.Is(err, store.ErrNotFound):
+				// Deleted by someone else between the listing and now.
+			case errors.Is(err, store.ErrConflict):
+				// Activated between the listing and now — a rollback landed on
+				// a deployment retention had picked. Correct outcome: the claim
+				// is refused and the deployment stays.
+				s.Log.InfoContext(ctx, "skipped a deployment that was activated during collection",
+					"project", p.Name, "deployment", dep.PublicID)
+			default:
+				return deleted, err
+			}
+			continue
+		}
+		if err := s.removeDeployment(ctx, dep); err != nil {
+			// The row is in the deleting state and committed, so recovery or
+			// the next sweep will finish it. Nothing serves it in the meantime.
+			s.Log.WarnContext(ctx, "could not finish deleting a deployment",
+				"project", p.Name, "deployment", dep.PublicID, "err", err)
+			continue
+		}
+		deleted++
+	}
+	return deleted, nil
+}
+
+// retiredAt is when a deployment stopped being served, or when it was created
+// if it never was. It is what the retention grace is measured from.
+func retiredAt(dep *store.Deployment) time.Time {
+	if dep.DeactivatedAt != nil {
+		return *dep.DeactivatedAt
+	}
+	return dep.CreatedAt
+}
+
+// claim marks a deployment as being deleted, which is what makes it safe to
+// start removing files.
+//
+// The project lock is held for exactly this statement. Activation takes the
+// same lock and re-reads the row under it, so the two orderings are the only
+// possible ones: either activation commits first and this fails on the
+// active = 0 condition, or this commits first and activation finds a deployment
+// in the deleting state and refuses it. There is no interleaving in which a
+// tree is removed from underneath a deployment that has just become active.
+func (s *Service) claim(ctx context.Context, p *store.Project, dep *store.Deployment) error {
+	unlock, err := s.locks.lock(ctx, p.ID)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+	return s.DB.MarkDeploymentDeleting(ctx, dep.ID)
+}
+
+// Delete removes one deployment on request. The active one cannot be deleted:
+// that is a conflict, not a permission problem, and the client is expected to
+// activate something else first.
+func (s *Service) Delete(ctx context.Context, p *store.Project, dep *store.Deployment) error {
+	if err := s.claim(ctx, p, dep); err != nil {
+		switch {
+		case errors.Is(err, store.ErrNotFound):
+			return api.Errorf(api.CodeNotFound, "no such deployment")
+		case errors.Is(err, store.ErrConflict):
+			return api.Errorf(api.CodeDeploymentActive,
+				"this deployment is the one the project is serving; activate another one first")
+		}
+		return err
+	}
+	if err := s.removeDeployment(ctx, dep); err != nil {
+		return err
+	}
+	s.Log.InfoContext(ctx, "deployment deleted", "project", p.Name, "deployment", dep.PublicID)
+	return nil
+}
+
+// RemoveProjectTrees deletes what a project left on disk once its rows are
+// gone. Its blobs are freed by the same cascade and collected on the next pass.
+func (s *Service) RemoveProjectTrees(projectID int64) error {
+	if s.Dir == "" {
+		return nil
+	}
+	return os.RemoveAll(filepath.Join(s.Dir, strconv.FormatInt(projectID, 10)))
+}
+
+// RunCollector collects on a timer until ctx is done. A failed pass is logged
+// and the next one runs as scheduled: everything the collector does is
+// idempotent, so there is nothing to unwind and no reason to stop.
+func (s *Service) RunCollector(ctx context.Context, every time.Duration) {
+	if every <= 0 {
+		return
+	}
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			stats, err := s.Collect(ctx, false)
+			if err != nil {
+				s.Log.WarnContext(ctx, "garbage collection did not finish", "err", err)
+			}
+			if stats.DeploymentsDeleted > 0 || stats.BlobsDeleted > 0 {
+				s.Log.InfoContext(ctx, "collected",
+					"deployments", stats.DeploymentsDeleted,
+					"blobs", stats.BlobsDeleted, "bytes", stats.BytesFreed)
+			}
+		}
+	}
+}
+
+func (s *Service) blobGrace() time.Duration {
+	if s.BlobGrace == 0 {
+		return defaultBlobGrace
+	}
+	return s.BlobGrace
+}

internal/deploy/gc_test.go

@@ -0,0 +1,617 @@
+package deploy
+
+import (
+	"database/sql"
+	"errors"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/iceBear67/simplepages/api"
+	"github.com/iceBear67/simplepages/internal/cas"
+	"github.com/iceBear67/simplepages/internal/store"
+)
+
+// These tests set BlobGrace negative. The grace exists so that a request which
+// has resolved a digest and is about to open it cannot lose the file underneath
+// it, and the default hour is far longer than any test wants to wait. Timestamps
+// are whole seconds, so a grace of zero would not collect a blob dereferenced in
+// the same second either — negative is the only value that means "now".
+const collectNow = -time.Minute
+
+// deployReady publishes a finished, inactive deployment holding contents.
+func (e *env) deployReady(t *testing.T, contents map[string]string) *store.Deployment {
+	t.Helper()
+	dep := e.create(t)
+	files := manifest(contents)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, files); err != nil {
+		t.Fatalf("SetManifest: %v", err)
+	}
+	paths := make([]string, 0, len(contents))
+	for p := range contents {
+		paths = append(paths, p)
+	}
+	e.upload(t, contents, paths...)
+	dep, err := e.svc.Finalize(t.Context(), e.p, dep)
+	if err != nil {
+		t.Fatalf("Finalize: %v", err)
+	}
+	return dep
+}
+
+// version publishes a deployment whose single file identifies it, which is
+// enough for retention tests: what matters is how many survive and which.
+func (e *env) version(t *testing.T, n int) *store.Deployment {
+	t.Helper()
+	return e.deployReady(t, map[string]string{"index.html": "v" + strconv.Itoa(n)})
+}
+
+func (e *env) activate(t *testing.T, dep *store.Deployment) {
+	t.Helper()
+	if _, err := e.svc.Activate(t.Context(), e.p, dep); err != nil {
+		t.Fatalf("Activate %s: %v", dep.PublicID, err)
+	}
+}
+
+// exec runs a statement the service has no method for. Retention tests need to
+// backdate rows, because the alternative is a test that sleeps for an hour.
+func (e *env) exec(t *testing.T, query string, args ...any) {
+	t.Helper()
+	err := e.db.Tx(t.Context(), func(tx *sql.Tx) error {
+		_, err := tx.ExecContext(t.Context(), query, args...)
+		return err
+	})
+	if err != nil {
+		t.Fatalf("%s: %v", query, err)
+	}
+}
+
+// setRetention rewrites the project's policy and re-reads it, because the
+// collector reads the row and not the struct the test is holding.
+func (e *env) setRetention(t *testing.T, count int, graceS int64) {
+	t.Helper()
+	e.exec(t, `UPDATE projects SET retention_count = ?, retention_grace_s = ? WHERE id = ?`,
+		count, graceS, e.p.ID)
+	p, err := e.db.ProjectByID(t.Context(), e.p.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	e.p = p
+}
+
+// alive reports whether a deployment still has a row and a tree.
+func (e *env) alive(t *testing.T, dep *store.Deployment) (row, tree bool) {
+	t.Helper()
+	_, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID)
+	switch {
+	case err == nil:
+		row = true
+	case errors.Is(err, store.ErrNotFound):
+	default:
+		t.Fatalf("DeploymentByPublicID: %v", err)
+	}
+	_, err = os.Stat(DeploymentDir(e.dir, e.p.ID, dep.PublicID))
+	switch {
+	case err == nil:
+		tree = true
+	case os.IsNotExist(err):
+	default:
+		t.Fatalf("stat deployment tree: %v", err)
+	}
+	return
+}
+
+func (e *env) mustSurvive(t *testing.T, dep *store.Deployment, why string) {
+	t.Helper()
+	row, tree := e.alive(t, dep)
+	if !row || !tree {
+		t.Errorf("%s (%s) was collected: row=%v tree=%v", why, dep.PublicID, row, tree)
+	}
+}
+
+func (e *env) mustBeGone(t *testing.T, dep *store.Deployment, why string) {
+	t.Helper()
+	row, tree := e.alive(t, dep)
+	if row || tree {
+		t.Errorf("%s (%s) survived: row=%v tree=%v", why, dep.PublicID, row, tree)
+	}
+}
+
+// blobCount is how many blobs the database knows about, which is the number the
+// milestone's manual check watches drop.
+func blobCount(t *testing.T, e *env) int64 {
+	t.Helper()
+	counts, err := e.db.Counts(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return counts.Blobs
+}
+
+func hasContent(t *testing.T, e *env, content string) bool {
+	t.Helper()
+	ok, err := e.cas.Has(cas.Sum([]byte(content)))
+	if err != nil {
+		t.Fatalf("cas.Has: %v", err)
+	}
+	return ok
+}
+
+// The headline retention rule from the milestone: deploy repeatedly, keep
+// retention_count of them plus whichever one is being served, and watch the
+// content of the rest go away.
+func TestCollectKeepsRetentionCountPlusTheActiveOne(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 10, 0)
+
+	var deps []*store.Deployment
+	for i := 1; i <= 15; i++ {
+		dep := e.version(t, i)
+		deps = append(deps, dep)
+		e.activate(t, dep)
+	}
+	// Roll back to the oldest one, so the deployment being served is also the
+	// one retention would otherwise drop first. Nothing may collect it.
+	e.activate(t, deps[0])
+
+	before := blobCount(t, e)
+	stats, err := e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 15 deployments, 10 kept by retention plus the active one: 4 collected.
+	if stats.DeploymentsDeleted != 4 {
+		t.Errorf("deleted %d deployments, want 4", stats.DeploymentsDeleted)
+	}
+	e.mustSurvive(t, deps[0], "the active deployment")
+	for _, dep := range deps[5:] {
+		e.mustSurvive(t, dep, "a deployment inside the retention window")
+	}
+	for _, dep := range deps[1:5] {
+		e.mustBeGone(t, dep, "a deployment past the retention window")
+	}
+
+	if after := blobCount(t, e); after != before-4 {
+		t.Errorf("blob count went from %d to %d, want %d", before, after, before-4)
+	}
+	if stats.BlobsDeleted != 4 {
+		t.Errorf("collected %d blobs, want the 4 the deleted deployments held", stats.BlobsDeleted)
+	}
+	if stats.BytesFreed <= 0 {
+		t.Errorf("BytesFreed = %d, want the size of what was removed", stats.BytesFreed)
+	}
+	for i := 2; i <= 5; i++ {
+		if hasContent(t, e, "v"+strconv.Itoa(i)) {
+			t.Errorf("content of the collected deployment v%d is still in the CAS", i)
+		}
+	}
+	// What the survivors reference is untouched, which is what makes a rollback
+	// to any of them still work.
+	if !hasContent(t, e, "v1") || !hasContent(t, e, "v15") {
+		t.Error("content a surviving deployment references was collected")
+	}
+
+	// Nothing left to do on a second pass.
+	stats, err = e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stats.DeploymentsDeleted != 0 || stats.BlobsDeleted != 0 {
+		t.Errorf("a second pass collected %+v, want nothing", stats)
+	}
+}
+
+// The grace period is measured from when a deployment stopped being served, so
+// a rollback that was a mistake can be undone for a while afterwards.
+func TestCollectHonoursTheRetentionGrace(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 0, 3600)
+
+	old := e.version(t, 1)
+	e.activate(t, old)
+	current := e.version(t, 2)
+	e.activate(t, current)
+
+	// retention_count is zero, so only the grace is protecting it.
+	stats, err := e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stats.DeploymentsDeleted != 0 {
+		t.Errorf("deleted %d deployments inside the grace period, want 0", stats.DeploymentsDeleted)
+	}
+	e.mustSurvive(t, old, "a deployment retired seconds ago")
+
+	// Backdate the deactivation past the grace and it becomes collectable.
+	e.exec(t, `UPDATE deployments SET deactivated_at = ? WHERE id = ?`,
+		time.Now().Add(-2*time.Hour).Unix(), old.ID)
+	if _, err := e.svc.Collect(t.Context(), false); err != nil {
+		t.Fatal(err)
+	}
+	e.mustBeGone(t, old, "a deployment retired before the grace period")
+	e.mustSurvive(t, current, "the active deployment")
+}
+
+// A deployment that never finished uploading is not retention's business: it
+// has no deactivated_at to measure from and someone may still be pushing to it.
+func TestCollectLeavesUnfinishedUploadsToTheExpiry(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 0, 0)
+
+	contents := map[string]string{"index.html": "in progress"}
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
+		t.Fatal(err)
+	}
+	e.upload(t, contents, "index.html")
+
+	if _, err := e.svc.Collect(t.Context(), false); err != nil {
+		t.Fatal(err)
+	}
+	got, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID)
+	if err != nil {
+		t.Fatalf("an in-progress upload was collected: %v", err)
+	}
+	if got.State != store.StateUploading {
+		t.Errorf("state = %q, want it left alone as %q", got.State, store.StateUploading)
+	}
+	// Its content is protected too, by the manifest rows that already reference
+	// it — which is the whole reason the manifest is written before the upload.
+	if !hasContent(t, e, "in progress") {
+		t.Error("the content of an in-progress upload was collected")
+	}
+
+	// Once it is old enough, one pass does the whole job: the expiry fails it
+	// and drops its manifest, retention finds a failed deployment older than it
+	// keeps failures for, and the blob pass then reaches what only it
+	// referenced. That the three run in that order is why it takes one pass and
+	// not three.
+	e.exec(t, `UPDATE deployments SET created_at = ? WHERE id = ?`,
+		time.Now().Add(-48*time.Hour).Unix(), dep.ID)
+	stats, err := e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stats.DeploymentsDeleted != 1 || stats.BlobsDeleted != 1 {
+		t.Errorf("collected %+v, want the abandoned upload and its content", stats)
+	}
+	if _, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID); !errors.Is(err, store.ErrNotFound) {
+		t.Errorf("the abandoned upload survived: %v", err)
+	}
+	if hasContent(t, e, "in progress") {
+		t.Error("content nothing references any more survived the sweep")
+	}
+}
+
+// A deployment that failed while being finalized keeps its row for a day, so an
+// operator looking into a broken CI job can still see that it failed and why.
+func TestCollectKeepsRecentFailuresForInspection(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 0, 0)
+
+	dep := e.create(t)
+	if err := e.db.MarkDeploymentFailed(t.Context(), dep.ID, "assembly failed"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := e.svc.Collect(t.Context(), false); err != nil {
+		t.Fatal(err)
+	}
+	got, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID)
+	if err != nil {
+		t.Fatalf("a deployment that failed moments ago was collected: %v", err)
+	}
+	if got.Error != "assembly failed" {
+		t.Errorf("error = %q, want the reason still readable", got.Error)
+	}
+
+	e.exec(t, `UPDATE deployments SET created_at = ? WHERE id = ?`,
+		time.Now().Add(-48*time.Hour).Unix(), dep.ID)
+	if _, err := e.svc.Collect(t.Context(), false); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID); !errors.Is(err, store.ErrNotFound) {
+		t.Errorf("a failed row older than the retention survived: %v", err)
+	}
+}
+
+// Content two deployments share outlives the first of them. This is what makes
+// cross-deployment deduplication safe to rely on.
+func TestCollectKeepsSharedContent(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 0, 0)
+
+	shared := "console.log(1)"
+	old := e.deployReady(t, map[string]string{"index.html": "v1", "app.js": shared})
+	e.activate(t, old)
+	current := e.deployReady(t, map[string]string{"index.html": "v2", "app.js": shared})
+	e.activate(t, current)
+
+	if _, err := e.svc.Collect(t.Context(), false); err != nil {
+		t.Fatal(err)
+	}
+	e.mustBeGone(t, old, "the superseded deployment")
+	if hasContent(t, e, "v1") {
+		t.Error("content only the collected deployment referenced survived")
+	}
+	if !hasContent(t, e, shared) {
+		t.Fatal("content the active deployment still references was collected")
+	}
+	// And it is still readable through the deployment that survived, which is
+	// the property the assembled tree shares an inode for.
+	body, err := os.ReadFile(filepath.Join(DeploymentDir(e.dir, e.p.ID, current.PublicID), "app.js"))
+	if err != nil || string(body) != shared {
+		t.Errorf("reading shared content from the surviving tree = %q, %v", body, err)
+	}
+}
+
+// The grace is what makes the read path safe without per-request reference
+// counting, so it has to actually hold content back.
+func TestCollectHoldsRecentlyDereferencedBlobs(t *testing.T) {
+	e := newEnv(t)
+	e.setRetention(t, 0, 0) // BlobGrace left at its default hour.
+
+	old := e.version(t, 1)
+	e.activate(t, old)
+	e.activate(t, e.version(t, 2))
+
+	stats, err := e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	e.mustBeGone(t, old, "the superseded deployment")
+	if stats.BlobsDeleted != 0 {
+		t.Errorf("collected %d blobs, want them held by the grace period", stats.BlobsDeleted)
+	}
+	if !hasContent(t, e, "v1") {
+		t.Error("content dereferenced moments ago was removed inside the grace period")
+	}
+}
+
+func TestCollectDryRunChangesNothing(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 0, 0)
+
+	old := e.version(t, 1)
+	e.activate(t, old)
+	current := e.version(t, 2)
+	e.activate(t, current)
+
+	stats, err := e.svc.Collect(t.Context(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !stats.DryRun {
+		t.Error("the report does not say it was a dry run")
+	}
+	if stats.DeploymentsDeleted != 1 {
+		t.Errorf("reported %d deployments, want the 1 that would be deleted", stats.DeploymentsDeleted)
+	}
+	e.mustSurvive(t, old, "a deployment a dry run only reported on")
+	e.mustSurvive(t, current, "the active deployment")
+	if !hasContent(t, e, "v1") {
+		t.Error("a dry run removed content")
+	}
+
+	// Blobs the reported deletions would free are not counted: nothing was
+	// deleted, so they are all still referenced. The number is a floor.
+	if stats.BlobsDeleted != 0 {
+		t.Errorf("a dry run reported %d collectable blobs, want 0 while everything is referenced",
+			stats.BlobsDeleted)
+	}
+
+	// The real pass then does what the dry run said it would.
+	stats, err = e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stats.DryRun {
+		t.Error("a real pass reported itself as a dry run")
+	}
+	if stats.DeploymentsDeleted != 1 {
+		t.Errorf("deleted %d deployments, want 1", stats.DeploymentsDeleted)
+	}
+	e.mustBeGone(t, old, "the superseded deployment")
+}
+
+func TestDeleteRemovesRowAndTree(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+
+	dep := e.version(t, 1)
+	current := e.version(t, 2)
+	e.activate(t, current)
+
+	if err := e.svc.Delete(t.Context(), e.p, dep); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	e.mustBeGone(t, dep, "the deleted deployment")
+
+	// Deleting it dropped the manifest, so the next collection reaches what only
+	// it referenced.
+	if _, err := e.svc.Collect(t.Context(), false); err != nil {
+		t.Fatal(err)
+	}
+	if hasContent(t, e, "v1") {
+		t.Error("content the deleted deployment held survived collection")
+	}
+
+	if err := e.svc.Delete(t.Context(), e.p, dep); apiCode(err) != api.CodeNotFound {
+		t.Errorf("deleting it again = %v, want %q", err, api.CodeNotFound)
+	}
+}
+
+// Deleting what a project is serving is a conflict, not a permission problem:
+// the client is told to activate something else first.
+func TestDeleteRefusesTheActiveDeployment(t *testing.T) {
+	e := newEnv(t)
+	dep := e.version(t, 1)
+	e.activate(t, dep)
+
+	err := e.svc.Delete(t.Context(), e.p, dep)
+	wantCode(t, err, api.CodeDeploymentActive)
+	e.mustSurvive(t, dep, "the active deployment")
+
+	// It becomes deletable the moment something else is being served, which is
+	// the sequence the error message describes.
+	e.activate(t, e.version(t, 2))
+	if err := e.svc.Delete(t.Context(), e.p, dep); err != nil {
+		t.Fatalf("Delete after activating another deployment: %v", err)
+	}
+	e.mustBeGone(t, dep, "the deployment that was superseded and then deleted")
+}
+
+func TestRemoveProjectTrees(t *testing.T) {
+	e := newEnv(t)
+	dep := e.version(t, 1)
+	e.activate(t, dep)
+
+	other := store.DefaultProject("other")
+	if err := e.db.CreateProject(t.Context(), other); err != nil {
+		t.Fatal(err)
+	}
+	otherDir := DeploymentDir(e.dir, other.ID, "dpl_0000000000000000")
+	if err := os.MkdirAll(otherDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := e.svc.RemoveProjectTrees(e.p.ID); err != nil {
+		t.Fatalf("RemoveProjectTrees: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(e.dir, strconv.FormatInt(e.p.ID, 10))); !os.IsNotExist(err) {
+		t.Errorf("the project's directory survived: %v", err)
+	}
+	if _, err := os.Stat(otherDir); err != nil {
+		t.Errorf("another project's directory was removed: %v", err)
+	}
+
+	// Idempotent: recovery may run it again after a crash partway through.
+	if err := e.svc.RemoveProjectTrees(e.p.ID); err != nil {
+		t.Errorf("a second removal: %v", err)
+	}
+}
+
+// Deleting the project takes its deployments with it, and the collector then
+// reclaims everything they referenced.
+func TestCollectAfterProjectDeletion(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+
+	dep := e.version(t, 1)
+	e.activate(t, dep)
+
+	if err := e.db.DeleteProject(t.Context(), e.p.ID); err != nil {
+		t.Fatalf("DeleteProject: %v", err)
+	}
+	if err := e.svc.RemoveProjectTrees(e.p.ID); err != nil {
+		t.Fatalf("RemoveProjectTrees: %v", err)
+	}
+
+	stats, err := e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stats.BlobsDeleted != 1 {
+		t.Errorf("collected %d blobs, want the 1 the deleted project held", stats.BlobsDeleted)
+	}
+	if hasContent(t, e, "v1") {
+		t.Error("content of a deleted project survived collection")
+	}
+	if n := blobCount(t, e); n != 0 {
+		t.Errorf("%d blob rows left after the project was deleted", n)
+	}
+}
+
+// A deployment claimed by a sweep that was interrupted is finished by the next
+// one, whatever its retention would otherwise have said.
+func TestCollectResumesAnInterruptedDeletion(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 10, 3600) // Generous: retention alone would keep it.
+
+	dep := e.version(t, 1)
+	e.activate(t, e.version(t, 2))
+	if err := e.db.MarkDeploymentDeleting(t.Context(), dep.ID); err != nil {
+		t.Fatal(err)
+	}
+
+	stats, err := e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stats.DeploymentsDeleted != 1 {
+		t.Errorf("deleted %d deployments, want the claimed one", stats.DeploymentsDeleted)
+	}
+	e.mustBeGone(t, dep, "a deployment a previous sweep had claimed")
+}
+
+// The activation path and the collector both take the project lock, and the
+// claim rechecks active = 0 under it. A deployment that becomes active between
+// being listed and being claimed is therefore refused rather than deleted.
+func TestCollectSkipsADeploymentActivatedUnderIt(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 0, 0)
+
+	old := e.version(t, 1)
+	e.activate(t, e.version(t, 2))
+
+	// Stand in for the interleaving: retention has decided to drop `old`, and a
+	// rollback activates it before the claim runs.
+	e.activate(t, old)
+	if err := e.svc.claim(t.Context(), e.p, old); !errors.Is(err, store.ErrConflict) {
+		t.Fatalf("claiming a deployment that became active = %v, want ErrConflict", err)
+	}
+	e.mustSurvive(t, old, "a deployment activated during collection")
+
+	stats, err := e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stats.DeploymentsDeleted != 1 {
+		t.Errorf("deleted %d deployments, want only the one that is no longer served", stats.DeploymentsDeleted)
+	}
+	e.mustSurvive(t, old, "the deployment the rollback made active")
+}
+
+// Collection walks every project, not just the one a request happened to name.
+func TestCollectSpansProjects(t *testing.T) {
+	e := newEnv(t)
+	e.svc.BlobGrace = collectNow
+	e.setRetention(t, 0, 0)
+
+	first := e.p
+	firstOld := e.version(t, 1)
+	e.activate(t, e.version(t, 2))
+
+	second := store.DefaultProject("second")
+	second.RetentionCount = 0
+	second.RetentionGraceS = 0
+	if err := e.db.CreateProject(t.Context(), second); err != nil {
+		t.Fatal(err)
+	}
+	e.p = second
+	secondOld := e.version(t, 3)
+	e.activate(t, e.version(t, 4))
+	e.p = first
+
+	stats, err := e.svc.Collect(t.Context(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stats.DeploymentsDeleted != 2 {
+		t.Errorf("deleted %d deployments, want one from each project", stats.DeploymentsDeleted)
+	}
+	e.mustBeGone(t, firstOld, "the first project's superseded deployment")
+	e.p = second
+	e.mustBeGone(t, secondOld, "the second project's superseded deployment")
+}

internal/deploy/recover.go

@@ -0,0 +1,213 @@
+package deploy
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/iceBear67/simplepages/internal/cas"
+	"github.com/iceBear67/simplepages/internal/store"
+)
+
+// staleUploadAge is how long a deployment may sit unfinished before recovery
+// gives up on it. Generous on purpose: the only thing separating a CI job that
+// died from one that is uploading a large site over a slow link is how long it
+// has been, and expiring the second kind turns a slow deploy into a failed one.
+const staleUploadAge = 24 * time.Hour
+
+// Recover makes the filesystem and the database agree again after a crash or an
+// unclean shutdown.
+//
+// It runs once at startup, before any listener exists and before the registry is
+// built, so it can assume it is the only thing touching either. Everything it
+// does is idempotent: being killed halfway through only means the next start
+// finds a little more to do.
+//
+// Nothing here is allowed to be fatal. A server that refuses to start because
+// one directory could not be swept is worse than one that starts and logs it —
+// the deployments themselves are already durable, and every inconsistency this
+// looks for is one the running system tolerates.
+func (s *Service) Recover(ctx context.Context) error {
+	// An upload that was in progress is unreferenced by construction: a blob only
+	// becomes reachable by being renamed out of the temp directory.
+	if n, err := s.CAS.PurgeTemp(); err != nil {
+		s.Log.Warn("could not clear interrupted uploads", "err", err)
+	} else if n > 0 {
+		s.Log.Info("cleared interrupted uploads", "count", n)
+	}
+
+	if err := s.recoverBlobs(ctx); err != nil {
+		return err
+	}
+
+	// Then the deployments that were still being written when the process went
+	// away, so their manifest rows are gone before the blob collector next runs
+	// and can reclaim whatever only they referenced.
+	n, err := s.DB.ExpireStaleDeployments(ctx, time.Now().Add(-staleUploadAge),
+		"abandoned: no activity for "+staleUploadAge.String())
+	if err != nil {
+		return err
+	}
+	if n > 0 {
+		s.Log.Info("expired unfinished deployments", "count", n)
+	}
+
+	if err := s.resumeDeletions(ctx); err != nil {
+		return err
+	}
+	return s.sweepTrees(ctx)
+}
+
+// recoverBlobs finds content the database believes is on disk and is not.
+//
+// This is the "restored the database, lost the disk" case, and also what a
+// half-finished collector sweep leaves behind. Marking the rows absent is
+// enough to fix it: the next deploy naming one of these digests is asked to
+// upload it, and every deployment that referenced it becomes deployable again as
+// soon as one does.
+func (s *Service) recoverBlobs(ctx context.Context) error {
+	var absent []cas.Digest
+	err := s.DB.EachPresentBlob(ctx, func(d cas.Digest) error {
+		has, err := s.CAS.Has(d)
+		if err != nil {
+			return err
+		}
+		if !has {
+			absent = append(absent, d)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if len(absent) == 0 {
+		return nil
+	}
+	if err := s.DB.MarkBlobsAbsent(ctx, absent); err != nil {
+		return err
+	}
+	// Loud, because on a healthy server this number is zero. Anything else means
+	// the content store lost data, and an operator wants to hear about it before
+	// a deploy fails for a reason that looks like the client's fault.
+	s.Log.Warn("content is missing from the store and was marked for re-upload",
+		"blobs", len(absent))
+	return nil
+}
+
+// resumeDeletions finishes what a collector sweep was doing when it stopped.
+//
+// A deployment enters the deleting state before anything of it is removed, so a
+// row still in that state is a tree that may be half gone. Half a tree is
+// exactly what nothing may serve, which is why the state is committed first: it
+// makes an interrupted deletion recognisable rather than indistinguishable from
+// a healthy deployment.
+func (s *Service) resumeDeletions(ctx context.Context) error {
+	deps, err := s.DB.DeploymentsInState(ctx, store.StateDeleting, 0)
+	if err != nil {
+		return err
+	}
+	for _, dep := range deps {
+		if err := s.removeDeployment(ctx, dep); err != nil {
+			s.Log.Warn("could not finish deleting a deployment",
+				"deployment", dep.PublicID, "err", err)
+			continue
+		}
+		s.Log.Info("finished deleting a deployment", "deployment", dep.PublicID)
+	}
+	return nil
+}
+
+// removeDeployment takes a claimed deployment the rest of the way: its tree
+// first, then its rows. Tree before rows, because a row without a tree is a
+// deployment that simply cannot be activated, whereas a tree without a row is
+// disk nobody will ever account for again.
+func (s *Service) removeDeployment(ctx context.Context, dep *store.Deployment) error {
+	if s.Dir != "" {
+		dir := DeploymentDir(s.Dir, dep.ProjectID, dep.PublicID)
+		if err := os.RemoveAll(dir); err != nil {
+			return err
+		}
+		if err := os.RemoveAll(dir + stagingSuffix); err != nil {
+			return err
+		}
+	}
+	return s.DB.DeleteDeployment(ctx, dep.ID)
+}
+
+// sweepTrees removes assembled trees that nothing refers to.
+//
+// Two kinds: staging directories, which are by definition a build that never
+// finished, and directories whose deployment row is gone — the reverse of the
+// blob audit above, and the residue of a deletion that removed rows before it
+// removed files, or of a database restored from an older backup than the disk.
+func (s *Service) sweepTrees(ctx context.Context) error {
+	if s.Dir == "" {
+		return nil
+	}
+	refs, err := s.DB.AllDeploymentRefs(ctx)
+	if err != nil {
+		return err
+	}
+	known := make(map[store.DeploymentRef]struct{}, len(refs))
+	for _, r := range refs {
+		known[r] = struct{}{}
+	}
+
+	projects, err := os.ReadDir(s.Dir)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil
+		}
+		return err
+	}
+
+	var staging, orphans int
+	for _, pe := range projects {
+		// Only the layout this package writes is ever considered for removal:
+		// one directory per project id, named by the id. Anything else under
+		// $DATA_DIR/deployments was put there by someone else and is left alone.
+		projectID, err := strconv.ParseInt(pe.Name(), 10, 64)
+		if err != nil || !pe.IsDir() {
+			continue
+		}
+		projectDir := filepath.Join(s.Dir, pe.Name())
+		entries, err := os.ReadDir(projectDir)
+		if err != nil {
+			s.Log.Warn("could not read a project's deployment directory", "dir", projectDir, "err", err)
+			continue
+		}
+		for _, e := range entries {
+			if !e.IsDir() {
+				continue
+			}
+			name := e.Name()
+			target := filepath.Join(projectDir, name)
+			switch {
+			case strings.HasSuffix(name, stagingSuffix):
+				if err := os.RemoveAll(target); err != nil {
+					s.Log.Warn("could not remove a staging directory", "dir", target, "err", err)
+					continue
+				}
+				staging++
+			default:
+				if _, ok := known[store.DeploymentRef{ProjectID: projectID, PublicID: name}]; ok {
+					continue
+				}
+				if err := os.RemoveAll(target); err != nil {
+					s.Log.Warn("could not remove an orphaned deployment tree", "dir", target, "err", err)
+					continue
+				}
+				orphans++
+			}
+		}
+	}
+	if staging > 0 || orphans > 0 {
+		s.Log.Info("swept unreferenced deployment trees", "staging", staging, "orphaned", orphans)
+	}
+	return nil
+}

internal/deploy/service.go

@@ -0,0 +1,311 @@
+package deploy
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/iceBear67/simplepages/api"
+	"github.com/iceBear67/simplepages/internal/cas"
+	"github.com/iceBear67/simplepages/internal/site"
+	"github.com/iceBear67/simplepages/internal/store"
+	"github.com/iceBear67/simplepages/internal/webroot"
+)
+
+// Service runs the deployment lifecycle.
+//
+// Client-visible failures are returned as *api.Error so the HTTP layer stays a
+// translation of shapes rather than a second copy of the rules; anything else
+// is an internal error and is rendered as an opaque 500 by httpx.WriteError.
+type Service struct {
+	DB  *store.DB
+	CAS *cas.Store
+	Log *slog.Logger
+
+	// Dir is the root of the assembled deployment trees. Empty means the
+	// operator chose assemble_mode=none: content is served straight from the
+	// CAS and nothing is built on disk.
+	Dir string
+
+	// Sites is the in-memory state the HTTP site handler reads. Activation
+	// publishes into it; nil leaves the service usable without a serving layer,
+	// which is what the store-level tests want.
+	Sites *site.Registry
+
+	// Webroot maintains the $WEBROOT/~project symlinks. Nil when the operator
+	// configured no webroot. Nothing here depends on it succeeding.
+	Webroot *webroot.Webroot
+
+	// BlobGrace is how long content must have been unreferenced before the
+	// collector removes it. Zero means defaultBlobGrace. Negative collects
+	// immediately, which is what the tests want and what an operator reclaiming
+	// space on a server they know is idle might ask for.
+	BlobGrace time.Duration
+
+	locks projectLocks
+}
+
+// Create starts a deployment. Nothing touches the filesystem until a manifest
+// arrives, so an abandoned create costs one row.
+func (s *Service) Create(ctx context.Context, p *store.Project, keyID string, meta map[string]string) (*store.Deployment, error) {
+	dep := &store.Deployment{ProjectID: p.ID, CreatedByKey: keyID, Meta: meta}
+	if err := s.DB.CreateDeployment(ctx, dep); err != nil {
+		return nil, err
+	}
+	return dep, nil
+}
+
+// SetManifest records the file list and reports which blobs still have to be
+// uploaded. files must already be validated: paths through pathutil and sizes
+// against the project's limits.
+func (s *Service) SetManifest(ctx context.Context, dep *store.Deployment, files []store.FileRow) (missing []cas.Digest, missingBytes int64, err error) {
+	missing, missingBytes, err = s.DB.SetManifest(ctx, dep.ID, files)
+	if err != nil {
+		switch {
+		case errors.Is(err, store.ErrConflict):
+			return nil, 0, api.Errorf(api.CodeConflict,
+				"this deployment can no longer accept a manifest; create a new one")
+		case errors.Is(err, cas.ErrSizeMismatch):
+			return nil, 0, api.Errorf(api.CodeSizeMismatch, "%s", err)
+		}
+		return nil, 0, err
+	}
+	return missing, missingBytes, nil
+}
+
+// Upload stores one blob's content.
+//
+// The digest must already be named by some manifest. That check is what keeps
+// the endpoint from being general-purpose storage: content nobody declared can
+// never be written, and the length it must have is the one the manifest agreed
+// on rather than whatever Content-Length claims.
+//
+// Reports whether the content was newly stored; a blob that is already present
+// is a success without reading the body, which is what makes a retried deploy
+// cheap.
+func (s *Service) Upload(ctx context.Context, digest cas.Digest, body io.Reader) (size int64, stored bool, err error) {
+	b, err := s.DB.Blob(ctx, digest)
+	if err != nil {
+		if errors.Is(err, store.ErrNotFound) {
+			return 0, false, api.Errorf(api.CodeNotFound,
+				"no manifest references this digest; send the manifest first")
+		}
+		return 0, false, err
+	}
+	if b.Present {
+		return b.Size, false, nil
+	}
+
+	// The blob's declared length is both the expectation and the ceiling: Put
+	// reads one byte past it and rejects anything longer, so a client cannot
+	// spend more of the disk than its manifest was accepted for. Put requires a
+	// positive ceiling, hence the floor of one byte for an empty blob.
+	limit := b.Size
+	if limit < 1 {
+		limit = 1
+	}
+	n, err := s.CAS.Put(ctx, digest, b.Size, limit, body)
+	if err != nil {
+		switch {
+		case errors.Is(err, cas.ErrDigestMismatch):
+			return 0, false, api.Errorf(api.CodeDigestMismatch, "%s", err)
+		case errors.Is(err, cas.ErrSizeMismatch):
+			return 0, false, api.Errorf(api.CodeSizeMismatch, "%s", err)
+		case errors.Is(err, cas.ErrTooLarge):
+			return 0, false, api.Errorf(api.CodeLimitExceeded, "%s", err)
+		}
+		return 0, false, err
+	}
+	if err := s.DB.MarkBlobPresent(ctx, digest, n); err != nil {
+		// The content is on disk and verified; only the row disagrees. A retry
+		// finds the blob already stored and updates the row then.
+		return 0, false, err
+	}
+	return n, true, nil
+}
+
+// Finalize checks that every blob arrived, assembles the tree, and marks the
+// deployment ready. It does not activate it: a ready deployment is one that
+// could be served, and choosing when to serve it is a separate decision.
+//
+// Retrying is safe. An assembled tree is left as it is, and a deployment that
+// is already ready simply stays ready.
+func (s *Service) Finalize(ctx context.Context, p *store.Project, dep *store.Deployment) (*store.Deployment, error) {
+	// One finalize per project at a time. Two concurrent CI jobs for one
+	// project are ordered rather than racing over the same directory.
+	unlock, err := s.locks.lock(ctx, p.ID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlock()
+
+	// Re-read under the lock: the state may have moved since the handler
+	// resolved it.
+	dep, err = s.DB.DeploymentByPublicID(ctx, p.ID, dep.PublicID)
+	if err != nil {
+		return nil, mapNotFound(err)
+	}
+	switch dep.State {
+	case store.StateUploading, store.StateReady:
+	default:
+		return nil, api.Errorf(api.CodeConflict, "cannot finalize a deployment that is %s", dep.State)
+	}
+
+	missing, err := s.DB.MissingBlobs(ctx, dep.ID)
+	if err != nil {
+		return nil, err
+	}
+	if len(missing) > 0 {
+		hex := make([]string, len(missing))
+		for i, d := range missing {
+			hex[i] = d.String()
+		}
+		return nil, api.Errorf(api.CodeBlobsMissing,
+			"%d blobs have not been uploaded", len(missing)).WithDetail("missing", hex)
+	}
+
+	if s.Dir != "" {
+		files, err := s.DB.DeploymentFiles(ctx, dep.ID)
+		if err != nil {
+			return nil, err
+		}
+		if err := Assemble(ctx, s.CAS, files, DeploymentDir(s.Dir, p.ID, dep.PublicID)); err != nil {
+			// A cancelled request is not a broken deployment: leave it uploading
+			// so the client can simply try again.
+			if ctx.Err() != nil {
+				return nil, err
+			}
+			s.Log.ErrorContext(ctx, "assembling deployment tree failed",
+				"project", p.Name, "deployment", dep.PublicID, "err", err)
+			if ferr := s.DB.MarkDeploymentFailed(context.WithoutCancel(ctx), dep.ID, err.Error()); ferr != nil {
+				s.Log.ErrorContext(ctx, "recording the failure failed too",
+					"deployment", dep.PublicID, "err", ferr)
+			}
+			return nil, api.Errorf(api.CodeInternal, "could not assemble the deployment tree")
+		}
+	}
+
+	if err := s.DB.MarkDeploymentReady(ctx, dep.ID); err != nil {
+		if errors.Is(err, store.ErrConflict) {
+			return nil, api.Errorf(api.CodeConflict, "%s", err)
+		}
+		return nil, err
+	}
+	return s.DB.DeploymentByPublicID(ctx, p.ID, dep.PublicID)
+}
+
+// Activate makes a ready deployment the one the project serves. Activating an
+// older deployment is how a rollback works, and costs exactly the same.
+//
+// The order of the steps is the entire correctness argument:
+//
+//  1. Take the project lock, so two activations of one project are ordered.
+//  2. Re-read the deployment under it and require that it is ready.
+//  3. Build the snapshot's index — the one step that reads the manifest and can
+//     fail — *before* anything has changed. A failure here leaves the currently
+//     served deployment exactly as it was.
+//  4. Commit the database transaction. From here on SQLite is the truth.
+//  5. Store the pointer. This single store is the switch: requests that started
+//     earlier finish on the old snapshot, later ones see the new one, and no
+//     request can ever observe a mixture of the two.
+//  6. Repoint the symlink, best effort.
+//
+// Database before memory matters: a crash between 4 and 5 restarts into a
+// process that serves what the database says. The reverse order would leave a
+// process serving something the database disagrees with.
+func (s *Service) Activate(ctx context.Context, p *store.Project, dep *store.Deployment) (*store.Deployment, error) {
+	unlock, err := s.locks.lock(ctx, p.ID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlock()
+
+	dep, err = s.DB.DeploymentByPublicID(ctx, p.ID, dep.PublicID)
+	if err != nil {
+		return nil, mapNotFound(err)
+	}
+	if dep.State != store.StateReady {
+		return nil, api.Errorf(api.CodeDeploymentNotReady,
+			"cannot activate a deployment that is %s", dep.State)
+	}
+
+	idx, err := s.index(ctx, dep)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := s.DB.ActivateDeployment(ctx, p.ID, dep.ID); err != nil {
+		switch {
+		case errors.Is(err, store.ErrNotFound):
+			return nil, api.Errorf(api.CodeNotFound, "no such deployment")
+		case errors.Is(err, store.ErrConflict):
+			return nil, api.Errorf(api.CodeConflict, "%s", err)
+		}
+		return nil, err
+	}
+
+	// Re-read so the snapshot and the response carry the timestamps the
+	// transaction actually wrote.
+	dep, err = s.DB.DeploymentByPublicID(ctx, p.ID, dep.PublicID)
+	if err != nil {
+		return nil, mapNotFound(err)
+	}
+
+	dir := s.deploymentDir(p, dep)
+	if s.Sites != nil {
+		s.Sites.Put(p).Activate(site.NewDeployment(dep, idx, dir))
+	}
+	if s.Webroot != nil && dir != "" {
+		if err := s.Webroot.Point(p.Name, dir); err != nil {
+			// The site is already being served from memory; the symlink is for
+			// everything else and the reconciler will fix it.
+			s.Log.ErrorContext(ctx, "could not repoint the webroot symlink",
+				"project", p.Name, "deployment", dep.PublicID, "err", err)
+		}
+	}
+	s.Log.InfoContext(ctx, "deployment activated",
+		"project", p.Name, "deployment", dep.PublicID,
+		"files", dep.FileCount, "bytes", dep.TotalBytes)
+	return dep, nil
+}
+
+func mapNotFound(err error) error {
+	if errors.Is(err, store.ErrNotFound) {
+		return api.Errorf(api.CodeNotFound, "no such deployment")
+	}
+	return err
+}
+
+// projectLocks serialises the mutating operations of one project against each
+// other. Entries are keyed by row id and are never evicted: there is one per
+// project that has ever been written to, which is bounded by the number of
+// projects.
+type projectLocks struct {
+	mu sync.Mutex
+	m  map[int64]chan struct{}
+}
+
+// lock acquires the project's lock, or gives up if ctx is done first — a
+// client that has already hung up should not keep a slow assembly waiting.
+func (l *projectLocks) lock(ctx context.Context, id int64) (func(), error) {
+	l.mu.Lock()
+	if l.m == nil {
+		l.m = make(map[int64]chan struct{})
+	}
+	ch, ok := l.m[id]
+	if !ok {
+		ch = make(chan struct{}, 1)
+		l.m[id] = ch
+	}
+	l.mu.Unlock()
+
+	select {
+	case ch <- struct{}{}:
+		return func() { <-ch }, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}

internal/deploy/service_test.go

@@ -0,0 +1,468 @@
+package deploy
+
+import (
+	"errors"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/iceBear67/simplepages/api"
+	"github.com/iceBear67/simplepages/internal/cas"
+	"github.com/iceBear67/simplepages/internal/store"
+)
+
+// env is a service on a real database and a real CAS, which is what these tests
+// are about: every interesting rule here is enforced by a trigger, a unique
+// index or the filesystem, and a fake would only assert that the fake agrees
+// with itself.
+type env struct {
+	svc *Service
+	db  *store.DB
+	cas *cas.Store
+	p   *store.Project
+	dir string // deployments root
+}
+
+func newEnv(t *testing.T) *env {
+	t.Helper()
+	base := t.TempDir()
+	log := slog.New(slog.DiscardHandler)
+
+	db, err := store.Open(t.Context(), filepath.Join(base, "pages.db"), log)
+	if err != nil {
+		t.Fatalf("store.Open: %v", err)
+	}
+	t.Cleanup(func() { db.Close() })
+
+	deployDir := filepath.Join(base, "deployments")
+	cs, err := cas.Open(filepath.Join(base, "cas"), cas.Options{ProbeDir: deployDir, Log: log})
+	if err != nil {
+		t.Fatalf("cas.Open: %v", err)
+	}
+	t.Cleanup(func() { cs.Close() })
+
+	p := store.DefaultProject("demo")
+	if err := db.CreateProject(t.Context(), p); err != nil {
+		t.Fatalf("CreateProject: %v", err)
+	}
+	return &env{
+		svc: &Service{DB: db, CAS: cs, Log: log, Dir: deployDir},
+		db:  db, cas: cs, p: p, dir: deployDir,
+	}
+}
+
+func (e *env) create(t *testing.T) *store.Deployment {
+	t.Helper()
+	dep, err := e.svc.Create(t.Context(), e.p, "", nil)
+	if err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+	return dep
+}
+
+// manifest turns path->content into the rows a client would have sent.
+func manifest(contents map[string]string) []store.FileRow {
+	files := make([]store.FileRow, 0, len(contents))
+	for p, c := range contents {
+		files = append(files, store.FileRow{Path: p, Digest: cas.Sum([]byte(c)), Size: int64(len(c))})
+	}
+	return files
+}
+
+// upload pushes every named blob through the service, as a client would.
+func (e *env) upload(t *testing.T, contents map[string]string, want ...string) {
+	t.Helper()
+	for _, p := range want {
+		c := contents[p]
+		if _, _, err := e.svc.Upload(t.Context(), cas.Sum([]byte(c)), strings.NewReader(c)); err != nil {
+			t.Fatalf("Upload %s: %v", p, err)
+		}
+	}
+}
+
+// apiCode returns the wire code of err, or "" if it is not a client error.
+func apiCode(err error) api.Code {
+	var e *api.Error
+	if errors.As(err, &e) {
+		return e.Code
+	}
+	return ""
+}
+
+func wantCode(t *testing.T, err error, want api.Code) {
+	t.Helper()
+	if got := apiCode(err); got != want {
+		t.Fatalf("error = %v (code %q), want code %q", err, got, want)
+	}
+}
+
+func TestFullDeploymentRoundTrip(t *testing.T) {
+	e := newEnv(t)
+	contents := map[string]string{
+		"index.html":    "<h1>hello</h1>",
+		"assets/app.js": "console.log(1)",
+		"copy.html":     "<h1>hello</h1>", // same blob as index.html
+	}
+	dep := e.create(t)
+	if dep.State != store.StatePending {
+		t.Fatalf("state = %s, want pending", dep.State)
+	}
+
+	files := manifest(contents)
+	missing, missingBytes, err := e.svc.SetManifest(t.Context(), dep, files)
+	if err != nil {
+		t.Fatalf("SetManifest: %v", err)
+	}
+	// Two unique blobs for three files: the shared one is only asked for once.
+	if len(missing) != 2 {
+		t.Fatalf("missing = %d digests, want 2", len(missing))
+	}
+	if want := int64(len("<h1>hello</h1>") + len("console.log(1)")); missingBytes != want {
+		t.Errorf("missingBytes = %d, want %d", missingBytes, want)
+	}
+
+	// Finalize before the content arrives names what is still outstanding rather
+	// than failing opaquely, because that list is what the client retries.
+	_, err = e.svc.Finalize(t.Context(), e.p, dep)
+	wantCode(t, err, api.CodeBlobsMissing)
+	var apiErr *api.Error
+	if errors.As(err, &apiErr) {
+		list, _ := apiErr.Details["missing"].([]string)
+		if len(list) != 2 {
+			t.Errorf("details.missing = %v, want 2 digests", apiErr.Details["missing"])
+		}
+	}
+
+	e.upload(t, contents, "index.html", "assets/app.js")
+
+	dep, err = e.svc.Finalize(t.Context(), e.p, dep)
+	if err != nil {
+		t.Fatalf("Finalize: %v", err)
+	}
+	if dep.State != store.StateReady {
+		t.Fatalf("state = %s, want ready", dep.State)
+	}
+	if dep.FileCount != 3 || dep.TotalBytes != int64(len(contents["index.html"])*2+len(contents["assets/app.js"])) {
+		t.Errorf("file_count = %d, total_bytes = %d", dep.FileCount, dep.TotalBytes)
+	}
+	if dep.FinalizedAt == nil {
+		t.Error("finalized_at was not recorded")
+	}
+	if dep.Active {
+		t.Error("finalize activated the deployment; that is a separate decision")
+	}
+
+	// The tree on disk is the deployment, byte for byte.
+	got := walk(t, DeploymentDir(e.dir, e.p.ID, dep.PublicID))
+	if len(got) != len(contents) {
+		t.Fatalf("assembled %v", got)
+	}
+	for p, c := range contents {
+		if got[p] != c {
+			t.Errorf("%s = %q, want %q", p, got[p], c)
+		}
+	}
+}
+
+// A second deployment of a mostly-unchanged site is the case the whole
+// content-addressed protocol exists for: only what actually changed is asked
+// for, across deployments and across projects.
+func TestASecondDeploymentOnlyAsksForWhatChanged(t *testing.T) {
+	e := newEnv(t)
+	first := map[string]string{"index.html": "v1", "assets/app.js": "shared"}
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(first)); err != nil {
+		t.Fatal(err)
+	}
+	e.upload(t, first, "index.html", "assets/app.js")
+	if _, err := e.svc.Finalize(t.Context(), e.p, dep); err != nil {
+		t.Fatal(err)
+	}
+
+	second := map[string]string{"index.html": "v2", "assets/app.js": "shared"}
+	dep2 := e.create(t)
+	missing, missingBytes, err := e.svc.SetManifest(t.Context(), dep2, manifest(second))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(missing) != 1 || missing[0] != cas.Sum([]byte("v2")) {
+		t.Fatalf("missing = %v, want just the changed index.html", digests(missing))
+	}
+	if missingBytes != 2 {
+		t.Errorf("missingBytes = %d, want 2", missingBytes)
+	}
+
+	e.upload(t, second, "index.html")
+	if _, err := e.svc.Finalize(t.Context(), e.p, dep2); err != nil {
+		t.Fatal(err)
+	}
+	// Both trees exist and disagree, which is what makes a rollback a rollback.
+	if got := walk(t, DeploymentDir(e.dir, e.p.ID, dep.PublicID)); got["index.html"] != "v1" {
+		t.Errorf("first deployment = %v, want v1 intact", got)
+	}
+	if got := walk(t, DeploymentDir(e.dir, e.p.ID, dep2.PublicID)); got["index.html"] != "v2" {
+		t.Errorf("second deployment = %v", got)
+	}
+}
+
+func digests(ds []cas.Digest) []string {
+	out := make([]string, len(ds))
+	for i, d := range ds {
+		out[i] = d.String()
+	}
+	return out
+}
+
+// The upload endpoint is not general-purpose storage. Content nobody declared
+// has no size the server agreed to and no deployment that would ever reference
+// it, so it is refused before a byte is read.
+func TestUploadRejectsContentNoManifestAskedFor(t *testing.T) {
+	e := newEnv(t)
+	body := &countingReader{r: strings.NewReader("unsolicited")}
+
+	_, _, err := e.svc.Upload(t.Context(), cas.Sum([]byte("unsolicited")), body)
+	wantCode(t, err, api.CodeNotFound)
+	if body.n != 0 {
+		t.Errorf("read %d bytes of a body it had already decided to refuse", body.n)
+	}
+	if has, _ := e.cas.Has(cas.Sum([]byte("unsolicited"))); has {
+		t.Error("the content was stored anyway")
+	}
+}
+
+// The claimed digest is only ever a claim. Without this check a client could
+// declare another project's digest and poison every project sharing that blob.
+func TestUploadRejectsContentThatDoesNotHashToItsDigest(t *testing.T) {
+	e := newEnv(t)
+	contents := map[string]string{"index.html": "honest"}
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
+		t.Fatal(err)
+	}
+
+	claimed := cas.Sum([]byte("honest"))
+	_, _, err := e.svc.Upload(t.Context(), claimed, strings.NewReader("forged"))
+	wantCode(t, err, api.CodeDigestMismatch)
+	if has, _ := e.cas.Has(claimed); has {
+		t.Fatal("the forged content was stored under the honest digest")
+	}
+
+	// And the deployment is still deployable once the real bytes arrive.
+	if _, _, err := e.svc.Upload(t.Context(), claimed, strings.NewReader("honest")); err != nil {
+		t.Fatalf("honest upload after a forged one: %v", err)
+	}
+	if _, err := e.svc.Finalize(t.Context(), e.p, dep); err != nil {
+		t.Fatalf("Finalize: %v", err)
+	}
+}
+
+// The manifest's size is the ceiling, so a client cannot spend more disk than
+// the manifest it got accepted for. Content-Length is never consulted.
+func TestUploadRejectsMoreBytesThanTheManifestDeclared(t *testing.T) {
+	e := newEnv(t)
+	dep := e.create(t)
+	body := strings.Repeat("x", 4096)
+	// Declare a small file, then send a large one under the same digest.
+	files := []store.FileRow{{Path: "a.txt", Digest: cas.Sum([]byte(body)), Size: 4}}
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, files); err != nil {
+		t.Fatal(err)
+	}
+
+	_, _, err := e.svc.Upload(t.Context(), files[0].Digest, strings.NewReader(body))
+	if code := apiCode(err); code != api.CodeSizeMismatch && code != api.CodeLimitExceeded {
+		t.Fatalf("error = %v (code %q), want a size rejection", err, code)
+	}
+	if has, _ := e.cas.Has(files[0].Digest); has {
+		t.Error("the oversized content was stored")
+	}
+}
+
+// Re-uploading a blob the server already has is the fast path a retried deploy
+// depends on: no body is read and nothing is rewritten.
+func TestUploadOfAPresentBlobDoesNotReadTheBody(t *testing.T) {
+	e := newEnv(t)
+	contents := map[string]string{"index.html": "hello"}
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
+		t.Fatal(err)
+	}
+	d := cas.Sum([]byte("hello"))
+
+	size, stored, err := e.svc.Upload(t.Context(), d, strings.NewReader("hello"))
+	if err != nil || !stored || size != 5 {
+		t.Fatalf("first upload: size=%d stored=%v err=%v", size, stored, err)
+	}
+	body := &countingReader{r: strings.NewReader("hello")}
+	size, stored, err = e.svc.Upload(t.Context(), d, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stored {
+		t.Error("the second upload claimed to have stored content the server already had")
+	}
+	if size != 5 {
+		t.Errorf("size = %d, want 5", size)
+	}
+	if body.n != 0 {
+		t.Errorf("read %d bytes of a blob it already had", body.n)
+	}
+}
+
+func TestUploadStoresAnEmptyBlob(t *testing.T) {
+	e := newEnv(t)
+	dep := e.create(t)
+	files := []store.FileRow{{Path: "empty", Digest: cas.Sum(nil), Size: 0}}
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, files); err != nil {
+		t.Fatal(err)
+	}
+	// Put insists on a positive ceiling; the service floors it at one byte so a
+	// legitimately empty file is still uploadable.
+	size, stored, err := e.svc.Upload(t.Context(), files[0].Digest, strings.NewReader(""))
+	if err != nil || !stored || size != 0 {
+		t.Fatalf("size=%d stored=%v err=%v", size, stored, err)
+	}
+	if _, err := e.svc.Finalize(t.Context(), e.p, dep); err != nil {
+		t.Fatalf("Finalize: %v", err)
+	}
+	got := walk(t, DeploymentDir(e.dir, e.p.ID, dep.PublicID))
+	if c, ok := got["empty"]; !ok || c != "" {
+		t.Errorf("tree = %v, want one empty file", got)
+	}
+}
+
+// Finalizing twice must be a no-op rather than a second assembly: the client
+// that lost its response to a timeout retries, and the tree may already be live.
+func TestFinalizeIsIdempotent(t *testing.T) {
+	e := newEnv(t)
+	contents := map[string]string{"index.html": "hello"}
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
+		t.Fatal(err)
+	}
+	e.upload(t, contents, "index.html")
+
+	first, err := e.svc.Finalize(t.Context(), e.p, dep)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := e.svc.Finalize(t.Context(), e.p, dep)
+	if err != nil {
+		t.Fatalf("second Finalize: %v", err)
+	}
+	if !first.FinalizedAt.Equal(*second.FinalizedAt) {
+		t.Errorf("finalized_at moved from %v to %v on a retry", first.FinalizedAt, second.FinalizedAt)
+	}
+}
+
+func TestFinalizeRejectsADeploymentWithNoManifest(t *testing.T) {
+	e := newEnv(t)
+	dep := e.create(t) // still pending
+
+	_, err := e.svc.Finalize(t.Context(), e.p, dep)
+	wantCode(t, err, api.CodeConflict)
+	if _, err := os.Stat(DeploymentDir(e.dir, e.p.ID, dep.PublicID)); !errors.Is(err, os.ErrNotExist) {
+		t.Error("a tree was built for a deployment that never had a manifest")
+	}
+}
+
+// A deployment belongs to exactly one project. Finalizing another project's
+// deployment must not work even when the caller knows its id.
+func TestFinalizeIsProjectScoped(t *testing.T) {
+	e := newEnv(t)
+	other := store.DefaultProject("other")
+	if err := e.db.CreateProject(t.Context(), other); err != nil {
+		t.Fatal(err)
+	}
+	contents := map[string]string{"index.html": "hello"}
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
+		t.Fatal(err)
+	}
+	e.upload(t, contents, "index.html")
+
+	_, err := e.svc.Finalize(t.Context(), other, dep)
+	wantCode(t, err, api.CodeNotFound)
+}
+
+// assemble_mode=none: content is served straight from the CAS, so finalize must
+// still succeed and must not build anything on disk.
+func TestFinalizeWithoutAssemblyBuildsNothing(t *testing.T) {
+	e := newEnv(t)
+	e.svc.Dir = ""
+	contents := map[string]string{"index.html": "hello"}
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
+		t.Fatal(err)
+	}
+	e.upload(t, contents, "index.html")
+
+	dep, err := e.svc.Finalize(t.Context(), e.p, dep)
+	if err != nil {
+		t.Fatalf("Finalize: %v", err)
+	}
+	if dep.State != store.StateReady {
+		t.Fatalf("state = %s, want ready", dep.State)
+	}
+	if _, err := os.Stat(DeploymentDir(e.dir, e.p.ID, dep.PublicID)); !errors.Is(err, os.ErrNotExist) {
+		t.Error("a tree was assembled despite assemble_mode=none")
+	}
+}
+
+// If assembly fails the deployment must end up failed rather than ready — a
+// ready deployment is one that could be served, and this one could not be.
+func TestFinalizeMarksTheDeploymentFailedWhenAssemblyCannotProceed(t *testing.T) {
+	e := newEnv(t)
+	contents := map[string]string{"index.html": "hello"}
+	dep := e.create(t)
+	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
+		t.Fatal(err)
+	}
+	e.upload(t, contents, "index.html")
+
+	// A plain file where the project's directory has to go: MkdirAll cannot get
+	// past it, so assembly fails for a reason that is not the client's fault.
+	if err := os.MkdirAll(e.dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Dir(DeploymentDir(e.dir, e.p.ID, dep.PublicID)), []byte("in the way"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := e.svc.Finalize(t.Context(), e.p, dep)
+	wantCode(t, err, api.CodeInternal)
+	// The reason stays server-side: the client is told nothing about the layout
+	// of the server's disk.
+	if strings.Contains(err.Error(), e.dir) {
+		t.Errorf("the error exposes a server path: %v", err)
+	}
+
+	after, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if after.State != store.StateFailed {
+		t.Fatalf("state = %s, want failed", after.State)
+	}
+	if after.Error == "" {
+		t.Error("no reason was recorded for the failure")
+	}
+	// And it stays failed: a failed deployment is never resurrected.
+	_, err = e.svc.Finalize(t.Context(), e.p, dep)
+	wantCode(t, err, api.CodeConflict)
+}
+
+// countingReader reports whether a body was read at all, which is how the tests
+// tell "refused up front" apart from "read and then discarded".
+type countingReader struct {
+	r io.Reader
+	n int
+}
+
+func (c *countingReader) Read(p []byte) (int, error) {
+	n, err := c.r.Read(p)
+	c.n += n
+	return n, err
+}

internal/deploy/sites.go

@@ -0,0 +1,127 @@
+package deploy
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/iceBear67/simplepages/internal/site"
+	"github.com/iceBear67/simplepages/internal/store"
+)
+
+// index reads a deployment's manifest and turns it into the lookup structures
+// the serving path needs. This is the expensive, failure-prone half of building
+// a snapshot, which is why it is separable: an activation runs it before it
+// changes anything.
+func (s *Service) index(ctx context.Context, dep *store.Deployment) (*site.Index, error) {
+	files, err := s.DB.DeploymentFiles(ctx, dep.ID)
+	if err != nil {
+		return nil, err
+	}
+	return site.NewIndex(files), nil
+}
+
+// deploymentDir is where a deployment's tree was assembled, or "" under
+// assemble_mode=none, where nothing was.
+func (s *Service) deploymentDir(p *store.Project, dep *store.Deployment) string {
+	if s.Dir == "" {
+		return ""
+	}
+	return DeploymentDir(s.Dir, p.ID, dep.PublicID)
+}
+
+// LoadSites builds the registry from the database and makes the webroot agree
+// with it. It runs once, before the listeners start.
+//
+// Without this a restart would serve 404 for every project until each one was
+// deployed again. It also has to complete before the registry can stand in as
+// the API's project resolver: a half-built registry would report a caller's own
+// project as one that does not exist.
+func (s *Service) LoadSites(ctx context.Context) error {
+	projects, err := s.DB.AllProjects(ctx)
+	if err != nil {
+		return err
+	}
+
+	entries := make([]*site.Project, 0, len(projects))
+	var active int
+	for _, p := range projects {
+		sp := site.NewProject(p)
+		entries = append(entries, sp)
+
+		dep, err := s.DB.ActiveDeployment(ctx, p.ID)
+		if err != nil {
+			if errors.Is(err, store.ErrNotFound) {
+				// A project that has never been deployed to. It exists, it
+				// resolves, and it answers 503 until something is activated.
+				continue
+			}
+			return err
+		}
+		idx, err := s.index(ctx, dep)
+		if err != nil {
+			return err
+		}
+		sp.Activate(site.NewDeployment(dep, idx, s.deploymentDir(p, dep)))
+		active++
+	}
+	s.Sites.Replace(entries)
+	s.Log.Info("site registry loaded", "projects", len(entries), "active", active)
+
+	// Best effort, like every other webroot operation: the server serves its own
+	// content and a stale symlink is not a reason to refuse to start.
+	if err := s.Reconcile(); err != nil {
+		s.Log.Warn("could not fully reconcile the webroot", "err", err)
+	}
+	return nil
+}
+
+// Reconcile makes $WEBROOT agree with what this process is serving: one symlink
+// per project with an active deployment, and nothing of ours left over for
+// projects that no longer have one.
+//
+// The wanted set comes from the registry rather than from the database, because
+// the registry is what requests are actually being answered from. The symlinks
+// exist so that an external reader — a reverse proxy, a backup job — sees the
+// same version this process does, and taking them from anywhere else would let
+// the two disagree.
+func (s *Service) Reconcile() error {
+	if s.Webroot == nil {
+		return nil
+	}
+	want := make(map[string]string)
+	for _, p := range s.Sites.Projects() {
+		// One load per project, and the value is used for both the name and the
+		// directory: reading Active() twice could straddle an activation and
+		// produce a link to a directory the other half of the pair disagrees with.
+		if d := p.Active(); d != nil && d.Dir != "" {
+			want[p.Name] = d.Dir
+		}
+	}
+	return s.Webroot.Reconcile(want)
+}
+
+// RunReconciler repairs the webroot on a timer until ctx is done.
+//
+// Activation repoints a symlink itself and logs when it cannot, so this is for
+// the cases where nothing was there to notice: a link an operator deleted or
+// edited, one whose repoint failed on a full disk, or one left pointing at a
+// deployment that has since been collected. Serving does not depend on any of
+// it, which is why a failure here is a warning and never stops the loop.
+func (s *Service) RunReconciler(ctx context.Context, every time.Duration) {
+	if s.Webroot == nil || every <= 0 {
+		return
+	}
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if err := s.Reconcile(); err != nil {
+				s.Log.Warn("could not fully reconcile the webroot", "err", err)
+			}
+		}
+	}
+}