pages/cmd/pages-server/integration_test.go

package main

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/iceBear67/simplepages/api"
	"github.com/iceBear67/simplepages/internal/client"
	"github.com/iceBear67/simplepages/internal/config"
)

// testenv is the whole server: the real app assembly, both handlers, and the
// pages CLI's own client library driving them. Nothing between the CI job and
// the bytes on the wire is stubbed, which is the point — the milestone's claim
// is that `pages deploy` followed by a GET returns the site, and only a test at
// this level can make that claim.
type testenv struct {
	app        *app
	cfg        config.Config
	site       *httptest.Server
	api        *httptest.Server
	admin      *client.Client
	adminToken string
	logBuf     *bytes.Buffer
}

func newTestenv(t *testing.T) *testenv {
	t.Helper()

	base := t.TempDir()
	cfg := config.Default()
	cfg.DataDir = filepath.Join(base, "data")
	cfg.Webroot = filepath.Join(base, "www")
	cfg.LogFormat = "text"
	cfg.LogLevel = "debug"
	if err := cfg.Validate(); err != nil {
		t.Fatalf("config: %v", err)
	}
	if err := cfg.EnsureDirs(); err != nil {
		t.Fatalf("ensure dirs: %v", err)
	}

	var buf bytes.Buffer
	log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))

	a, err := newApp(t.Context(), cfg, log)
	if err != nil {
		t.Fatalf("newApp: %v", err)
	}
	t.Cleanup(a.close)
	a.ready.Store(true)

	e := &testenv{app: a, cfg: cfg, logBuf: &buf}
	e.site = httptest.NewServer(a.siteHandler())
	t.Cleanup(e.site.Close)
	e.api = httptest.NewServer(a.apiHandler())
	t.Cleanup(e.api.Close)

	// The bootstrap token is how an operator gets their first credential, and
	// the only place the server ever writes one to disk.
	raw, err := os.ReadFile(cfg.BootstrapTokenPath())
	if err != nil {
		t.Fatalf("read bootstrap token: %v", err)
	}
	fi, err := os.Stat(cfg.BootstrapTokenPath())
	if err != nil {
		t.Fatal(err)
	}
	if fi.Mode().Perm() != 0o600 {
		t.Errorf("bootstrap-token mode = %v, want 0600", fi.Mode().Perm())
	}
	e.adminToken = strings.TrimSpace(string(raw))
	e.admin = e.client(t, e.adminToken)
	return e
}

func (e *testenv) client(t *testing.T, token string) *client.Client {
	t.Helper()
	c, err := client.New(client.Config{BaseURL: e.api.URL, Token: token, HTTP: e.api.Client()})
	if err != nil {
		t.Fatalf("client: %v", err)
	}
	return c
}

// project creates a project and returns a client holding a key scoped to it,
// which is what a CI job would be given.
func (e *testenv) project(t *testing.T, name string) *client.Client {
	t.Helper()
	if _, err := e.admin.CreateProject(t.Context(), api.CreateProjectRequest{Name: name}); err != nil {
		t.Fatalf("create project %s: %v", name, err)
	}
	key, err := e.admin.CreateProjectKey(t.Context(), name, api.CreateKeyRequest{Name: "ci"})
	if err != nil {
		t.Fatalf("create key for %s: %v", name, err)
	}
	return e.client(t, key.Token)
}

// deploy scans a directory of literal file contents and pushes it, exactly as
// `pages deploy` does.
func (e *testenv) deploy(t *testing.T, c *client.Client, project string, files map[string]string) *client.DeployResult {
	t.Helper()
	dir := t.TempDir()
	for name, content := range files {
		p := filepath.Join(dir, filepath.FromSlash(name))
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
			t.Fatal(err)
		}
	}

	src, err := client.Scan(t.Context(), dir, client.ScanOptions{})
	if err != nil {
		t.Fatalf("scan: %v", err)
	}
	defer src.Close()

	res, err := c.Deploy(t.Context(), client.DeployOptions{
		Project: project, Source: src, Activate: true,
	})
	if err != nil {
		t.Fatalf("deploy: %v", err)
	}
	return res
}

// get fetches a site URL.
func (e *testenv) get(t *testing.T, path string, header http.Header) *http.Response {
	t.Helper()
	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, e.site.URL+path, nil)
	if err != nil {
		t.Fatal(err)
	}
	req.Header = header
	if req.Header == nil {
		req.Header = http.Header{}
	}
	// Redirects are part of what is under test, so they are never followed.
	c := *e.site.Client()
	c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
	resp, err := c.Do(req)
	if err != nil {
		t.Fatalf("GET %s: %v", path, err)
	}
	t.Cleanup(func() { resp.Body.Close() })
	return resp
}

func (e *testenv) body(t *testing.T, resp *http.Response) string {
	t.Helper()
	b, err := io.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

// TestDeployThenServe is the milestone's acceptance test, in the order the
// manual walkthrough in the plan does it.
func TestDeployThenServe(t *testing.T) {
	e := newTestenv(t)
	ci := e.project(t, "demo")

	v1 := map[string]string{
		"index.html":      "<h1>v1</h1><script src=\"assets/app.js\"></script>",
		"assets/app.js":   "console.log(1)",
		"docs/index.html": "<h1>docs</h1>",
	}
	res := e.deploy(t, ci, "demo", v1)
	if !res.Activated || res.Deployment.State != api.StateReady {
		t.Fatalf("deploy result = %+v", res)
	}
	if res.FileCount != 3 || res.Uploaded != 3 || res.Deduplicated != 0 {
		t.Fatalf("first deploy uploaded %d of %d files, deduplicated %d",
			res.Uploaded, res.FileCount, res.Deduplicated)
	}

	// --- the site is being served
	resp := e.get(t, "/~demo/", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("GET /~demo/ = %d", resp.StatusCode)
	}
	if got := e.body(t, resp); got != v1["index.html"] {
		t.Errorf("body = %q, want %q", got, v1["index.html"])
	}
	if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/html") {
		t.Errorf("Content-Type = %q", ct)
	}
	if resp.Header.Get("X-Content-Type-Options") != "nosniff" {
		t.Error("the nosniff header is missing")
	}

	resp = e.get(t, "/~demo/assets/app.js", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("GET the script = %d", resp.StatusCode)
	}
	etag := resp.Header.Get("ETag")
	if !strings.HasPrefix(etag, `"sha256:`) {
		t.Errorf("ETag = %q, want a content digest", etag)
	}
	if got := e.body(t, resp); got != v1["assets/app.js"] {
		t.Errorf("script = %q", got)
	}

	// A repeat request with the validator is the cheap 304 the default
	// Cache-Control is chosen to produce.
	resp = e.get(t, "/~demo/assets/app.js", http.Header{"If-None-Match": {etag}})
	if resp.StatusCode != http.StatusNotModified {
		t.Errorf("conditional GET = %d, want 304", resp.StatusCode)
	}

	// Range and conditional handling come from http.ServeContent, which is the
	// reason the handler hands it an *os.File rather than writing bytes itself.
	resp = e.get(t, "/~demo/assets/app.js", http.Header{"Range": {"bytes=8-12"}})
	if resp.StatusCode != http.StatusPartialContent {
		t.Errorf("ranged GET = %d, want 206", resp.StatusCode)
	}
	if got, want := e.body(t, resp), v1["assets/app.js"][8:13]; got != want {
		t.Errorf("range bytes=8-12 gave %q, want %q", got, want)
	}
	if cr := resp.Header.Get("Content-Range"); cr != "bytes 8-12/14" {
		t.Errorf("Content-Range = %q", cr)
	}
	// If-Range with a matching validator serves the range; a stale one serves
	// the whole file, which is what keeps a mid-deploy resume correct.
	resp = e.get(t, "/~demo/assets/app.js", http.Header{
		"Range": {"bytes=0-3"}, "If-Range": {`"sha256:` + strings.Repeat("0", 64) + `"`},
	})
	if resp.StatusCode != http.StatusOK {
		t.Errorf("If-Range with a stale validator = %d, want 200", resp.StatusCode)
	}

	// Directory handling.
	resp = e.get(t, "/~demo/docs", nil)
	if resp.StatusCode != http.StatusMovedPermanently {
		t.Errorf("GET /~demo/docs = %d, want 301", resp.StatusCode)
	}
	if loc := resp.Header.Get("Location"); loc != "/~demo/docs/" {
		t.Errorf("Location = %q", loc)
	}
	resp = e.get(t, "/~demo/docs/", nil)
	if got := e.body(t, resp); got != v1["docs/index.html"] {
		t.Errorf("GET /~demo/docs/ = %q", got)
	}
	resp = e.get(t, "/~demo/nope.html", nil)
	if resp.StatusCode != http.StatusNotFound {
		t.Errorf("a missing file = %d, want 404", resp.StatusCode)
	}

	// --- the symlink is live
	dir, err := os.Readlink(filepath.Join(e.cfg.Webroot, "~demo"))
	if err != nil {
		t.Fatalf("readlink ~demo: %v", err)
	}
	if !strings.HasSuffix(dir, res.Deployment.ID) {
		t.Errorf("~demo -> %q, want the active deployment", dir)
	}
	onDisk, err := os.ReadFile(filepath.Join(dir, "index.html"))
	if err != nil {
		t.Fatalf("read through the symlink: %v", err)
	}
	if string(onDisk) != v1["index.html"] {
		t.Errorf("the assembled tree holds %q", onDisk)
	}

	// --- a second deploy uploads only what changed
	v2 := map[string]string{
		"index.html":      "<h1>v2</h1><script src=\"assets/app.js\"></script>",
		"assets/app.js":   v1["assets/app.js"],
		"docs/index.html": v1["docs/index.html"],
	}
	res2 := e.deploy(t, ci, "demo", v2)
	if res2.Uploaded != 1 || res2.Deduplicated != 2 {
		t.Errorf("second deploy uploaded %d and reused %d; want 1 and 2",
			res2.Uploaded, res2.Deduplicated)
	}
	if got := e.body(t, e.get(t, "/~demo/", nil)); got != v2["index.html"] {
		t.Errorf("after the second deploy the site serves %q", got)
	}

	// --- rollback is one activation of the older deployment
	if _, err := ci.Activate(t.Context(), "demo", res.Deployment.ID); err != nil {
		t.Fatalf("rollback: %v", err)
	}
	if got := e.body(t, e.get(t, "/~demo/", nil)); got != v1["index.html"] {
		t.Errorf("after the rollback the site serves %q", got)
	}
	dir, err = os.Readlink(filepath.Join(e.cfg.Webroot, "~demo"))
	if err != nil {
		t.Fatal(err)
	}
	if !strings.HasSuffix(dir, res.Deployment.ID) {
		t.Errorf("the rollback left ~demo -> %q", dir)
	}
}

// TestRetentionAndRollback is M5's acceptance check, in the order the plan's
// manual walkthrough does it: deploy fifteen times, confirm that collection
// keeps ten plus the one being served and that the blob count falls, then roll
// back to an old deployment and see the old site.
//
// It drives the endpoints through the CLI's own client, so `pages deployment
// list`, `pages deployment activate` and `pages system gc` are covered by the
// same run.
func TestRetentionAndRollback(t *testing.T) {
	e := newTestenv(t)
	ci := e.project(t, "demo")

	// Ten to keep, and no retention grace: the grace exists so that a rollback
	// target is not collected out from under an operator who is still deciding,
	// and any real value for it would outlast the test.
	ten, zero := 10, 0
	if _, err := e.admin.PatchProject(t.Context(), "demo",
		api.ProjectPatch{RetentionCount: &ten, RetentionGrace: &zero}); err != nil {
		t.Fatalf("configure retention: %v", err)
	}
	// Likewise the blob grace, which protects content a request has resolved and
	// is about to open.
	e.app.deploy.BlobGrace = -time.Minute

	const versions = 15
	var deps []string
	for i := 1; i <= versions; i++ {
		body := fmt.Sprintf("<h1>v%d</h1>", i)
		res := e.deploy(t, ci, "demo", map[string]string{
			"index.html": body,
			// Shared by every version, so the blob count below is a statement
			// about content that fell out of use, not about content churning.
			"assets/app.js": "console.log(1)",
		})
		deps = append(deps, res.Deployment.ID)
	}
	if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v15</h1>" {
		t.Fatalf("after %d deploys the site serves %q", versions, got)
	}

	// --- roll back to the first deployment
	//
	// Deliberately the oldest one: it is also the one retention would drop
	// first, so this proves the active deployment is exempt rather than merely
	// young enough to survive.
	if _, err := ci.Activate(t.Context(), "demo", deps[0]); err != nil {
		t.Fatalf("rollback: %v", err)
	}
	if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v1</h1>" {
		t.Errorf("after the rollback the site serves %q, want the old version", got)
	}
	link, err := os.Readlink(filepath.Join(e.cfg.Webroot, "~demo"))
	if err != nil {
		t.Fatal(err)
	}
	if !strings.HasSuffix(link, deps[0]) {
		t.Errorf("~demo -> %q, want the rolled-back deployment", link)
	}

	before, err := e.admin.SystemInfo(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	if before.Deployments != versions {
		t.Fatalf("%d deployments before collection, want %d", before.Deployments, versions)
	}

	// --- collect
	stats, err := e.admin.Collect(t.Context(), false)
	if err != nil {
		t.Fatalf("gc: %v", err)
	}
	// Fifteen deployments, one of them active and therefore not considered at
	// all, ten of the remaining fourteen kept by retention: four to delete.
	if stats.DeploymentsDeleted != 4 {
		t.Errorf("collected %d deployments, want 4", stats.DeploymentsDeleted)
	}
	if stats.BlobsDeleted != 4 || stats.BytesFreed == 0 {
		t.Errorf("collected %+v, want the four index pages those deployments held", stats)
	}

	after, err := e.admin.SystemInfo(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	if after.Deployments != 11 {
		t.Errorf("%d deployments after collection, want the 10 kept plus the active one", after.Deployments)
	}
	if after.Blobs >= before.Blobs {
		t.Errorf("blobs went from %d to %d, want the count to fall", before.Blobs, after.Blobs)
	}
	if after.Blobs != 12 {
		// Eleven surviving index pages plus the script every version shares.
		t.Errorf("%d blobs after collection, want 12", after.Blobs)
	}

	// The survivors are the active one and the ten newest, and a listing says so.
	list, err := ci.ListDeployments(t.Context(), "demo", client.DeploymentListOptions{})
	if err != nil {
		t.Fatalf("list: %v", err)
	}
	live := make(map[string]bool, len(list.Deployments))
	for _, d := range list.Deployments {
		live[d.ID] = true
		if d.Active != (d.ID == deps[0]) {
			t.Errorf("deployment %s: active = %v", d.ID, d.Active)
		}
	}
	if len(live) != 11 {
		t.Fatalf("%d deployments listed, want 11", len(live))
	}
	for i, id := range deps {
		want := i == 0 || i >= versions-10
		if live[id] != want {
			t.Errorf("v%d (%s): present = %v, want %v", i+1, id, live[id], want)
		}
	}

	// The site is still what the rollback made it, and the content of a
	// collected deployment is what was reclaimed — not the active one's.
	if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v1</h1>" {
		t.Errorf("after collection the site serves %q", got)
	}
	if got := e.body(t, e.get(t, "/~demo/assets/app.js", nil)); got != "console.log(1)" {
		t.Errorf("the shared script is now %q", got)
	}

	// A collected deployment is gone from the API too, and a second pass has
	// nothing left to do.
	if _, err := ci.GetDeployment(t.Context(), "demo", deps[1], false); err == nil {
		t.Errorf("deployment v2 (%s) still readable after collection", deps[1])
	}
	again, err := e.admin.Collect(t.Context(), false)
	if err != nil {
		t.Fatal(err)
	}
	if again.DeploymentsDeleted != 0 || again.BlobsDeleted != 0 {
		t.Errorf("a second pass collected %+v", again)
	}

	// --- deleting on request
	//
	// The one being served is refused; another one goes immediately.
	if err := ci.DeleteDeployment(t.Context(), "demo", deps[0]); err == nil {
		t.Error("the active deployment was deleted on request")
	} else if code := api.CodeOf(err); code != api.CodeDeploymentActive {
		t.Errorf("delete active: code = %q, want %q", code, api.CodeDeploymentActive)
	}
	spare := deps[versions-2]
	if err := ci.DeleteDeployment(t.Context(), "demo", spare); err != nil {
		t.Fatalf("delete %s: %v", spare, err)
	}
	if _, err := ci.GetDeployment(t.Context(), "demo", spare, false); err == nil {
		t.Errorf("deployment %s still readable after being deleted", spare)
	}
	if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v1</h1>" {
		t.Errorf("deleting a spare disturbed the site: %q", got)
	}

	// Reference counts and manifests agree throughout: the collector trusts
	// those counters, and a count that reads low is the one way this design can
	// lose content a deployment still needs.
	rep, err := e.admin.Fsck(t.Context(), false)
	if err != nil {
		t.Fatalf("fsck: %v", err)
	}
	if rep.DriftCount != 0 {
		t.Errorf("fsck reports %d drifted blobs: %+v", rep.DriftCount, rep.Drift)
	}
}

// A project that exists but has never activated anything is a different answer
// from one that does not exist: 503 says "come back", 404 says "wrong URL".
func TestServeBeforeAnythingIsDeployed(t *testing.T) {
	e := newTestenv(t)
	e.project(t, "demo")

	if resp := e.get(t, "/~demo/", nil); resp.StatusCode != http.StatusServiceUnavailable {
		t.Errorf("GET an undeployed project = %d, want 503", resp.StatusCode)
	}
	if resp := e.get(t, "/~ghost/", nil); resp.StatusCode != http.StatusNotFound {
		t.Errorf("GET an unknown project = %d, want 404", resp.StatusCode)
	}
	if resp := e.get(t, "/", nil); resp.StatusCode != http.StatusNotFound {
		t.Errorf("GET the site root = %d, want 404", resp.StatusCode)
	}
}

// The management API and the site content are separate listeners on purpose:
// neither surface may answer for the other, whatever the request looks like.
func TestTheTwoListenersDoNotOverlap(t *testing.T) {
	e := newTestenv(t)
	ci := e.project(t, "demo")
	e.deploy(t, ci, "demo", map[string]string{"index.html": "<h1>hi</h1>"})

	// The site listener knows nothing about the API.
	if resp := e.get(t, api.PathProjects(), nil); resp.StatusCode != http.StatusNotFound {
		t.Errorf("the site listener answered %s with %d", api.PathProjects(), resp.StatusCode)
	}

	// And the API listener serves no content.
	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, e.api.URL+"/~demo/", nil)
	if err != nil {
		t.Fatal(err)
	}
	resp, err := e.api.Client().Do(req)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusNotFound {
		t.Errorf("the API listener answered /~demo/ with %d", resp.StatusCode)
	}

	// Both carry the health probes, so either can be the one a load balancer
	// targets.
	for _, base := range []string{e.site.URL, e.api.URL} {
		for _, path := range []string{"/healthz", "/readyz"} {
			req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, base+path, nil)
			if err != nil {
				t.Fatal(err)
			}
			resp, err := e.site.Client().Do(req)
			if err != nil {
				t.Fatal(err)
			}
			resp.Body.Close()
			if resp.StatusCode != http.StatusOK {
				t.Errorf("GET %s%s = %d", base, path, resp.StatusCode)
			}
		}
	}
}

// A project key reaches its own project and nothing else, end to end.
func TestProjectKeysAreConfinedToTheirProject(t *testing.T) {
	e := newTestenv(t)
	mine := e.project(t, "mine")
	theirs := e.project(t, "theirs")

	e.deploy(t, theirs, "theirs", map[string]string{"index.html": "<h1>secret</h1>"})

	if _, err := mine.CreateDeployment(t.Context(), "theirs", nil); err == nil {
		t.Fatal("a project key created a deployment in another project")
	}
	if _, err := mine.GetProject(t.Context(), "theirs"); err == nil {
		t.Fatal("a project key read another project")
	}

	// Path routing means both sites share an origin, which is exactly the
	// property documented as the reason not to host mutually untrusting
	// projects this way. Reading a neighbour's content over HTTP is expected;
	// reaching its management API is not.
	if resp := e.get(t, "/~theirs/", nil); resp.StatusCode != http.StatusOK {
		t.Errorf("GET the neighbour's site = %d", resp.StatusCode)
	}
}

// The one assertion that has to hold no matter what else is logged: no part of
// a token ever reaches the log, on either listener.
func TestTokensNeverReachTheLog(t *testing.T) {
	e := newTestenv(t)
	ci := e.project(t, "demo")
	e.deploy(t, ci, "demo", map[string]string{"index.html": "<h1>hi</h1>"})

	// A bad token in every place a client could put one.
	bad := "pgs_aaaaaaaaaaaaaaaa_" + strings.Repeat("b", 43)
	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
		e.api.URL+api.PathProjects()+"?token="+bad, nil)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Authorization", "Bearer "+bad)
	resp, err := e.api.Client().Do(req)
	if err != nil {
		t.Fatal(err)
	}
	resp.Body.Close()

	logged := e.logBuf.String()
	for _, secret := range []string{bad, strings.Repeat("b", 43), "aaaaaaaaaaaaaaaa"} {
		if strings.Contains(logged, secret) {
			t.Fatalf("the log contains %q", secret)
		}
	}
	if strings.Contains(strings.ToLower(logged), "authorization") {
		t.Fatal("the log mentions the Authorization header")
	}
	// The log is not empty, so the assertions above mean something.
	if !strings.Contains(logged, "deployment activated") {
		t.Fatal("nothing was logged at all; the assertions above prove nothing")
	}
}

// Restarting the process must serve exactly what it served before: the registry
// is rebuilt from the database, which is the reason activation commits there
// before it stores the pointer.
func TestRestartServesTheSameDeployment(t *testing.T) {
	e := newTestenv(t)
	ci := e.project(t, "demo")
	res := e.deploy(t, ci, "demo", map[string]string{
		"index.html":    "<h1>v1</h1>",
		"assets/app.js": "console.log(1)",
	})
	if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v1</h1>" {
		t.Fatalf("before restart: %q", got)
	}

	// Tear the process down and build a second one on the same directories.
	e.site.Close()
	e.api.Close()
	e.app.close()

	var buf bytes.Buffer
	log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
	a2, err := newApp(context.Background(), e.cfg, log)
	if err != nil {
		t.Fatalf("second newApp: %v", err)
	}
	t.Cleanup(a2.close)
	a2.ready.Store(true)
	e.app, e.logBuf = a2, &buf
	e.site = httptest.NewServer(a2.siteHandler())
	t.Cleanup(e.site.Close)
	e.api = httptest.NewServer(a2.apiHandler())
	t.Cleanup(e.api.Close)
	e.admin = e.client(t, e.adminToken)

	resp := e.get(t, "/~demo/", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("after restart: status = %d", resp.StatusCode)
	}
	if got := e.body(t, resp); got != "<h1>v1</h1>" {
		t.Errorf("after restart the site serves %q", got)
	}
	if got := e.body(t, e.get(t, "/~demo/assets/app.js", nil)); got != "console.log(1)" {
		t.Errorf("after restart the script is %q", got)
	}

	// And the second process did not mint a second bootstrap admin key.
	keys, err := e.admin.ListKeys(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	admins := 0
	for _, k := range keys.Keys {
		if k.Scope == string(api.ScopeAdmin) && k.RevokedAt == nil {
			admins++
		}
	}
	if admins != 1 {
		t.Errorf("%d admin keys after a restart, want 1", admins)
	}

	// The database is the source of truth the registry was rebuilt from, so the
	// deployment being served is the same row, not merely the same bytes.
	p, err := e.admin.GetProject(t.Context(), "demo")
	if err != nil {
		t.Fatal(err)
	}
	if p.ActiveDeployment == nil || p.ActiveDeployment.ID != res.Deployment.ID {
		t.Errorf("active deployment after restart = %+v, want %s", p.ActiveDeployment, res.Deployment.ID)
	}
}