pages/internal/deploy/atomicity_test.go

package deploy

import (
	"bytes"
	"context"
	"database/sql"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"testing"

	"github.com/iceBear67/simplepages/internal/cas"
	"github.com/iceBear67/simplepages/internal/site"
	"github.com/iceBear67/simplepages/internal/store"
	"github.com/iceBear67/simplepages/internal/webroot"
)

// These are the tests the whole program exists for.
//
// The failure they are written against is the one rsync has: during a
// deployment a visitor sees the new HTML with the old JavaScript, or an asset
// that is not there yet. Everything else in this repository — the immutable
// snapshot, the single pointer store, the database-before-memory ordering — is
// a means to making that state unobservable, and unobservable is a claim about
// concurrent behaviour that only a concurrent test can support.
//
// What is actually asserted, and why it is the strongest true statement:
//
//   - Every single response is internally consistent. big.bin is half a
//     megabyte of one repeated byte, so it spans many writes and a switch
//     landing mid-body would show up as a seam. Every byte of it equal to the
//     same version is the claim "no request ever saw a half-updated site".
//   - A reader's successive responses never go backwards. Requests within one
//     reader are strictly ordered — the next is not sent until the previous has
//     been read to completion — so the version it observes may only rise.
//
// It would be tempting to also demand that three separate GETs issued around
// the same time report the same version. That is not a property this or any
// design has: they are three requests, an activation may legitimately land
// between any two of them, and asserting otherwise would be asserting that the
// switch never happens. The per-response guarantee above is what "atomic
// deployment" means.

const (
	// Large enough that a response spans many socket writes, so a switch has
	// somewhere to land mid-body.
	stormFileSize = 512 << 10
	// The in-flight tests need the server to still be blocked writing when the
	// test does something underneath it, which means comfortably more than a
	// loopback socket will buffer for a client that has stopped reading.
	inflightFileSize = 8 << 20
)

// switchEnv is an env with the serving layer attached: a registry, a webroot
// and an HTTP server, which is the only configuration in which the switch is
// observable from the outside.
type switchEnv struct {
	*env
	reg   *site.Registry
	wrDir string
	srv   *httptest.Server
	cl    *http.Client
}

func newSwitchEnv(t *testing.T) *switchEnv {
	t.Helper()
	e := newEnv(t)
	log := slog.New(slog.DiscardHandler)

	reg := site.NewRegistry()
	wrDir := t.TempDir()
	wr, err := webroot.Open(wrDir, e.dir)
	if err != nil {
		t.Fatalf("webroot.Open: %v", err)
	}
	e.svc.Sites = reg
	e.svc.Webroot = wr
	reg.Put(e.p)

	srv := httptest.NewServer(&site.Handler{Registry: reg, CAS: e.cas, Log: log})
	t.Cleanup(srv.Close)

	// The default transport keeps two idle connections per host, which would
	// turn 64 readers into a connection churn benchmark instead of a switching
	// one.
	cl := &http.Client{Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512}}
	t.Cleanup(cl.CloseIdleConnections)

	return &switchEnv{env: e, reg: reg, wrDir: wrDir, srv: srv, cl: cl}
}

// versionFiles is deployment n: three small files that name their version, and
// one large one filled with the single byte n so that any part of it identifies
// the whole.
func versionFiles(n, size int) map[string]string {
	v := strconv.Itoa(n)
	return map[string]string{
		"marker.txt": v,
		"a.txt":      v,
		"b.txt":      v,
		"big.bin":    string(bytes.Repeat([]byte{byte(n)}, size)),
	}
}

// publish takes version n all the way to ready without activating it.
func (e *switchEnv) publish(t *testing.T, n, size int) *store.Deployment {
	t.Helper()
	contents := versionFiles(n, size)
	dep := e.create(t)
	if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
		t.Fatalf("SetManifest v%d: %v", n, err)
	}
	names := make([]string, 0, len(contents))
	for p := range contents {
		names = append(names, p)
	}
	e.upload(t, contents, names...)
	dep, err := e.svc.Finalize(t.Context(), e.p, dep)
	if err != nil {
		t.Fatalf("Finalize v%d: %v", n, err)
	}
	return dep
}

func (e *switchEnv) activate(t *testing.T, dep *store.Deployment) *store.Deployment {
	t.Helper()
	out, err := e.svc.Activate(t.Context(), e.p, dep)
	if err != nil {
		t.Fatalf("Activate %s: %v", dep.PublicID, err)
	}
	return out
}

// get fetches one file from the served site. It returns errors rather than
// failing the test, because most of its callers are goroutines.
func (e *switchEnv) get(name string) ([]byte, error) {
	resp, err := e.cl.Get(e.srv.URL + "/~demo/" + name)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		io.Copy(io.Discard, resp.Body)
		return nil, fmt.Errorf("GET %s: status %d", name, resp.StatusCode)
	}
	return io.ReadAll(resp.Body)
}

// bigVersion is the assertion that matters: it reports which version a big.bin
// body came from, and fails if the body is not entirely from one version.
func bigVersion(b []byte, size int) (int, error) {
	if len(b) != size {
		return 0, fmt.Errorf("big.bin is %d bytes, want %d", len(b), size)
	}
	first := b[0]
	for i, c := range b {
		if c != first {
			return 0, fmt.Errorf(
				"big.bin mixes two deployments: byte 0 is from version %d but byte %d is from version %d",
				first, i, c)
		}
	}
	return int(first), nil
}

func smallVersion(b []byte) (int, error) {
	n, err := strconv.Atoi(string(b))
	if err != nil {
		return 0, fmt.Errorf("file body %q is not a version number: %v", b, err)
	}
	return n, nil
}

func TestActivationAtomicity(t *testing.T) {
	const versions = 50
	const readers = 64

	e := newSwitchEnv(t)
	deps := make([]*store.Deployment, versions)
	valid := make(map[string]bool, versions)
	for i := range deps {
		deps[i] = e.publish(t, i+1, stormFileSize)
		valid[deps[i].PublicID] = true
	}
	e.activate(t, deps[0])

	ctx, stop := context.WithCancel(t.Context())
	var wg sync.WaitGroup

	// Anti-vacuity: if every reader only ever saw the last version the
	// invariants above hold trivially and prove nothing.
	var mu sync.Mutex
	lowest, highest := versions+1, 0
	record := func(n int) {
		mu.Lock()
		defer mu.Unlock()
		lowest = min(lowest, n)
		highest = max(highest, n)
	}

	for range readers {
		wg.Add(1)
		go func() {
			defer wg.Done()
			seen := 0
			for ctx.Err() == nil {
				for _, name := range []string{"a.txt", "b.txt", "big.bin"} {
					body, err := e.get(name)
					if err != nil {
						if ctx.Err() == nil {
							t.Errorf("%v", err)
						}
						return
					}
					var n int
					if name == "big.bin" {
						n, err = bigVersion(body, stormFileSize)
					} else {
						n, err = smallVersion(body)
					}
					if err != nil {
						t.Errorf("%s: %v", name, err)
						return
					}
					if n < seen {
						t.Errorf("%s reported version %d after version %d had already been "+
							"served to this reader: the switch was observed running backwards",
							name, n, seen)
						return
					}
					seen = n
					record(n)
				}
			}
		}()
	}

	// The symlink is not what serves the site, but an external reader — a
	// reverse proxy, a backup job — follows it, and it must never be missing or
	// dangling while the switch runs.
	link := filepath.Join(e.wrDir, "~demo")
	wg.Add(1)
	go func() {
		defer wg.Done()
		for ctx.Err() == nil {
			target, err := os.Readlink(link)
			if err != nil {
				t.Errorf("$WEBROOT/~demo: %v", err)
				return
			}
			// Stat follows the link, so a dangling one fails here.
			fi, err := os.Stat(target)
			if err != nil {
				t.Errorf("$WEBROOT/~demo -> %s: %v", target, err)
				return
			}
			if !fi.IsDir() {
				t.Errorf("$WEBROOT/~demo -> %s is not a directory", target)
				return
			}
			if !valid[filepath.Base(target)] {
				t.Errorf("$WEBROOT/~demo -> %s, which is not a deployment of this project", target)
				return
			}
		}
	}()

	for _, dep := range deps[1:] {
		if _, err := e.svc.Activate(t.Context(), e.p, dep); err != nil {
			t.Errorf("Activate %s: %v", dep.PublicID, err)
			break
		}
	}
	stop()
	wg.Wait()

	if t.Failed() {
		return
	}
	if lowest >= versions {
		t.Fatalf("every observation was of version %d or later: the readers never "+
			"overlapped the switching, so this test proved nothing", lowest)
	}
	if lowest == highest {
		t.Fatalf("every observation was of version %d: no switch was observed", lowest)
	}
	for _, name := range []string{"marker.txt", "a.txt", "b.txt"} {
		body, err := e.get(name)
		if err != nil {
			t.Fatalf("%v", err)
		}
		if got, err := smallVersion(body); err != nil || got != versions {
			t.Errorf("after the storm %s = %q (%v), want version %d", name, body, err, versions)
		}
	}
}

// openBig starts a request for big.bin and reads only its first kilobyte, so
// the response is still open and the server is still blocked writing it.
func (e *switchEnv) openBig(t *testing.T) (*http.Response, []byte) {
	t.Helper()
	resp, err := e.cl.Get(e.srv.URL + "/~demo/big.bin")
	if err != nil {
		t.Fatalf("GET big.bin: %v", err)
	}
	if resp.StatusCode != http.StatusOK {
		resp.Body.Close()
		t.Fatalf("GET big.bin: status %d", resp.StatusCode)
	}
	head := make([]byte, 1024)
	if _, err := io.ReadFull(resp.Body, head); err != nil {
		resp.Body.Close()
		t.Fatalf("reading the start of big.bin: %v", err)
	}
	return resp, head
}

// drain finishes a response opened by openBig and reports which version the
// whole body came from.
func drain(t *testing.T, resp *http.Response, head []byte) int {
	t.Helper()
	defer resp.Body.Close()
	rest, err := io.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("reading the rest of big.bin: %v", err)
	}
	n, err := bigVersion(append(head, rest...), inflightFileSize)
	if err != nil {
		t.Fatalf("%v", err)
	}
	return n
}

func TestAnInFlightRequestKeepsReadingTheDeploymentItStartedOn(t *testing.T) {
	e := newSwitchEnv(t)
	v1 := e.publish(t, 1, inflightFileSize)
	v2 := e.publish(t, 2, inflightFileSize)
	e.activate(t, v1)

	resp, head := e.openBig(t)
	e.activate(t, v2)

	// This is the property a symlink rename cannot give you: the switch has
	// already happened, and this response is still the one it started as.
	if n := drain(t, resp, head); n != 1 {
		t.Errorf("a request that started before the switch finished on version %d, want 1", n)
	}
	body, err := e.get("marker.txt")
	if err != nil {
		t.Fatalf("%v", err)
	}
	if n, _ := smallVersion(body); n != 2 {
		t.Errorf("a request that started after the switch got version %d, want 2", n)
	}
}

func TestAnInFlightRequestSurvivesTheContentBeingCollected(t *testing.T) {
	e := newSwitchEnv(t)
	v1 := e.publish(t, 1, inflightFileSize)
	v2 := e.publish(t, 2, inflightFileSize)
	e.activate(t, v1)

	resp, head := e.openBig(t)
	e.activate(t, v2)

	// Everything a collector could possibly remove, with no grace period at
	// all: the assembled tree and the content it was hardlinked from. Both,
	// because removing only one of them leaves the inode alive through the
	// other and the test would prove nothing.
	if err := os.RemoveAll(DeploymentDir(e.dir, e.p.ID, v1.PublicID)); err != nil {
		t.Fatalf("removing the old tree: %v", err)
	}
	for _, c := range versionFiles(1, inflightFileSize) {
		if err := e.cas.Remove(cas.Sum([]byte(c))); err != nil {
			t.Fatalf("removing old content: %v", err)
		}
	}

	// The handler is holding an open descriptor, and POSIX keeps the inode
	// alive until it closes. The grace period in the collector exists so this
	// never has to be relied on, but relying on it has to work.
	if n := drain(t, resp, head); n != 1 {
		t.Errorf("a request whose content was deleted under it finished on version %d, want 1", n)
	}
	if _, err := e.get("marker.txt"); err != nil {
		t.Errorf("the live deployment stopped serving after the old one was collected: %v", err)
	}
}

func TestAFailedActivationChangesNothing(t *testing.T) {
	e := newSwitchEnv(t)
	v1 := e.publish(t, 1, 64)
	v2 := e.publish(t, 2, 64)
	e.activate(t, v1)

	link := filepath.Join(e.wrDir, "~demo")
	before, err := os.Readlink(link)
	if err != nil {
		t.Fatalf("$WEBROOT/~demo: %v", err)
	}

	// Make v2's manifest unreadable, which fails Activate inside index() —
	// before the transaction, before the pointer store, before the symlink. The
	// bogus blobs row exists only to satisfy the foreign key; the point is the
	// one-byte digest, which cas.FromBytes refuses.
	err = e.db.Tx(t.Context(), func(tx *sql.Tx) error {
		if _, err := tx.ExecContext(t.Context(), `
			INSERT INTO blobs (digest, size, present, created_at, last_ref_at)
			VALUES (x'00', 1, 1, 0, 0) ON CONFLICT(digest) DO NOTHING`); err != nil {
			return err
		}
		_, err := tx.ExecContext(t.Context(),
			`UPDATE deployment_files SET digest = x'00' WHERE deployment_id = ?`, v2.ID)
		return err
	})
	if err != nil {
		t.Fatalf("corrupting the manifest: %v", err)
	}

	if _, err := e.svc.Activate(t.Context(), e.p, v2); err == nil {
		t.Fatal("Activate succeeded on a deployment whose manifest cannot be read")
	}

	// In memory.
	body, err := e.get("marker.txt")
	if err != nil {
		t.Fatalf("%v", err)
	}
	if n, _ := smallVersion(body); n != 1 {
		t.Errorf("after the failed activation the site serves version %d, want 1", n)
	}
	// In the database, which is what a restart would come back to.
	active, err := e.db.ActiveDeployment(t.Context(), e.p.ID)
	if err != nil {
		t.Fatalf("ActiveDeployment: %v", err)
	}
	if active.PublicID != v1.PublicID {
		t.Errorf("the database says %s is active, want %s", active.PublicID, v1.PublicID)
	}
	// And on disk.
	after, err := os.Readlink(link)
	if err != nil {
		t.Fatalf("$WEBROOT/~demo: %v", err)
	}
	if after != before {
		t.Errorf("$WEBROOT/~demo moved to %s, want it left at %s", after, before)
	}
}