init
This commit is contained in:
iceBear67
@@ -0,0 +1,454 @@
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/iceBear67/simplepages/internal/cas"
|
||||
"github.com/iceBear67/simplepages/internal/site"
|
||||
"github.com/iceBear67/simplepages/internal/store"
|
||||
"github.com/iceBear67/simplepages/internal/webroot"
|
||||
)
|
||||
|
||||
// These are the tests the whole program exists for.
|
||||
//
|
||||
// The failure they are written against is the one rsync has: during a
|
||||
// deployment a visitor sees the new HTML with the old JavaScript, or an asset
|
||||
// that is not there yet. Everything else in this repository — the immutable
|
||||
// snapshot, the single pointer store, the database-before-memory ordering — is
|
||||
// a means to making that state unobservable, and unobservable is a claim about
|
||||
// concurrent behaviour that only a concurrent test can support.
|
||||
//
|
||||
// What is actually asserted, and why it is the strongest true statement:
|
||||
//
|
||||
// - Every single response is internally consistent. big.bin is half a
|
||||
// megabyte of one repeated byte, so it spans many writes and a switch
|
||||
// landing mid-body would show up as a seam. Every byte of it equal to the
|
||||
// same version is the claim "no request ever saw a half-updated site".
|
||||
// - A reader's successive responses never go backwards. Requests within one
|
||||
// reader are strictly ordered — the next is not sent until the previous has
|
||||
// been read to completion — so the version it observes may only rise.
|
||||
//
|
||||
// It would be tempting to also demand that three separate GETs issued around
|
||||
// the same time report the same version. That is not a property this or any
|
||||
// design has: they are three requests, an activation may legitimately land
|
||||
// between any two of them, and asserting otherwise would be asserting that the
|
||||
// switch never happens. The per-response guarantee above is what "atomic
|
||||
// deployment" means.
|
||||
|
||||
const (
|
||||
// Large enough that a response spans many socket writes, so a switch has
|
||||
// somewhere to land mid-body.
|
||||
stormFileSize = 512 << 10
|
||||
// The in-flight tests need the server to still be blocked writing when the
|
||||
// test does something underneath it, which means comfortably more than a
|
||||
// loopback socket will buffer for a client that has stopped reading.
|
||||
inflightFileSize = 8 << 20
|
||||
)
|
||||
|
||||
// switchEnv is an env with the serving layer attached: a registry, a webroot
|
||||
// and an HTTP server, which is the only configuration in which the switch is
|
||||
// observable from the outside.
|
||||
type switchEnv struct {
|
||||
*env
|
||||
reg *site.Registry
|
||||
wrDir string
|
||||
srv *httptest.Server
|
||||
cl *http.Client
|
||||
}
|
||||
|
||||
func newSwitchEnv(t *testing.T) *switchEnv {
|
||||
t.Helper()
|
||||
e := newEnv(t)
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
|
||||
reg := site.NewRegistry()
|
||||
wrDir := t.TempDir()
|
||||
wr, err := webroot.Open(wrDir, e.dir)
|
||||
if err != nil {
|
||||
t.Fatalf("webroot.Open: %v", err)
|
||||
}
|
||||
e.svc.Sites = reg
|
||||
e.svc.Webroot = wr
|
||||
reg.Put(e.p)
|
||||
|
||||
srv := httptest.NewServer(&site.Handler{Registry: reg, CAS: e.cas, Log: log})
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
// The default transport keeps two idle connections per host, which would
|
||||
// turn 64 readers into a connection churn benchmark instead of a switching
|
||||
// one.
|
||||
cl := &http.Client{Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512}}
|
||||
t.Cleanup(cl.CloseIdleConnections)
|
||||
|
||||
return &switchEnv{env: e, reg: reg, wrDir: wrDir, srv: srv, cl: cl}
|
||||
}
|
||||
|
||||
// versionFiles is deployment n: three small files that name their version, and
|
||||
// one large one filled with the single byte n so that any part of it identifies
|
||||
// the whole.
|
||||
func versionFiles(n, size int) map[string]string {
|
||||
v := strconv.Itoa(n)
|
||||
return map[string]string{
|
||||
"marker.txt": v,
|
||||
"a.txt": v,
|
||||
"b.txt": v,
|
||||
"big.bin": string(bytes.Repeat([]byte{byte(n)}, size)),
|
||||
}
|
||||
}
|
||||
|
||||
// publish takes version n all the way to ready without activating it.
|
||||
func (e *switchEnv) publish(t *testing.T, n, size int) *store.Deployment {
|
||||
t.Helper()
|
||||
contents := versionFiles(n, size)
|
||||
dep := e.create(t)
|
||||
if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil {
|
||||
t.Fatalf("SetManifest v%d: %v", n, err)
|
||||
}
|
||||
names := make([]string, 0, len(contents))
|
||||
for p := range contents {
|
||||
names = append(names, p)
|
||||
}
|
||||
e.upload(t, contents, names...)
|
||||
dep, err := e.svc.Finalize(t.Context(), e.p, dep)
|
||||
if err != nil {
|
||||
t.Fatalf("Finalize v%d: %v", n, err)
|
||||
}
|
||||
return dep
|
||||
}
|
||||
|
||||
func (e *switchEnv) activate(t *testing.T, dep *store.Deployment) *store.Deployment {
|
||||
t.Helper()
|
||||
out, err := e.svc.Activate(t.Context(), e.p, dep)
|
||||
if err != nil {
|
||||
t.Fatalf("Activate %s: %v", dep.PublicID, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// get fetches one file from the served site. It returns errors rather than
|
||||
// failing the test, because most of its callers are goroutines.
|
||||
func (e *switchEnv) get(name string) ([]byte, error) {
|
||||
resp, err := e.cl.Get(e.srv.URL + "/~demo/" + name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("GET %s: status %d", name, resp.StatusCode)
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
// bigVersion is the assertion that matters: it reports which version a big.bin
|
||||
// body came from, and fails if the body is not entirely from one version.
|
||||
func bigVersion(b []byte, size int) (int, error) {
|
||||
if len(b) != size {
|
||||
return 0, fmt.Errorf("big.bin is %d bytes, want %d", len(b), size)
|
||||
}
|
||||
first := b[0]
|
||||
for i, c := range b {
|
||||
if c != first {
|
||||
return 0, fmt.Errorf(
|
||||
"big.bin mixes two deployments: byte 0 is from version %d but byte %d is from version %d",
|
||||
first, i, c)
|
||||
}
|
||||
}
|
||||
return int(first), nil
|
||||
}
|
||||
|
||||
func smallVersion(b []byte) (int, error) {
|
||||
n, err := strconv.Atoi(string(b))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("file body %q is not a version number: %v", b, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func TestActivationAtomicity(t *testing.T) {
|
||||
const versions = 50
|
||||
const readers = 64
|
||||
|
||||
e := newSwitchEnv(t)
|
||||
deps := make([]*store.Deployment, versions)
|
||||
valid := make(map[string]bool, versions)
|
||||
for i := range deps {
|
||||
deps[i] = e.publish(t, i+1, stormFileSize)
|
||||
valid[deps[i].PublicID] = true
|
||||
}
|
||||
e.activate(t, deps[0])
|
||||
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Anti-vacuity: if every reader only ever saw the last version the
|
||||
// invariants above hold trivially and prove nothing.
|
||||
var mu sync.Mutex
|
||||
lowest, highest := versions+1, 0
|
||||
record := func(n int) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
lowest = min(lowest, n)
|
||||
highest = max(highest, n)
|
||||
}
|
||||
|
||||
for range readers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
seen := 0
|
||||
for ctx.Err() == nil {
|
||||
for _, name := range []string{"a.txt", "b.txt", "big.bin"} {
|
||||
body, err := e.get(name)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
var n int
|
||||
if name == "big.bin" {
|
||||
n, err = bigVersion(body, stormFileSize)
|
||||
} else {
|
||||
n, err = smallVersion(body)
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
return
|
||||
}
|
||||
if n < seen {
|
||||
t.Errorf("%s reported version %d after version %d had already been "+
|
||||
"served to this reader: the switch was observed running backwards",
|
||||
name, n, seen)
|
||||
return
|
||||
}
|
||||
seen = n
|
||||
record(n)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// The symlink is not what serves the site, but an external reader — a
|
||||
// reverse proxy, a backup job — follows it, and it must never be missing or
|
||||
// dangling while the switch runs.
|
||||
link := filepath.Join(e.wrDir, "~demo")
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for ctx.Err() == nil {
|
||||
target, err := os.Readlink(link)
|
||||
if err != nil {
|
||||
t.Errorf("$WEBROOT/~demo: %v", err)
|
||||
return
|
||||
}
|
||||
// Stat follows the link, so a dangling one fails here.
|
||||
fi, err := os.Stat(target)
|
||||
if err != nil {
|
||||
t.Errorf("$WEBROOT/~demo -> %s: %v", target, err)
|
||||
return
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
t.Errorf("$WEBROOT/~demo -> %s is not a directory", target)
|
||||
return
|
||||
}
|
||||
if !valid[filepath.Base(target)] {
|
||||
t.Errorf("$WEBROOT/~demo -> %s, which is not a deployment of this project", target)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for _, dep := range deps[1:] {
|
||||
if _, err := e.svc.Activate(t.Context(), e.p, dep); err != nil {
|
||||
t.Errorf("Activate %s: %v", dep.PublicID, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
stop()
|
||||
wg.Wait()
|
||||
|
||||
if t.Failed() {
|
||||
return
|
||||
}
|
||||
if lowest >= versions {
|
||||
t.Fatalf("every observation was of version %d or later: the readers never "+
|
||||
"overlapped the switching, so this test proved nothing", lowest)
|
||||
}
|
||||
if lowest == highest {
|
||||
t.Fatalf("every observation was of version %d: no switch was observed", lowest)
|
||||
}
|
||||
for _, name := range []string{"marker.txt", "a.txt", "b.txt"} {
|
||||
body, err := e.get(name)
|
||||
if err != nil {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if got, err := smallVersion(body); err != nil || got != versions {
|
||||
t.Errorf("after the storm %s = %q (%v), want version %d", name, body, err, versions)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// openBig starts a request for big.bin and reads only its first kilobyte, so
|
||||
// the response is still open and the server is still blocked writing it.
|
||||
func (e *switchEnv) openBig(t *testing.T) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
resp, err := e.cl.Get(e.srv.URL + "/~demo/big.bin")
|
||||
if err != nil {
|
||||
t.Fatalf("GET big.bin: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
t.Fatalf("GET big.bin: status %d", resp.StatusCode)
|
||||
}
|
||||
head := make([]byte, 1024)
|
||||
if _, err := io.ReadFull(resp.Body, head); err != nil {
|
||||
resp.Body.Close()
|
||||
t.Fatalf("reading the start of big.bin: %v", err)
|
||||
}
|
||||
return resp, head
|
||||
}
|
||||
|
||||
// drain finishes a response opened by openBig and reports which version the
|
||||
// whole body came from.
|
||||
func drain(t *testing.T, resp *http.Response, head []byte) int {
|
||||
t.Helper()
|
||||
defer resp.Body.Close()
|
||||
rest, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("reading the rest of big.bin: %v", err)
|
||||
}
|
||||
n, err := bigVersion(append(head, rest...), inflightFileSize)
|
||||
if err != nil {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestAnInFlightRequestKeepsReadingTheDeploymentItStartedOn(t *testing.T) {
|
||||
e := newSwitchEnv(t)
|
||||
v1 := e.publish(t, 1, inflightFileSize)
|
||||
v2 := e.publish(t, 2, inflightFileSize)
|
||||
e.activate(t, v1)
|
||||
|
||||
resp, head := e.openBig(t)
|
||||
e.activate(t, v2)
|
||||
|
||||
// This is the property a symlink rename cannot give you: the switch has
|
||||
// already happened, and this response is still the one it started as.
|
||||
if n := drain(t, resp, head); n != 1 {
|
||||
t.Errorf("a request that started before the switch finished on version %d, want 1", n)
|
||||
}
|
||||
body, err := e.get("marker.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if n, _ := smallVersion(body); n != 2 {
|
||||
t.Errorf("a request that started after the switch got version %d, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnInFlightRequestSurvivesTheContentBeingCollected(t *testing.T) {
|
||||
e := newSwitchEnv(t)
|
||||
v1 := e.publish(t, 1, inflightFileSize)
|
||||
v2 := e.publish(t, 2, inflightFileSize)
|
||||
e.activate(t, v1)
|
||||
|
||||
resp, head := e.openBig(t)
|
||||
e.activate(t, v2)
|
||||
|
||||
// Everything a collector could possibly remove, with no grace period at
|
||||
// all: the assembled tree and the content it was hardlinked from. Both,
|
||||
// because removing only one of them leaves the inode alive through the
|
||||
// other and the test would prove nothing.
|
||||
if err := os.RemoveAll(DeploymentDir(e.dir, e.p.ID, v1.PublicID)); err != nil {
|
||||
t.Fatalf("removing the old tree: %v", err)
|
||||
}
|
||||
for _, c := range versionFiles(1, inflightFileSize) {
|
||||
if err := e.cas.Remove(cas.Sum([]byte(c))); err != nil {
|
||||
t.Fatalf("removing old content: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The handler is holding an open descriptor, and POSIX keeps the inode
|
||||
// alive until it closes. The grace period in the collector exists so this
|
||||
// never has to be relied on, but relying on it has to work.
|
||||
if n := drain(t, resp, head); n != 1 {
|
||||
t.Errorf("a request whose content was deleted under it finished on version %d, want 1", n)
|
||||
}
|
||||
if _, err := e.get("marker.txt"); err != nil {
|
||||
t.Errorf("the live deployment stopped serving after the old one was collected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedActivationChangesNothing(t *testing.T) {
|
||||
e := newSwitchEnv(t)
|
||||
v1 := e.publish(t, 1, 64)
|
||||
v2 := e.publish(t, 2, 64)
|
||||
e.activate(t, v1)
|
||||
|
||||
link := filepath.Join(e.wrDir, "~demo")
|
||||
before, err := os.Readlink(link)
|
||||
if err != nil {
|
||||
t.Fatalf("$WEBROOT/~demo: %v", err)
|
||||
}
|
||||
|
||||
// Make v2's manifest unreadable, which fails Activate inside index() —
|
||||
// before the transaction, before the pointer store, before the symlink. The
|
||||
// bogus blobs row exists only to satisfy the foreign key; the point is the
|
||||
// one-byte digest, which cas.FromBytes refuses.
|
||||
err = e.db.Tx(t.Context(), func(tx *sql.Tx) error {
|
||||
if _, err := tx.ExecContext(t.Context(), `
|
||||
INSERT INTO blobs (digest, size, present, created_at, last_ref_at)
|
||||
VALUES (x'00', 1, 1, 0, 0) ON CONFLICT(digest) DO NOTHING`); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.ExecContext(t.Context(),
|
||||
`UPDATE deployment_files SET digest = x'00' WHERE deployment_id = ?`, v2.ID)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("corrupting the manifest: %v", err)
|
||||
}
|
||||
|
||||
if _, err := e.svc.Activate(t.Context(), e.p, v2); err == nil {
|
||||
t.Fatal("Activate succeeded on a deployment whose manifest cannot be read")
|
||||
}
|
||||
|
||||
// In memory.
|
||||
body, err := e.get("marker.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if n, _ := smallVersion(body); n != 1 {
|
||||
t.Errorf("after the failed activation the site serves version %d, want 1", n)
|
||||
}
|
||||
// In the database, which is what a restart would come back to.
|
||||
active, err := e.db.ActiveDeployment(t.Context(), e.p.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ActiveDeployment: %v", err)
|
||||
}
|
||||
if active.PublicID != v1.PublicID {
|
||||
t.Errorf("the database says %s is active, want %s", active.PublicID, v1.PublicID)
|
||||
}
|
||||
// And on disk.
|
||||
after, err := os.Readlink(link)
|
||||
if err != nil {
|
||||
t.Fatalf("$WEBROOT/~demo: %v", err)
|
||||
}
|
||||
if after != before {
|
||||
t.Errorf("$WEBROOT/~demo moved to %s, want it left at %s", after, before)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user