package deploy import ( "bytes" "context" "database/sql" "fmt" "io" "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" "sync" "testing" "github.com/iceBear67/simplepages/internal/cas" "github.com/iceBear67/simplepages/internal/site" "github.com/iceBear67/simplepages/internal/store" "github.com/iceBear67/simplepages/internal/webroot" ) // These are the tests the whole program exists for. // // The failure they are written against is the one rsync has: during a // deployment a visitor sees the new HTML with the old JavaScript, or an asset // that is not there yet. Everything else in this repository — the immutable // snapshot, the single pointer store, the database-before-memory ordering — is // a means to making that state unobservable, and unobservable is a claim about // concurrent behaviour that only a concurrent test can support. // // What is actually asserted, and why it is the strongest true statement: // // - Every single response is internally consistent. big.bin is half a // megabyte of one repeated byte, so it spans many writes and a switch // landing mid-body would show up as a seam. Every byte of it equal to the // same version is the claim "no request ever saw a half-updated site". // - A reader's successive responses never go backwards. Requests within one // reader are strictly ordered — the next is not sent until the previous has // been read to completion — so the version it observes may only rise. // // It would be tempting to also demand that three separate GETs issued around // the same time report the same version. That is not a property this or any // design has: they are three requests, an activation may legitimately land // between any two of them, and asserting otherwise would be asserting that the // switch never happens. The per-response guarantee above is what "atomic // deployment" means. const ( // Large enough that a response spans many socket writes, so a switch has // somewhere to land mid-body. stormFileSize = 512 << 10 // The in-flight tests need the server to still be blocked writing when the // test does something underneath it, which means comfortably more than a // loopback socket will buffer for a client that has stopped reading. inflightFileSize = 8 << 20 ) // switchEnv is an env with the serving layer attached: a registry, a webroot // and an HTTP server, which is the only configuration in which the switch is // observable from the outside. type switchEnv struct { *env reg *site.Registry wrDir string srv *httptest.Server cl *http.Client } func newSwitchEnv(t *testing.T) *switchEnv { t.Helper() e := newEnv(t) log := slog.New(slog.DiscardHandler) reg := site.NewRegistry() wrDir := t.TempDir() wr, err := webroot.Open(wrDir, e.dir) if err != nil { t.Fatalf("webroot.Open: %v", err) } e.svc.Sites = reg e.svc.Webroot = wr reg.Put(e.p) srv := httptest.NewServer(&site.Handler{Registry: reg, CAS: e.cas, Log: log}) t.Cleanup(srv.Close) // The default transport keeps two idle connections per host, which would // turn 64 readers into a connection churn benchmark instead of a switching // one. cl := &http.Client{Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512}} t.Cleanup(cl.CloseIdleConnections) return &switchEnv{env: e, reg: reg, wrDir: wrDir, srv: srv, cl: cl} } // versionFiles is deployment n: three small files that name their version, and // one large one filled with the single byte n so that any part of it identifies // the whole. func versionFiles(n, size int) map[string]string { v := strconv.Itoa(n) return map[string]string{ "marker.txt": v, "a.txt": v, "b.txt": v, "big.bin": string(bytes.Repeat([]byte{byte(n)}, size)), } } // publish takes version n all the way to ready without activating it. func (e *switchEnv) publish(t *testing.T, n, size int) *store.Deployment { t.Helper() contents := versionFiles(n, size) dep := e.create(t) if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { t.Fatalf("SetManifest v%d: %v", n, err) } names := make([]string, 0, len(contents)) for p := range contents { names = append(names, p) } e.upload(t, contents, names...) dep, err := e.svc.Finalize(t.Context(), e.p, dep) if err != nil { t.Fatalf("Finalize v%d: %v", n, err) } return dep } func (e *switchEnv) activate(t *testing.T, dep *store.Deployment) *store.Deployment { t.Helper() out, err := e.svc.Activate(t.Context(), e.p, dep) if err != nil { t.Fatalf("Activate %s: %v", dep.PublicID, err) } return out } // get fetches one file from the served site. It returns errors rather than // failing the test, because most of its callers are goroutines. func (e *switchEnv) get(name string) ([]byte, error) { resp, err := e.cl.Get(e.srv.URL + "/~demo/" + name) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { io.Copy(io.Discard, resp.Body) return nil, fmt.Errorf("GET %s: status %d", name, resp.StatusCode) } return io.ReadAll(resp.Body) } // bigVersion is the assertion that matters: it reports which version a big.bin // body came from, and fails if the body is not entirely from one version. func bigVersion(b []byte, size int) (int, error) { if len(b) != size { return 0, fmt.Errorf("big.bin is %d bytes, want %d", len(b), size) } first := b[0] for i, c := range b { if c != first { return 0, fmt.Errorf( "big.bin mixes two deployments: byte 0 is from version %d but byte %d is from version %d", first, i, c) } } return int(first), nil } func smallVersion(b []byte) (int, error) { n, err := strconv.Atoi(string(b)) if err != nil { return 0, fmt.Errorf("file body %q is not a version number: %v", b, err) } return n, nil } func TestActivationAtomicity(t *testing.T) { const versions = 50 const readers = 64 e := newSwitchEnv(t) deps := make([]*store.Deployment, versions) valid := make(map[string]bool, versions) for i := range deps { deps[i] = e.publish(t, i+1, stormFileSize) valid[deps[i].PublicID] = true } e.activate(t, deps[0]) ctx, stop := context.WithCancel(t.Context()) var wg sync.WaitGroup // Anti-vacuity: if every reader only ever saw the last version the // invariants above hold trivially and prove nothing. var mu sync.Mutex lowest, highest := versions+1, 0 record := func(n int) { mu.Lock() defer mu.Unlock() lowest = min(lowest, n) highest = max(highest, n) } for range readers { wg.Add(1) go func() { defer wg.Done() seen := 0 for ctx.Err() == nil { for _, name := range []string{"a.txt", "b.txt", "big.bin"} { body, err := e.get(name) if err != nil { if ctx.Err() == nil { t.Errorf("%v", err) } return } var n int if name == "big.bin" { n, err = bigVersion(body, stormFileSize) } else { n, err = smallVersion(body) } if err != nil { t.Errorf("%s: %v", name, err) return } if n < seen { t.Errorf("%s reported version %d after version %d had already been "+ "served to this reader: the switch was observed running backwards", name, n, seen) return } seen = n record(n) } } }() } // The symlink is not what serves the site, but an external reader — a // reverse proxy, a backup job — follows it, and it must never be missing or // dangling while the switch runs. link := filepath.Join(e.wrDir, "~demo") wg.Add(1) go func() { defer wg.Done() for ctx.Err() == nil { target, err := os.Readlink(link) if err != nil { t.Errorf("$WEBROOT/~demo: %v", err) return } // Stat follows the link, so a dangling one fails here. fi, err := os.Stat(target) if err != nil { t.Errorf("$WEBROOT/~demo -> %s: %v", target, err) return } if !fi.IsDir() { t.Errorf("$WEBROOT/~demo -> %s is not a directory", target) return } if !valid[filepath.Base(target)] { t.Errorf("$WEBROOT/~demo -> %s, which is not a deployment of this project", target) return } } }() for _, dep := range deps[1:] { if _, err := e.svc.Activate(t.Context(), e.p, dep); err != nil { t.Errorf("Activate %s: %v", dep.PublicID, err) break } } stop() wg.Wait() if t.Failed() { return } if lowest >= versions { t.Fatalf("every observation was of version %d or later: the readers never "+ "overlapped the switching, so this test proved nothing", lowest) } if lowest == highest { t.Fatalf("every observation was of version %d: no switch was observed", lowest) } for _, name := range []string{"marker.txt", "a.txt", "b.txt"} { body, err := e.get(name) if err != nil { t.Fatalf("%v", err) } if got, err := smallVersion(body); err != nil || got != versions { t.Errorf("after the storm %s = %q (%v), want version %d", name, body, err, versions) } } } // openBig starts a request for big.bin and reads only its first kilobyte, so // the response is still open and the server is still blocked writing it. func (e *switchEnv) openBig(t *testing.T) (*http.Response, []byte) { t.Helper() resp, err := e.cl.Get(e.srv.URL + "/~demo/big.bin") if err != nil { t.Fatalf("GET big.bin: %v", err) } if resp.StatusCode != http.StatusOK { resp.Body.Close() t.Fatalf("GET big.bin: status %d", resp.StatusCode) } head := make([]byte, 1024) if _, err := io.ReadFull(resp.Body, head); err != nil { resp.Body.Close() t.Fatalf("reading the start of big.bin: %v", err) } return resp, head } // drain finishes a response opened by openBig and reports which version the // whole body came from. func drain(t *testing.T, resp *http.Response, head []byte) int { t.Helper() defer resp.Body.Close() rest, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("reading the rest of big.bin: %v", err) } n, err := bigVersion(append(head, rest...), inflightFileSize) if err != nil { t.Fatalf("%v", err) } return n } func TestAnInFlightRequestKeepsReadingTheDeploymentItStartedOn(t *testing.T) { e := newSwitchEnv(t) v1 := e.publish(t, 1, inflightFileSize) v2 := e.publish(t, 2, inflightFileSize) e.activate(t, v1) resp, head := e.openBig(t) e.activate(t, v2) // This is the property a symlink rename cannot give you: the switch has // already happened, and this response is still the one it started as. if n := drain(t, resp, head); n != 1 { t.Errorf("a request that started before the switch finished on version %d, want 1", n) } body, err := e.get("marker.txt") if err != nil { t.Fatalf("%v", err) } if n, _ := smallVersion(body); n != 2 { t.Errorf("a request that started after the switch got version %d, want 2", n) } } func TestAnInFlightRequestSurvivesTheContentBeingCollected(t *testing.T) { e := newSwitchEnv(t) v1 := e.publish(t, 1, inflightFileSize) v2 := e.publish(t, 2, inflightFileSize) e.activate(t, v1) resp, head := e.openBig(t) e.activate(t, v2) // Everything a collector could possibly remove, with no grace period at // all: the assembled tree and the content it was hardlinked from. Both, // because removing only one of them leaves the inode alive through the // other and the test would prove nothing. if err := os.RemoveAll(DeploymentDir(e.dir, e.p.ID, v1.PublicID)); err != nil { t.Fatalf("removing the old tree: %v", err) } for _, c := range versionFiles(1, inflightFileSize) { if err := e.cas.Remove(cas.Sum([]byte(c))); err != nil { t.Fatalf("removing old content: %v", err) } } // The handler is holding an open descriptor, and POSIX keeps the inode // alive until it closes. The grace period in the collector exists so this // never has to be relied on, but relying on it has to work. if n := drain(t, resp, head); n != 1 { t.Errorf("a request whose content was deleted under it finished on version %d, want 1", n) } if _, err := e.get("marker.txt"); err != nil { t.Errorf("the live deployment stopped serving after the old one was collected: %v", err) } } func TestAFailedActivationChangesNothing(t *testing.T) { e := newSwitchEnv(t) v1 := e.publish(t, 1, 64) v2 := e.publish(t, 2, 64) e.activate(t, v1) link := filepath.Join(e.wrDir, "~demo") before, err := os.Readlink(link) if err != nil { t.Fatalf("$WEBROOT/~demo: %v", err) } // Make v2's manifest unreadable, which fails Activate inside index() — // before the transaction, before the pointer store, before the symlink. The // bogus blobs row exists only to satisfy the foreign key; the point is the // one-byte digest, which cas.FromBytes refuses. err = e.db.Tx(t.Context(), func(tx *sql.Tx) error { if _, err := tx.ExecContext(t.Context(), ` INSERT INTO blobs (digest, size, present, created_at, last_ref_at) VALUES (x'00', 1, 1, 0, 0) ON CONFLICT(digest) DO NOTHING`); err != nil { return err } _, err := tx.ExecContext(t.Context(), `UPDATE deployment_files SET digest = x'00' WHERE deployment_id = ?`, v2.ID) return err }) if err != nil { t.Fatalf("corrupting the manifest: %v", err) } if _, err := e.svc.Activate(t.Context(), e.p, v2); err == nil { t.Fatal("Activate succeeded on a deployment whose manifest cannot be read") } // In memory. body, err := e.get("marker.txt") if err != nil { t.Fatalf("%v", err) } if n, _ := smallVersion(body); n != 1 { t.Errorf("after the failed activation the site serves version %d, want 1", n) } // In the database, which is what a restart would come back to. active, err := e.db.ActiveDeployment(t.Context(), e.p.ID) if err != nil { t.Fatalf("ActiveDeployment: %v", err) } if active.PublicID != v1.PublicID { t.Errorf("the database says %s is active, want %s", active.PublicID, v1.PublicID) } // And on disk. after, err := os.Readlink(link) if err != nil { t.Fatalf("$WEBROOT/~demo: %v", err) } if after != before { t.Errorf("$WEBROOT/~demo moved to %s, want it left at %s", after, before) } }