This commit is contained in:
iceBear67
2026-08-15 07:13:00 +00:00
commit dd50674fdc
114 changed files with 26865 additions and 0 deletions
+673
View File
@@ -0,0 +1,673 @@
package main
import (
"bytes"
"context"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/iceBear67/simplepages/api"
"github.com/iceBear67/simplepages/internal/client"
"github.com/iceBear67/simplepages/internal/config"
)
// testenv is the whole server: the real app assembly, both handlers, and the
// pages CLI's own client library driving them. Nothing between the CI job and
// the bytes on the wire is stubbed, which is the point — the milestone's claim
// is that `pages deploy` followed by a GET returns the site, and only a test at
// this level can make that claim.
type testenv struct {
app *app
cfg config.Config
site *httptest.Server
api *httptest.Server
admin *client.Client
adminToken string
logBuf *bytes.Buffer
}
func newTestenv(t *testing.T) *testenv {
t.Helper()
base := t.TempDir()
cfg := config.Default()
cfg.DataDir = filepath.Join(base, "data")
cfg.Webroot = filepath.Join(base, "www")
cfg.LogFormat = "text"
cfg.LogLevel = "debug"
if err := cfg.Validate(); err != nil {
t.Fatalf("config: %v", err)
}
if err := cfg.EnsureDirs(); err != nil {
t.Fatalf("ensure dirs: %v", err)
}
var buf bytes.Buffer
log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
a, err := newApp(t.Context(), cfg, log)
if err != nil {
t.Fatalf("newApp: %v", err)
}
t.Cleanup(a.close)
a.ready.Store(true)
e := &testenv{app: a, cfg: cfg, logBuf: &buf}
e.site = httptest.NewServer(a.siteHandler())
t.Cleanup(e.site.Close)
e.api = httptest.NewServer(a.apiHandler())
t.Cleanup(e.api.Close)
// The bootstrap token is how an operator gets their first credential, and
// the only place the server ever writes one to disk.
raw, err := os.ReadFile(cfg.BootstrapTokenPath())
if err != nil {
t.Fatalf("read bootstrap token: %v", err)
}
fi, err := os.Stat(cfg.BootstrapTokenPath())
if err != nil {
t.Fatal(err)
}
if fi.Mode().Perm() != 0o600 {
t.Errorf("bootstrap-token mode = %v, want 0600", fi.Mode().Perm())
}
e.adminToken = strings.TrimSpace(string(raw))
e.admin = e.client(t, e.adminToken)
return e
}
func (e *testenv) client(t *testing.T, token string) *client.Client {
t.Helper()
c, err := client.New(client.Config{BaseURL: e.api.URL, Token: token, HTTP: e.api.Client()})
if err != nil {
t.Fatalf("client: %v", err)
}
return c
}
// project creates a project and returns a client holding a key scoped to it,
// which is what a CI job would be given.
func (e *testenv) project(t *testing.T, name string) *client.Client {
t.Helper()
if _, err := e.admin.CreateProject(t.Context(), api.CreateProjectRequest{Name: name}); err != nil {
t.Fatalf("create project %s: %v", name, err)
}
key, err := e.admin.CreateProjectKey(t.Context(), name, api.CreateKeyRequest{Name: "ci"})
if err != nil {
t.Fatalf("create key for %s: %v", name, err)
}
return e.client(t, key.Token)
}
// deploy scans a directory of literal file contents and pushes it, exactly as
// `pages deploy` does.
func (e *testenv) deploy(t *testing.T, c *client.Client, project string, files map[string]string) *client.DeployResult {
t.Helper()
dir := t.TempDir()
for name, content := range files {
p := filepath.Join(dir, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
src, err := client.Scan(t.Context(), dir, client.ScanOptions{})
if err != nil {
t.Fatalf("scan: %v", err)
}
defer src.Close()
res, err := c.Deploy(t.Context(), client.DeployOptions{
Project: project, Source: src, Activate: true,
})
if err != nil {
t.Fatalf("deploy: %v", err)
}
return res
}
// get fetches a site URL.
func (e *testenv) get(t *testing.T, path string, header http.Header) *http.Response {
t.Helper()
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, e.site.URL+path, nil)
if err != nil {
t.Fatal(err)
}
req.Header = header
if req.Header == nil {
req.Header = http.Header{}
}
// Redirects are part of what is under test, so they are never followed.
c := *e.site.Client()
c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
resp, err := c.Do(req)
if err != nil {
t.Fatalf("GET %s: %v", path, err)
}
t.Cleanup(func() { resp.Body.Close() })
return resp
}
func (e *testenv) body(t *testing.T, resp *http.Response) string {
t.Helper()
b, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
return string(b)
}
// TestDeployThenServe is the milestone's acceptance test, in the order the
// manual walkthrough in the plan does it.
func TestDeployThenServe(t *testing.T) {
e := newTestenv(t)
ci := e.project(t, "demo")
v1 := map[string]string{
"index.html": "<h1>v1</h1><script src=\"assets/app.js\"></script>",
"assets/app.js": "console.log(1)",
"docs/index.html": "<h1>docs</h1>",
}
res := e.deploy(t, ci, "demo", v1)
if !res.Activated || res.Deployment.State != api.StateReady {
t.Fatalf("deploy result = %+v", res)
}
if res.FileCount != 3 || res.Uploaded != 3 || res.Deduplicated != 0 {
t.Fatalf("first deploy uploaded %d of %d files, deduplicated %d",
res.Uploaded, res.FileCount, res.Deduplicated)
}
// --- the site is being served
resp := e.get(t, "/~demo/", nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("GET /~demo/ = %d", resp.StatusCode)
}
if got := e.body(t, resp); got != v1["index.html"] {
t.Errorf("body = %q, want %q", got, v1["index.html"])
}
if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/html") {
t.Errorf("Content-Type = %q", ct)
}
if resp.Header.Get("X-Content-Type-Options") != "nosniff" {
t.Error("the nosniff header is missing")
}
resp = e.get(t, "/~demo/assets/app.js", nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("GET the script = %d", resp.StatusCode)
}
etag := resp.Header.Get("ETag")
if !strings.HasPrefix(etag, `"sha256:`) {
t.Errorf("ETag = %q, want a content digest", etag)
}
if got := e.body(t, resp); got != v1["assets/app.js"] {
t.Errorf("script = %q", got)
}
// A repeat request with the validator is the cheap 304 the default
// Cache-Control is chosen to produce.
resp = e.get(t, "/~demo/assets/app.js", http.Header{"If-None-Match": {etag}})
if resp.StatusCode != http.StatusNotModified {
t.Errorf("conditional GET = %d, want 304", resp.StatusCode)
}
// Range and conditional handling come from http.ServeContent, which is the
// reason the handler hands it an *os.File rather than writing bytes itself.
resp = e.get(t, "/~demo/assets/app.js", http.Header{"Range": {"bytes=8-12"}})
if resp.StatusCode != http.StatusPartialContent {
t.Errorf("ranged GET = %d, want 206", resp.StatusCode)
}
if got, want := e.body(t, resp), v1["assets/app.js"][8:13]; got != want {
t.Errorf("range bytes=8-12 gave %q, want %q", got, want)
}
if cr := resp.Header.Get("Content-Range"); cr != "bytes 8-12/14" {
t.Errorf("Content-Range = %q", cr)
}
// If-Range with a matching validator serves the range; a stale one serves
// the whole file, which is what keeps a mid-deploy resume correct.
resp = e.get(t, "/~demo/assets/app.js", http.Header{
"Range": {"bytes=0-3"}, "If-Range": {`"sha256:` + strings.Repeat("0", 64) + `"`},
})
if resp.StatusCode != http.StatusOK {
t.Errorf("If-Range with a stale validator = %d, want 200", resp.StatusCode)
}
// Directory handling.
resp = e.get(t, "/~demo/docs", nil)
if resp.StatusCode != http.StatusMovedPermanently {
t.Errorf("GET /~demo/docs = %d, want 301", resp.StatusCode)
}
if loc := resp.Header.Get("Location"); loc != "/~demo/docs/" {
t.Errorf("Location = %q", loc)
}
resp = e.get(t, "/~demo/docs/", nil)
if got := e.body(t, resp); got != v1["docs/index.html"] {
t.Errorf("GET /~demo/docs/ = %q", got)
}
resp = e.get(t, "/~demo/nope.html", nil)
if resp.StatusCode != http.StatusNotFound {
t.Errorf("a missing file = %d, want 404", resp.StatusCode)
}
// --- the symlink is live
dir, err := os.Readlink(filepath.Join(e.cfg.Webroot, "~demo"))
if err != nil {
t.Fatalf("readlink ~demo: %v", err)
}
if !strings.HasSuffix(dir, res.Deployment.ID) {
t.Errorf("~demo -> %q, want the active deployment", dir)
}
onDisk, err := os.ReadFile(filepath.Join(dir, "index.html"))
if err != nil {
t.Fatalf("read through the symlink: %v", err)
}
if string(onDisk) != v1["index.html"] {
t.Errorf("the assembled tree holds %q", onDisk)
}
// --- a second deploy uploads only what changed
v2 := map[string]string{
"index.html": "<h1>v2</h1><script src=\"assets/app.js\"></script>",
"assets/app.js": v1["assets/app.js"],
"docs/index.html": v1["docs/index.html"],
}
res2 := e.deploy(t, ci, "demo", v2)
if res2.Uploaded != 1 || res2.Deduplicated != 2 {
t.Errorf("second deploy uploaded %d and reused %d; want 1 and 2",
res2.Uploaded, res2.Deduplicated)
}
if got := e.body(t, e.get(t, "/~demo/", nil)); got != v2["index.html"] {
t.Errorf("after the second deploy the site serves %q", got)
}
// --- rollback is one activation of the older deployment
if _, err := ci.Activate(t.Context(), "demo", res.Deployment.ID); err != nil {
t.Fatalf("rollback: %v", err)
}
if got := e.body(t, e.get(t, "/~demo/", nil)); got != v1["index.html"] {
t.Errorf("after the rollback the site serves %q", got)
}
dir, err = os.Readlink(filepath.Join(e.cfg.Webroot, "~demo"))
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(dir, res.Deployment.ID) {
t.Errorf("the rollback left ~demo -> %q", dir)
}
}
// TestRetentionAndRollback is M5's acceptance check, in the order the plan's
// manual walkthrough does it: deploy fifteen times, confirm that collection
// keeps ten plus the one being served and that the blob count falls, then roll
// back to an old deployment and see the old site.
//
// It drives the endpoints through the CLI's own client, so `pages deployment
// list`, `pages deployment activate` and `pages system gc` are covered by the
// same run.
func TestRetentionAndRollback(t *testing.T) {
e := newTestenv(t)
ci := e.project(t, "demo")
// Ten to keep, and no retention grace: the grace exists so that a rollback
// target is not collected out from under an operator who is still deciding,
// and any real value for it would outlast the test.
ten, zero := 10, 0
if _, err := e.admin.PatchProject(t.Context(), "demo",
api.ProjectPatch{RetentionCount: &ten, RetentionGrace: &zero}); err != nil {
t.Fatalf("configure retention: %v", err)
}
// Likewise the blob grace, which protects content a request has resolved and
// is about to open.
e.app.deploy.BlobGrace = -time.Minute
const versions = 15
var deps []string
for i := 1; i <= versions; i++ {
body := fmt.Sprintf("<h1>v%d</h1>", i)
res := e.deploy(t, ci, "demo", map[string]string{
"index.html": body,
// Shared by every version, so the blob count below is a statement
// about content that fell out of use, not about content churning.
"assets/app.js": "console.log(1)",
})
deps = append(deps, res.Deployment.ID)
}
if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v15</h1>" {
t.Fatalf("after %d deploys the site serves %q", versions, got)
}
// --- roll back to the first deployment
//
// Deliberately the oldest one: it is also the one retention would drop
// first, so this proves the active deployment is exempt rather than merely
// young enough to survive.
if _, err := ci.Activate(t.Context(), "demo", deps[0]); err != nil {
t.Fatalf("rollback: %v", err)
}
if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v1</h1>" {
t.Errorf("after the rollback the site serves %q, want the old version", got)
}
link, err := os.Readlink(filepath.Join(e.cfg.Webroot, "~demo"))
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(link, deps[0]) {
t.Errorf("~demo -> %q, want the rolled-back deployment", link)
}
before, err := e.admin.SystemInfo(t.Context())
if err != nil {
t.Fatal(err)
}
if before.Deployments != versions {
t.Fatalf("%d deployments before collection, want %d", before.Deployments, versions)
}
// --- collect
stats, err := e.admin.Collect(t.Context(), false)
if err != nil {
t.Fatalf("gc: %v", err)
}
// Fifteen deployments, one of them active and therefore not considered at
// all, ten of the remaining fourteen kept by retention: four to delete.
if stats.DeploymentsDeleted != 4 {
t.Errorf("collected %d deployments, want 4", stats.DeploymentsDeleted)
}
if stats.BlobsDeleted != 4 || stats.BytesFreed == 0 {
t.Errorf("collected %+v, want the four index pages those deployments held", stats)
}
after, err := e.admin.SystemInfo(t.Context())
if err != nil {
t.Fatal(err)
}
if after.Deployments != 11 {
t.Errorf("%d deployments after collection, want the 10 kept plus the active one", after.Deployments)
}
if after.Blobs >= before.Blobs {
t.Errorf("blobs went from %d to %d, want the count to fall", before.Blobs, after.Blobs)
}
if after.Blobs != 12 {
// Eleven surviving index pages plus the script every version shares.
t.Errorf("%d blobs after collection, want 12", after.Blobs)
}
// The survivors are the active one and the ten newest, and a listing says so.
list, err := ci.ListDeployments(t.Context(), "demo", client.DeploymentListOptions{})
if err != nil {
t.Fatalf("list: %v", err)
}
live := make(map[string]bool, len(list.Deployments))
for _, d := range list.Deployments {
live[d.ID] = true
if d.Active != (d.ID == deps[0]) {
t.Errorf("deployment %s: active = %v", d.ID, d.Active)
}
}
if len(live) != 11 {
t.Fatalf("%d deployments listed, want 11", len(live))
}
for i, id := range deps {
want := i == 0 || i >= versions-10
if live[id] != want {
t.Errorf("v%d (%s): present = %v, want %v", i+1, id, live[id], want)
}
}
// The site is still what the rollback made it, and the content of a
// collected deployment is what was reclaimed — not the active one's.
if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v1</h1>" {
t.Errorf("after collection the site serves %q", got)
}
if got := e.body(t, e.get(t, "/~demo/assets/app.js", nil)); got != "console.log(1)" {
t.Errorf("the shared script is now %q", got)
}
// A collected deployment is gone from the API too, and a second pass has
// nothing left to do.
if _, err := ci.GetDeployment(t.Context(), "demo", deps[1], false); err == nil {
t.Errorf("deployment v2 (%s) still readable after collection", deps[1])
}
again, err := e.admin.Collect(t.Context(), false)
if err != nil {
t.Fatal(err)
}
if again.DeploymentsDeleted != 0 || again.BlobsDeleted != 0 {
t.Errorf("a second pass collected %+v", again)
}
// --- deleting on request
//
// The one being served is refused; another one goes immediately.
if err := ci.DeleteDeployment(t.Context(), "demo", deps[0]); err == nil {
t.Error("the active deployment was deleted on request")
} else if code := api.CodeOf(err); code != api.CodeDeploymentActive {
t.Errorf("delete active: code = %q, want %q", code, api.CodeDeploymentActive)
}
spare := deps[versions-2]
if err := ci.DeleteDeployment(t.Context(), "demo", spare); err != nil {
t.Fatalf("delete %s: %v", spare, err)
}
if _, err := ci.GetDeployment(t.Context(), "demo", spare, false); err == nil {
t.Errorf("deployment %s still readable after being deleted", spare)
}
if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v1</h1>" {
t.Errorf("deleting a spare disturbed the site: %q", got)
}
// Reference counts and manifests agree throughout: the collector trusts
// those counters, and a count that reads low is the one way this design can
// lose content a deployment still needs.
rep, err := e.admin.Fsck(t.Context(), false)
if err != nil {
t.Fatalf("fsck: %v", err)
}
if rep.DriftCount != 0 {
t.Errorf("fsck reports %d drifted blobs: %+v", rep.DriftCount, rep.Drift)
}
}
// A project that exists but has never activated anything is a different answer
// from one that does not exist: 503 says "come back", 404 says "wrong URL".
func TestServeBeforeAnythingIsDeployed(t *testing.T) {
e := newTestenv(t)
e.project(t, "demo")
if resp := e.get(t, "/~demo/", nil); resp.StatusCode != http.StatusServiceUnavailable {
t.Errorf("GET an undeployed project = %d, want 503", resp.StatusCode)
}
if resp := e.get(t, "/~ghost/", nil); resp.StatusCode != http.StatusNotFound {
t.Errorf("GET an unknown project = %d, want 404", resp.StatusCode)
}
if resp := e.get(t, "/", nil); resp.StatusCode != http.StatusNotFound {
t.Errorf("GET the site root = %d, want 404", resp.StatusCode)
}
}
// The management API and the site content are separate listeners on purpose:
// neither surface may answer for the other, whatever the request looks like.
func TestTheTwoListenersDoNotOverlap(t *testing.T) {
e := newTestenv(t)
ci := e.project(t, "demo")
e.deploy(t, ci, "demo", map[string]string{"index.html": "<h1>hi</h1>"})
// The site listener knows nothing about the API.
if resp := e.get(t, api.PathProjects(), nil); resp.StatusCode != http.StatusNotFound {
t.Errorf("the site listener answered %s with %d", api.PathProjects(), resp.StatusCode)
}
// And the API listener serves no content.
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, e.api.URL+"/~demo/", nil)
if err != nil {
t.Fatal(err)
}
resp, err := e.api.Client().Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("the API listener answered /~demo/ with %d", resp.StatusCode)
}
// Both carry the health probes, so either can be the one a load balancer
// targets.
for _, base := range []string{e.site.URL, e.api.URL} {
for _, path := range []string{"/healthz", "/readyz"} {
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, base+path, nil)
if err != nil {
t.Fatal(err)
}
resp, err := e.site.Client().Do(req)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("GET %s%s = %d", base, path, resp.StatusCode)
}
}
}
}
// A project key reaches its own project and nothing else, end to end.
func TestProjectKeysAreConfinedToTheirProject(t *testing.T) {
e := newTestenv(t)
mine := e.project(t, "mine")
theirs := e.project(t, "theirs")
e.deploy(t, theirs, "theirs", map[string]string{"index.html": "<h1>secret</h1>"})
if _, err := mine.CreateDeployment(t.Context(), "theirs", nil); err == nil {
t.Fatal("a project key created a deployment in another project")
}
if _, err := mine.GetProject(t.Context(), "theirs"); err == nil {
t.Fatal("a project key read another project")
}
// Path routing means both sites share an origin, which is exactly the
// property documented as the reason not to host mutually untrusting
// projects this way. Reading a neighbour's content over HTTP is expected;
// reaching its management API is not.
if resp := e.get(t, "/~theirs/", nil); resp.StatusCode != http.StatusOK {
t.Errorf("GET the neighbour's site = %d", resp.StatusCode)
}
}
// The one assertion that has to hold no matter what else is logged: no part of
// a token ever reaches the log, on either listener.
func TestTokensNeverReachTheLog(t *testing.T) {
e := newTestenv(t)
ci := e.project(t, "demo")
e.deploy(t, ci, "demo", map[string]string{"index.html": "<h1>hi</h1>"})
// A bad token in every place a client could put one.
bad := "pgs_aaaaaaaaaaaaaaaa_" + strings.Repeat("b", 43)
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
e.api.URL+api.PathProjects()+"?token="+bad, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", "Bearer "+bad)
resp, err := e.api.Client().Do(req)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
logged := e.logBuf.String()
for _, secret := range []string{bad, strings.Repeat("b", 43), "aaaaaaaaaaaaaaaa"} {
if strings.Contains(logged, secret) {
t.Fatalf("the log contains %q", secret)
}
}
if strings.Contains(strings.ToLower(logged), "authorization") {
t.Fatal("the log mentions the Authorization header")
}
// The log is not empty, so the assertions above mean something.
if !strings.Contains(logged, "deployment activated") {
t.Fatal("nothing was logged at all; the assertions above prove nothing")
}
}
// Restarting the process must serve exactly what it served before: the registry
// is rebuilt from the database, which is the reason activation commits there
// before it stores the pointer.
func TestRestartServesTheSameDeployment(t *testing.T) {
e := newTestenv(t)
ci := e.project(t, "demo")
res := e.deploy(t, ci, "demo", map[string]string{
"index.html": "<h1>v1</h1>",
"assets/app.js": "console.log(1)",
})
if got := e.body(t, e.get(t, "/~demo/", nil)); got != "<h1>v1</h1>" {
t.Fatalf("before restart: %q", got)
}
// Tear the process down and build a second one on the same directories.
e.site.Close()
e.api.Close()
e.app.close()
var buf bytes.Buffer
log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
a2, err := newApp(context.Background(), e.cfg, log)
if err != nil {
t.Fatalf("second newApp: %v", err)
}
t.Cleanup(a2.close)
a2.ready.Store(true)
e.app, e.logBuf = a2, &buf
e.site = httptest.NewServer(a2.siteHandler())
t.Cleanup(e.site.Close)
e.api = httptest.NewServer(a2.apiHandler())
t.Cleanup(e.api.Close)
e.admin = e.client(t, e.adminToken)
resp := e.get(t, "/~demo/", nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("after restart: status = %d", resp.StatusCode)
}
if got := e.body(t, resp); got != "<h1>v1</h1>" {
t.Errorf("after restart the site serves %q", got)
}
if got := e.body(t, e.get(t, "/~demo/assets/app.js", nil)); got != "console.log(1)" {
t.Errorf("after restart the script is %q", got)
}
// And the second process did not mint a second bootstrap admin key.
keys, err := e.admin.ListKeys(t.Context())
if err != nil {
t.Fatal(err)
}
admins := 0
for _, k := range keys.Keys {
if k.Scope == string(api.ScopeAdmin) && k.RevokedAt == nil {
admins++
}
}
if admins != 1 {
t.Errorf("%d admin keys after a restart, want 1", admins)
}
// The database is the source of truth the registry was rebuilt from, so the
// deployment being served is the same row, not merely the same bytes.
p, err := e.admin.GetProject(t.Context(), "demo")
if err != nil {
t.Fatal(err)
}
if p.ActiveDeployment == nil || p.ActiveDeployment.ID != res.Deployment.ID {
t.Errorf("active deployment after restart = %+v, want %s", p.ActiveDeployment, res.Deployment.ID)
}
}
+406
View File
@@ -0,0 +1,406 @@
// Command pages-server serves static sites deployed through the pages CLI and
// exposes the management API used to drive those deployments.
//
// It binds two listeners: a public one that only ever serves site content, and a
// management one (loopback by default) that only ever serves the API. Keeping
// them apart means the management surface never shares an origin with content
// that projects control.
package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"os/signal"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/iceBear67/simplepages/internal/adminapi"
"github.com/iceBear67/simplepages/internal/auth"
"github.com/iceBear67/simplepages/internal/cas"
"github.com/iceBear67/simplepages/internal/config"
"github.com/iceBear67/simplepages/internal/deploy"
"github.com/iceBear67/simplepages/internal/httpx"
"github.com/iceBear67/simplepages/internal/site"
"github.com/iceBear67/simplepages/internal/store"
"github.com/iceBear67/simplepages/internal/version"
"github.com/iceBear67/simplepages/internal/webroot"
)
// Background worker cadences. The auth flusher batches last_used_at updates;
// writing one per request would funnel every authenticated read through the
// single write connection.
const (
touchFlushInterval = 60 * time.Second
// failedAuthBurst is how many failed authentications one client address may
// make before it is throttled, and failedAuthPeriod is how long a full
// budget takes to refill. Successful requests cost nothing, so a busy CI
// fleet never meets these numbers.
failedAuthBurst = 10
failedAuthPeriod = time.Minute
failedAuthClients = 10000
)
func main() {
if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil {
if errors.Is(err, flag.ErrHelp) {
os.Exit(2)
}
fmt.Fprintf(os.Stderr, "pages-server: %v\n", err)
os.Exit(1)
}
}
func run(args []string, stdout, stderr io.Writer) error {
opts, err := config.Load(args, stderr)
if err != nil {
return err
}
if opts.ShowVersion {
fmt.Fprintf(stdout, "pages-server %s\n", version.String())
return nil
}
if opts.CheckOnly {
fmt.Fprintln(stdout, "configuration ok")
return nil
}
cfg := opts.Config
log := cfg.Logger(stderr)
log.Info("starting", "version", version.Short(),
"data_dir", cfg.DataDir, "webroot", cfg.Webroot,
"assemble_mode", string(cfg.AssembleMode))
if err := cfg.EnsureDirs(); err != nil {
return err
}
// The first signal starts a graceful shutdown; a second one gives up on the
// in-flight requests, which is what an operator means by pressing Ctrl-C
// twice.
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
go func() {
<-ctx.Done()
hard := make(chan os.Signal, 1)
signal.Notify(hard, os.Interrupt, syscall.SIGTERM)
<-hard
fmt.Fprintln(stderr, "pages-server: second signal, exiting immediately")
os.Exit(130)
}()
app, err := newApp(ctx, cfg, log)
if err != nil {
return err
}
defer app.close()
// Background workers get their own cancellable context so they are stopped
// *after* the listeners have drained: the auth flusher's final write should
// include the last requests the server handled.
workerCtx, stopWorkers := context.WithCancel(context.Background())
var workers sync.WaitGroup
// Registered after the app's own cleanup so it runs before it: the workers
// must be finished with the database before anything closes it.
defer func() {
stopWorkers()
workers.Wait()
}()
for _, worker := range []func(context.Context){
func(ctx context.Context) { app.verifier.RunFlusher(ctx, touchFlushInterval) },
func(ctx context.Context) { app.deploy.RunReconciler(ctx, cfg.ReconcileInterval.D()) },
func(ctx context.Context) { app.deploy.RunCollector(ctx, cfg.GCInterval.D()) },
} {
workers.Add(1)
go func() {
defer workers.Done()
worker(workerCtx)
}()
}
siteSrv, err := httpx.Listen("site", cfg.Listen, app.siteHandler(), httpx.Timeouts{
ReadHeader: cfg.ReadHeaderTimeout.D(),
Read: cfg.ReadTimeout.D(),
Idle: cfg.IdleTimeout.D(),
// No Write timeout: see httpx.Timeouts.
}, log)
if err != nil {
return fmt.Errorf("listen %s: %w", cfg.Listen, err)
}
apiSrv, err := httpx.Listen("api", cfg.APIListen, app.apiHandler(), httpx.Timeouts{
ReadHeader: cfg.ReadHeaderTimeout.D(),
Read: cfg.ReadTimeout.D(),
Idle: cfg.IdleTimeout.D(),
}, log)
if err != nil {
return fmt.Errorf("listen %s: %w", cfg.APIListen, err)
}
app.ready.Store(true)
g := &httpx.Group{
Servers: []*httpx.Server{siteSrv, apiSrv},
Grace: cfg.ShutdownGrace.D(),
Log: log,
}
if err := g.Run(ctx); err != nil {
return err
}
log.Info("stopped")
return nil
}
// app holds the process-wide state shared by both listeners.
type app struct {
cfg config.Config
log *slog.Logger
started time.Time
db *store.DB
cas *cas.Store
sites *site.Registry
webroot *webroot.Webroot
deploy *deploy.Service
verifier *auth.Verifier
authmw *auth.Middleware
admin *adminapi.Server
// ready gates /readyz: the process may be accepting connections before it
// can actually answer for content, and a load balancer needs to know.
ready atomic.Bool
}
func newApp(ctx context.Context, cfg config.Config, log *slog.Logger) (*app, error) {
db, err := store.Open(ctx, cfg.DBPath(), log)
if err != nil {
return nil, fmt.Errorf("open store: %w", err)
}
if _, err := auth.EnsureAdminKey(ctx, db, cfg.BootstrapTokenPath(), log); err != nil {
db.Close()
return nil, err
}
// The probe directory must be the one trees are assembled in: hardlinks
// cannot cross filesystems, so probing anywhere else answers a different
// question. With assemble_mode=none nothing is assembled and the mode is
// irrelevant, so ask for copy rather than probe for something unused.
casOpts := cas.Options{ProbeDir: cfg.DeploymentsDir(), Log: log}
switch cfg.AssembleMode {
case config.AssembleHardlink:
casOpts.Mode = cas.LinkHard
case config.AssembleCopy, config.AssembleNone:
casOpts.Mode = cas.LinkCopy
}
cs, err := cas.Open(cfg.CASDir(), casOpts)
if err != nil {
db.Close()
return nil, err
}
verifier := auth.NewVerifier(db, log, auth.DefaultCacheTTL)
// An empty Dir is how the service is told to skip on-disk assembly.
deployDir := cfg.DeploymentsDir()
if cfg.AssembleMode == config.AssembleNone {
deployDir = ""
}
// With nothing assembled on disk there is nothing for a symlink to point at,
// so assemble_mode=none leaves the webroot unmanaged. Failing to open it
// otherwise is a permissions or layout problem the operator asked for and
// should hear about now, rather than as a warning on every activation.
var wr *webroot.Webroot
if deployDir != "" && cfg.Webroot != "" {
wr, err = webroot.Open(cfg.Webroot, deployDir)
if err != nil {
cs.Close()
db.Close()
return nil, err
}
}
a := &app{
cfg: cfg,
log: log,
started: time.Now(),
db: db,
cas: cs,
sites: site.NewRegistry(),
webroot: wr,
verifier: verifier,
authmw: &auth.Middleware{
V: verifier,
Limiter: auth.NewLimiter(failedAuthBurst, failedAuthPeriod, failedAuthClients),
Trusted: cfg.TrustedProxies(),
Log: log,
},
}
a.deploy = &deploy.Service{
DB: db, CAS: cs, Log: log, Dir: deployDir,
Sites: a.sites, Webroot: wr,
}
a.admin = &adminapi.Server{
DB: db,
Auth: a.authmw,
Deploy: a.deploy,
Log: log,
Limits: cfg.Limits,
// The registry answers ownership checks and "what is this project
// serving" from memory. Both are only correct once LoadSites below has
// finished, which is why it runs before any listener exists.
Resolver: a.sites,
Sites: a.sites,
BaseURL: cfg.SiteURL,
LinkMode: string(cs.LinkMode()),
Started: a.started,
}
a.admin.Hooks = adminapi.Hooks{
ProjectChanged: func(_ context.Context, p *store.Project) {
// A new project starts with nothing activated, so it resolves and
// then answers 503 until something is deployed to it. A changed one
// keeps serving what it was serving, with new settings.
a.sites.Put(p)
},
ProjectDeleted: func(ctx context.Context, p *store.Project) {
a.sites.Delete(p.Name)
if wr != nil {
if err := wr.Unpoint(p.Name); err != nil {
log.WarnContext(ctx, "could not remove the webroot symlink",
"project", p.Name, "err", err)
}
}
// The rows are already gone and their blobs are already
// unreferenced; this is only the assembled trees, which nothing
// would otherwise account for. A failure here is not worth failing
// the request over — the startup sweep removes them as orphans.
if err := a.deploy.RemoveProjectTrees(p.ID); err != nil {
log.WarnContext(ctx, "could not remove the project's deployment trees",
"project", p.Name, "err", err)
}
},
}
// Recovery runs before the registry is built, and both run before any
// listener exists. That order is what lets recovery assume it is alone with
// the data directory, and it means the first request is answered from state
// that has already been reconciled rather than from whatever the last crash
// left behind.
if err := a.deploy.Recover(ctx); err != nil {
a.close()
return nil, fmt.Errorf("recover: %w", err)
}
if err := a.deploy.LoadSites(ctx); err != nil {
a.close()
return nil, fmt.Errorf("load sites: %w", err)
}
return a, nil
}
// pingDB checks that the read pool can still reach the database, with a
// deadline of its own so a wedged store cannot hold a probe open indefinitely.
func (a *app) pingDB(ctx context.Context) error {
if a.db == nil {
return errors.New("store not open")
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
return a.db.Reader().PingContext(ctx)
}
func (a *app) close() {
if a.cas != nil {
if err := a.cas.Close(); err != nil {
a.log.Error("closing content store", "err", err)
}
}
if a.db != nil {
if err := a.db.Close(); err != nil {
a.log.Error("closing store", "err", err)
}
}
}
// middleware is the chain shared by both listeners, outermost first.
//
// The order is load-bearing. Recover must sit *inside* AccessLog: a panic that
// unwound past AccessLog would skip its logging entirely, so the one request
// that most needs a log line would be the one without one. With Recover
// innermost the panic becomes an ordinary 500 return that AccessLog then records
// normally, and Recover can see the shared request id and the recorder that
// tells it whether a response has already begun. Panics in the middleware
// itself are left to net/http, which closes the connection.
func (a *app) middleware() []httpx.Middleware {
return []httpx.Middleware{
httpx.WithRequestID(a.cfg.TrustedProxies()),
httpx.AccessLog(a.log, a.cfg.TrustedProxies()),
httpx.Recover(a.log),
}
}
func (a *app) siteHandler() http.Handler {
mux := http.NewServeMux()
a.registerHealth(mux)
// Site routing is hand-parsed rather than expressed as a ServeMux pattern:
// "/~{project}/{path...}" is rejected by net/http, whose wildcards must start
// at the beginning of a path segment. Registering "/" still gets us the mux's
// built-in ".."/"//" normalisation redirects.
//
// Those redirects are a convenience, not a defence: a percent-encoded
// "/~a/%2e%2e/%2e%2e/etc/passwd" reaches the handler with r.URL.Path already
// decoded to "/~a/../../etc/passwd" and no redirect issued (verified against
// this server). The resolver therefore does its own path.Clean and validation
// rather than assume the mux normalised anything.
mux.Handle("/", &site.Handler{Registry: a.sites, CAS: a.cas, Log: a.log})
return httpx.Chain(mux, a.middleware()...)
}
func (a *app) apiHandler() http.Handler {
mux := http.NewServeMux()
a.registerHealth(mux)
a.admin.Register(mux)
return httpx.Chain(mux, a.middleware()...)
}
// registerHealth adds the probe endpoints to a mux. They live on both listeners
// so a probe can target whichever one the deployment exposes.
func (a *app) registerHealth(mux *http.ServeMux) {
// Liveness: answers as long as the process can schedule a goroutine. It must
// never touch the database, or a slow query would get the process killed.
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprintln(w, "ok")
})
// Readiness: reports whether this process can serve real traffic yet.
mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) {
status := http.StatusOK
body := map[string]any{
"status": "ready",
"version": version.Short(),
"uptime_s": int64(time.Since(a.started).Seconds()),
}
if !a.ready.Load() {
status = http.StatusServiceUnavailable
body["status"] = "starting"
} else if err := a.pingDB(r.Context()); err != nil {
// Unlike /healthz this may touch the database: a process that
// cannot read its own store should be taken out of rotation, not
// restarted.
a.log.WarnContext(r.Context(), "readiness probe: store unreachable", "err", err)
status = http.StatusServiceUnavailable
body["status"] = "degraded"
}
w.Header().Set("Cache-Control", "no-store")
httpx.WriteJSON(w, status, body)
})
}
+78
View File
@@ -0,0 +1,78 @@
package main
import (
"errors"
"os/exec"
"strings"
"testing"
)
// forbidden names packages that must never reach the CLI binary, with the
// reason, because a future contributor will hit this test and need to know
// which of the two possible fixes applies: move the code, or widen the CLI's
// remit deliberately.
var forbidden = map[string]string{
"modernc.org/sqlite": "the SQLite driver is megabytes of translated C; " +
"the CLI must never link a database",
"modernc.org/libc": "pulled in by the SQLite driver",
"database/sql": "a probe: nothing the CLI does needs a database, so if this " +
"appears, a server package leaked in through an import",
"github.com/BurntSushi/toml": "server configuration only; the CLI's config " +
"file is JSON precisely so this stays out",
"github.com/iceBear67/simplepages/internal/store": "server-side storage",
"github.com/iceBear67/simplepages/internal/site": "server-side serving",
"github.com/iceBear67/simplepages/internal/deploy": "server-side deployment lifecycle",
"github.com/iceBear67/simplepages/internal/cas": "server-side content store",
"github.com/iceBear67/simplepages/internal/auth": "server-side verification; the CLI only carries a token",
"github.com/iceBear67/simplepages/internal/adminapi": "server-side handlers; the wire types " +
"the CLI needs live in api/",
"net/http/httptest": "test-only helper that would bloat the shipped binary",
}
// TestCLIImportGraph is the mechanism behind the "two binaries" decision: the
// CLI is downloaded on every CI run, so its size and its build requirements
// (no cgo, no C toolchain) are user-visible properties. A stray import is easy
// to add and invisible until someone notices the binary doubled, so the rule is
// enforced here rather than written down.
func TestCLIImportGraph(t *testing.T) {
for _, pkg := range depsOf(t, "./cmd/pages") {
if why, bad := forbidden[pkg]; bad {
t.Errorf("cmd/pages depends on %s\n %s", pkg, why)
}
}
}
// TestAPIPackageIsStdlibOnly guards the other half of the arrangement: api/ is
// imported by both binaries, so anything it depends on is automatically part of
// the CLI. Keeping it to the standard library is what lets the server and the
// client share wire types at all.
func TestAPIPackageIsStdlibOnly(t *testing.T) {
const self = "github.com/iceBear67/simplepages/api"
for _, pkg := range depsOf(t, self) {
// go list -deps includes the package itself.
if pkg == self {
continue
}
// A standard library import path never has a dot in its first segment.
if first, _, _ := strings.Cut(pkg, "/"); strings.Contains(first, ".") {
t.Errorf("api imports %s, but it must depend on the standard library only", pkg)
}
}
}
func depsOf(t *testing.T, pkg string) []string {
t.Helper()
// The test's working directory is cmd/pages; a relative package pattern has
// to be resolved from the module root two levels up.
cmd := exec.Command("go", "list", "-deps", pkg)
cmd.Dir = "../.."
out, err := cmd.Output()
if err != nil {
var ee *exec.ExitError
if errors.As(err, &ee) {
t.Fatalf("go list -deps %s: %v\n%s", pkg, err, ee.Stderr)
}
t.Fatalf("go list -deps %s: %v", pkg, err)
}
return strings.Fields(string(out))
}
+86
View File
@@ -0,0 +1,86 @@
// Command pages is the client for a pages-server installation.
//
// It is a plain HTTP client on purpose: no SQLite, no server packages, nothing
// that needs cgo. CI jobs download this binary on every run, so its size and
// its dependency graph are features — see deps_test.go, which enforces both.
package main
import (
"context"
"errors"
"fmt"
"os"
"os/signal"
"syscall"
"github.com/iceBear67/simplepages/api"
"github.com/iceBear67/simplepages/internal/clicmd"
"github.com/iceBear67/simplepages/internal/cliutil"
)
// Exit codes. A CI step distinguishes "the command was wrong" from "the server
// said no" without parsing messages.
const (
exitOK = 0
exitError = 1
exitUsage = 2
)
func main() {
// Ctrl-C cancels in-flight requests rather than killing the process
// mid-upload, which would leave a deployment stuck in "uploading" until it
// expires.
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
os.Exit(run(ctx, os.Args[1:]))
}
func run(ctx context.Context, args []string) int {
g := clicmd.NewGlobals()
root := clicmd.Root(g)
err := cliutil.Run(ctx, root, args, g.Err, g.Register)
switch {
case err == nil:
return exitOK
case errors.Is(err, cliutil.ErrUsage):
// Run has already printed the usage text. The bare sentinel carries no
// message of its own — flag or the command tree printed one.
if err != cliutil.ErrUsage {
fmt.Fprintf(g.Err, "pages: %v\n", err)
}
return exitUsage
case errors.Is(err, cliutil.ErrAborted):
fmt.Fprintln(g.Err, "pages: aborted")
return exitError
case errors.Is(err, context.Canceled):
fmt.Fprintln(g.Err, "pages: interrupted")
return exitError
default:
fmt.Fprintf(g.Err, "pages: %v\n", err)
hint(g, err)
return exitError
}
}
// hint turns the codes the server uses into the one sentence that usually
// resolves the problem.
func hint(g *clicmd.Globals, err error) {
var apiErr *api.Error
if !errors.As(err, &apiErr) {
return
}
switch apiErr.Code {
case api.CodeUnauthorized:
fmt.Fprintln(g.Err, "hint: the token is missing, malformed, expired or revoked; check PAGES_TOKEN")
case api.CodeForbidden:
fmt.Fprintln(g.Err, "hint: this key is not allowed here; a project key can only act on its own project")
case api.CodeNotFound:
fmt.Fprintln(g.Err, "hint: check the name, and that the key you are using can see it")
}
}