From dd50674fdc48d22129a795dd3b3c4b8db3a337a0 Mon Sep 17 00:00:00 2001 From: iceBear67 Date: Sat, 15 Aug 2026 07:13:00 +0000 Subject: [PATCH] init --- AGENTS.md | 166 +++++ CLAUDE.md | 1 + Makefile | 40 + README.md | 153 ++++ api/errors.go | 84 +++ api/paths.go | 70 ++ api/types.go | 258 +++++++ cmd/pages-server/integration_test.go | 673 +++++++++++++++++ cmd/pages-server/main.go | 406 ++++++++++ cmd/pages/deps_test.go | 78 ++ cmd/pages/main.go | 86 +++ docs/operations.md | 478 ++++++++++++ go.mod | 18 + go.sum | 24 + internal/adminapi/activate_test.go | 307 ++++++++ internal/adminapi/adminapi_test.go | 252 +++++++ internal/adminapi/blobs.go | 39 + internal/adminapi/convert.go | 199 +++++ internal/adminapi/deployments.go | 377 ++++++++++ internal/adminapi/deployments_test.go | 589 +++++++++++++++ internal/adminapi/keys.go | 172 +++++ internal/adminapi/keys_test.go | 361 +++++++++ internal/adminapi/maintenance_test.go | 405 ++++++++++ internal/adminapi/projects.go | 169 +++++ internal/adminapi/projects_test.go | 439 +++++++++++ internal/adminapi/router_test.go | 141 ++++ internal/adminapi/server.go | 265 +++++++ internal/adminapi/system.go | 125 ++++ internal/adminapi/system_test.go | 101 +++ internal/adminapi/validate.go | 107 +++ internal/adminapi/validate_test.go | 114 +++ internal/auth/bootstrap.go | 110 +++ internal/auth/bootstrap_test.go | 285 +++++++ internal/auth/middleware.go | 216 ++++++ internal/auth/middleware_test.go | 407 ++++++++++ internal/auth/ratelimit.go | 118 +++ internal/auth/ratelimit_test.go | 176 +++++ internal/auth/token.go | 134 ++++ internal/auth/token_test.go | 135 ++++ internal/auth/verify.go | 247 ++++++ internal/auth/verify_test.go | 407 ++++++++++ internal/cache/cache.go | 179 +++++ internal/cache/cache_test.go | 223 ++++++ internal/cas/cas.go | 414 +++++++++++ internal/cas/cas_test.go | 498 +++++++++++++ internal/cas/cas_unix_test.go | 115 +++ internal/cas/digest.go | 88 +++ internal/clicmd/config.go | 118 +++ internal/clicmd/config_test.go | 136 ++++ internal/clicmd/deploy.go | 220 ++++++ internal/clicmd/deployment.go | 357 +++++++++ internal/clicmd/deployment_test.go | 326 ++++++++ internal/clicmd/globals.go | 216 ++++++ internal/clicmd/globals_test.go | 341 +++++++++ internal/clicmd/key.go | 208 ++++++ internal/clicmd/project.go | 278 +++++++ internal/clicmd/root.go | 272 +++++++ internal/client/calls.go | 210 ++++++ internal/client/client.go | 224 ++++++ internal/client/client_test.go | 363 +++++++++ internal/client/deploy.go | 492 ++++++++++++ internal/client/scan.go | 323 ++++++++ internal/cliutil/command.go | 237 ++++++ internal/cliutil/command_test.go | 197 +++++ internal/cliutil/input.go | 131 ++++ internal/cliutil/optflag.go | 169 +++++ internal/cliutil/output.go | 159 ++++ internal/config/config.go | 311 ++++++++ internal/config/config_test.go | 266 +++++++ internal/config/load.go | 200 +++++ internal/deploy/assemble.go | 147 ++++ internal/deploy/assemble_test.go | 228 ++++++ internal/deploy/atomicity_test.go | 454 +++++++++++ internal/deploy/gc.go | 259 +++++++ internal/deploy/gc_test.go | 617 +++++++++++++++ internal/deploy/recover.go | 213 ++++++ internal/deploy/service.go | 311 ++++++++ internal/deploy/service_test.go | 468 ++++++++++++ internal/deploy/sites.go | 127 ++++ internal/httpx/httpx_test.go | 287 +++++++ internal/httpx/json.go | 127 ++++ internal/httpx/middleware.go | 332 +++++++++ internal/httpx/server.go | 143 ++++ internal/pathutil/pathutil.go | 171 +++++ internal/pathutil/pathutil_test.go | 180 +++++ internal/site/deployment.go | 115 +++ internal/site/registry.go | 176 +++++ internal/site/registry_test.go | 149 ++++ internal/site/resolve.go | 129 ++++ internal/site/resolve_test.go | 445 +++++++++++ internal/site/route.go | 36 + internal/site/route_test.go | 56 ++ internal/site/serve.go | 143 ++++ internal/site/serve_test.go | 377 ++++++++++ .../fuzz/FuzzResolve/dee8cb812bd7d77f | 3 + internal/store/blobs.go | 94 +++ internal/store/db.go | 244 ++++++ internal/store/deployments.go | 453 +++++++++++ internal/store/deployments_test.go | 703 ++++++++++++++++++ internal/store/errors.go | 73 ++ internal/store/fsck.go | 117 +++ internal/store/keys.go | 181 +++++ internal/store/keys_test.go | 329 ++++++++ internal/store/maintenance.go | 344 +++++++++ internal/store/maintenance_test.go | 623 ++++++++++++++++ internal/store/migrate.go | 182 +++++ internal/store/migrations/0001_init.sql | 109 +++ internal/store/projects.go | 230 ++++++ internal/store/projects_test.go | 285 +++++++ internal/store/stats.go | 30 + internal/store/store_test.go | 301 ++++++++ internal/version/version.go | 69 ++ internal/webroot/webroot.go | 207 ++++++ internal/webroot/webroot_test.go | 395 ++++++++++ 114 files changed, 26865 insertions(+) create mode 100644 AGENTS.md create mode 120000 CLAUDE.md create mode 100644 Makefile create mode 100644 README.md create mode 100644 api/errors.go create mode 100644 api/paths.go create mode 100644 api/types.go create mode 100644 cmd/pages-server/integration_test.go create mode 100644 cmd/pages-server/main.go create mode 100644 cmd/pages/deps_test.go create mode 100644 cmd/pages/main.go create mode 100644 docs/operations.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/adminapi/activate_test.go create mode 100644 internal/adminapi/adminapi_test.go create mode 100644 internal/adminapi/blobs.go create mode 100644 internal/adminapi/convert.go create mode 100644 internal/adminapi/deployments.go create mode 100644 internal/adminapi/deployments_test.go create mode 100644 internal/adminapi/keys.go create mode 100644 internal/adminapi/keys_test.go create mode 100644 internal/adminapi/maintenance_test.go create mode 100644 internal/adminapi/projects.go create mode 100644 internal/adminapi/projects_test.go create mode 100644 internal/adminapi/router_test.go create mode 100644 internal/adminapi/server.go create mode 100644 internal/adminapi/system.go create mode 100644 internal/adminapi/system_test.go create mode 100644 internal/adminapi/validate.go create mode 100644 internal/adminapi/validate_test.go create mode 100644 internal/auth/bootstrap.go create mode 100644 internal/auth/bootstrap_test.go create mode 100644 internal/auth/middleware.go create mode 100644 internal/auth/middleware_test.go create mode 100644 internal/auth/ratelimit.go create mode 100644 internal/auth/ratelimit_test.go create mode 100644 internal/auth/token.go create mode 100644 internal/auth/token_test.go create mode 100644 internal/auth/verify.go create mode 100644 internal/auth/verify_test.go create mode 100644 internal/cache/cache.go create mode 100644 internal/cache/cache_test.go create mode 100644 internal/cas/cas.go create mode 100644 internal/cas/cas_test.go create mode 100644 internal/cas/cas_unix_test.go create mode 100644 internal/cas/digest.go create mode 100644 internal/clicmd/config.go create mode 100644 internal/clicmd/config_test.go create mode 100644 internal/clicmd/deploy.go create mode 100644 internal/clicmd/deployment.go create mode 100644 internal/clicmd/deployment_test.go create mode 100644 internal/clicmd/globals.go create mode 100644 internal/clicmd/globals_test.go create mode 100644 internal/clicmd/key.go create mode 100644 internal/clicmd/project.go create mode 100644 internal/clicmd/root.go create mode 100644 internal/client/calls.go create mode 100644 internal/client/client.go create mode 100644 internal/client/client_test.go create mode 100644 internal/client/deploy.go create mode 100644 internal/client/scan.go create mode 100644 internal/cliutil/command.go create mode 100644 internal/cliutil/command_test.go create mode 100644 internal/cliutil/input.go create mode 100644 internal/cliutil/optflag.go create mode 100644 internal/cliutil/output.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/config/load.go create mode 100644 internal/deploy/assemble.go create mode 100644 internal/deploy/assemble_test.go create mode 100644 internal/deploy/atomicity_test.go create mode 100644 internal/deploy/gc.go create mode 100644 internal/deploy/gc_test.go create mode 100644 internal/deploy/recover.go create mode 100644 internal/deploy/service.go create mode 100644 internal/deploy/service_test.go create mode 100644 internal/deploy/sites.go create mode 100644 internal/httpx/httpx_test.go create mode 100644 internal/httpx/json.go create mode 100644 internal/httpx/middleware.go create mode 100644 internal/httpx/server.go create mode 100644 internal/pathutil/pathutil.go create mode 100644 internal/pathutil/pathutil_test.go create mode 100644 internal/site/deployment.go create mode 100644 internal/site/registry.go create mode 100644 internal/site/registry_test.go create mode 100644 internal/site/resolve.go create mode 100644 internal/site/resolve_test.go create mode 100644 internal/site/route.go create mode 100644 internal/site/route_test.go create mode 100644 internal/site/serve.go create mode 100644 internal/site/serve_test.go create mode 100644 internal/site/testdata/fuzz/FuzzResolve/dee8cb812bd7d77f create mode 100644 internal/store/blobs.go create mode 100644 internal/store/db.go create mode 100644 internal/store/deployments.go create mode 100644 internal/store/deployments_test.go create mode 100644 internal/store/errors.go create mode 100644 internal/store/fsck.go create mode 100644 internal/store/keys.go create mode 100644 internal/store/keys_test.go create mode 100644 internal/store/maintenance.go create mode 100644 internal/store/maintenance_test.go create mode 100644 internal/store/migrate.go create mode 100644 internal/store/migrations/0001_init.sql create mode 100644 internal/store/projects.go create mode 100644 internal/store/projects_test.go create mode 100644 internal/store/stats.go create mode 100644 internal/store/store_test.go create mode 100644 internal/version/version.go create mode 100644 internal/webroot/webroot.go create mode 100644 internal/webroot/webroot_test.go diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..109d65f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,166 @@ +# Working on simplepages + +Read [`README.md`](README.md) first for what this is, and +[`docs/operations.md`](docs/operations.md) for how it is run. This file is for +whoever — human or agent — is changing the code. + +Everything here exists to protect one property: **a request sees the whole old +deployment or the whole new one, never a mixture.** If a change makes that +property harder to reason about, it is the wrong change however much code it +saves. + +## Before you call a change done + +```sh +make fmt vet test +make race +make atomicity # -race -count=20, takes ~80s +go test ./cmd/pages -run TestCLIImportGraph +``` + +Run the last two whenever you touch `internal/site`, `internal/deploy`, or +anything the CLI imports. `make race` is not optional for changes in those +packages — the interesting failures here are all interleavings. + +## Layout + +| Path | | +|---|---| +| `api/` | Wire types, error codes, path builders. **Stdlib only** — it is shared with the CLI, and that is the whole reason the CLI stays dependency-free. | +| `cmd/pages-server/` | Assembly: config, storage, recovery, listeners, background workers, signals. Also holds the end-to-end tests. | +| `cmd/pages/` | CLI entry point plus `deps_test.go`, which enforces the import graph. | +| `internal/store/` | SQLite: dual pool, migrations, queries, fsck. | +| `internal/cas/` | Content-addressed blob store. | +| `internal/site/` | ★ The read path: registry, immutable deployment snapshots, URL resolution, the HTTP handler. | +| `internal/deploy/` | Deployment lifecycle, assembly, startup recovery, reconciler, GC. | +| `internal/webroot/` | `$WEBROOT/~name` symlink maintenance. | +| `internal/auth/` | Token mint/parse/verify, middleware, failed-auth rate limiting. | +| `internal/adminapi/` | Management API routes and handlers. | +| `internal/config/`, `internal/httpx/`, `internal/cache/`, `internal/pathutil/`, `internal/version/` | Support. | +| `internal/client/`, `internal/clicmd/`, `internal/cliutil/` | CLI: HTTP client and deploy flow, commands, command tree and rendering. | + +## Invariants + +Each of these is load-bearing. Changing one is a design decision, not a +refactor. + +**1. Load `p.Active()` exactly once per request.** `internal/site/serve.go` +takes the snapshot at the top and every later access uses that value. A second +`Active()` call in the same request is the bug this whole design prevents: the +two loads could straddle an activation and serve a mixed page. `Deployment` is +immutable after construction, so holding the pointer is free and correct. + +**2. Activation order is fixed.** Project lock → require `ready` → build the +snapshot → **commit to SQLite** → `atomic.Pointer.Store` → best-effort +`webroot.Point`. Database before memory, so a crash between the two restarts +into the state the database already recorded. The reverse order leaves a +process serving A while the database says B. Building the snapshot before the +commit means a failure there changes nothing at all. + +**3. Never trust a client-supplied hash.** `cas.Store.Put` rehashes the received +stream and compares against the claimed digest before linking the blob into +place; a mismatch is a 400 and the temp file is discarded. Without this a client +could claim another project's digest, upload arbitrary bytes, and poison that +blob for every project referencing it. This check is what makes cross-project +dedup safe — do not move it, skip it for "already known" digests, or trust a +digest because the transport was TLS. + +**4. Tokens never appear in logs, query strings, or error messages.** There is a +test that runs a request through the middleware into a buffer and greps for the +token; keep it passing. The CLI prefers `PAGES_TOKEN` and `--token-file` over +`--token` because argv is world-readable via `/proc//cmdline` on a shared +runner, and the help text says so. The bootstrap token file and the CLI config +file are 0600. + +**5. All writes go through `db.Tx`.** `store.DB` exposes `Reader()` for queries +and `Tx(ctx, fn)` for everything else. There is deliberately no `Writer()` — the +single write connection plus `BEGIN IMMEDIATE` plus busy retries is the only +thing keeping `SQLITE_BUSY` off the hot path. Tests write through `db.Tx` too. + +**6. Upload paths pass both `fs.ValidPath` and `filepath.Localize`,** and are +additionally rejected for NUL or control bytes, a segment over 255 bytes, a +total over 4096, duplicates, and case-insensitive collisions (the assembled tree +may land on a case-insensitive filesystem, where a collision silently +overwrites). One check is not enough: `fs.ValidPath` allows backslashes and +Windows reserved names, `Localize` allows things `ValidPath` rejects. + +**7. The read path serves from the CAS by digest.** The only filesystem path +constructed while serving is `cas/ab/cd/<64 hex>`, derived from a `[32]byte` +that came out of a map lookup. No user-controlled string reaches the filesystem, +so read-path traversal is not defended against — it is structurally impossible. +Serving from the assembled directory instead would give that back. + +**8. No archive extraction, ever.** Files arrive one at a time; the server never +unpacks or decompresses anything. That is what makes zip bombs and tar-slip +inapplicable rather than mitigated. "Just accept a tarball, it's fewer round +trips" reopens both — it needs a fresh security review, not a patch. + +**9. `RequireProject` compares resolved project IDs,** never the name string +from the URL. Names can be reused after a delete; IDs cannot. Same rule for +deployments: confirm `{id}` really belongs to `{name}` before changing anything. + +**10. `webroot.Reconcile` only removes entries that are symlinks pointing inside +`$DATA_DIR/deployments`.** `$WEBROOT` belongs to the operator and will contain +files that are none of our business. + +**11. Assembled files and blobs are 0444, directories 0755, and client-supplied +modes are ignored.** There is no `mode` column in the schema on purpose, so a +setuid bit has nowhere to be stored even if someone tries to send one. On a +hardlinking filesystem the assembled file *is* the blob's inode — writing to a +file under `$WEBROOT` corrupts it for every project that references it. + +**12. The CLI's dependency graph is enforced, not documented.** +`cmd/pages/deps_test.go` fails if `go list -deps ./cmd/pages` reaches +`modernc.org/sqlite`, `modernc.org/libc`, `database/sql`, `github.com/BurntSushi/toml`, +`net/http/httptest`, or `internal/{store,site,deploy,cas,auth,adminapi}`. +`database/sql` is a probe: it can only appear if server code leaked in. If you +need a type in both the CLI and the server, it belongs in `api/` (stdlib only) +or `internal/pathutil` (stdlib only). + +## Conventions + +**Comments explain why, not what.** The code already says what it does. A +comment earns its place by recording the reasoning that is not recoverable from +the code — why this order, why not the obvious alternative, what breaks if +someone changes it. Match the density of the surrounding file; several of the +sharper decisions are commented in place precisely so a later reader does not +"optimize" them away (`internal/cache`'s note about not caching file handles is +the canonical example). + +**Do not over-design.** No metrics nobody reads, no abstraction with one +implementation, no configuration knob without a caller who needs it. If a +simpler thing works, ship the simpler thing. + +**Dependencies.** Three direct ones, on purpose: `modernc.org/sqlite` (pure Go, +so `CGO_ENABLED=0` cross-compiles), `github.com/BurntSushi/toml` (server config +only), `golang.org/x/sync/errgroup` (bounded concurrency, both sides). No web +framework, router, ORM, migration tool, CLI framework, logging library +(`log/slog`), or UUID library (`crypto/rand` + hex). Adding a fourth needs a +reason that survives being written down. + +**Tests** are table-driven with names that read as sentences +(`TestAnInFlightRequestSurvivesTheContentBeingCollected`). Two gotchas worth +knowing before you debug an unexpected pass: retention tests must patch +`retention_grace_s = 0`, because a deployment that was never activated is aged +from `created_at` and is otherwise protected for the full grace period; and blob +collection tests need a negative `Service.BlobGrace`, since zero means "use the +one-hour default", not "collect now". + +**CLI rendering:** `cliutil.Bool` prints `yes`/`no`, and `cliutil.Truncate(s, n)` +returns `n-1` characters plus an ellipsis. Both have caught tests out. + +## Scope + +v1 (M0–M5) is complete: deploy, atomic switch, rollback, crash recovery, +retention and GC, and the atomicity proof tests. + +Deferred, deliberately — do not build these speculatively: precompressed +`br`/`gzip` variants (the `deployment_files.encoding` column is already there +for it), per-project header rules, host routing (one domain per project, which +is also the fix for the same-origin limit), autoindex, a metrics endpoint, a +resident-manifest cap, and packaging. + +**Known limit:** one server process per database. The in-memory registry is +updated by the process that wrote the change, so a second process on the same +SQLite file serves stale content until it restarts. Horizontal scaling needs a +change-notification mechanism first. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..afb2ef9 --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +BIN ?= bin +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +PKG := github.com/iceBear67/simplepages +LDFLAGS := -s -w -X $(PKG)/internal/version.Version=$(VERSION) +GOFLAGS := -trimpath + +.PHONY: all build server cli test race atomicity vet fmt tidy clean + +all: build + +build: server cli + +server: + go build $(GOFLAGS) -ldflags '$(LDFLAGS)' -o $(BIN)/pages-server ./cmd/pages-server + +# The CLI is downloaded by CI runners, so its size is a feature: keep it small +# and keep server-side dependencies out of it (enforced by cmd/pages/deps_test.go). +cli: + CGO_ENABLED=0 go build $(GOFLAGS) -ldflags '$(LDFLAGS)' -o $(BIN)/pages ./cmd/pages + +test: + go test ./... + +race: + go test -race ./... + +atomicity: + go test -race -count=20 ./internal/deploy -run TestActivationAtomicity + +vet: + go vet ./... + +fmt: + gofmt -l -w . + +tidy: + go mod tidy + +clean: + rm -rf $(BIN) diff --git a/README.md b/README.md new file mode 100644 index 0000000..a8f79e6 --- /dev/null +++ b/README.md @@ -0,0 +1,153 @@ +# simplepages + +A static-site deployment target for CI. Your build job pushes a directory; the +site switches over to it in one step. + +The problem it exists to solve is the one `rsync` has: rsync writes the new +build into the live directory file by file, and for the length of that transfer +visitors get a **half-updated site** — new HTML asking for JS that is not there +yet, hashed asset names that do not resolve, images 404ing. simplepages closes +that window. A deployment is uploaded and verified in full first, and only then +does one atomic pointer store make it live. + +**The guarantee:** every request sees the whole old deployment or the whole new +one, never a mixture. That includes requests already in flight — a download that +started before the switch keeps reading the deployment it started on, all the +way to its last byte, even if that deployment is deleted meanwhile. + +Rolling back is the same operation in reverse, so it costs one pointer store +too: the old deployment's files are still on disk. + +## How it fits together + +``` +CI runner pages-server visitor +───────── ──────────── ─────── +pages deploy ./dist ──manifest──▶ "I am missing 11 of these 142" + ──blobs─────▶ content-addressed store + ──finalize──▶ assemble tree, fsync, rename + ──activate──▶ ┌──────────────────────────┐ + │ atomic.Pointer[Deployment]│◀── GET /~demo/ + └──────────────────────────┘ + $WEBROOT/~demo -> …/dpl_a1b2… +``` + +Uploads are **content-addressed and incremental**: the CLI sends a manifest of +paths, SHA-256 digests and sizes, and the server answers with the digests it +does not already have. A rebuild that changes one file uploads one file, no +matter how big the site is, and identical files are stored once across every +deployment and every project. + +`$WEBROOT/~PROJECT` is kept pointing at the live deployment's directory so nginx +or any other external consumer can serve it directly, but the server serves the +site itself and does not depend on that symlink being correct. + +## Quick start + +```sh +make build # -> bin/pages-server and bin/pages + +export PAGES_DATA=$(mktemp -d) PAGES_WEBROOT=$(mktemp -d) +bin/pages-server --data-dir "$PAGES_DATA" --webroot "$PAGES_WEBROOT" & +``` + +On first start the server mints an admin key and writes the token to +`$DATA_DIR/bootstrap-token`, mode 0600. That is the only place it ever writes a +token to disk; delete the file once you have taken the token. + +```sh +export PAGES_SERVER=http://127.0.0.1:8081 +export PAGES_TOKEN=$(cat "$PAGES_DATA/bootstrap-token") + +bin/pages project create demo +export PAGES_TOKEN=$(bin/pages key create --project demo -o json | jq -r .token) + +mkdir -p dist/assets +echo '

v1

' > dist/index.html +echo 'console.log(1)' > dist/assets/app.js + +bin/pages deploy ./dist --project demo +curl -s localhost:8080/~demo/ #

v1

… +``` + +Change one file and deploy again — the CLI reports how much the manifest +negotiation saved: + +``` +142 files, 3.1 MiB; 1 new blob, 402 KiB to upload +``` + +Give CI a **project-scoped** key, never the admin one. A project key can create, +upload, finalize, activate and delete deployments in its own project and read +that project's settings, and nothing else anywhere. + +## Two binaries + +| | | +|---|---| +| `pages-server` | The server: management API on `--api-listen` (default `127.0.0.1:8081`), static content on `--listen` (default `:8080`). Storage is SQLite plus a content-addressed blob store under `--data-dir`. | +| `pages` | The CLI. Pure HTTP client — no SQLite, no cgo, no server packages linked in. It is downloaded by CI runners, so keeping it that way is a feature, and `cmd/pages/deps_test.go` fails the build if a server dependency creeps in. | + +## CLI + +``` +pages deploy upload a directory and switch to it +pages project create|list|show|update|delete +pages deployment list|show|activate|delete # activate an older one = rollback +pages key create|list|revoke +pages whoami which key am I actually using +pages system info|gc|fsck admin only +pages config show|set|path +pages version +``` + +Settings resolve flag → `PAGES_*` environment variable → config file → +built-in default. In CI, pass the token as `PAGES_TOKEN` or `--token-file`; +`--token` puts the secret in argv, which on a shared runner is world-readable +through `/proc//cmdline`. + +Every `pages deployment` command works with a project key, so a CI job can roll +its own project back without an admin credential. + +## Building and testing + +```sh +make build # both binaries, -trimpath -ldflags '-s -w' +make test # go test ./... +make race # go test -race ./... +make atomicity # the switch-is-atomic proof, -race -count=20 +make vet fmt tidy +``` + +`make atomicity` is the test that guards the one thing this project sells: 64 +readers hammer three files plus a 512 KiB body while a writer activates 50 +deployments back to back, and any reader that observes two versions at once +fails the run. + +## Security + +Read [`docs/operations.md`](docs/operations.md) §7 before hosting anything you +care about. The headlines: + +- **Path routing is not a security boundary.** `/~a/` and `/~b/` share one + origin, so project A's JavaScript can read project B's files, cookies and + `localStorage`, and can register a service worker that intercepts *every* + project on the host. This is fine for mutually trusting projects, which is + what v1 targets. Hosting untrusted sites needs one domain per project. +- Client-supplied hashes are never trusted: the server rehashes every uploaded + byte and rejects a mismatch, which is what makes cross-project dedup safe. +- Tokens appear once, at creation. They are never logged, never accepted in a + query string, and never included in an error message. +- Blobs and assembled files are `0444`, directories `0755`; upload permissions + are ignored entirely, so a setuid bit cannot reach disk. +- A project key can tell whether a digest it already knows exists on the server + (a blob existence oracle). Known, accepted, and documented in §7. + +## Documentation + +[`docs/operations.md`](docs/operations.md) — installation, configuration, +deploying, garbage collection and retention, the security model and its limits, +scaling limits, troubleshooting and backups. + +[`AGENTS.md`](AGENTS.md) — repository map, the invariants that hold the +guarantee up, and what to run before calling a change done. diff --git a/api/errors.go b/api/errors.go new file mode 100644 index 0000000..df2a287 --- /dev/null +++ b/api/errors.go @@ -0,0 +1,84 @@ +// Package api holds the wire format shared by pages-server and the pages CLI. +// +// It must depend on the standard library only. cmd/pages imports this package, +// and cmd/pages/deps_test.go fails the build if any server-side dependency +// (SQLite driver, database/sql, internal/store, ...) reaches the CLI through it. +package api + +import ( + "errors" + "fmt" +) + +// Code is a machine-readable error code. Clients switch on these; the +// accompanying message is for humans and may change. +type Code string + +const ( + CodeBadRequest Code = "bad_request" + CodeUnauthorized Code = "unauthorized" + CodeForbidden Code = "forbidden" + CodeNotFound Code = "not_found" + CodeMethodNotAllowed Code = "method_not_allowed" + CodeConflict Code = "conflict" + CodePayloadTooLarge Code = "payload_too_large" + CodeRateLimited Code = "rate_limited" + CodeInternal Code = "internal" + CodeUnavailable Code = "unavailable" + + // Domain-specific codes. + CodeProjectExists Code = "project_exists" + CodeInvalidProjectName Code = "invalid_project_name" + CodeInvalidPath Code = "invalid_path" + CodeDigestMismatch Code = "digest_mismatch" + CodeSizeMismatch Code = "size_mismatch" + CodeDeploymentNotReady Code = "deployment_not_ready" + CodeDeploymentActive Code = "deployment_active" + CodeBlobsMissing Code = "blobs_missing" + CodeLimitExceeded Code = "limit_exceeded" +) + +// Error is the body of every non-2xx management API response: +// +// {"error":{"code":"deployment_not_ready","message":"...","details":{...}}} +type Error struct { + Code Code `json:"code"` + Message string `json:"message"` + Details map[string]any `json:"details,omitempty"` +} + +// ErrorEnvelope wraps Error so the JSON has a single top-level "error" key. +type ErrorEnvelope struct { + Error Error `json:"error"` +} + +func (e *Error) Error() string { + if e.Message == "" { + return string(e.Code) + } + return fmt.Sprintf("%s: %s", e.Code, e.Message) +} + +// Errorf builds an *Error with a formatted message. +func Errorf(code Code, format string, args ...any) *Error { + return &Error{Code: code, Message: fmt.Sprintf(format, args...)} +} + +// WithDetail attaches a detail key and returns the receiver for chaining. +func (e *Error) WithDetail(key string, val any) *Error { + if e.Details == nil { + e.Details = make(map[string]any, 1) + } + e.Details[key] = val + return e +} + +// CodeOf reports the Code carried by err, or CodeInternal when err is not an +// *Error. It unwraps, so a wrapped *Error is still recognised. +func CodeOf(err error) Code { + var apiErr *Error + if errors.As(err, &apiErr) { + return apiErr.Code + } + return CodeInternal +} diff --git a/api/paths.go b/api/paths.go new file mode 100644 index 0000000..a2c58df --- /dev/null +++ b/api/paths.go @@ -0,0 +1,70 @@ +package api + +import "net/url" + +// Version is the API path prefix. It is a constant rather than a client option +// because the CLI and the server are released together; a mismatch is a bug, +// not a configuration. +const Version = "/api/v1" + +// Path builders, shared so the client and the server's route table cannot drift +// apart. Every segment that comes from user input is escaped: a project named +// with a slash could otherwise rewrite the request into a different endpoint. +// (Project names are pattern-checked on creation, so this is defence in depth +// against a name that predates a stricter pattern.) + +func PathProjects() string { return Version + "/projects" } + +func PathProject(name string) string { + return Version + "/projects/" + url.PathEscape(name) +} + +func PathProjectKeys(name string) string { + return PathProject(name) + "/keys" +} + +func PathKeys() string { return Version + "/keys" } + +func PathKey(id string) string { + return Version + "/keys/" + url.PathEscape(id) +} + +func PathWhoAmI() string { return Version + "/whoami" } + +func PathSystemInfo() string { return Version + "/system/info" } + +func PathGC() string { return Version + "/gc" } + +func PathFsck() string { return Version + "/fsck" } + +func PathDeployments(project string) string { + return PathProject(project) + "/deployments" +} + +func PathDeployment(project, id string) string { + return PathDeployments(project) + "/" + url.PathEscape(id) +} + +func PathManifest(project, id string) string { + return PathDeployment(project, id) + "/manifest" +} + +func PathFinalize(project, id string) string { + return PathDeployment(project, id) + "/finalize" +} + +func PathActivate(project, id string) string { + return PathDeployment(project, id) + "/activate" +} + +// PathBlob addresses a blob by lowercase hex digest. Blobs are global rather +// than per-project because the content-addressed store deduplicates across +// projects; see the security notes on the blob existence oracle. +func PathBlob(hexDigest string) string { + return Version + "/blobs/" + url.PathEscape(hexDigest) +} + +// SiteURL returns where a project is served under path routing. +func SiteURL(base, project string) string { + return base + "/~" + url.PathEscape(project) + "/" +} diff --git a/api/types.go b/api/types.go new file mode 100644 index 0000000..074646c --- /dev/null +++ b/api/types.go @@ -0,0 +1,258 @@ +package api + +import "time" + +// Deployment states, as they appear on the wire. +const ( + StatePending = "pending" + StateUploading = "uploading" + StateReady = "ready" + StateFailed = "failed" + StateDeleting = "deleting" +) + +// Key scopes, as they appear on the wire. +const ( + ScopeAdmin = "admin" + ScopeProject = "project" +) + +// ---------------------------------------------------------------- projects + +// Project is the server's view of a project. +// +// Every mutable setting also appears in ProjectPatch. Adding a field here that +// cannot be changed afterwards is a deliberate choice, not an oversight: Name +// is immutable because renaming would invalidate every deployed URL and every +// webroot symlink pointing at it. +type Project struct { + Name string `json:"name"` + DisplayName string `json:"display_name,omitempty"` + IndexFile string `json:"index_file"` + NotFoundFile string `json:"not_found_file,omitempty"` + SPAFallback bool `json:"spa_fallback"` + CacheControl string `json:"cache_control"` + + RetentionCount int `json:"retention_count"` + RetentionGrace int `json:"retention_grace_s"` + MaxFiles int `json:"max_files"` + MaxFileBytes int64 `json:"max_file_bytes"` + MaxTotalBytes int64 `json:"max_total_bytes"` + + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + + // ActiveDeployment is nil when the project has never been deployed, which + // is why it is a pointer rather than a zero-valued struct. + ActiveDeployment *Deployment `json:"active_deployment,omitempty"` + // URL is where the active deployment is served, when the server knows its + // public base URL. + URL string `json:"url,omitempty"` +} + +// CreateProjectRequest creates a project. Everything except Name is optional +// and falls back to the server's defaults. +type CreateProjectRequest struct { + Name string `json:"name"` + Patch *ProjectPatch `json:"config,omitempty"` +} + +// ProjectPatch is a partial update. +// +// Every field is a pointer so the server can tell "leave this alone" from "set +// this to the zero value" — without that, PATCH could never clear a custom 404 +// document or turn the SPA fallback off. +type ProjectPatch struct { + DisplayName *string `json:"display_name,omitempty"` + IndexFile *string `json:"index_file,omitempty"` + NotFoundFile *string `json:"not_found_file,omitempty"` + SPAFallback *bool `json:"spa_fallback,omitempty"` + CacheControl *string `json:"cache_control,omitempty"` + RetentionCount *int `json:"retention_count,omitempty"` + RetentionGrace *int `json:"retention_grace_s,omitempty"` + MaxFiles *int `json:"max_files,omitempty"` + MaxFileBytes *int64 `json:"max_file_bytes,omitempty"` + MaxTotalBytes *int64 `json:"max_total_bytes,omitempty"` +} + +// ProjectList is the paged response for GET /api/v1/projects. +type ProjectList struct { + Projects []Project `json:"projects"` + NextCursor string `json:"next_cursor,omitempty"` +} + +// -------------------------------------------------------------------- keys + +// Key describes an API key. It never carries the secret: the full token exists +// on the wire exactly once, in CreateKeyResponse. +type Key struct { + ID string `json:"id"` + Scope string `json:"scope"` + Project string `json:"project,omitempty"` + Name string `json:"name,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + LastUsed *time.Time `json:"last_used_at,omitempty"` + RevokedAt *time.Time `json:"revoked_at,omitempty"` +} + +// Revoked reports whether the key has been revoked. +func (k Key) Revoked() bool { return k.RevokedAt != nil } + +// CreateKeyRequest mints a key. Project is set by the URL for the +// project-scoped endpoint and must be empty otherwise. +type CreateKeyRequest struct { + Name string `json:"name,omitempty"` + // ExpiresAt is absolute, not a duration: the CLI parses "90d" locally so a + // clock skew between client and server cannot silently shift expiry. + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// CreateKeyResponse is the only place a full token ever appears. +type CreateKeyResponse struct { + Key Key `json:"key"` + // Token is shown once and never retrievable again. Clients must not log it. + Token string `json:"token"` +} + +// KeyList is the response for the key listing endpoints. +type KeyList struct { + Keys []Key `json:"keys"` +} + +// WhoAmI describes the caller's own credential. +type WhoAmI struct { + KeyID string `json:"key_id"` + Scope string `json:"scope"` + Project string `json:"project,omitempty"` + Name string `json:"name,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// ------------------------------------------------------------- deployments + +// Deployment is the server's view of one upload. +type Deployment struct { + ID string `json:"id"` + Project string `json:"project"` + State string `json:"state"` + Active bool `json:"active"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` + Meta map[string]string `json:"meta,omitempty"` + Error string `json:"error,omitempty"` + + CreatedAt time.Time `json:"created_at"` + FinalizedAt *time.Time `json:"finalized_at,omitempty"` + ActivatedAt *time.Time `json:"activated_at,omitempty"` + + // URL is where this deployment is served, set on the response to an + // activation when the server knows its public base URL. It is the project's + // URL: only the active deployment has one, since there are no per-version + // preview addresses. + URL string `json:"url,omitempty"` + + // Files is populated only by GET .../deployments/{id}?files=true. + Files []FileEntry `json:"files,omitempty"` +} + +// FileEntry is one line of a manifest. Digest is lowercase hex; the server +// stores the raw 32 bytes, and hex exists only at this boundary. +type FileEntry struct { + Path string `json:"path"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// CreateDeploymentRequest starts a deployment. +type CreateDeploymentRequest struct { + Meta map[string]string `json:"meta,omitempty"` +} + +// ManifestRequest declares the complete file list of a deployment. +type ManifestRequest struct { + Files []FileEntry `json:"files"` +} + +// ManifestResponse tells the client which blobs the server does not have yet. +// +// Missing is the number that makes content-addressed upload worth having, so +// the CLI prints it: "142 files, 3.1 MiB; 11 new blobs, 402 KiB to upload". +type ManifestResponse struct { + Missing []string `json:"missing"` + MissingBytes int64 `json:"missing_bytes"` + Have int `json:"have"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` +} + +// BlobResponse acknowledges an uploaded blob. +type BlobResponse struct { + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// DeploymentList is the paged response for the deployment listing endpoint. +type DeploymentList struct { + Deployments []Deployment `json:"deployments"` + NextCursor string `json:"next_cursor,omitempty"` +} + +// ------------------------------------------------------------------ system + +// SystemInfo is the response for GET /api/v1/system/info. +type SystemInfo struct { + Version string `json:"version"` + UptimeS int64 `json:"uptime_s"` + Projects int64 `json:"projects"` + Deployments int64 `json:"deployments"` + Blobs int64 `json:"blobs"` + CASBytes int64 `json:"cas_bytes"` + LinkMode string `json:"link_mode"` + SchemaVer int `json:"schema_version"` +} + +// GCRequest asks for a garbage collection pass. +type GCRequest struct { + DryRun bool `json:"dry_run,omitempty"` +} + +// GCStats reports what a collection pass did, or would have done. +// +// On a dry run the blob numbers count what is collectable right now, not what +// deleting the listed deployments would additionally free: nothing was deleted, +// so those blobs are still referenced. The figures are a floor. +type GCStats struct { + DryRun bool `json:"dry_run"` + DeploymentsDeleted int `json:"deployments_deleted"` + BlobsDeleted int `json:"blobs_deleted"` + BytesFreed int64 `json:"bytes_freed"` +} + +// FsckRequest asks for a consistency check, optionally correcting what it +// finds. +type FsckRequest struct { + Repair bool `json:"repair,omitempty"` +} + +// FsckReport is the result of a consistency check. +type FsckReport struct { + // Blobs is how many were examined, DriftCount how many disagreed with the + // manifests that reference them. Drift lists the first hundred of them, + // because the list is for a person to read. + Blobs int64 `json:"blobs"` + DriftCount int `json:"drift_count"` + Drift []BlobDrift `json:"drift,omitempty"` + Repaired int `json:"repaired"` +} + +// BlobDrift is one blob whose stored reference count is not the number of +// manifest entries that name it. +type BlobDrift struct { + Digest string `json:"digest"` + // Stored above Actual only wastes disk. Stored below Actual is the + // dangerous direction: the collector may remove content a deployment still + // needs. + Stored int64 `json:"stored"` + Actual int64 `json:"actual"` +} diff --git a/cmd/pages-server/integration_test.go b/cmd/pages-server/integration_test.go new file mode 100644 index 0000000..9291cc8 --- /dev/null +++ b/cmd/pages-server/integration_test.go @@ -0,0 +1,673 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/client" + "github.com/iceBear67/simplepages/internal/config" +) + +// testenv is the whole server: the real app assembly, both handlers, and the +// pages CLI's own client library driving them. Nothing between the CI job and +// the bytes on the wire is stubbed, which is the point — the milestone's claim +// is that `pages deploy` followed by a GET returns the site, and only a test at +// this level can make that claim. +type testenv struct { + app *app + cfg config.Config + site *httptest.Server + api *httptest.Server + admin *client.Client + adminToken string + logBuf *bytes.Buffer +} + +func newTestenv(t *testing.T) *testenv { + t.Helper() + + base := t.TempDir() + cfg := config.Default() + cfg.DataDir = filepath.Join(base, "data") + cfg.Webroot = filepath.Join(base, "www") + cfg.LogFormat = "text" + cfg.LogLevel = "debug" + if err := cfg.Validate(); err != nil { + t.Fatalf("config: %v", err) + } + if err := cfg.EnsureDirs(); err != nil { + t.Fatalf("ensure dirs: %v", err) + } + + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + a, err := newApp(t.Context(), cfg, log) + if err != nil { + t.Fatalf("newApp: %v", err) + } + t.Cleanup(a.close) + a.ready.Store(true) + + e := &testenv{app: a, cfg: cfg, logBuf: &buf} + e.site = httptest.NewServer(a.siteHandler()) + t.Cleanup(e.site.Close) + e.api = httptest.NewServer(a.apiHandler()) + t.Cleanup(e.api.Close) + + // The bootstrap token is how an operator gets their first credential, and + // the only place the server ever writes one to disk. + raw, err := os.ReadFile(cfg.BootstrapTokenPath()) + if err != nil { + t.Fatalf("read bootstrap token: %v", err) + } + fi, err := os.Stat(cfg.BootstrapTokenPath()) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Errorf("bootstrap-token mode = %v, want 0600", fi.Mode().Perm()) + } + e.adminToken = strings.TrimSpace(string(raw)) + e.admin = e.client(t, e.adminToken) + return e +} + +func (e *testenv) client(t *testing.T, token string) *client.Client { + t.Helper() + c, err := client.New(client.Config{BaseURL: e.api.URL, Token: token, HTTP: e.api.Client()}) + if err != nil { + t.Fatalf("client: %v", err) + } + return c +} + +// project creates a project and returns a client holding a key scoped to it, +// which is what a CI job would be given. +func (e *testenv) project(t *testing.T, name string) *client.Client { + t.Helper() + if _, err := e.admin.CreateProject(t.Context(), api.CreateProjectRequest{Name: name}); err != nil { + t.Fatalf("create project %s: %v", name, err) + } + key, err := e.admin.CreateProjectKey(t.Context(), name, api.CreateKeyRequest{Name: "ci"}) + if err != nil { + t.Fatalf("create key for %s: %v", name, err) + } + return e.client(t, key.Token) +} + +// deploy scans a directory of literal file contents and pushes it, exactly as +// `pages deploy` does. +func (e *testenv) deploy(t *testing.T, c *client.Client, project string, files map[string]string) *client.DeployResult { + t.Helper() + dir := t.TempDir() + for name, content := range files { + p := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + + src, err := client.Scan(t.Context(), dir, client.ScanOptions{}) + if err != nil { + t.Fatalf("scan: %v", err) + } + defer src.Close() + + res, err := c.Deploy(t.Context(), client.DeployOptions{ + Project: project, Source: src, Activate: true, + }) + if err != nil { + t.Fatalf("deploy: %v", err) + } + return res +} + +// get fetches a site URL. +func (e *testenv) get(t *testing.T, path string, header http.Header) *http.Response { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, e.site.URL+path, nil) + if err != nil { + t.Fatal(err) + } + req.Header = header + if req.Header == nil { + req.Header = http.Header{} + } + // Redirects are part of what is under test, so they are never followed. + c := *e.site.Client() + c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + resp, err := c.Do(req) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + t.Cleanup(func() { resp.Body.Close() }) + return resp +} + +func (e *testenv) body(t *testing.T, resp *http.Response) string { + t.Helper() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// TestDeployThenServe is the milestone's acceptance test, in the order the +// manual walkthrough in the plan does it. +func TestDeployThenServe(t *testing.T) { + e := newTestenv(t) + ci := e.project(t, "demo") + + v1 := map[string]string{ + "index.html": "

v1

", + "assets/app.js": "console.log(1)", + "docs/index.html": "

docs

", + } + res := e.deploy(t, ci, "demo", v1) + if !res.Activated || res.Deployment.State != api.StateReady { + t.Fatalf("deploy result = %+v", res) + } + if res.FileCount != 3 || res.Uploaded != 3 || res.Deduplicated != 0 { + t.Fatalf("first deploy uploaded %d of %d files, deduplicated %d", + res.Uploaded, res.FileCount, res.Deduplicated) + } + + // --- the site is being served + resp := e.get(t, "/~demo/", nil) + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET /~demo/ = %d", resp.StatusCode) + } + if got := e.body(t, resp); got != v1["index.html"] { + t.Errorf("body = %q, want %q", got, v1["index.html"]) + } + if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/html") { + t.Errorf("Content-Type = %q", ct) + } + if resp.Header.Get("X-Content-Type-Options") != "nosniff" { + t.Error("the nosniff header is missing") + } + + resp = e.get(t, "/~demo/assets/app.js", nil) + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET the script = %d", resp.StatusCode) + } + etag := resp.Header.Get("ETag") + if !strings.HasPrefix(etag, `"sha256:`) { + t.Errorf("ETag = %q, want a content digest", etag) + } + if got := e.body(t, resp); got != v1["assets/app.js"] { + t.Errorf("script = %q", got) + } + + // A repeat request with the validator is the cheap 304 the default + // Cache-Control is chosen to produce. + resp = e.get(t, "/~demo/assets/app.js", http.Header{"If-None-Match": {etag}}) + if resp.StatusCode != http.StatusNotModified { + t.Errorf("conditional GET = %d, want 304", resp.StatusCode) + } + + // Range and conditional handling come from http.ServeContent, which is the + // reason the handler hands it an *os.File rather than writing bytes itself. + resp = e.get(t, "/~demo/assets/app.js", http.Header{"Range": {"bytes=8-12"}}) + if resp.StatusCode != http.StatusPartialContent { + t.Errorf("ranged GET = %d, want 206", resp.StatusCode) + } + if got, want := e.body(t, resp), v1["assets/app.js"][8:13]; got != want { + t.Errorf("range bytes=8-12 gave %q, want %q", got, want) + } + if cr := resp.Header.Get("Content-Range"); cr != "bytes 8-12/14" { + t.Errorf("Content-Range = %q", cr) + } + // If-Range with a matching validator serves the range; a stale one serves + // the whole file, which is what keeps a mid-deploy resume correct. + resp = e.get(t, "/~demo/assets/app.js", http.Header{ + "Range": {"bytes=0-3"}, "If-Range": {`"sha256:` + strings.Repeat("0", 64) + `"`}, + }) + if resp.StatusCode != http.StatusOK { + t.Errorf("If-Range with a stale validator = %d, want 200", resp.StatusCode) + } + + // Directory handling. + resp = e.get(t, "/~demo/docs", nil) + if resp.StatusCode != http.StatusMovedPermanently { + t.Errorf("GET /~demo/docs = %d, want 301", resp.StatusCode) + } + if loc := resp.Header.Get("Location"); loc != "/~demo/docs/" { + t.Errorf("Location = %q", loc) + } + resp = e.get(t, "/~demo/docs/", nil) + if got := e.body(t, resp); got != v1["docs/index.html"] { + t.Errorf("GET /~demo/docs/ = %q", got) + } + resp = e.get(t, "/~demo/nope.html", nil) + if resp.StatusCode != http.StatusNotFound { + t.Errorf("a missing file = %d, want 404", resp.StatusCode) + } + + // --- the symlink is live + dir, err := os.Readlink(filepath.Join(e.cfg.Webroot, "~demo")) + if err != nil { + t.Fatalf("readlink ~demo: %v", err) + } + if !strings.HasSuffix(dir, res.Deployment.ID) { + t.Errorf("~demo -> %q, want the active deployment", dir) + } + onDisk, err := os.ReadFile(filepath.Join(dir, "index.html")) + if err != nil { + t.Fatalf("read through the symlink: %v", err) + } + if string(onDisk) != v1["index.html"] { + t.Errorf("the assembled tree holds %q", onDisk) + } + + // --- a second deploy uploads only what changed + v2 := map[string]string{ + "index.html": "

v2

", + "assets/app.js": v1["assets/app.js"], + "docs/index.html": v1["docs/index.html"], + } + res2 := e.deploy(t, ci, "demo", v2) + if res2.Uploaded != 1 || res2.Deduplicated != 2 { + t.Errorf("second deploy uploaded %d and reused %d; want 1 and 2", + res2.Uploaded, res2.Deduplicated) + } + if got := e.body(t, e.get(t, "/~demo/", nil)); got != v2["index.html"] { + t.Errorf("after the second deploy the site serves %q", got) + } + + // --- rollback is one activation of the older deployment + if _, err := ci.Activate(t.Context(), "demo", res.Deployment.ID); err != nil { + t.Fatalf("rollback: %v", err) + } + if got := e.body(t, e.get(t, "/~demo/", nil)); got != v1["index.html"] { + t.Errorf("after the rollback the site serves %q", got) + } + dir, err = os.Readlink(filepath.Join(e.cfg.Webroot, "~demo")) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(dir, res.Deployment.ID) { + t.Errorf("the rollback left ~demo -> %q", dir) + } +} + +// TestRetentionAndRollback is M5's acceptance check, in the order the plan's +// manual walkthrough does it: deploy fifteen times, confirm that collection +// keeps ten plus the one being served and that the blob count falls, then roll +// back to an old deployment and see the old site. +// +// It drives the endpoints through the CLI's own client, so `pages deployment +// list`, `pages deployment activate` and `pages system gc` are covered by the +// same run. +func TestRetentionAndRollback(t *testing.T) { + e := newTestenv(t) + ci := e.project(t, "demo") + + // Ten to keep, and no retention grace: the grace exists so that a rollback + // target is not collected out from under an operator who is still deciding, + // and any real value for it would outlast the test. + ten, zero := 10, 0 + if _, err := e.admin.PatchProject(t.Context(), "demo", + api.ProjectPatch{RetentionCount: &ten, RetentionGrace: &zero}); err != nil { + t.Fatalf("configure retention: %v", err) + } + // Likewise the blob grace, which protects content a request has resolved and + // is about to open. + e.app.deploy.BlobGrace = -time.Minute + + const versions = 15 + var deps []string + for i := 1; i <= versions; i++ { + body := fmt.Sprintf("

v%d

", i) + res := e.deploy(t, ci, "demo", map[string]string{ + "index.html": body, + // Shared by every version, so the blob count below is a statement + // about content that fell out of use, not about content churning. + "assets/app.js": "console.log(1)", + }) + deps = append(deps, res.Deployment.ID) + } + if got := e.body(t, e.get(t, "/~demo/", nil)); got != "

v15

" { + t.Fatalf("after %d deploys the site serves %q", versions, got) + } + + // --- roll back to the first deployment + // + // Deliberately the oldest one: it is also the one retention would drop + // first, so this proves the active deployment is exempt rather than merely + // young enough to survive. + if _, err := ci.Activate(t.Context(), "demo", deps[0]); err != nil { + t.Fatalf("rollback: %v", err) + } + if got := e.body(t, e.get(t, "/~demo/", nil)); got != "

v1

" { + t.Errorf("after the rollback the site serves %q, want the old version", got) + } + link, err := os.Readlink(filepath.Join(e.cfg.Webroot, "~demo")) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(link, deps[0]) { + t.Errorf("~demo -> %q, want the rolled-back deployment", link) + } + + before, err := e.admin.SystemInfo(t.Context()) + if err != nil { + t.Fatal(err) + } + if before.Deployments != versions { + t.Fatalf("%d deployments before collection, want %d", before.Deployments, versions) + } + + // --- collect + stats, err := e.admin.Collect(t.Context(), false) + if err != nil { + t.Fatalf("gc: %v", err) + } + // Fifteen deployments, one of them active and therefore not considered at + // all, ten of the remaining fourteen kept by retention: four to delete. + if stats.DeploymentsDeleted != 4 { + t.Errorf("collected %d deployments, want 4", stats.DeploymentsDeleted) + } + if stats.BlobsDeleted != 4 || stats.BytesFreed == 0 { + t.Errorf("collected %+v, want the four index pages those deployments held", stats) + } + + after, err := e.admin.SystemInfo(t.Context()) + if err != nil { + t.Fatal(err) + } + if after.Deployments != 11 { + t.Errorf("%d deployments after collection, want the 10 kept plus the active one", after.Deployments) + } + if after.Blobs >= before.Blobs { + t.Errorf("blobs went from %d to %d, want the count to fall", before.Blobs, after.Blobs) + } + if after.Blobs != 12 { + // Eleven surviving index pages plus the script every version shares. + t.Errorf("%d blobs after collection, want 12", after.Blobs) + } + + // The survivors are the active one and the ten newest, and a listing says so. + list, err := ci.ListDeployments(t.Context(), "demo", client.DeploymentListOptions{}) + if err != nil { + t.Fatalf("list: %v", err) + } + live := make(map[string]bool, len(list.Deployments)) + for _, d := range list.Deployments { + live[d.ID] = true + if d.Active != (d.ID == deps[0]) { + t.Errorf("deployment %s: active = %v", d.ID, d.Active) + } + } + if len(live) != 11 { + t.Fatalf("%d deployments listed, want 11", len(live)) + } + for i, id := range deps { + want := i == 0 || i >= versions-10 + if live[id] != want { + t.Errorf("v%d (%s): present = %v, want %v", i+1, id, live[id], want) + } + } + + // The site is still what the rollback made it, and the content of a + // collected deployment is what was reclaimed — not the active one's. + if got := e.body(t, e.get(t, "/~demo/", nil)); got != "

v1

" { + t.Errorf("after collection the site serves %q", got) + } + if got := e.body(t, e.get(t, "/~demo/assets/app.js", nil)); got != "console.log(1)" { + t.Errorf("the shared script is now %q", got) + } + + // A collected deployment is gone from the API too, and a second pass has + // nothing left to do. + if _, err := ci.GetDeployment(t.Context(), "demo", deps[1], false); err == nil { + t.Errorf("deployment v2 (%s) still readable after collection", deps[1]) + } + again, err := e.admin.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if again.DeploymentsDeleted != 0 || again.BlobsDeleted != 0 { + t.Errorf("a second pass collected %+v", again) + } + + // --- deleting on request + // + // The one being served is refused; another one goes immediately. + if err := ci.DeleteDeployment(t.Context(), "demo", deps[0]); err == nil { + t.Error("the active deployment was deleted on request") + } else if code := api.CodeOf(err); code != api.CodeDeploymentActive { + t.Errorf("delete active: code = %q, want %q", code, api.CodeDeploymentActive) + } + spare := deps[versions-2] + if err := ci.DeleteDeployment(t.Context(), "demo", spare); err != nil { + t.Fatalf("delete %s: %v", spare, err) + } + if _, err := ci.GetDeployment(t.Context(), "demo", spare, false); err == nil { + t.Errorf("deployment %s still readable after being deleted", spare) + } + if got := e.body(t, e.get(t, "/~demo/", nil)); got != "

v1

" { + t.Errorf("deleting a spare disturbed the site: %q", got) + } + + // Reference counts and manifests agree throughout: the collector trusts + // those counters, and a count that reads low is the one way this design can + // lose content a deployment still needs. + rep, err := e.admin.Fsck(t.Context(), false) + if err != nil { + t.Fatalf("fsck: %v", err) + } + if rep.DriftCount != 0 { + t.Errorf("fsck reports %d drifted blobs: %+v", rep.DriftCount, rep.Drift) + } +} + +// A project that exists but has never activated anything is a different answer +// from one that does not exist: 503 says "come back", 404 says "wrong URL". +func TestServeBeforeAnythingIsDeployed(t *testing.T) { + e := newTestenv(t) + e.project(t, "demo") + + if resp := e.get(t, "/~demo/", nil); resp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("GET an undeployed project = %d, want 503", resp.StatusCode) + } + if resp := e.get(t, "/~ghost/", nil); resp.StatusCode != http.StatusNotFound { + t.Errorf("GET an unknown project = %d, want 404", resp.StatusCode) + } + if resp := e.get(t, "/", nil); resp.StatusCode != http.StatusNotFound { + t.Errorf("GET the site root = %d, want 404", resp.StatusCode) + } +} + +// The management API and the site content are separate listeners on purpose: +// neither surface may answer for the other, whatever the request looks like. +func TestTheTwoListenersDoNotOverlap(t *testing.T) { + e := newTestenv(t) + ci := e.project(t, "demo") + e.deploy(t, ci, "demo", map[string]string{"index.html": "

hi

"}) + + // The site listener knows nothing about the API. + if resp := e.get(t, api.PathProjects(), nil); resp.StatusCode != http.StatusNotFound { + t.Errorf("the site listener answered %s with %d", api.PathProjects(), resp.StatusCode) + } + + // And the API listener serves no content. + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, e.api.URL+"/~demo/", nil) + if err != nil { + t.Fatal(err) + } + resp, err := e.api.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("the API listener answered /~demo/ with %d", resp.StatusCode) + } + + // Both carry the health probes, so either can be the one a load balancer + // targets. + for _, base := range []string{e.site.URL, e.api.URL} { + for _, path := range []string{"/healthz", "/readyz"} { + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, base+path, nil) + if err != nil { + t.Fatal(err) + } + resp, err := e.site.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("GET %s%s = %d", base, path, resp.StatusCode) + } + } + } +} + +// A project key reaches its own project and nothing else, end to end. +func TestProjectKeysAreConfinedToTheirProject(t *testing.T) { + e := newTestenv(t) + mine := e.project(t, "mine") + theirs := e.project(t, "theirs") + + e.deploy(t, theirs, "theirs", map[string]string{"index.html": "

secret

"}) + + if _, err := mine.CreateDeployment(t.Context(), "theirs", nil); err == nil { + t.Fatal("a project key created a deployment in another project") + } + if _, err := mine.GetProject(t.Context(), "theirs"); err == nil { + t.Fatal("a project key read another project") + } + + // Path routing means both sites share an origin, which is exactly the + // property documented as the reason not to host mutually untrusting + // projects this way. Reading a neighbour's content over HTTP is expected; + // reaching its management API is not. + if resp := e.get(t, "/~theirs/", nil); resp.StatusCode != http.StatusOK { + t.Errorf("GET the neighbour's site = %d", resp.StatusCode) + } +} + +// The one assertion that has to hold no matter what else is logged: no part of +// a token ever reaches the log, on either listener. +func TestTokensNeverReachTheLog(t *testing.T) { + e := newTestenv(t) + ci := e.project(t, "demo") + e.deploy(t, ci, "demo", map[string]string{"index.html": "

hi

"}) + + // A bad token in every place a client could put one. + bad := "pgs_aaaaaaaaaaaaaaaa_" + strings.Repeat("b", 43) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, + e.api.URL+api.PathProjects()+"?token="+bad, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+bad) + resp, err := e.api.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + + logged := e.logBuf.String() + for _, secret := range []string{bad, strings.Repeat("b", 43), "aaaaaaaaaaaaaaaa"} { + if strings.Contains(logged, secret) { + t.Fatalf("the log contains %q", secret) + } + } + if strings.Contains(strings.ToLower(logged), "authorization") { + t.Fatal("the log mentions the Authorization header") + } + // The log is not empty, so the assertions above mean something. + if !strings.Contains(logged, "deployment activated") { + t.Fatal("nothing was logged at all; the assertions above prove nothing") + } +} + +// Restarting the process must serve exactly what it served before: the registry +// is rebuilt from the database, which is the reason activation commits there +// before it stores the pointer. +func TestRestartServesTheSameDeployment(t *testing.T) { + e := newTestenv(t) + ci := e.project(t, "demo") + res := e.deploy(t, ci, "demo", map[string]string{ + "index.html": "

v1

", + "assets/app.js": "console.log(1)", + }) + if got := e.body(t, e.get(t, "/~demo/", nil)); got != "

v1

" { + t.Fatalf("before restart: %q", got) + } + + // Tear the process down and build a second one on the same directories. + e.site.Close() + e.api.Close() + e.app.close() + + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + a2, err := newApp(context.Background(), e.cfg, log) + if err != nil { + t.Fatalf("second newApp: %v", err) + } + t.Cleanup(a2.close) + a2.ready.Store(true) + e.app, e.logBuf = a2, &buf + e.site = httptest.NewServer(a2.siteHandler()) + t.Cleanup(e.site.Close) + e.api = httptest.NewServer(a2.apiHandler()) + t.Cleanup(e.api.Close) + e.admin = e.client(t, e.adminToken) + + resp := e.get(t, "/~demo/", nil) + if resp.StatusCode != http.StatusOK { + t.Fatalf("after restart: status = %d", resp.StatusCode) + } + if got := e.body(t, resp); got != "

v1

" { + t.Errorf("after restart the site serves %q", got) + } + if got := e.body(t, e.get(t, "/~demo/assets/app.js", nil)); got != "console.log(1)" { + t.Errorf("after restart the script is %q", got) + } + + // And the second process did not mint a second bootstrap admin key. + keys, err := e.admin.ListKeys(t.Context()) + if err != nil { + t.Fatal(err) + } + admins := 0 + for _, k := range keys.Keys { + if k.Scope == string(api.ScopeAdmin) && k.RevokedAt == nil { + admins++ + } + } + if admins != 1 { + t.Errorf("%d admin keys after a restart, want 1", admins) + } + + // The database is the source of truth the registry was rebuilt from, so the + // deployment being served is the same row, not merely the same bytes. + p, err := e.admin.GetProject(t.Context(), "demo") + if err != nil { + t.Fatal(err) + } + if p.ActiveDeployment == nil || p.ActiveDeployment.ID != res.Deployment.ID { + t.Errorf("active deployment after restart = %+v, want %s", p.ActiveDeployment, res.Deployment.ID) + } +} diff --git a/cmd/pages-server/main.go b/cmd/pages-server/main.go new file mode 100644 index 0000000..fe4d333 --- /dev/null +++ b/cmd/pages-server/main.go @@ -0,0 +1,406 @@ +// Command pages-server serves static sites deployed through the pages CLI and +// exposes the management API used to drive those deployments. +// +// It binds two listeners: a public one that only ever serves site content, and a +// management one (loopback by default) that only ever serves the API. Keeping +// them apart means the management surface never shares an origin with content +// that projects control. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "os/signal" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/iceBear67/simplepages/internal/adminapi" + "github.com/iceBear67/simplepages/internal/auth" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/config" + "github.com/iceBear67/simplepages/internal/deploy" + "github.com/iceBear67/simplepages/internal/httpx" + "github.com/iceBear67/simplepages/internal/site" + "github.com/iceBear67/simplepages/internal/store" + "github.com/iceBear67/simplepages/internal/version" + "github.com/iceBear67/simplepages/internal/webroot" +) + +// Background worker cadences. The auth flusher batches last_used_at updates; +// writing one per request would funnel every authenticated read through the +// single write connection. +const ( + touchFlushInterval = 60 * time.Second + + // failedAuthBurst is how many failed authentications one client address may + // make before it is throttled, and failedAuthPeriod is how long a full + // budget takes to refill. Successful requests cost nothing, so a busy CI + // fleet never meets these numbers. + failedAuthBurst = 10 + failedAuthPeriod = time.Minute + failedAuthClients = 10000 +) + +func main() { + if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "pages-server: %v\n", err) + os.Exit(1) + } +} + +func run(args []string, stdout, stderr io.Writer) error { + opts, err := config.Load(args, stderr) + if err != nil { + return err + } + if opts.ShowVersion { + fmt.Fprintf(stdout, "pages-server %s\n", version.String()) + return nil + } + if opts.CheckOnly { + fmt.Fprintln(stdout, "configuration ok") + return nil + } + + cfg := opts.Config + log := cfg.Logger(stderr) + log.Info("starting", "version", version.Short(), + "data_dir", cfg.DataDir, "webroot", cfg.Webroot, + "assemble_mode", string(cfg.AssembleMode)) + + if err := cfg.EnsureDirs(); err != nil { + return err + } + + // The first signal starts a graceful shutdown; a second one gives up on the + // in-flight requests, which is what an operator means by pressing Ctrl-C + // twice. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + go func() { + <-ctx.Done() + hard := make(chan os.Signal, 1) + signal.Notify(hard, os.Interrupt, syscall.SIGTERM) + <-hard + fmt.Fprintln(stderr, "pages-server: second signal, exiting immediately") + os.Exit(130) + }() + + app, err := newApp(ctx, cfg, log) + if err != nil { + return err + } + defer app.close() + + // Background workers get their own cancellable context so they are stopped + // *after* the listeners have drained: the auth flusher's final write should + // include the last requests the server handled. + workerCtx, stopWorkers := context.WithCancel(context.Background()) + var workers sync.WaitGroup + // Registered after the app's own cleanup so it runs before it: the workers + // must be finished with the database before anything closes it. + defer func() { + stopWorkers() + workers.Wait() + }() + for _, worker := range []func(context.Context){ + func(ctx context.Context) { app.verifier.RunFlusher(ctx, touchFlushInterval) }, + func(ctx context.Context) { app.deploy.RunReconciler(ctx, cfg.ReconcileInterval.D()) }, + func(ctx context.Context) { app.deploy.RunCollector(ctx, cfg.GCInterval.D()) }, + } { + workers.Add(1) + go func() { + defer workers.Done() + worker(workerCtx) + }() + } + + siteSrv, err := httpx.Listen("site", cfg.Listen, app.siteHandler(), httpx.Timeouts{ + ReadHeader: cfg.ReadHeaderTimeout.D(), + Read: cfg.ReadTimeout.D(), + Idle: cfg.IdleTimeout.D(), + // No Write timeout: see httpx.Timeouts. + }, log) + if err != nil { + return fmt.Errorf("listen %s: %w", cfg.Listen, err) + } + apiSrv, err := httpx.Listen("api", cfg.APIListen, app.apiHandler(), httpx.Timeouts{ + ReadHeader: cfg.ReadHeaderTimeout.D(), + Read: cfg.ReadTimeout.D(), + Idle: cfg.IdleTimeout.D(), + }, log) + if err != nil { + return fmt.Errorf("listen %s: %w", cfg.APIListen, err) + } + + app.ready.Store(true) + + g := &httpx.Group{ + Servers: []*httpx.Server{siteSrv, apiSrv}, + Grace: cfg.ShutdownGrace.D(), + Log: log, + } + if err := g.Run(ctx); err != nil { + return err + } + log.Info("stopped") + return nil +} + +// app holds the process-wide state shared by both listeners. +type app struct { + cfg config.Config + log *slog.Logger + started time.Time + + db *store.DB + cas *cas.Store + sites *site.Registry + webroot *webroot.Webroot + deploy *deploy.Service + verifier *auth.Verifier + authmw *auth.Middleware + admin *adminapi.Server + + // ready gates /readyz: the process may be accepting connections before it + // can actually answer for content, and a load balancer needs to know. + ready atomic.Bool +} + +func newApp(ctx context.Context, cfg config.Config, log *slog.Logger) (*app, error) { + db, err := store.Open(ctx, cfg.DBPath(), log) + if err != nil { + return nil, fmt.Errorf("open store: %w", err) + } + + if _, err := auth.EnsureAdminKey(ctx, db, cfg.BootstrapTokenPath(), log); err != nil { + db.Close() + return nil, err + } + + // The probe directory must be the one trees are assembled in: hardlinks + // cannot cross filesystems, so probing anywhere else answers a different + // question. With assemble_mode=none nothing is assembled and the mode is + // irrelevant, so ask for copy rather than probe for something unused. + casOpts := cas.Options{ProbeDir: cfg.DeploymentsDir(), Log: log} + switch cfg.AssembleMode { + case config.AssembleHardlink: + casOpts.Mode = cas.LinkHard + case config.AssembleCopy, config.AssembleNone: + casOpts.Mode = cas.LinkCopy + } + cs, err := cas.Open(cfg.CASDir(), casOpts) + if err != nil { + db.Close() + return nil, err + } + + verifier := auth.NewVerifier(db, log, auth.DefaultCacheTTL) + // An empty Dir is how the service is told to skip on-disk assembly. + deployDir := cfg.DeploymentsDir() + if cfg.AssembleMode == config.AssembleNone { + deployDir = "" + } + + // With nothing assembled on disk there is nothing for a symlink to point at, + // so assemble_mode=none leaves the webroot unmanaged. Failing to open it + // otherwise is a permissions or layout problem the operator asked for and + // should hear about now, rather than as a warning on every activation. + var wr *webroot.Webroot + if deployDir != "" && cfg.Webroot != "" { + wr, err = webroot.Open(cfg.Webroot, deployDir) + if err != nil { + cs.Close() + db.Close() + return nil, err + } + } + + a := &app{ + cfg: cfg, + log: log, + started: time.Now(), + db: db, + cas: cs, + sites: site.NewRegistry(), + webroot: wr, + verifier: verifier, + authmw: &auth.Middleware{ + V: verifier, + Limiter: auth.NewLimiter(failedAuthBurst, failedAuthPeriod, failedAuthClients), + Trusted: cfg.TrustedProxies(), + Log: log, + }, + } + a.deploy = &deploy.Service{ + DB: db, CAS: cs, Log: log, Dir: deployDir, + Sites: a.sites, Webroot: wr, + } + a.admin = &adminapi.Server{ + DB: db, + Auth: a.authmw, + Deploy: a.deploy, + Log: log, + Limits: cfg.Limits, + // The registry answers ownership checks and "what is this project + // serving" from memory. Both are only correct once LoadSites below has + // finished, which is why it runs before any listener exists. + Resolver: a.sites, + Sites: a.sites, + BaseURL: cfg.SiteURL, + LinkMode: string(cs.LinkMode()), + Started: a.started, + } + a.admin.Hooks = adminapi.Hooks{ + ProjectChanged: func(_ context.Context, p *store.Project) { + // A new project starts with nothing activated, so it resolves and + // then answers 503 until something is deployed to it. A changed one + // keeps serving what it was serving, with new settings. + a.sites.Put(p) + }, + ProjectDeleted: func(ctx context.Context, p *store.Project) { + a.sites.Delete(p.Name) + if wr != nil { + if err := wr.Unpoint(p.Name); err != nil { + log.WarnContext(ctx, "could not remove the webroot symlink", + "project", p.Name, "err", err) + } + } + // The rows are already gone and their blobs are already + // unreferenced; this is only the assembled trees, which nothing + // would otherwise account for. A failure here is not worth failing + // the request over — the startup sweep removes them as orphans. + if err := a.deploy.RemoveProjectTrees(p.ID); err != nil { + log.WarnContext(ctx, "could not remove the project's deployment trees", + "project", p.Name, "err", err) + } + }, + } + + // Recovery runs before the registry is built, and both run before any + // listener exists. That order is what lets recovery assume it is alone with + // the data directory, and it means the first request is answered from state + // that has already been reconciled rather than from whatever the last crash + // left behind. + if err := a.deploy.Recover(ctx); err != nil { + a.close() + return nil, fmt.Errorf("recover: %w", err) + } + if err := a.deploy.LoadSites(ctx); err != nil { + a.close() + return nil, fmt.Errorf("load sites: %w", err) + } + return a, nil +} + +// pingDB checks that the read pool can still reach the database, with a +// deadline of its own so a wedged store cannot hold a probe open indefinitely. +func (a *app) pingDB(ctx context.Context) error { + if a.db == nil { + return errors.New("store not open") + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + return a.db.Reader().PingContext(ctx) +} + +func (a *app) close() { + if a.cas != nil { + if err := a.cas.Close(); err != nil { + a.log.Error("closing content store", "err", err) + } + } + if a.db != nil { + if err := a.db.Close(); err != nil { + a.log.Error("closing store", "err", err) + } + } +} + +// middleware is the chain shared by both listeners, outermost first. +// +// The order is load-bearing. Recover must sit *inside* AccessLog: a panic that +// unwound past AccessLog would skip its logging entirely, so the one request +// that most needs a log line would be the one without one. With Recover +// innermost the panic becomes an ordinary 500 return that AccessLog then records +// normally, and Recover can see the shared request id and the recorder that +// tells it whether a response has already begun. Panics in the middleware +// itself are left to net/http, which closes the connection. +func (a *app) middleware() []httpx.Middleware { + return []httpx.Middleware{ + httpx.WithRequestID(a.cfg.TrustedProxies()), + httpx.AccessLog(a.log, a.cfg.TrustedProxies()), + httpx.Recover(a.log), + } +} + +func (a *app) siteHandler() http.Handler { + mux := http.NewServeMux() + a.registerHealth(mux) + + // Site routing is hand-parsed rather than expressed as a ServeMux pattern: + // "/~{project}/{path...}" is rejected by net/http, whose wildcards must start + // at the beginning of a path segment. Registering "/" still gets us the mux's + // built-in ".."/"//" normalisation redirects. + // + // Those redirects are a convenience, not a defence: a percent-encoded + // "/~a/%2e%2e/%2e%2e/etc/passwd" reaches the handler with r.URL.Path already + // decoded to "/~a/../../etc/passwd" and no redirect issued (verified against + // this server). The resolver therefore does its own path.Clean and validation + // rather than assume the mux normalised anything. + mux.Handle("/", &site.Handler{Registry: a.sites, CAS: a.cas, Log: a.log}) + return httpx.Chain(mux, a.middleware()...) +} + +func (a *app) apiHandler() http.Handler { + mux := http.NewServeMux() + a.registerHealth(mux) + a.admin.Register(mux) + return httpx.Chain(mux, a.middleware()...) +} + +// registerHealth adds the probe endpoints to a mux. They live on both listeners +// so a probe can target whichever one the deployment exposes. +func (a *app) registerHealth(mux *http.ServeMux) { + // Liveness: answers as long as the process can schedule a goroutine. It must + // never touch the database, or a slow query would get the process killed. + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + fmt.Fprintln(w, "ok") + }) + + // Readiness: reports whether this process can serve real traffic yet. + mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { + status := http.StatusOK + body := map[string]any{ + "status": "ready", + "version": version.Short(), + "uptime_s": int64(time.Since(a.started).Seconds()), + } + if !a.ready.Load() { + status = http.StatusServiceUnavailable + body["status"] = "starting" + } else if err := a.pingDB(r.Context()); err != nil { + // Unlike /healthz this may touch the database: a process that + // cannot read its own store should be taken out of rotation, not + // restarted. + a.log.WarnContext(r.Context(), "readiness probe: store unreachable", "err", err) + status = http.StatusServiceUnavailable + body["status"] = "degraded" + } + w.Header().Set("Cache-Control", "no-store") + httpx.WriteJSON(w, status, body) + }) +} diff --git a/cmd/pages/deps_test.go b/cmd/pages/deps_test.go new file mode 100644 index 0000000..4907afc --- /dev/null +++ b/cmd/pages/deps_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "errors" + "os/exec" + "strings" + "testing" +) + +// forbidden names packages that must never reach the CLI binary, with the +// reason, because a future contributor will hit this test and need to know +// which of the two possible fixes applies: move the code, or widen the CLI's +// remit deliberately. +var forbidden = map[string]string{ + "modernc.org/sqlite": "the SQLite driver is megabytes of translated C; " + + "the CLI must never link a database", + "modernc.org/libc": "pulled in by the SQLite driver", + "database/sql": "a probe: nothing the CLI does needs a database, so if this " + + "appears, a server package leaked in through an import", + "github.com/BurntSushi/toml": "server configuration only; the CLI's config " + + "file is JSON precisely so this stays out", + "github.com/iceBear67/simplepages/internal/store": "server-side storage", + "github.com/iceBear67/simplepages/internal/site": "server-side serving", + "github.com/iceBear67/simplepages/internal/deploy": "server-side deployment lifecycle", + "github.com/iceBear67/simplepages/internal/cas": "server-side content store", + "github.com/iceBear67/simplepages/internal/auth": "server-side verification; the CLI only carries a token", + "github.com/iceBear67/simplepages/internal/adminapi": "server-side handlers; the wire types " + + "the CLI needs live in api/", + "net/http/httptest": "test-only helper that would bloat the shipped binary", +} + +// TestCLIImportGraph is the mechanism behind the "two binaries" decision: the +// CLI is downloaded on every CI run, so its size and its build requirements +// (no cgo, no C toolchain) are user-visible properties. A stray import is easy +// to add and invisible until someone notices the binary doubled, so the rule is +// enforced here rather than written down. +func TestCLIImportGraph(t *testing.T) { + for _, pkg := range depsOf(t, "./cmd/pages") { + if why, bad := forbidden[pkg]; bad { + t.Errorf("cmd/pages depends on %s\n %s", pkg, why) + } + } +} + +// TestAPIPackageIsStdlibOnly guards the other half of the arrangement: api/ is +// imported by both binaries, so anything it depends on is automatically part of +// the CLI. Keeping it to the standard library is what lets the server and the +// client share wire types at all. +func TestAPIPackageIsStdlibOnly(t *testing.T) { + const self = "github.com/iceBear67/simplepages/api" + for _, pkg := range depsOf(t, self) { + // go list -deps includes the package itself. + if pkg == self { + continue + } + // A standard library import path never has a dot in its first segment. + if first, _, _ := strings.Cut(pkg, "/"); strings.Contains(first, ".") { + t.Errorf("api imports %s, but it must depend on the standard library only", pkg) + } + } +} + +func depsOf(t *testing.T, pkg string) []string { + t.Helper() + // The test's working directory is cmd/pages; a relative package pattern has + // to be resolved from the module root two levels up. + cmd := exec.Command("go", "list", "-deps", pkg) + cmd.Dir = "../.." + out, err := cmd.Output() + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + t.Fatalf("go list -deps %s: %v\n%s", pkg, err, ee.Stderr) + } + t.Fatalf("go list -deps %s: %v", pkg, err) + } + return strings.Fields(string(out)) +} diff --git a/cmd/pages/main.go b/cmd/pages/main.go new file mode 100644 index 0000000..d649384 --- /dev/null +++ b/cmd/pages/main.go @@ -0,0 +1,86 @@ +// Command pages is the client for a pages-server installation. +// +// It is a plain HTTP client on purpose: no SQLite, no server packages, nothing +// that needs cgo. CI jobs download this binary on every run, so its size and +// its dependency graph are features — see deps_test.go, which enforces both. +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/clicmd" + "github.com/iceBear67/simplepages/internal/cliutil" +) + +// Exit codes. A CI step distinguishes "the command was wrong" from "the server +// said no" without parsing messages. +const ( + exitOK = 0 + exitError = 1 + exitUsage = 2 +) + +func main() { + // Ctrl-C cancels in-flight requests rather than killing the process + // mid-upload, which would leave a deployment stuck in "uploading" until it + // expires. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + os.Exit(run(ctx, os.Args[1:])) +} + +func run(ctx context.Context, args []string) int { + g := clicmd.NewGlobals() + root := clicmd.Root(g) + + err := cliutil.Run(ctx, root, args, g.Err, g.Register) + switch { + case err == nil: + return exitOK + + case errors.Is(err, cliutil.ErrUsage): + // Run has already printed the usage text. The bare sentinel carries no + // message of its own — flag or the command tree printed one. + if err != cliutil.ErrUsage { + fmt.Fprintf(g.Err, "pages: %v\n", err) + } + return exitUsage + + case errors.Is(err, cliutil.ErrAborted): + fmt.Fprintln(g.Err, "pages: aborted") + return exitError + + case errors.Is(err, context.Canceled): + fmt.Fprintln(g.Err, "pages: interrupted") + return exitError + + default: + fmt.Fprintf(g.Err, "pages: %v\n", err) + hint(g, err) + return exitError + } +} + +// hint turns the codes the server uses into the one sentence that usually +// resolves the problem. +func hint(g *clicmd.Globals, err error) { + var apiErr *api.Error + if !errors.As(err, &apiErr) { + return + } + switch apiErr.Code { + case api.CodeUnauthorized: + fmt.Fprintln(g.Err, "hint: the token is missing, malformed, expired or revoked; check PAGES_TOKEN") + case api.CodeForbidden: + fmt.Fprintln(g.Err, "hint: this key is not allowed here; a project key can only act on its own project") + case api.CodeNotFound: + fmt.Fprintln(g.Err, "hint: check the name, and that the key you are using can see it") + } +} diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..00b3e21 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,478 @@ +# Operating simplepages + +`pages-server` is a static-site host built around one guarantee: **a request +either sees the whole old deployment or the whole new one, never a mixture.** +Everything below either explains how to run it or tells you where that +guarantee stops. + +Read [Security model and its limits](#security-model-and-its-limits) before you +put this on a network. Three of the items there are not bugs to be fixed later — +they are properties of the design, and one of them (same-origin) decides whether +this server is suitable for your workload at all. + +--- + +## 1. What it does + +``` +CI job pages-server visitor +------ ------------ ------- +pages deploy ./dist ──► upload only the changed blobs + assemble a directory tree + ┌─────────────────────────┐ + │ ONE atomic pointer swap │ ◄── GET /~demo/ + └─────────────────────────┘ + repoint $WEBROOT/~demo +``` + +Unlike `rsync`, no file of the new version is visible until every file of it is +present and verified. The switch itself is a single `atomic.Pointer` store, so a +request that started before the swap keeps reading the old snapshot to +completion — including a large file already halfway down the wire. + +Rollback is the same operation pointed at an older deployment id, and costs the +same single store. + +--- + +## 2. Installation + +Build both binaries. The server needs SQLite (pure Go, so no cgo and no C +toolchain); the CLI deliberately shares no server code at all. + +```sh +CGO_ENABLED=0 go build -o /usr/local/bin/pages-server ./cmd/pages-server +CGO_ENABLED=0 go build -o /usr/local/bin/pages ./cmd/pages +``` + +`pages` is what goes into CI runners. Its dependency graph is enforced by a +test (`go test ./cmd/pages -run TestCLIImportGraph`), so it can never +accidentally start dragging in the database driver. + +### Directories + +| Path | Contents | Who writes it | +|---|---|---| +| `$DATA_DIR/pages.db` | projects, deployments, keys, blob refcounts | server only | +| `$DATA_DIR/cas/` | content-addressed blobs, `0444` | server only | +| `$DATA_DIR/deployments///` | assembled trees | server only | +| `$WEBROOT/~` | symlink to the active deployment | server only | + +`$WEBROOT` and `$DATA_DIR` must not overlap; the server refuses to start if they +do. + +### systemd + +```ini +[Unit] +Description=simplepages static site server +After=network-online.target + +[Service] +Type=exec +ExecStartPre=/usr/local/bin/pages-server --config /etc/pages-server/config.toml --check-config +ExecStart=/usr/local/bin/pages-server --config /etc/pages-server/config.toml +Restart=on-failure +StateDirectory=pages-server +ReadWritePaths=/var/lib/pages-server /srv/www +ProtectSystem=strict +PrivateTmp=yes +NoNewPrivileges=yes +UMask=0022 + +[Install] +WantedBy=multi-user.target +``` + +`--check-config` validates and exits without binding a port, which is why it is +safe in `ExecStartPre`. + +**Do not use `DynamicUser=yes`.** The UID it allocates rotates, and the +assembled trees are hardlinks whose ownership must stay stable across restarts. + +--- + +## 3. Configuration + +Precedence is **flag > `PAGES_*` environment variable > config file > default**. + +```toml +# /etc/pages-server/config.toml +data_dir = "/var/lib/pages-server" +webroot = "/srv/www" +listen = ":8080" # static content +api_listen = "127.0.0.1:8081" # management API +site_url = "https://pages.example.com" + +trusted_proxy_cidrs = ["127.0.0.1/32", "::1/128"] +assemble_mode = "auto" # auto | hardlink | copy | none + +log_level = "info" +log_format = "json" + +gc_interval = "15m" +reconcile_interval = "5m" +shutdown_grace = "30s" + +read_header_timeout = "10s" +read_timeout = "5m" +idle_timeout = "120s" + +[limits] +max_file_bytes = 268435456 # 256 MiB +max_manifest_files = 50000 +max_concurrent_uploads = 32 +max_manifest_bytes = 67108864 # 64 MiB of manifest JSON +max_json_bytes = 1048576 # 1 MiB for ordinary API bodies +``` + +Every key above has a matching `PAGES_*` variable (`PAGES_DATA_DIR`, +`PAGES_API_LISTEN`, `PAGES_ASSEMBLE_MODE`, `PAGES_RECONCILE_INTERVAL`, …). The +ones worth overriding on the command line also have flags: `--data-dir`, +`--webroot`, `--listen`, `--api-listen`, `--site-url`, `--assemble-mode`, +`--log-level`, `--log-format`, `--gc-interval`, `--shutdown-grace`, plus +`--config`, `--check-config` and `--version`. Run `pages-server -h` for the +authoritative list. + +`listen` and `api_listen` must differ, and `webroot` may be neither equal to nor +nested inside `data_dir` (in either direction) — the reconciler prunes symlinks +from one and the GC deletes trees from the other, so overlapping them would let +each destroy the other's state. + +### The two listeners are separate on purpose + +`listen` serves nothing but site content. `api_listen` serves nothing but the +management API, and defaults to loopback. Neither answers for the other — there +is a test that asserts exactly this. Expose the API through your reverse proxy on +its own hostname, or not at all; nothing forces a management surface onto the +origin your sites are served from. + +TLS is the reverse proxy's job in both cases. + +### `site_url` + +Purely cosmetic: it is the origin API responses quote back to a CI job so the +job can print where its build landed. Behind a proxy the server cannot learn +this name, so you have to tell it. Leaving it empty just omits the `url` field. + +### `assemble_mode` + +- `auto` (default) — probe for hardlink support at startup, fall back to copy. +- `hardlink` — require hardlinks; fail loudly if the filesystem refuses. +- `copy` — always copy. Correct everywhere, uses disk proportional to content. +- `none` — skip on-disk assembly entirely. Content is served straight from the + CAS and **`$WEBROOT` symlinks are not maintained**. Choose this only if + nothing outside `pages-server` reads the files. + +On overlayfs (Docker's default upper layer, among others) cross-layer `link()` +either triggers a copy-up or fails outright, so `auto` may silently land on +copy. Check the startup log line reporting the detected mode if disk usage +surprises you. + +### `trusted_proxy_cidrs` + +`X-Forwarded-For` is honoured **only** when the direct peer address falls inside +one of these prefixes. The client IP is used for the failed-authentication rate +limiter, so getting this wrong either lets one proxy IP absorb everyone's +budget (too narrow) or lets a client forge its own identity (too wide). Set it +to your proxy's address and nothing else. + +--- + +## 4. First run + +On first start, if no unrevoked admin key exists, the server mints one and +writes the token to `$DATA_DIR/bootstrap-token` with mode `0600`, logging a +warning. **This is the only place the server ever writes a token to disk.** + +```fish +set -x PAGES_SERVER http://127.0.0.1:8081 +set -x PAGES_TOKEN (cat /var/lib/pages-server/bootstrap-token) + +pages project create demo +pages key create --project demo --name github-actions -o json # token shown ONCE +``` + +Then delete the bootstrap file. It is not recreated as long as one unrevoked +admin key exists. + +Give CI a **project-scoped** key, never the admin one. A project key can create, +upload, finalize and activate deployments in its own project and read that +project's settings — and nothing else, in any other project. + +--- + +## 5. Deploying + +```sh +pages deploy ./dist --project demo +``` + +What happens, and what each step buys you: + +1. **Scan.** Walk the directory, hash every file with SHA-256 in parallel. + Symlinks are refused by default (`--follow-symlinks` opts in, and still + refuses targets outside the tree). Devices, sockets and FIFOs are always + refused. +2. **Create.** A `pending` deployment row. No filesystem is touched. +3. **Manifest.** Send every path + digest + size; the server answers with the + digests it does *not* already have. This is where the incremental win shows + up — a rebuild that changes one file uploads one blob, regardless of how big + the site is. The CLI prints the saving. +4. **Upload.** Only the missing blobs, bounded concurrency (`--concurrency`), + exponential backoff honouring `Retry-After` (`--retries`). +5. **Finalize.** The server verifies nothing is missing, then assembles the tree + into `…/.staging`, fsyncs it, and renames it into place. +6. **Activate** (unless `--activate=false`) — the atomic swap. + +**Interrupted deploys need no special handling.** Re-run `pages deploy`. The +blobs that made it are still in the CAS, so the new deployment's manifest +negotiation returns a much smaller missing set. There is no resume protocol +because none is needed. + +### Rollback + +```sh +pages deployment list --project demo # newest first; ACTIVE marks what is live +pages deployment activate dpl_… # one pointer store +``` + +The superseded deployment stays `ready` on disk until retention expires it, so +rollback costs nothing but the swap. `pages deployment show dpl_… --files` gives +the manifest of any of them if you need to see what a version contained before +switching to it, and `pages deployment delete dpl_…` removes one — except the +active one, which answers `409 deployment_active` until something else is +activated. + +Every `pages deployment` command accepts a project key, so a CI job can roll its +own project back without an admin credential. `pages system gc` and `pages +system fsck` are server-wide and need an admin key. + +### Base paths — the one thing that will bite you + +Sites are served under `/~PROJECT/`. A build that assumes it lives at `/` will +request `/assets/app.js`, get a 404, and look broken. **Set the base path at +build time:** + +| Tool | Setting | +|---|---| +| Vite | `vite build --base=/~demo/` | +| Create React App | `PUBLIC_URL=/~demo` | +| Next.js (static export) | `basePath: '/~demo'` | +| Hugo | `--baseURL=/~demo/` | +| plain HTML | `` | + +This is inherent to prefix hosting; the server cannot fix it. If you need sites +at the root of their own domain, you need host routing (§8). + +### Project settings + +```sh +pages project update demo --spa --not-found-file 404.html --retention 20 +``` + +- `--index-file` (default `index.html`) — what a directory URL serves. +- `--not-found-file` — served *with* a 404 status. Without it, 404s are bare. +- `--spa` — serve the index document for unknown paths, **but only when the + request's `Accept` header includes `text/html`.** That condition matters: it + is why a missing `/assets/app.js` still returns a real 404 instead of an HTML + page with status 200 and an afternoon of `Unexpected token '<'`. +- `--cache-control` (default `public, max-age=0, must-revalidate`) — correct by + construction here, because every response carries a strong content-hash ETag. + Repeat visits are cheap 304s and deploys are visible immediately. Only raise + `max-age` for content served under hashed filenames. +- `--retention` / `--retention-grace` — see §6. + +--- + +## 6. Garbage collection and retention + +The sweep runs every `gc_interval`, and on demand via `pages system gc` (admin), +which is `POST /api/v1/gc`. `pages system gc --dry-run` reports what would go +without deleting anything — it cannot report the bytes, because nothing was +deleted and the content is all still referenced. + +Per project, a deployment is deleted when **all** of these hold: + +- it is not active; +- it was superseded more than `retention_grace` ago (default 1 h) — a + deployment that was never activated is measured from when it was created, so + an uploaded-but-unused one is kept for the grace period too; +- it is not among the newest `retention_count` finished deployments (default 10). + +Failed deployments are dropped after 24 h. Blobs are removed once their +refcount reaches zero and they have been unreferenced for an hour. + +**Why the grace period exists.** A request that has already taken its snapshot +and is about to open a blob gets an hour of slack. Combined with POSIX's +guarantee that an open file descriptor keeps an inode alive, an in-flight +download survives even a forced GC. If the server ever logs a missing blob on +the read path, that is a genuine bug — it is logged at ERROR precisely so it +cannot be mistaken for normal behaviour. + +`pages system fsck` (`POST /api/v1/fsck`, admin) recomputes every blob refcount +from the manifests and reports drift; `--repair` (`{"repair": true}`) corrects +it. On a healthy server it always reports none — the counts are maintained by +database triggers — so it is for the cases outside normal operation: a restored +backup, or a database edited by hand. A count that reads *low* is the dangerous +one, because it is content the collector will delete while a deployment still +needs it. + +--- + +## 7. Security model and its limits + +Three of these are accepted properties of the v1 design, not deferred work. +They are stated here so the decision is yours and not a surprise. + +### 7.1 ⚠️ Path routing is not a security boundary between projects + +`/~a/` and `/~b/` are the **same origin**. That means project A's JavaScript +can: + +- `fetch('/~b/secret.json')` and read the response; +- read and write cookies and `localStorage` for the whole host; +- register a service worker scoped to `/`, intercepting **every request to + every project on that host**, including future ones. + +There is no browser mechanism that prevents this. `X-Content-Type-Options: +nosniff` is always sent, and it does not help here. + +**Only host projects that trust each other.** If you need to host sites from +mutually distrusting tenants, give each project its own domain so each gets its +own origin. That is host routing, which v1 does not implement. + +### 7.2 ⚠️ Assembled files share inodes with CAS blobs + +When hardlink assembly is in use (the default when supported), a file under +`$WEBROOT/~demo/` and its blob in `$DATA_DIR/cas/` are **the same inode**. +Editing that file in place corrupts the blob — and therefore corrupts every +other project and every other deployment referencing the same content. + +Mitigations in place: blobs and assembled files are `0444`, directories `0755`, +and client-supplied file modes are ignored entirely (there is no `mode` column +in the schema, so a setuid bit can never reach disk). + +**Treat `$WEBROOT` as read-only to everything except `pages-server`.** Do not +point `rsync --delete` at it, do not let a deploy script `chmod -R` it, do not +edit a file "just to check something". If you must modify content, deploy it. + +Setting `assemble_mode = "copy"` removes the shared inode at the cost of disk. + +### 7.3 ⚠️ The manifest endpoint is a blob existence oracle + +A project-scoped key can put any digest in a manifest and learn from the +`missing` response whether the server already holds that content — including +content belonging to another project. + +The attacker must already know the exact SHA-256, so this only ever confirms +"somebody here hosts this file I already have". It cannot be used to read +content or to enumerate anything. + +This is the price of cross-project deduplication, which is the feature that +makes repeated CI deploys fast. If it matters to you, the fixes are: partition +the CAS per project (losing all dedup), or always report digests the calling +project has never referenced as missing (losing cross-project dedup only). + +### 7.4 What is structurally prevented + +- **Read-path traversal.** The only filesystem path built while serving is + `cas/ab/cd/<64 hex>`, derived from a 32-byte digest that came out of an + in-memory map lookup. No user-controlled byte reaches the filesystem on the + read path, so traversal is not blocked — it is impossible. +- **Write-path traversal.** Every manifest path must satisfy both + `fs.ValidPath` and `filepath.Localize`, and NUL bytes, control characters, + segments over 255 bytes, total lengths over 4096, duplicates, and + case-insensitively colliding paths are all rejected. +- **Poisoned blobs.** The server never trusts a client's digest: it recomputes + SHA-256 over the received stream and rejects a mismatch. Without this, a + client could claim another project's digest and overwrite shared content. +- **Zip bombs and tar-slip.** There is no archive format and no server-side + decompression. Files arrive individually. *Do not "improve" this into tarball + upload without re-reviewing this section.* +- **Symlink escape from uploaded content.** The server creates only directories + and hardlinks inside a deployment tree; it never creates a symlink there. + +### 7.5 Tokens + +Format `pgs__`: an 80-bit key id and a 256-bit random secret. +Stored as SHA-256 (not bcrypt — the secret is uniformly random, so there is no +dictionary to defend against, and a KDF per request would hand every +unauthenticated client a CPU-exhaustion lever), compared in constant time. + +- Tokens are returned **once**, at creation. +- They are **never logged, never accepted in a query string, never included in + an error message.** A test asserts this by running a request through the + middleware and grepping the log output. +- Prefer `PAGES_TOKEN` or `--token-file` over `--token`. On a shared CI runner + `argv` is world-readable through `/proc//cmdline`. +- Revocation is immediate: the auth cache is invalidated by a generation bump. +- The CLI config file is written `0600`. + +Failed authentications are rate limited per client IP (see +`trusted_proxy_cidrs`). Successful ones are not. + +--- + +## 8. Scaling limits + +**Run exactly one `pages-server` process per database.** The in-memory registry +is updated synchronously by the process that performed the write; a second +process on the same SQLite file would keep serving stale content indefinitely, +because nothing tells it the active deployment changed. + +Vertically there is plenty of room — serving is a map lookup plus a file read, +and reads never touch SQLite. The bound worth knowing is memory: resident +manifests cost roughly 100 bytes per file, and only active deployments are +resident. A thousand projects of fifty thousand files each would be about 6 GB. + +Horizontal scaling would need a change-notification mechanism (polling +`deployments.updated_at`, or external pub/sub). It is not in v1. + +--- + +## 9. Monitoring and troubleshooting + +`/healthz` and `/readyz` are served on **both** listeners, so either can be the +one your load balancer targets. `/healthz` never touches the database; +`/readyz` returns 503 until the database is reachable and the registry is +loaded. + +`GET /api/v1/system/info` (admin) reports version, uptime, project and +deployment counts, CAS size and the detected link mode. `pages system info` +formats it. + +Access logs are one line per request with `method`, `path`, `status`, `bytes`, +`dur_ms`, `project`, `deployment`, `key_id` and `req_id`. + +| Symptom | Cause | Fix | +|---|---|---| +| Site returns 503 | project exists, nothing activated yet | deploy, or activate an existing deployment | +| Site returns 404 at `/~name/` | no such project (404 and 503 are deliberately different answers) | check the name | +| Assets 404, page loads | build has the wrong base path | rebuild with `--base=/~name/` (§5) | +| `$WEBROOT/~name` missing | external tool removed it, or `assemble_mode = "none"` | the reconciler restores it within `reconcile_interval` | +| Disk grows steadily | retention too generous, or copy-mode assembly | lower `--retention`, run `pages system gc`, check the link mode in the startup log | +| ERROR "blob missing from the content store" | genuine bug, or somebody deleted CAS files | run `pages system fsck`; redeploy affected projects | +| Deploy hangs at upload | `max_concurrent_uploads` saturated by another job | wait, or raise the limit | + +**Recovering a database with an intact CAS** is normal: the server marks blobs +whose files are gone as absent at startup and the next deploy re-uploads them. +Losing both means redeploying, which for a CI-driven site is one pipeline run. + +### Graceful shutdown + +`SIGINT`/`SIGTERM` stops accepting connections, gives in-flight requests +`shutdown_grace` to finish, cancels background workers, checkpoints the WAL and +closes cleanly. A second signal forces an immediate close. + +--- + +## 10. Backups + +Stop the server, or use `sqlite3 pages.db ".backup"` — a plain `cp` of a live +WAL database is not safe. Then copy `$DATA_DIR/cas/`. + +`$DATA_DIR/deployments/` does **not** need backing up: it is reconstructible +from the CAS and the database. `$WEBROOT` does not need backing up either; it +holds only symlinks, regenerated on the next activation and by the reconciler +within `reconcile_interval`. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f4fce54 --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module github.com/iceBear67/simplepages + +go 1.25.0 + +require ( + github.com/BurntSushi/toml v1.6.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.56.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..362bf65 --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= diff --git a/internal/adminapi/activate_test.go b/internal/adminapi/activate_test.go new file mode 100644 index 0000000..bd9c045 --- /dev/null +++ b/internal/adminapi/activate_test.go @@ -0,0 +1,307 @@ +package adminapi + +import ( + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/deploy" +) + +// readyDeployment runs a whole deployment through create, manifest, upload and +// finalize, so an activation test starts from the only state activation accepts. +func (e *env) readyDeployment(t *testing.T, token, project string, contents map[string]string) api.Deployment { + t.Helper() + dep := e.startDeployment(t, token, project) + + files := make([]api.FileEntry, 0, len(contents)) + for path, content := range contents { + files = append(files, entry(path, content)) + } + status, body := e.do(t, http.MethodPost, api.PathManifest(project, dep.ID), token, + api.ManifestRequest{Files: files}) + var mr api.ManifestResponse + mustJSON(t, status, http.StatusOK, body, &mr) + + for _, content := range contents { + if status, body := e.putBlob(t, token, content); status != http.StatusCreated && status != http.StatusOK { + t.Fatalf("upload: status = %d; body: %s", status, body) + } + } + status, body = e.do(t, http.MethodPost, api.PathFinalize(project, dep.ID), token, nil) + var out api.Deployment + mustJSON(t, status, http.StatusOK, body, &out) + return out +} + +// symlink reads $WEBROOT/~project, failing if it is not a link. +func (e *env) symlink(t *testing.T, project string) string { + t.Helper() + target, err := os.Readlink(filepath.Join(e.webrootDir, "~"+project)) + if err != nil { + t.Fatalf("readlink ~%s: %v", project, err) + } + return target +} + +// TestActivate covers the endpoint the whole product turns on: what it answers, +// what it publishes into the serving registry, and what it leaves on disk. +func TestActivate(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + pid := e.projectID(t, "demo") + + // Nothing is served before the first activation. + sp, ok := e.sites.Lookup("demo") + if !ok { + t.Fatal("creating a project did not register it") + } + if sp.Active() != nil { + t.Fatal("a project with no deployment is serving something") + } + + first := e.readyDeployment(t, token, "demo", map[string]string{ + "index.html": "

v1

", + "assets/app.js": "console.log(1)", + }) + + status, body := e.do(t, http.MethodPost, api.PathActivate("demo", first.ID), token, nil) + var got api.Deployment + mustJSON(t, status, http.StatusOK, body, &got) + if got.ID != first.ID || !got.Active || got.State != api.StateReady { + t.Fatalf("activated = %+v", got) + } + if got.ActivatedAt == nil { + t.Error("activated_at is missing from the response") + } + if got.URL != "" { + t.Errorf("url = %q, want it omitted when no base URL is configured", got.URL) + } + + // The registry is what the site handler reads, so this is the assertion that + // the deployment is actually being served and not merely recorded. + d := sp.Active() + if d == nil { + t.Fatal("activation did not publish into the registry") + } + if d.ID != first.ID || d.FileCount != 2 { + t.Fatalf("published snapshot = %+v", d) + } + if _, ok := d.Lookup("assets/app.js"); !ok { + t.Error("the published snapshot does not know a file the manifest declared") + } + + wantDir := deploy.DeploymentDir(e.deployDir, pid, first.ID) + if got := e.symlink(t, "demo"); got != wantDir { + t.Errorf("~demo -> %q, want %q", got, wantDir) + } + + // The project listing now reports what is being served, without a query per + // project. + status, body = e.do(t, http.MethodGet, api.PathProject("demo"), token, nil) + var p api.Project + mustJSON(t, status, http.StatusOK, body, &p) + if p.ActiveDeployment == nil || p.ActiveDeployment.ID != first.ID { + t.Fatalf("active_deployment = %+v", p.ActiveDeployment) + } + if !p.ActiveDeployment.Active || p.ActiveDeployment.State != api.StateReady { + t.Errorf("active_deployment = %+v", p.ActiveDeployment) + } + + // --- a second deployment takes over + second := e.readyDeployment(t, token, "demo", map[string]string{ + "index.html": "

v2

", + "assets/app.js": "console.log(1)", // unchanged, so it shares a blob + }) + status, body = e.do(t, http.MethodPost, api.PathActivate("demo", second.ID), token, nil) + mustJSON(t, status, http.StatusOK, body, &got) + if got.ID != second.ID || !got.Active { + t.Fatalf("second activation = %+v", got) + } + if d := sp.Active(); d == nil || d.ID != second.ID { + t.Fatalf("the registry still serves %v", d) + } + if got := e.symlink(t, "demo"); got != deploy.DeploymentDir(e.deployDir, pid, second.ID) { + t.Errorf("~demo -> %q, want the second deployment", got) + } + + // --- rollback is the same endpoint on an older deployment + status, body = e.do(t, http.MethodPost, api.PathActivate("demo", first.ID), token, nil) + mustJSON(t, status, http.StatusOK, body, &got) + if got.ID != first.ID || !got.Active { + t.Fatalf("rollback = %+v", got) + } + if d := sp.Active(); d == nil || d.ID != first.ID { + t.Fatalf("the registry did not roll back: %v", d) + } + if got := e.symlink(t, "demo"); got != wantDir { + t.Errorf("~demo -> %q, want the first deployment again", got) + } + + // Re-activating what is already active is a no-op that still answers. + status, body = e.do(t, http.MethodPost, api.PathActivate("demo", first.ID), token, nil) + mustJSON(t, status, http.StatusOK, body, &got) + if !got.Active { + t.Errorf("re-activation = %+v", got) + } +} + +func TestActivateReportsTheSiteURL(t *testing.T) { + e := newEnv(t) + e.server.BaseURL = "https://pages.example.com" + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + dep := e.readyDeployment(t, token, "demo", map[string]string{"index.html": "

hi

"}) + + status, body := e.do(t, http.MethodPost, api.PathActivate("demo", dep.ID), token, nil) + var got api.Deployment + mustJSON(t, status, http.StatusOK, body, &got) + if want := api.SiteURL(e.server.BaseURL, "demo"); got.URL != want { + t.Errorf("url = %q, want %q", got.URL, want) + } +} + +// Activation is the one operation that changes what the world sees, so every +// deployment that is not finished must be refused — and refused without +// disturbing whatever is being served. +func TestActivateRequiresAReadyDeployment(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + serving := e.readyDeployment(t, token, "demo", map[string]string{"index.html": "

v1

"}) + status, body := e.do(t, http.MethodPost, api.PathActivate("demo", serving.ID), token, nil) + mustJSON(t, status, http.StatusOK, body, nil) + + t.Run("pending", func(t *testing.T) { + dep := e.startDeployment(t, token, "demo") + status, body := e.do(t, http.MethodPost, api.PathActivate("demo", dep.ID), token, nil) + if status != http.StatusConflict { + t.Fatalf("status = %d, want 409; body: %s", status, body) + } + if code := errCode(t, body); code != api.CodeDeploymentNotReady { + t.Errorf("code = %q, want %q", code, api.CodeDeploymentNotReady) + } + }) + + t.Run("uploading", func(t *testing.T) { + dep := e.startDeployment(t, token, "demo") + status, body := e.do(t, http.MethodPost, api.PathManifest("demo", dep.ID), token, + api.ManifestRequest{Files: []api.FileEntry{entry("index.html", "never uploaded")}}) + mustJSON(t, status, http.StatusOK, body, nil) + + status, body = e.do(t, http.MethodPost, api.PathActivate("demo", dep.ID), token, nil) + if status != http.StatusConflict { + t.Fatalf("status = %d, want 409; body: %s", status, body) + } + }) + + t.Run("unknown", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathActivate("demo", "dpl_ffffffffffffffff"), token, nil) + if status != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body: %s", status, body) + } + if code := errCode(t, body); code != api.CodeNotFound { + t.Errorf("code = %q, want %q", code, api.CodeNotFound) + } + }) + + // Every refusal above left the site alone. + sp, _ := e.sites.Lookup("demo") + if d := sp.Active(); d == nil || d.ID != serving.ID { + t.Errorf("a refused activation changed what is served: %v", d) + } +} + +// The security property: naming another project's deployment must not activate +// it, whether the caller routes through their own project or the victim's. +func TestActivateIsProjectScoped(t *testing.T) { + e := newEnv(t) + e.createProject(t, "victim") + e.createProject(t, "attacker") + victimToken := e.mintProject(t, e.projectID(t, "victim"), "ci") + attackerToken := e.mintProject(t, e.projectID(t, "attacker"), "ci") + + target := e.readyDeployment(t, victimToken, "victim", map[string]string{"index.html": "

v1

"}) + newer := e.readyDeployment(t, victimToken, "victim", map[string]string{"index.html": "

v2

"}) + status, body := e.do(t, http.MethodPost, api.PathActivate("victim", newer.ID), victimToken, nil) + mustJSON(t, status, http.StatusOK, body, nil) + + t.Run("through the victim's project", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathActivate("victim", target.ID), attackerToken, nil) + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body: %s", status, body) + } + }) + + t.Run("through the attacker's own project", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathActivate("attacker", target.ID), attackerToken, nil) + if status != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body: %s", status, body) + } + }) + + sp, _ := e.sites.Lookup("victim") + if d := sp.Active(); d == nil || d.ID != newer.ID { + t.Errorf("the victim is now serving %v", d) + } + if _, ok := e.sites.Lookup("attacker"); !ok { + t.Fatal("the attacker's project vanished") + } + if sp, _ := e.sites.Lookup("attacker"); sp.Active() != nil { + t.Error("the attacker ended up serving the victim's deployment") + } +} + +func TestActivateRejectsABodyAndWrongMethods(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + dep := e.readyDeployment(t, token, "demo", map[string]string{"index.html": "

hi

"}) + + status, body := e.do(t, http.MethodPost, api.PathActivate("demo", dep.ID), token, + map[string]string{"unexpected": "field"}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + + for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodDelete} { + resp := e.doResp(t, method, api.PathActivate("demo", dep.ID), token, nil) + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Errorf("%s: status = %d, want 405", method, resp.StatusCode) + } + if allow := resp.Header.Get("Allow"); allow != http.MethodPost { + t.Errorf("%s: Allow = %q, want POST", method, allow) + } + } + + if status, body := e.do(t, http.MethodPost, api.PathActivate("demo", dep.ID), "", nil); status != http.StatusUnauthorized { + t.Errorf("unauthenticated: status = %d, want 401; body: %s", status, body) + } +} + +// Deleting a project takes its symlink with it: the deployment tree is about to +// be GC'd, and a link left pointing at it would dangle. +func TestDeleteProjectUnpointsTheWebroot(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + dep := e.readyDeployment(t, token, "demo", map[string]string{"index.html": "

hi

"}) + status, body := e.do(t, http.MethodPost, api.PathActivate("demo", dep.ID), token, nil) + mustJSON(t, status, http.StatusOK, body, nil) + e.symlink(t, "demo") + + status, body = e.do(t, http.MethodDelete, api.PathProject("demo"), e.adminToken, nil) + if status != http.StatusNoContent { + t.Fatalf("delete: status = %d; body: %s", status, body) + } + if _, err := os.Lstat(filepath.Join(e.webrootDir, "~demo")); err == nil { + t.Error("the symlink outlived the project") + } + if _, ok := e.sites.Lookup("demo"); ok { + t.Error("the deleted project is still in the serving registry") + } +} diff --git a/internal/adminapi/adminapi_test.go b/internal/adminapi/adminapi_test.go new file mode 100644 index 0000000..f9c9c85 --- /dev/null +++ b/internal/adminapi/adminapi_test.go @@ -0,0 +1,252 @@ +package adminapi + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/auth" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/config" + "github.com/iceBear67/simplepages/internal/deploy" + "github.com/iceBear67/simplepages/internal/site" + "github.com/iceBear67/simplepages/internal/store" + "github.com/iceBear67/simplepages/internal/webroot" +) + +// env is a management API running against a real SQLite file in a temp dir. +// Nothing here is faked below the HTTP boundary: the tests exercise the same +// store, verifier and route table the server assembles. +type env struct { + db *store.DB + cas *cas.Store + deployDir string + webrootDir string + sites *site.Registry + verifier *auth.Verifier + server *Server + ts *httptest.Server + adminToken string + logBuf *bytes.Buffer +} + +func newEnv(t *testing.T) *env { + t.Helper() + ctx := t.Context() + + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + base := t.TempDir() + db, err := store.Open(ctx, filepath.Join(base, "pages.db"), log) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { db.Close() }) + + deployDir := filepath.Join(base, "deployments") + cs, err := cas.Open(filepath.Join(base, "cas"), cas.Options{ProbeDir: deployDir, Log: log}) + if err != nil { + t.Fatalf("open cas: %v", err) + } + t.Cleanup(func() { cs.Close() }) + + webrootDir := filepath.Join(base, "www") + wr, err := webroot.Open(webrootDir, deployDir) + if err != nil { + t.Fatalf("open webroot: %v", err) + } + + // The serving layer is wired up exactly as cmd/pages-server does it, so the + // endpoints that publish into it — activate above all — are exercised against + // the same collaborators they have in production. + verifier := auth.NewVerifier(db, log, auth.DefaultCacheTTL) + sites := site.NewRegistry() + e := &env{ + db: db, cas: cs, deployDir: deployDir, webrootDir: webrootDir, + sites: sites, verifier: verifier, logBuf: &buf, + } + e.server = &Server{ + DB: db, + Deploy: &deploy.Service{ + DB: db, CAS: cs, Log: log, Dir: deployDir, + Sites: sites, Webroot: wr, + }, + Auth: &auth.Middleware{ + V: verifier, + // A burst high enough that the limiter never fires by accident; the + // throttling behaviour itself is tested in internal/auth. + Limiter: auth.NewLimiter(10000, time.Minute, 128), + Log: log, + }, + Log: log, + Limits: config.Default().Limits, + Sites: sites, + Started: time.Now().Add(-time.Minute), + Hooks: Hooks{ + ProjectChanged: func(_ context.Context, p *store.Project) { sites.Put(p) }, + ProjectDeleted: func(_ context.Context, p *store.Project) { + sites.Delete(p.Name) + if err := wr.Unpoint(p.Name); err != nil { + t.Errorf("unpoint %s: %v", p.Name, err) + } + }, + }, + } + + mux := http.NewServeMux() + e.server.Register(mux) + e.ts = httptest.NewServer(mux) + t.Cleanup(e.ts.Close) + + e.adminToken = e.mintAdmin(t, "test-admin") + return e +} + +// mintAdmin creates an admin key directly in the store, the way the bootstrap +// path does, and returns its token. +func (e *env) mintAdmin(t *testing.T, name string) string { + t.Helper() + return e.mint(t, store.ScopeAdmin, nil, name) +} + +func (e *env) mintProject(t *testing.T, projectID int64, name string) string { + t.Helper() + return e.mint(t, store.ScopeProject, &projectID, name) +} + +func (e *env) mint(t *testing.T, scope store.Scope, projectID *int64, name string) string { + t.Helper() + return e.mintWith(t, scope, projectID, name, nil) +} + +// mintWith goes around the API so a test can create a key the API would refuse +// to mint — an already-expired one, for instance. +func (e *env) mintWith(t *testing.T, scope store.Scope, projectID *int64, name string, expires *time.Time) string { + t.Helper() + token, keyID, hash, err := auth.Mint() + if err != nil { + t.Fatalf("mint: %v", err) + } + k := &store.APIKey{ + ID: keyID, SecretHash: hash[:], Scope: scope, + ProjectID: projectID, Name: name, ExpiresAt: expires, + } + if err := e.db.CreateKey(t.Context(), k); err != nil { + t.Fatalf("create key: %v", err) + } + return token +} + +// do issues a request and returns the status and raw body. The body is returned +// undecoded so tests can assert on what is literally on the wire. +func (e *env) do(t *testing.T, method, path, token string, body any) (int, []byte) { + t.Helper() + var r io.Reader + if body != nil { + buf, err := json.Marshal(body) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + r = bytes.NewReader(buf) + } + req, err := http.NewRequestWithContext(t.Context(), method, e.ts.URL+path, r) + if err != nil { + t.Fatalf("new request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := e.ts.Client().Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + return resp.StatusCode, raw +} + +// doResp is do() for the tests that need to inspect response headers. +func (e *env) doResp(t *testing.T, method, path, token string, body any) *http.Response { + t.Helper() + var r io.Reader + if body != nil { + buf, err := json.Marshal(body) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + r = bytes.NewReader(buf) + } + req, err := http.NewRequestWithContext(t.Context(), method, e.ts.URL+path, r) + if err != nil { + t.Fatalf("new request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := e.ts.Client().Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + t.Cleanup(func() { resp.Body.Close() }) + return resp +} + +// mustJSON decodes body into v, failing the test if the status is unexpected. +func mustJSON(t *testing.T, status, want int, body []byte, v any) { + t.Helper() + if status != want { + t.Fatalf("status = %d, want %d; body: %s", status, want, body) + } + if v == nil { + return + } + if err := json.Unmarshal(body, v); err != nil { + t.Fatalf("decode %T: %v; body: %s", v, err, body) + } +} + +// errCode extracts the machine-readable code from an error envelope. +func errCode(t *testing.T, body []byte) api.Code { + t.Helper() + var env api.ErrorEnvelope + if err := json.Unmarshal(body, &env); err != nil { + t.Fatalf("decode error envelope: %v; body: %s", err, body) + } + return env.Error.Code +} + +// createProject is the fixture most tests start from. +func (e *env) createProject(t *testing.T, name string) api.Project { + t.Helper() + status, body := e.do(t, http.MethodPost, api.PathProjects(), e.adminToken, + api.CreateProjectRequest{Name: name}) + var p api.Project + mustJSON(t, status, http.StatusCreated, body, &p) + return p +} + +// projectID looks up the row id, which the wire format deliberately does not +// carry. +func (e *env) projectID(t *testing.T, name string) int64 { + t.Helper() + p, err := e.db.ProjectByName(context.Background(), name) + if err != nil { + t.Fatalf("project %q: %v", name, err) + } + return p.ID +} diff --git a/internal/adminapi/blobs.go b/internal/adminapi/blobs.go new file mode 100644 index 0000000..dd2325a --- /dev/null +++ b/internal/adminapi/blobs.go @@ -0,0 +1,39 @@ +package adminapi + +import ( + "net/http" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/httpx" +) + +// putBlob handles PUT /api/v1/blobs/{digest}. +// +// Blobs are global rather than per-project because the store deduplicates +// across projects, so any authenticated key may upload one — but only content +// some manifest already declared, and only bytes that really hash to the digest +// in the URL. Both checks live in deploy.Service and cas.Store; the digest here +// is a claim until then. +// +// The route is deliberately not owner-guarded: there is no project in the path +// to guard against. What a caller can do with it is bounded by the manifest +// requirement, and the resulting cross-project existence oracle is the known, +// documented trade-off of shared deduplication. +func (s *Server) putBlob(w http.ResponseWriter, r *http.Request) error { + digest, err := cas.ParseDigest(r.PathValue("digest")) + if err != nil { + return api.Errorf(api.CodeBadRequest, "%s", err) + } + + size, stored, err := s.Deploy.Upload(r.Context(), digest, r.Body) + if err != nil { + return err + } + status := http.StatusOK // already had it + if stored { + status = http.StatusCreated + } + httpx.WriteJSON(w, status, api.BlobResponse{Digest: digest.String(), Size: size}) + return nil +} diff --git a/internal/adminapi/convert.go b/internal/adminapi/convert.go new file mode 100644 index 0000000..6ff023d --- /dev/null +++ b/internal/adminapi/convert.go @@ -0,0 +1,199 @@ +package adminapi + +import ( + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/store" +) + +// projectOf renders a stored project for the wire. +func (s *Server) projectOf(p *store.Project) api.Project { + out := api.Project{ + Name: p.Name, + DisplayName: p.DisplayName, + IndexFile: p.IndexFile, + NotFoundFile: p.NotFoundFile, + SPAFallback: p.SPAFallback, + CacheControl: p.CacheControl, + RetentionCount: p.RetentionCount, + RetentionGrace: p.RetentionGraceS, + MaxFiles: p.MaxFiles, + MaxFileBytes: p.MaxFileBytes, + MaxTotalBytes: p.MaxTotalBytes, + CreatedAt: p.CreatedAt, + UpdatedAt: p.UpdatedAt, + } + if s.BaseURL != "" { + out.URL = api.SiteURL(s.BaseURL, p.Name) + } + out.ActiveDeployment = s.activeOf(p) + return out +} + +// activeOf summarises what a project is serving right now. +// +// It reads the registry rather than the database so listing a hundred projects +// stays one query. The summary carries what is being served — id, size, +// timestamps — and not the row's metadata or error text; those come from +// fetching the deployment itself. +func (s *Server) activeOf(p *store.Project) *api.Deployment { + if s.Sites == nil { + return nil + } + sp, ok := s.Sites.Lookup(p.Name) + if !ok { + return nil + } + d := sp.Active() + if d == nil { + return nil + } + out := &api.Deployment{ + ID: d.ID, + Project: p.Name, + State: api.StateReady, // only a ready deployment can be active + Active: true, + FileCount: d.FileCount, + TotalBytes: d.TotalBytes, + CreatedAt: d.CreatedAt, + } + if !d.ActivatedAt.IsZero() { + t := d.ActivatedAt + out.ActivatedAt = &t + } + return out +} + +// deploymentOf renders a stored deployment for the wire. +// +// The row id stays behind: the wire only ever names a deployment by its public +// id, so nothing a client holds can be walked to a neighbouring row. +func deploymentOf(p *store.Project, d *store.Deployment) api.Deployment { + return api.Deployment{ + ID: d.PublicID, + Project: p.Name, + State: string(d.State), + Active: d.Active, + FileCount: d.FileCount, + TotalBytes: d.TotalBytes, + Meta: d.Meta, + Error: d.Error, + CreatedAt: d.CreatedAt, + FinalizedAt: copyTime(d.FinalizedAt), + ActivatedAt: copyTime(d.ActivatedAt), + } +} + +// keyOf renders a stored key. It deliberately has no access to the secret: the +// store never loads one in a form that could be rendered, only the hash. +func keyOf(k *store.APIKey, projectName string) api.Key { + return api.Key{ + ID: k.ID, + Scope: string(k.Scope), + Project: projectName, + Name: k.Name, + CreatedAt: k.CreatedAt, + ExpiresAt: copyTime(k.ExpiresAt), + LastUsed: copyTime(k.LastUsedAt), + RevokedAt: copyTime(k.RevokedAt), + } +} + +// copyTime defensively copies an optional timestamp so a response value cannot +// alias a cached store row. +func copyTime(t *time.Time) *time.Time { + if t == nil { + return nil + } + v := *t + return &v +} + +// applyPatch folds a partial update into p, validating as it goes. +// +// A patch is all-or-nothing: it is applied to a copy by the caller, so a +// rejected field leaves the stored project untouched rather than half-updated. +func (s *Server) applyPatch(p *store.Project, patch *api.ProjectPatch) error { + if patch == nil { + return nil + } + if v := patch.DisplayName; v != nil { + if err := checkText("display_name", *v, maxDisplayNameLen); err != nil { + return err + } + p.DisplayName = *v + } + if v := patch.IndexFile; v != nil { + if err := checkSitePath("index_file", *v); err != nil { + return err + } + p.IndexFile = *v + } + if v := patch.NotFoundFile; v != nil { + // The empty string is how a patch clears the custom 404 document; every + // other value must name a real relative path. + if *v != "" { + if err := checkSitePath("not_found_file", *v); err != nil { + return err + } + } + p.NotFoundFile = *v + } + if v := patch.SPAFallback; v != nil { + p.SPAFallback = *v + } + if v := patch.CacheControl; v != nil { + if err := checkHeaderValue("cache_control", *v); err != nil { + return err + } + p.CacheControl = *v + } + if v := patch.RetentionCount; v != nil { + // At least one: retaining zero deployments would delete the active one. + if *v < 1 || *v > 1000 { + return api.Errorf(api.CodeBadRequest, "retention_count must be between 1 and 1000") + } + p.RetentionCount = *v + } + if v := patch.RetentionGrace; v != nil { + if *v < 0 || *v > 30*24*3600 { + return api.Errorf(api.CodeBadRequest, + "retention_grace_s must be between 0 and %d", 30*24*3600) + } + p.RetentionGraceS = *v + } + if v := patch.MaxFiles; v != nil { + if err := checkCeiling("max_files", int64(*v), int64(s.Limits.MaxManifestFiles)); err != nil { + return err + } + p.MaxFiles = *v + } + if v := patch.MaxFileBytes; v != nil { + if err := checkCeiling("max_file_bytes", *v, s.Limits.MaxFileBytes); err != nil { + return err + } + p.MaxFileBytes = *v + } + if v := patch.MaxTotalBytes; v != nil { + if *v < 1 { + return api.Errorf(api.CodeBadRequest, "max_total_bytes must be positive") + } + p.MaxTotalBytes = *v + } + return nil +} + +// checkCeiling enforces that a per-project limit is positive and does not +// exceed the server-wide one. A project may lower its own ceiling but never +// raise it past what the operator configured. +func checkCeiling(field string, v, ceiling int64) error { + if v < 1 { + return api.Errorf(api.CodeBadRequest, "%s must be positive", field) + } + if ceiling > 0 && v > ceiling { + return api.Errorf(api.CodeBadRequest, + "%s must be at most the server limit of %d", field, ceiling) + } + return nil +} diff --git a/internal/adminapi/deployments.go b/internal/adminapi/deployments.go new file mode 100644 index 0000000..a70681a --- /dev/null +++ b/internal/adminapi/deployments.go @@ -0,0 +1,377 @@ +package adminapi + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/httpx" + "github.com/iceBear67/simplepages/internal/pathutil" + "github.com/iceBear67/simplepages/internal/store" +) + +// Caps on the deployment metadata a CI job may attach. Generous enough for a +// commit sha, a branch, a run URL and an actor; small enough that the column +// cannot become a place to store things. +const ( + maxMetaEntries = 32 + maxMetaKeyLen = 64 + maxMetaValueLen = 512 +) + +// createDeployment handles POST /api/v1/projects/{name}/deployments. +func (s *Server) createDeployment(w http.ResponseWriter, r *http.Request) error { + p, err := s.project(r) + if err != nil { + return err + } + // The body is optional: a deployment that carries no metadata is a bare POST, + // which is what `curl -X POST` and any shell-driven CI job send. + var req api.CreateDeploymentRequest + if r.ContentLength != 0 { + if err := httpx.DecodeJSON(w, r, s.maxJSON(), &req); err != nil { + return err + } + } + if err := checkMeta(req.Meta); err != nil { + return err + } + id, err := s.identity(r) + if err != nil { + return err + } + + dep, err := s.Deploy.Create(r.Context(), p, id.KeyID, req.Meta) + if err != nil { + return err + } + httpx.LogAttr(r.Context(), "deployment", dep.PublicID) + w.Header().Set("Location", api.PathDeployment(p.Name, dep.PublicID)) + httpx.WriteJSON(w, http.StatusCreated, deploymentOf(p, dep)) + return nil +} + +// setManifest handles POST .../deployments/{id}/manifest. +// +// The body is walked one entry at a time rather than unmarshalled whole: a +// 50,000-file manifest would otherwise be resident twice over, once as raw JSON +// and once as structs, for no benefit — nothing here needs to see the entries +// together. +func (s *Server) setManifest(w http.ResponseWriter, r *http.Request) error { + p, dep, err := s.deployment(r) + if err != nil { + return err + } + + r.Body = http.MaxBytesReader(w, r.Body, s.maxManifest()) + var ( + files []store.FileRow + paths = pathutil.NewSet(0) + unique = make(map[cas.Digest]struct{}) + totalBytes int64 + ) + err = decodeManifest(json.NewDecoder(r.Body), func(f api.FileEntry) error { + if err := paths.Add(f.Path); err != nil { + return api.Errorf(api.CodeInvalidPath, "%s", err) + } + digest, err := cas.ParseDigest(f.Digest) + if err != nil { + return api.Errorf(api.CodeBadRequest, "%s: %s", f.Path, err) + } + if f.Size < 0 { + return api.Errorf(api.CodeBadRequest, "%s: size must not be negative", f.Path) + } + if f.Size > p.MaxFileBytes { + return api.Errorf(api.CodeLimitExceeded, + "%s is %d bytes; this project allows at most %d per file", f.Path, f.Size, p.MaxFileBytes) + } + if len(files) >= p.MaxFiles { + return api.Errorf(api.CodeLimitExceeded, + "a deployment of this project may contain at most %d files", p.MaxFiles) + } + totalBytes += f.Size + if totalBytes > p.MaxTotalBytes { + return api.Errorf(api.CodeLimitExceeded, + "a deployment of this project may total at most %d bytes", p.MaxTotalBytes) + } + unique[digest] = struct{}{} + files = append(files, store.FileRow{Path: f.Path, Digest: digest, Size: f.Size}) + return nil + }) + if err != nil { + return err + } + if len(files) == 0 { + return api.Errorf(api.CodeBadRequest, "a manifest must list at least one file") + } + + missing, missingBytes, err := s.Deploy.SetManifest(r.Context(), dep, files) + if err != nil { + return err + } + out := api.ManifestResponse{ + Missing: make([]string, 0, len(missing)), + MissingBytes: missingBytes, + Have: len(unique) - len(missing), + FileCount: len(files), + TotalBytes: totalBytes, + } + for _, d := range missing { + out.Missing = append(out.Missing, d.String()) + } + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} + +// finalize handles POST .../deployments/{id}/finalize. +func (s *Server) finalize(w http.ResponseWriter, r *http.Request) error { + p, dep, err := s.deployment(r) + if err != nil { + return err + } + if err := httpx.NoBody(r); err != nil { + return err + } + dep, err = s.Deploy.Finalize(r.Context(), p, dep) + if err != nil { + return err + } + httpx.WriteJSON(w, http.StatusOK, deploymentOf(p, dep)) + return nil +} + +// activate handles POST .../deployments/{id}/activate. +// +// This is also the rollback endpoint: activating an older ready deployment is +// the same operation, and costs the same single pointer store. +func (s *Server) activate(w http.ResponseWriter, r *http.Request) error { + p, dep, err := s.deployment(r) + if err != nil { + return err + } + if err := httpx.NoBody(r); err != nil { + return err + } + dep, err = s.Deploy.Activate(r.Context(), p, dep) + if err != nil { + return err + } + out := deploymentOf(p, dep) + if s.BaseURL != "" { + out.URL = api.SiteURL(s.BaseURL, p.Name) + } + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} + +// listDeployments handles GET .../deployments. +func (s *Server) listDeployments(w http.ResponseWriter, r *http.Request) error { + p, err := s.project(r) + if err != nil { + return err + } + limit, err := intQuery(r, "limit", 100, 1, 500) + if err != nil { + return err + } + state, err := parseState(r.URL.Query().Get("state")) + if err != nil { + return err + } + deps, next, err := s.DB.ListDeployments(r.Context(), p.ID, state, limit, r.URL.Query().Get("cursor")) + if err != nil { + return err + } + out := api.DeploymentList{Deployments: make([]api.Deployment, 0, len(deps)), NextCursor: next} + for _, dep := range deps { + out.Deployments = append(out.Deployments, deploymentOf(p, dep)) + } + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} + +// getDeployment handles GET .../deployments/{id}, with ?files=true adding the +// manifest. +// +// The manifest is opt-in because it can be fifty thousand entries: a listing +// that carried it by default would make `pages deployment list` unusable on a +// large site for information nobody asked for. +func (s *Server) getDeployment(w http.ResponseWriter, r *http.Request) error { + p, dep, err := s.deployment(r) + if err != nil { + return err + } + out := deploymentOf(p, dep) + if r.URL.Query().Get("files") == "true" { + files, err := s.DB.DeploymentFiles(r.Context(), dep.ID) + if err != nil { + return err + } + out.Files = make([]api.FileEntry, 0, len(files)) + for _, f := range files { + out.Files = append(out.Files, api.FileEntry{ + Path: f.Path, Digest: f.Digest.String(), Size: f.Size, + }) + } + } + if dep.Active && s.BaseURL != "" { + out.URL = api.SiteURL(s.BaseURL, p.Name) + } + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} + +// deleteDeployment handles DELETE .../deployments/{id}. Deleting the active one +// is a 409: the client is expected to activate something else first, so that +// the project is never left with nothing to serve by accident. +func (s *Server) deleteDeployment(w http.ResponseWriter, r *http.Request) error { + if err := httpx.NoBody(r); err != nil { + return err + } + p, dep, err := s.deployment(r) + if err != nil { + return err + } + if err := s.Deploy.Delete(r.Context(), p, dep); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +// parseState validates the ?state= filter. The empty string means no filter; +// anything else must be a state that exists, so a typo is a 400 rather than a +// silently empty list. +func parseState(raw string) (store.State, error) { + switch raw { + case "": + return "", nil + case api.StatePending, api.StateUploading, api.StateReady, api.StateFailed, api.StateDeleting: + return store.State(raw), nil + } + return "", api.Errorf(api.CodeBadRequest, "unknown deployment state %q", raw) +} + +// deployment resolves both the {name} and {id} wildcards. +// +// The deployment is looked up within the project, never on its own: a +// project-scoped caller that guessed another project's deployment id gets the +// same "no such deployment" as one that guessed a nonexistent one. +func (s *Server) deployment(r *http.Request) (*store.Project, *store.Deployment, error) { + p, err := s.project(r) + if err != nil { + return nil, nil, err + } + dep, err := s.DB.DeploymentByPublicID(r.Context(), p.ID, r.PathValue("id")) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, nil, api.Errorf(api.CodeNotFound, "no such deployment") + } + return nil, nil, err + } + httpx.LogAttr(r.Context(), "deployment", dep.PublicID) + return p, dep, nil +} + +// decodeManifest streams {"files":[...]} and calls onFile for each entry. +func decodeManifest(dec *json.Decoder, onFile func(api.FileEntry) error) error { + if err := expectDelim(dec, '{', "manifest must be a JSON object"); err != nil { + return err + } + seen := false + for dec.More() { + tok, err := dec.Token() + if err != nil { + return badJSON(err) + } + key, _ := tok.(string) + if key != "files" { + // Unknown keys are skipped rather than rejected: a newer CLI may send + // a field this server predates, and the body is bounded anyway. + var skip json.RawMessage + if err := dec.Decode(&skip); err != nil { + return badJSON(err) + } + continue + } + seen = true + if err := expectDelim(dec, '[', "files must be an array"); err != nil { + return err + } + for dec.More() { + var f api.FileEntry + if err := dec.Decode(&f); err != nil { + return badJSON(err) + } + if err := onFile(f); err != nil { + return err + } + } + if _, err := dec.Token(); err != nil { // closing ] + return badJSON(err) + } + } + if _, err := dec.Token(); err != nil { // closing } + return badJSON(err) + } + if !seen { + return api.Errorf(api.CodeBadRequest, "manifest is missing the files array") + } + return nil +} + +func expectDelim(dec *json.Decoder, want json.Delim, msg string) error { + tok, err := dec.Token() + if err != nil { + return badJSON(err) + } + if d, ok := tok.(json.Delim); !ok || d != want { + return api.Errorf(api.CodeBadRequest, "%s", msg) + } + return nil +} + +// badJSON turns a decoder failure into a client error, keeping the body-size +// rejection distinguishable from a syntax one. +func badJSON(err error) error { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + return api.Errorf(api.CodePayloadTooLarge, "manifest exceeds %d bytes", maxErr.Limit) + } + var syn *json.SyntaxError + if errors.As(err, &syn) { + return api.Errorf(api.CodeBadRequest, "malformed JSON at byte %d", syn.Offset) + } + var typeErr *json.UnmarshalTypeError + if errors.As(err, &typeErr) { + return api.Errorf(api.CodeBadRequest, "field %q: want %s", typeErr.Field, typeErr.Type) + } + return api.Errorf(api.CodeBadRequest, "malformed manifest") +} + +func checkMeta(meta map[string]string) error { + if len(meta) > maxMetaEntries { + return api.Errorf(api.CodeBadRequest, "meta may hold at most %d entries", maxMetaEntries) + } + for k, v := range meta { + if k == "" { + return api.Errorf(api.CodeBadRequest, "meta keys must not be empty") + } + if err := checkText("meta key", k, maxMetaKeyLen); err != nil { + return err + } + if err := checkText("meta value of "+k, v, maxMetaValueLen); err != nil { + return err + } + } + return nil +} + +func (s *Server) maxManifest() int64 { + if s.Limits.MaxManifestBytes > 0 { + return s.Limits.MaxManifestBytes + } + return 64 << 20 +} diff --git a/internal/adminapi/deployments_test.go b/internal/adminapi/deployments_test.go new file mode 100644 index 0000000..0e5dfa5 --- /dev/null +++ b/internal/adminapi/deployments_test.go @@ -0,0 +1,589 @@ +package adminapi + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/deploy" +) + +// entry is a manifest line for content the test holds. +func entry(path, content string) api.FileEntry { + return api.FileEntry{Path: path, Digest: cas.Sum([]byte(content)).String(), Size: int64(len(content))} +} + +// putBlob uploads raw bytes, which is the one endpoint that is not JSON in and +// so cannot go through env.do. +func (e *env) putBlob(t *testing.T, token, content string) (int, []byte) { + t.Helper() + return e.putBlobAs(t, token, cas.Sum([]byte(content)).String(), content) +} + +func (e *env) putBlobAs(t *testing.T, token, hexDigest, content string) (int, []byte) { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, + e.ts.URL+api.PathBlob(hexDigest), strings.NewReader(content)) + if err != nil { + t.Fatal(err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := e.ts.Client().Do(req) + if err != nil { + t.Fatalf("PUT blob: %v", err) + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, raw +} + +// startDeployment is the fixture the manifest and finalize tests build on. +func (e *env) startDeployment(t *testing.T, token, project string) api.Deployment { + t.Helper() + status, body := e.do(t, http.MethodPost, api.PathDeployments(project), token, + api.CreateDeploymentRequest{}) + var dep api.Deployment + mustJSON(t, status, http.StatusCreated, body, &dep) + return dep +} + +// TestDeploymentFlow walks the three endpoints in the order a CI job does and +// asserts on what a client can actually see at each step. +func TestDeploymentFlow(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + // --- create + status, body := e.do(t, http.MethodPost, api.PathDeployments(p.Name), token, + api.CreateDeploymentRequest{Meta: map[string]string{"git_sha": "abc123", "branch": "main"}}) + var dep api.Deployment + mustJSON(t, status, http.StatusCreated, body, &dep) + if !strings.HasPrefix(dep.ID, "dpl_") { + t.Errorf("id = %q, want a dpl_ prefix", dep.ID) + } + if dep.State != api.StatePending || dep.Project != "demo" { + t.Errorf("deployment = %+v", dep) + } + if dep.Meta["git_sha"] != "abc123" { + t.Errorf("meta = %v, want the submitted values back", dep.Meta) + } + // A bare POST with no body is a deployment with no metadata, and the response + // points at where the new deployment lives. + resp := e.doResp(t, http.MethodPost, api.PathDeployments(p.Name), token, nil) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("bodyless create: status = %d, want 201", resp.StatusCode) + } + if loc := resp.Header.Get("Location"); !strings.Contains(loc, api.PathDeployments(p.Name)+"/dpl_") { + t.Errorf("Location = %q, want the new deployment's path", loc) + } + + // --- manifest + contents := map[string]string{ + "index.html": "

hello

", + "assets/app.js": "console.log(1)", + "copy.html": "

hello

", // shares a blob with index.html + } + status, body = e.do(t, http.MethodPost, api.PathManifest(p.Name, dep.ID), token, + api.ManifestRequest{Files: []api.FileEntry{ + entry("index.html", contents["index.html"]), + entry("assets/app.js", contents["assets/app.js"]), + entry("copy.html", contents["copy.html"]), + }}) + var mr api.ManifestResponse + mustJSON(t, status, http.StatusOK, body, &mr) + if len(mr.Missing) != 2 { + t.Fatalf("missing = %v, want the 2 distinct blobs", mr.Missing) + } + if mr.Have != 0 || mr.FileCount != 3 { + t.Errorf("have = %d, file_count = %d, want 0 and 3", mr.Have, mr.FileCount) + } + if want := int64(len(contents["index.html"])*2 + len(contents["assets/app.js"])); mr.TotalBytes != want { + t.Errorf("total_bytes = %d, want %d", mr.TotalBytes, want) + } + if mr.MissingBytes != int64(len(contents["index.html"])+len(contents["assets/app.js"])) { + t.Errorf("missing_bytes = %d counts the shared blob twice", mr.MissingBytes) + } + + // --- finalize before the content arrives names what is outstanding + status, body = e.do(t, http.MethodPost, api.PathFinalize(p.Name, dep.ID), token, nil) + if status != http.StatusConflict { + t.Fatalf("premature finalize: status = %d, body: %s", status, body) + } + if code := errCode(t, body); code != api.CodeBlobsMissing { + t.Errorf("code = %q, want %q", code, api.CodeBlobsMissing) + } + var env api.ErrorEnvelope + if err := json.Unmarshal(body, &env); err != nil { + t.Fatal(err) + } + if missing, _ := env.Error.Details["missing"].([]any); len(missing) != 2 { + t.Errorf("details.missing = %v, want the 2 digests to retry", env.Error.Details["missing"]) + } + + // --- upload + for _, name := range []string{"index.html", "assets/app.js"} { + status, body := e.putBlob(t, token, contents[name]) + var br api.BlobResponse + mustJSON(t, status, http.StatusCreated, body, &br) + if br.Digest != cas.Sum([]byte(contents[name])).String() || br.Size != int64(len(contents[name])) { + t.Errorf("%s: response = %+v", name, br) + } + } + // A re-upload is a cheap 200 rather than a 201: this is what makes a retried + // deploy fast, so the distinction is part of the contract. + status, body = e.putBlob(t, token, contents["index.html"]) + mustJSON(t, status, http.StatusOK, body, nil) + + // --- a second manifest now reports what the server already has + dep2 := e.startDeployment(t, token, p.Name) + status, body = e.do(t, http.MethodPost, api.PathManifest(p.Name, dep2.ID), token, + api.ManifestRequest{Files: []api.FileEntry{ + entry("index.html", contents["index.html"]), + entry("assets/app.js", contents["assets/app.js"]), + entry("new.txt", "brand new"), + }}) + mr = api.ManifestResponse{} + mustJSON(t, status, http.StatusOK, body, &mr) + if len(mr.Missing) != 1 || mr.Have != 2 { + t.Errorf("missing = %v, have = %d; want only the new file to be asked for", mr.Missing, mr.Have) + } + + // --- finalize + status, body = e.do(t, http.MethodPost, api.PathFinalize(p.Name, dep.ID), token, nil) + var done api.Deployment + mustJSON(t, status, http.StatusOK, body, &done) + if done.State != api.StateReady || done.FileCount != 3 { + t.Fatalf("finalized = %+v", done) + } + if done.Active { + t.Error("finalize activated the deployment") + } + if done.FinalizedAt == nil { + t.Error("finalized_at is missing from the response") + } + + // The tree really is on disk, and byte-identical to what was uploaded. + dir := deploy.DeploymentDir(e.deployDir, e.projectID(t, "demo"), dep.ID) + for name, want := range contents { + got, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + if string(got) != want { + t.Errorf("%s = %q, want %q", name, got, want) + } + } + + // Finalizing again is a no-op, not a second assembly. + status, body = e.do(t, http.MethodPost, api.PathFinalize(p.Name, dep.ID), token, nil) + mustJSON(t, status, http.StatusOK, body, nil) +} + +func TestManifestRejectsBadEntries(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + good := entry("index.html", "hello") + cases := []struct { + name string + files []api.FileEntry + want api.Code + }{ + {"escaping path", []api.FileEntry{entry("../etc/passwd", "x")}, api.CodeInvalidPath}, + {"absolute path", []api.FileEntry{entry("/etc/passwd", "x")}, api.CodeInvalidPath}, + {"empty path", []api.FileEntry{entry("", "x")}, api.CodeInvalidPath}, + {"trailing slash", []api.FileEntry{entry("dir/", "x")}, api.CodeInvalidPath}, + {"backslash", []api.FileEntry{entry(`a\b`, "x")}, api.CodeInvalidPath}, + {"NUL byte", []api.FileEntry{entry("a\x00b", "x")}, api.CodeInvalidPath}, + {"duplicate path", []api.FileEntry{good, good}, api.CodeInvalidPath}, + // A case-insensitive filesystem would silently collapse these two into + // one file, so the manifest is refused rather than assembled wrongly. + {"case collision", []api.FileEntry{entry("Index.html", "a"), entry("index.html", "b")}, api.CodeInvalidPath}, + {"file used as a directory", []api.FileEntry{entry("a", "x"), entry("a/b", "y")}, api.CodeInvalidPath}, + {"uppercase hex digest", []api.FileEntry{{Path: "a", Digest: strings.ToUpper(good.Digest), Size: 1}}, api.CodeBadRequest}, + {"short digest", []api.FileEntry{{Path: "a", Digest: "abc", Size: 1}}, api.CodeBadRequest}, + {"negative size", []api.FileEntry{{Path: "a", Digest: good.Digest, Size: -1}}, api.CodeBadRequest}, + {"no files", []api.FileEntry{}, api.CodeBadRequest}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dep := e.startDeployment(t, token, p.Name) + status, body := e.do(t, http.MethodPost, api.PathManifest(p.Name, dep.ID), token, + api.ManifestRequest{Files: tc.files}) + if status < 400 || status >= 500 { + t.Fatalf("status = %d, want a 4xx; body: %s", status, body) + } + if code := errCode(t, body); code != tc.want { + t.Errorf("code = %q, want %q; body: %s", code, tc.want, body) + } + }) + } +} + +func TestManifestEnforcesProjectLimits(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + id := e.projectID(t, "demo") + token := e.mintProject(t, id, "ci") + + // Lower the project's own ceilings; a project may tighten but never raise + // them, and these are the values the manifest is checked against. + two := 2 + small := int64(4) + status, body := e.do(t, http.MethodPatch, api.PathProject(p.Name), e.adminToken, + api.ProjectPatch{MaxFiles: &two, MaxFileBytes: &small}) + mustJSON(t, status, http.StatusOK, body, nil) + + for _, tc := range []struct { + name string + files []api.FileEntry + }{ + {"too many files", []api.FileEntry{entry("a", "1"), entry("b", "2"), entry("c", "3")}}, + {"file too large", []api.FileEntry{entry("a", "much too long")}}, + } { + t.Run(tc.name, func(t *testing.T) { + dep := e.startDeployment(t, token, p.Name) + status, body := e.do(t, http.MethodPost, api.PathManifest(p.Name, dep.ID), token, + api.ManifestRequest{Files: tc.files}) + if code := errCode(t, body); code != api.CodeLimitExceeded { + t.Errorf("status %d, code = %q, want %q; body: %s", status, code, api.CodeLimitExceeded, body) + } + }) + } + + t.Run("total too large", func(t *testing.T) { + big := int64(3) + status, body := e.do(t, http.MethodPatch, api.PathProject(p.Name), e.adminToken, + api.ProjectPatch{MaxTotalBytes: &big}) + mustJSON(t, status, http.StatusOK, body, nil) + + dep := e.startDeployment(t, token, p.Name) + status, body = e.do(t, http.MethodPost, api.PathManifest(p.Name, dep.ID), token, + api.ManifestRequest{Files: []api.FileEntry{entry("a", "12"), entry("b", "34")}}) + if code := errCode(t, body); code != api.CodeLimitExceeded { + t.Errorf("status %d, code = %q, want %q; body: %s", status, code, api.CodeLimitExceeded, body) + } + }) +} + +// The manifest body is bounded before it is parsed, so a 50,000-entry manifest +// cannot be turned into an unbounded allocation. +func TestManifestBodyIsBounded(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + e.server.Limits.MaxManifestBytes = 256 + + dep := e.startDeployment(t, token, p.Name) + files := make([]api.FileEntry, 32) + for i := range files { + files[i] = entry(string(rune('a'+i%26))+strings.Repeat("x", i), "content") + } + status, body := e.do(t, http.MethodPost, api.PathManifest(p.Name, dep.ID), token, + api.ManifestRequest{Files: files}) + if code := errCode(t, body); code != api.CodePayloadTooLarge { + t.Errorf("status %d, code = %q, want %q; body: %s", status, code, api.CodePayloadTooLarge, body) + } +} + +func TestManifestRejectsMalformedBodies(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + for _, tc := range []struct{ name, body string }{ + {"not an object", `[]`}, + {"files is not an array", `{"files":{}}`}, + {"entry is not an object", `{"files":["index.html"]}`}, + {"truncated", `{"files":[{"path":"a"`}, + {"no files key", `{"meta":{}}`}, + {"empty body", ``}, + } { + t.Run(tc.name, func(t *testing.T) { + dep := e.startDeployment(t, token, p.Name) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, + e.ts.URL+api.PathManifest(p.Name, dep.ID), strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + resp, err := e.ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", resp.StatusCode, raw) + } + if code := errCode(t, raw); code != api.CodeBadRequest { + t.Errorf("code = %q, want %q", code, api.CodeBadRequest) + } + }) + } +} + +// Unknown top-level keys are skipped so a newer CLI can add a field without +// every older server rejecting its deployments. +func TestManifestIgnoresUnknownTopLevelKeys(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + dep := e.startDeployment(t, token, p.Name) + + body := `{"future_field":{"a":[1,2,3]},"files":[` + + `{"path":"index.html","digest":"` + cas.Sum([]byte("hi")).String() + `","size":2}` + + `],"another":"ignored"}` + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, + e.ts.URL+api.PathManifest(p.Name, dep.ID), strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+token) + resp, err := e.ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + var mr api.ManifestResponse + mustJSON(t, resp.StatusCode, http.StatusOK, raw, &mr) + if mr.FileCount != 1 { + t.Errorf("file_count = %d, want 1", mr.FileCount) + } +} + +func TestCreateDeploymentRejectsOversizedMeta(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + cases := map[string]map[string]string{ + "too many entries": func() map[string]string { + m := make(map[string]string, maxMetaEntries+1) + for i := range maxMetaEntries + 1 { + m[string(rune('a'+i%26))+strings.Repeat("k", i)] = "v" + } + return m + }(), + "key too long": {strings.Repeat("k", maxMetaKeyLen+1): "v"}, + "value too long": {"k": strings.Repeat("v", maxMetaValueLen+1)}, + "empty key": {"": "v"}, + } + for name, meta := range cases { + t.Run(name, func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathDeployments(p.Name), token, + api.CreateDeploymentRequest{Meta: meta}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + }) + } +} + +// A project-scoped key reaches its own project's deployments and nothing else. +// The check compares resolved project ids, so it holds for the deployment id in +// the path as well as for the project name. +func TestDeploymentEndpointsAreProjectScoped(t *testing.T) { + e := newEnv(t) + e.createProject(t, "mine") + e.createProject(t, "theirs") + mine := e.mintProject(t, e.projectID(t, "mine"), "ci") + theirs := e.mintProject(t, e.projectID(t, "theirs"), "ci") + + // Their deployment, created with their own key. + dep := e.startDeployment(t, theirs, "theirs") + + t.Run("another project's endpoint is forbidden", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathDeployments("theirs"), mine, api.CreateDeploymentRequest{}) + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body: %s", status, body) + } + }) + + t.Run("another project's deployment id is not found", func(t *testing.T) { + // Named under the caller's own project, so the ownership guard passes and + // the scoping that matters is the one in the lookup itself. + status, body := e.do(t, http.MethodPost, api.PathFinalize("mine", dep.ID), mine, nil) + if status != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body: %s", status, body) + } + if code := errCode(t, body); code != api.CodeNotFound { + t.Errorf("code = %q, want %q", code, api.CodeNotFound) + } + }) + + t.Run("an admin reaches every project", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathDeployments("theirs"), e.adminToken, + api.CreateDeploymentRequest{}) + mustJSON(t, status, http.StatusCreated, body, nil) + }) +} + +func TestBlobUpload(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + dep := e.startDeployment(t, token, "demo") + + content := "hello" + digest := cas.Sum([]byte(content)).String() + + t.Run("content no manifest declared is refused", func(t *testing.T) { + status, body := e.putBlob(t, token, "nobody asked for this") + if status != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body: %s", status, body) + } + if code := errCode(t, body); code != api.CodeNotFound { + t.Errorf("code = %q, want %q", code, api.CodeNotFound) + } + }) + + status, body := e.do(t, http.MethodPost, api.PathManifest("demo", dep.ID), token, + api.ManifestRequest{Files: []api.FileEntry{entry("index.html", content)}}) + mustJSON(t, status, http.StatusOK, body, nil) + + t.Run("a digest that is not one is a 400", func(t *testing.T) { + for _, bad := range []string{"nothex", strings.ToUpper(digest), digest + "00", "../../etc/passwd"} { + status, body := e.putBlobAs(t, token, bad, content) + if status != http.StatusBadRequest { + t.Errorf("%q: status = %d, want 400; body: %s", bad, status, body) + } + } + }) + + t.Run("content that does not hash to its digest is rejected", func(t *testing.T) { + // The same length the manifest declared, so this is a digest rejection + // and not the length check catching it first. + status, body := e.putBlobAs(t, token, digest, "forge") + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + if code := errCode(t, body); code != api.CodeDigestMismatch { + t.Errorf("code = %q, want %q", code, api.CodeDigestMismatch) + } + if has, _ := e.cas.Has(cas.Sum([]byte(content))); has { + t.Fatal("the forged content was stored under the honest digest") + } + }) + + t.Run("more bytes than the manifest declared are rejected", func(t *testing.T) { + dep := e.startDeployment(t, token, "demo") + long := strings.Repeat("x", 1024) + status, body := e.do(t, http.MethodPost, api.PathManifest("demo", dep.ID), token, + api.ManifestRequest{Files: []api.FileEntry{{Path: "a", Digest: cas.Sum([]byte(long)).String(), Size: 4}}}) + mustJSON(t, status, http.StatusOK, body, nil) + + status, body = e.putBlobAs(t, token, cas.Sum([]byte(long)).String(), long) + if status < 400 || status >= 500 { + t.Fatalf("status = %d, want a 4xx; body: %s", status, body) + } + }) + + t.Run("the honest content is accepted", func(t *testing.T) { + status, body := e.putBlob(t, token, content) + var br api.BlobResponse + mustJSON(t, status, http.StatusCreated, body, &br) + if br.Digest != digest || br.Size != int64(len(content)) { + t.Errorf("response = %+v", br) + } + }) +} + +// Every deployment route is authenticated, including the blob one that has no +// project in its path to guard against. +func TestDeploymentRoutesRequireAuthentication(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + dep := e.startDeployment(t, token, "demo") + + for _, path := range []string{ + api.PathDeployments("demo"), + api.PathManifest("demo", dep.ID), + api.PathFinalize("demo", dep.ID), + } { + if status, body := e.do(t, http.MethodPost, path, "", nil); status != http.StatusUnauthorized { + t.Errorf("POST %s unauthenticated: status = %d, want 401; body: %s", path, status, body) + } + } + if status, body := e.putBlob(t, "", "anything"); status != http.StatusUnauthorized { + t.Errorf("PUT blob unauthenticated: status = %d, want 401; body: %s", status, body) + } +} + +func TestDeploymentRoutesRejectWrongMethods(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + dep := e.startDeployment(t, token, "demo") + + for _, tc := range []struct{ method, path, allow string }{ + {http.MethodDelete, api.PathDeployments("demo"), "GET, POST"}, + {http.MethodPatch, api.PathDeployment("demo", dep.ID), "GET, DELETE"}, + {http.MethodGet, api.PathManifest("demo", dep.ID), "POST"}, + {http.MethodGet, api.PathFinalize("demo", dep.ID), "POST"}, + {http.MethodGet, api.PathBlob(cas.Sum([]byte("x")).String()), "PUT"}, + } { + resp := e.doResp(t, tc.method, tc.path, token, nil) + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Errorf("%s %s: status = %d, want 405", tc.method, tc.path, resp.StatusCode) + } + if got := resp.Header.Get("Allow"); got != tc.allow { + t.Errorf("%s %s: Allow = %q, want %q", tc.method, tc.path, got, tc.allow) + } + } +} + +func TestFinalizeRejectsARequestBody(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + dep := e.startDeployment(t, token, "demo") + + status, body := e.do(t, http.MethodPost, api.PathFinalize("demo", dep.ID), token, + map[string]string{"activate": "true"}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } +} + +// Nothing on these routes may put a credential on the wire or in the log, the +// same guarantee the project and key endpoints carry. +func TestDeploymentEndpointsNeverEchoTheToken(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + secret := token[strings.LastIndex(token, "_")+1:] + + dep := e.startDeployment(t, token, "demo") + _, manifestBody := e.do(t, http.MethodPost, api.PathManifest("demo", dep.ID), token, + api.ManifestRequest{Files: []api.FileEntry{entry("index.html", "hi")}}) + _, blobBody := e.putBlob(t, token, "hi") + _, finalizeBody := e.do(t, http.MethodPost, api.PathFinalize("demo", dep.ID), token, nil) + + for name, body := range map[string][]byte{ + "manifest": manifestBody, "blob": blobBody, "finalize": finalizeBody, + "log": e.logBuf.Bytes(), + } { + if bytes.Contains(body, []byte(secret)) || bytes.Contains(body, []byte(token)) { + t.Errorf("the %s output contains the token", name) + } + } +} diff --git a/internal/adminapi/keys.go b/internal/adminapi/keys.go new file mode 100644 index 0000000..d5d645f --- /dev/null +++ b/internal/adminapi/keys.go @@ -0,0 +1,172 @@ +package adminapi + +import ( + "errors" + "net/http" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/auth" + "github.com/iceBear67/simplepages/internal/httpx" + "github.com/iceBear67/simplepages/internal/store" +) + +// createAdminKey handles POST /api/v1/keys. +func (s *Server) createAdminKey(w http.ResponseWriter, r *http.Request) error { + return s.createKey(w, r, nil) +} + +// createProjectKey handles POST /api/v1/projects/{name}/keys. +func (s *Server) createProjectKey(w http.ResponseWriter, r *http.Request) error { + p, err := s.project(r) + if err != nil { + return err + } + return s.createKey(w, r, p) +} + +// createKey mints a key and returns it with its token. +// +// This is the only response in the whole API that carries a credential. It is +// marked no-store, and the token is never written anywhere else: not to the +// log, not to the database (only its SHA-256 is), and not to any later +// response. Losing it means minting a new key. +func (s *Server) createKey(w http.ResponseWriter, r *http.Request, project *store.Project) error { + var req api.CreateKeyRequest + if err := httpx.DecodeJSON(w, r, s.maxJSON(), &req); err != nil { + return err + } + if err := checkText("name", req.Name, maxKeyNameLen); err != nil { + return err + } + if req.ExpiresAt != nil && !req.ExpiresAt.After(time.Now()) { + return api.Errorf(api.CodeBadRequest, "expires_at is in the past") + } + + token, keyID, hash, err := auth.Mint() + if err != nil { + return err + } + k := &store.APIKey{ + ID: keyID, + SecretHash: hash[:], + Scope: store.ScopeAdmin, + Name: req.Name, + ExpiresAt: copyTime(req.ExpiresAt), + } + var projectName string + if project != nil { + k.Scope = store.ScopeProject + k.ProjectID = &project.ID + projectName = project.Name + } + if err := s.DB.CreateKey(r.Context(), k); err != nil { + return err + } + + httpx.LogAttr(r.Context(), "created_key_id", k.ID) + noStore(w) + httpx.WriteJSON(w, http.StatusCreated, api.CreateKeyResponse{ + Key: keyOf(k, projectName), + Token: token, + }) + return nil +} + +// listKeys handles GET /api/v1/keys, admin only. +func (s *Server) listKeys(w http.ResponseWriter, r *http.Request) error { + ks, err := s.DB.ListKeys(r.Context(), nil) + if err != nil { + return err + } + names, err := s.projectNames(r.Context()) + if err != nil { + return err + } + out := api.KeyList{Keys: make([]api.Key, 0, len(ks))} + for _, k := range ks { + var name string + if k.ProjectID != nil { + name = names[*k.ProjectID] + } + out.Keys = append(out.Keys, keyOf(k, name)) + } + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} + +// listProjectKeys handles GET /api/v1/projects/{name}/keys. +func (s *Server) listProjectKeys(w http.ResponseWriter, r *http.Request) error { + p, err := s.project(r) + if err != nil { + return err + } + ks, err := s.DB.ListKeys(r.Context(), &p.ID) + if err != nil { + return err + } + out := api.KeyList{Keys: make([]api.Key, 0, len(ks))} + for _, k := range ks { + out.Keys = append(out.Keys, keyOf(k, p.Name)) + } + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} + +// revokeKey handles DELETE /api/v1/keys/{key_id}. +// +// Admins may revoke anything; a project key may revoke keys belonging to its +// own project, which includes itself. That last case is deliberate: a CI runner +// that believes its token leaked should be able to burn it without waiting for +// an operator. +func (s *Server) revokeKey(w http.ResponseWriter, r *http.Request) error { + if err := httpx.NoBody(r); err != nil { + return err + } + ident, err := s.identity(r) + if err != nil { + return err + } + id := r.PathValue("key_id") + if !auth.ValidKeyID(id) { + // Malformed ids are rejected before the query so the endpoint cannot be + // used to probe the table with arbitrary strings. + return api.Errorf(api.CodeBadRequest, "malformed key id") + } + + k, err := s.DB.KeyByID(r.Context(), id) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + // A non-admin must not be able to tell "no such key" from "someone + // else's key": that would turn this endpoint into an oracle for + // which key ids exist. + if !ident.IsAdmin() { + return errNotYourKey() + } + return api.Errorf(api.CodeNotFound, "no such key") + } + return err + } + if !ident.IsAdmin() { + if k.ProjectID == nil || !ident.Owns(*k.ProjectID) { + return errNotYourKey() + } + } + + if err := s.DB.RevokeKey(r.Context(), id); err != nil { + if errors.Is(err, store.ErrNotFound) { + return api.Errorf(api.CodeNotFound, "no such key") + } + return err + } + + // Revocation must take effect now, not when the auth cache entry expires. + s.Auth.V.Invalidate() + httpx.LogAttr(r.Context(), "revoked_key_id", id) + w.WriteHeader(http.StatusNoContent) + return nil +} + +func errNotYourKey() error { + return api.Errorf(api.CodeForbidden, "this key does not have access to that key") +} diff --git a/internal/adminapi/keys_test.go b/internal/adminapi/keys_test.go new file mode 100644 index 0000000..c26119a --- /dev/null +++ b/internal/adminapi/keys_test.go @@ -0,0 +1,361 @@ +package adminapi + +import ( + "bytes" + "net/http" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/store" +) + +// createKey posts to path and returns the minted credential. +func (e *env) createKey(t *testing.T, path, token string, req api.CreateKeyRequest) api.CreateKeyResponse { + t.Helper() + status, body := e.do(t, http.MethodPost, path, token, req) + var out api.CreateKeyResponse + mustJSON(t, status, http.StatusCreated, body, &out) + return out +} + +func TestCreateAdminKeyReturnsWorkingToken(t *testing.T) { + e := newEnv(t) + out := e.createKey(t, api.PathKeys(), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + if out.Key.Scope != api.ScopeAdmin { + t.Errorf("scope = %q, want %q", out.Key.Scope, api.ScopeAdmin) + } + if out.Key.Project != "" { + t.Errorf("admin key reports project %q", out.Key.Project) + } + if !strings.HasPrefix(out.Token, "pgs_"+out.Key.ID+"_") { + t.Errorf("token does not carry its own key id %q", out.Key.ID) + } + + // The new key authenticates and reports itself. + status, body := e.do(t, http.MethodGet, api.PathWhoAmI(), out.Token, nil) + var who api.WhoAmI + mustJSON(t, status, http.StatusOK, body, &who) + if who.KeyID != out.Key.ID || who.Scope != api.ScopeAdmin || who.Name != "ci" { + t.Errorf("whoami = %+v, want key %q scope admin name ci", who, out.Key.ID) + } +} + +// The token exists on the wire exactly once. If it were cacheable, a shared +// proxy in front of the management API would keep a live credential on disk. +func TestCreateKeyResponseIsNoStore(t *testing.T) { + e := newEnv(t) + resp := e.doResp(t, http.MethodPost, api.PathKeys(), e.adminToken, api.CreateKeyRequest{}) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + if cc := resp.Header.Get("Cache-Control"); !strings.Contains(cc, "no-store") { + t.Errorf("Cache-Control = %q, want it to contain no-store", cc) + } +} + +// The one invariant that matters most in this package: no endpoint other than +// creation may ever put a secret on the wire. Asserted against the raw bytes, +// not a decoded struct, so a field added to api.Key later cannot leak one past +// this test. +func TestNoEndpointEverReturnsASecret(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + admin := e.createKey(t, api.PathKeys(), e.adminToken, api.CreateKeyRequest{Name: "admin-2"}) + proj := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + + secrets := []string{ + admin.Token, secretOf(t, admin.Token), + proj.Token, secretOf(t, proj.Token), + e.adminToken, secretOf(t, e.adminToken), + } + + for _, tc := range []struct{ method, path, token string }{ + {http.MethodGet, api.PathKeys(), e.adminToken}, + {http.MethodGet, api.PathProjectKeys("demo"), e.adminToken}, + {http.MethodGet, api.PathProjectKeys("demo"), proj.Token}, + {http.MethodGet, api.PathWhoAmI(), proj.Token}, + {http.MethodGet, api.PathProject("demo"), e.adminToken}, + {http.MethodGet, api.PathProjects(), e.adminToken}, + } { + t.Run(tc.method+" "+tc.path, func(t *testing.T) { + status, body := e.do(t, tc.method, tc.path, tc.token, nil) + if status != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", status, body) + } + for _, s := range secrets { + if bytes.Contains(body, []byte(s)) { + t.Fatalf("response contains a credential") + } + } + // The public half is fine to return, and the listings would be + // useless without it — check the test is actually looking at keys. + if strings.HasSuffix(tc.path, "/keys") && !bytes.Contains(body, []byte(proj.Key.ID)) && + !bytes.Contains(body, []byte(admin.Key.ID)) { + t.Errorf("key listing mentions no key id at all: %s", body) + } + }) + } + + // And the log, which sees every request, never saw one either. + if logged := e.logBuf.String(); logged != "" { + for _, s := range secrets { + if strings.Contains(logged, s) { + t.Fatal("a credential reached the log") + } + } + } +} + +// secretOf returns the half of a token that must never be seen again. The split +// is bounded at three because the base64url secret may itself contain "_". +func secretOf(t *testing.T, token string) string { + t.Helper() + parts := strings.SplitN(token, "_", 3) + if len(parts) != 3 { + t.Fatalf("token has %d parts, want 3", len(parts)) + } + return parts[2] +} + +func TestCreateProjectKey(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + out := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + if out.Key.Scope != api.ScopeProject { + t.Errorf("scope = %q, want %q", out.Key.Scope, api.ScopeProject) + } + if out.Key.Project != "demo" { + t.Errorf("project = %q, want demo", out.Key.Project) + } + + status, body := e.do(t, http.MethodGet, api.PathWhoAmI(), out.Token, nil) + var who api.WhoAmI + mustJSON(t, status, http.StatusOK, body, &who) + if who.Scope != api.ScopeProject || who.Project != "demo" { + t.Errorf("whoami = %+v, want scope project on demo", who) + } +} + +func TestCreateProjectKeyForUnknownProject(t *testing.T) { + e := newEnv(t) + status, body := e.do(t, http.MethodPost, api.PathProjectKeys("ghost"), e.adminToken, + api.CreateKeyRequest{}) + if status != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body: %s", status, body) + } +} + +func TestCreateKeyRejectsBadInput(t *testing.T) { + e := newEnv(t) + past := time.Now().Add(-time.Minute) + + t.Run("expired on arrival", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathKeys(), e.adminToken, + api.CreateKeyRequest{ExpiresAt: &past}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + }) + t.Run("control characters in name", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathKeys(), e.adminToken, + api.CreateKeyRequest{Name: "ci\x1b[2Jrunner"}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + }) + t.Run("oversized name", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathKeys(), e.adminToken, + api.CreateKeyRequest{Name: strings.Repeat("a", maxKeyNameLen+1)}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + }) +} + +func TestExpiredKeyDoesNotAuthenticate(t *testing.T) { + e := newEnv(t) + // The API refuses to mint one already expired, so this goes in through the + // store to exercise the verifier's expiry check rather than the validator's. + future := time.Now().Add(time.Hour) + live := e.createKey(t, api.PathKeys(), e.adminToken, api.CreateKeyRequest{ExpiresAt: &future}) + if status, body := e.do(t, http.MethodGet, api.PathWhoAmI(), live.Token, nil); status != http.StatusOK { + t.Fatalf("key expiring in an hour: status = %d, body: %s", status, body) + } + + past := time.Now().Add(-time.Hour) + token := e.mintWith(t, store.ScopeAdmin, nil, "stale", &past) + if status, _ := e.do(t, http.MethodGet, api.PathWhoAmI(), token, nil); status != http.StatusUnauthorized { + t.Errorf("expired key authenticated: status = %d", status) + } +} + +// Revocation must be visible on the next request, not when a cache entry ages +// out — an operator revoking a leaked token is racing an attacker who has it. +func TestRevokeTakesEffectImmediately(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + victim := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + + // Warm the auth cache: without Invalidate() the revocation would not be + // observed until the entry expired. + if status, _ := e.do(t, http.MethodGet, api.PathWhoAmI(), victim.Token, nil); status != http.StatusOK { + t.Fatalf("key did not work before revocation") + } + + status, body := e.do(t, http.MethodDelete, api.PathKey(victim.Key.ID), e.adminToken, nil) + if status != http.StatusNoContent { + t.Fatalf("revoke: status = %d, want 204; body: %s", status, body) + } + if status, _ := e.do(t, http.MethodGet, api.PathWhoAmI(), victim.Token, nil); status != http.StatusUnauthorized { + t.Errorf("revoked key still authenticates: status = %d", status) + } + + // Revoking again is a no-op rather than an error: a retrying CI step must + // not fail on the second attempt. + if status, body := e.do(t, http.MethodDelete, api.PathKey(victim.Key.ID), e.adminToken, nil); status != http.StatusNoContent { + t.Errorf("second revoke: status = %d, want 204; body: %s", status, body) + } + + // The listing keeps it, with a revocation timestamp, so an operator can see + // what happened. + status, body = e.do(t, http.MethodGet, api.PathKeys(), e.adminToken, nil) + var list api.KeyList + mustJSON(t, status, http.StatusOK, body, &list) + var found bool + for _, k := range list.Keys { + if k.ID == victim.Key.ID { + found = true + if !k.Revoked() { + t.Errorf("key %s is listed without revoked_at", k.ID) + } + } + } + if !found { + t.Errorf("revoked key vanished from the listing") + } +} + +// A CI runner that believes its token leaked should be able to burn it without +// waiting for an operator. +func TestProjectKeyMayRevokeItself(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + k := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + + status, body := e.do(t, http.MethodDelete, api.PathKey(k.Key.ID), k.Token, nil) + if status != http.StatusNoContent { + t.Fatalf("status = %d, want 204; body: %s", status, body) + } + if status, _ := e.do(t, http.MethodGet, api.PathWhoAmI(), k.Token, nil); status != http.StatusUnauthorized { + t.Errorf("key survived revoking itself: status = %d", status) + } +} + +func TestRevokeAuthorisation(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + e.createProject(t, "other") + + mine := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + sibling := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, api.CreateKeyRequest{Name: "ci-2"}) + foreign := e.createKey(t, api.PathProjectKeys("other"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + adminKey := e.createKey(t, api.PathKeys(), e.adminToken, api.CreateKeyRequest{Name: "admin-2"}) + + t.Run("sibling in the same project", func(t *testing.T) { + status, body := e.do(t, http.MethodDelete, api.PathKey(sibling.Key.ID), mine.Token, nil) + if status != http.StatusNoContent { + t.Fatalf("status = %d, want 204; body: %s", status, body) + } + }) + t.Run("another project's key", func(t *testing.T) { + status, body := e.do(t, http.MethodDelete, api.PathKey(foreign.Key.ID), mine.Token, nil) + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body: %s", status, body) + } + }) + t.Run("an admin key", func(t *testing.T) { + // An admin key has no project, so a project-scoped caller can never own + // it. Escalating by revoking the operator's credentials is the attack + // this closes. + status, body := e.do(t, http.MethodDelete, api.PathKey(adminKey.Key.ID), mine.Token, nil) + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body: %s", status, body) + } + if status, _ := e.do(t, http.MethodGet, api.PathWhoAmI(), adminKey.Token, nil); status != http.StatusOK { + t.Errorf("the admin key stopped working: status = %d", status) + } + }) +} + +// 404 versus 403 is an oracle: a project key must not be able to walk the key +// id space and learn which ones exist. +func TestRevokeUnknownKey(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + proj := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + const unknown = "abcdefghijklmnop" // well-formed, never minted + + t.Run("admin", func(t *testing.T) { + status, body := e.do(t, http.MethodDelete, api.PathKey(unknown), e.adminToken, nil) + if status != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body: %s", status, body) + } + }) + t.Run("project", func(t *testing.T) { + status, body := e.do(t, http.MethodDelete, api.PathKey(unknown), proj.Token, nil) + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body: %s", status, body) + } + }) +} + +func TestRevokeMalformedKeyID(t *testing.T) { + e := newEnv(t) + for _, id := range []string{"nope", "ABCDEFGHIJKLMNOP", "abcdefghijklmno1", strings.Repeat("a", 17)} { + t.Run(id, func(t *testing.T) { + status, body := e.do(t, http.MethodDelete, api.PathKey(id), e.adminToken, nil) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + }) + } +} + +func TestListProjectKeysIsScoped(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + e.createProject(t, "other") + mine := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + foreign := e.createKey(t, api.PathProjectKeys("other"), e.adminToken, api.CreateKeyRequest{Name: "ci"}) + + status, body := e.do(t, http.MethodGet, api.PathProjectKeys("demo"), mine.Token, nil) + var list api.KeyList + mustJSON(t, status, http.StatusOK, body, &list) + if len(list.Keys) != 1 || list.Keys[0].ID != mine.Key.ID { + t.Fatalf("listing = %+v, want just %s", list.Keys, mine.Key.ID) + } + if list.Keys[0].Project != "demo" { + t.Errorf("project = %q, want demo", list.Keys[0].Project) + } + + if status, _ := e.do(t, http.MethodGet, api.PathProjectKeys("other"), mine.Token, nil); status != http.StatusForbidden { + t.Errorf("read another project's keys: status = %d, want 403", status) + } + + // The admin listing spans projects and names each key's project. + status, body = e.do(t, http.MethodGet, api.PathKeys(), e.adminToken, nil) + var all api.KeyList + mustJSON(t, status, http.StatusOK, body, &all) + byID := make(map[string]api.Key, len(all.Keys)) + for _, k := range all.Keys { + byID[k.ID] = k + } + if got := byID[foreign.Key.ID].Project; got != "other" { + t.Errorf("foreign key's project = %q, want other", got) + } + if _, ok := byID[mine.Key.ID]; !ok { + t.Errorf("admin listing is missing %s", mine.Key.ID) + } +} diff --git a/internal/adminapi/maintenance_test.go b/internal/adminapi/maintenance_test.go new file mode 100644 index 0000000..d24ef00 --- /dev/null +++ b/internal/adminapi/maintenance_test.go @@ -0,0 +1,405 @@ +package adminapi + +import ( + "database/sql" + "net/http" + "os" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/deploy" +) + +// exec runs a statement against the store. Tests use it to put the database +// into a state the API cannot produce — drifted reference counts, above all, +// which is the only thing fsck exists to find. +func (e *env) exec(t *testing.T, query string, args ...any) { + t.Helper() + err := e.db.Tx(t.Context(), func(tx *sql.Tx) error { + _, err := tx.ExecContext(t.Context(), query, args...) + return err + }) + if err != nil { + t.Fatalf("%s: %v", query, err) + } +} + +// oneFile is a single-page deployment, enough to give each version content no +// other version shares — which is what makes the collector's effect visible. +func oneFile(version string) map[string]string { + return map[string]string{"index.html": "

" + version + "

"} +} + +// hasContent reports whether the content store still holds these bytes. +func (e *env) hasContent(t *testing.T, content string) bool { + t.Helper() + ok, err := e.cas.Has(cas.Sum([]byte(content))) + if err != nil { + t.Fatalf("cas has: %v", err) + } + return ok +} + +func (e *env) activate(t *testing.T, token, project, id string) { + t.Helper() + status, body := e.do(t, http.MethodPost, api.PathActivate(project, id), token, nil) + mustJSON(t, status, http.StatusOK, body, nil) +} + +func (e *env) listDeployments(t *testing.T, token, project, query string) api.DeploymentList { + t.Helper() + status, body := e.do(t, http.MethodGet, api.PathDeployments(project)+query, token, nil) + var out api.DeploymentList + mustJSON(t, status, http.StatusOK, body, &out) + return out +} + +func ids(list api.DeploymentList) []string { + out := make([]string, 0, len(list.Deployments)) + for _, d := range list.Deployments { + out = append(out, d.ID) + } + return out +} + +func TestListDeployments(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + d1 := e.readyDeployment(t, token, "demo", oneFile("v1")) + d2 := e.readyDeployment(t, token, "demo", oneFile("v2")) + // Left pending: a listing exists partly so an operator can see the + // deployments that never finished. + d3 := e.startDeployment(t, token, "demo") + e.activate(t, token, "demo", d1.ID) + + all := e.listDeployments(t, token, "demo", "") + if got, want := ids(all), []string{d3.ID, d2.ID, d1.ID}; !equalStrings(got, want) { + t.Errorf("ids = %v, want newest first %v", got, want) + } + if all.NextCursor != "" { + t.Errorf("next_cursor = %q, want empty when a page is the whole list", all.NextCursor) + } + for _, d := range all.Deployments { + if d.Project != "demo" { + t.Errorf("deployment %s: project = %q", d.ID, d.Project) + } + if (d.ID == d1.ID) != d.Active { + t.Errorf("deployment %s: active = %v, want it only for the activated one", d.ID, d.Active) + } + if len(d.Files) != 0 { + t.Errorf("deployment %s: a listing carried %d manifest entries", d.ID, len(d.Files)) + } + } + + ready := e.listDeployments(t, token, "demo", "?state=ready") + if got, want := ids(ready), []string{d2.ID, d1.ID}; !equalStrings(got, want) { + t.Errorf("state=ready = %v, want %v", got, want) + } + if got := ids(e.listDeployments(t, token, "demo", "?state=failed")); len(got) != 0 { + t.Errorf("state=failed = %v, want none", got) + } + + // Paging: one at a time, following the cursor, visits the same list. + var paged []string + cursor := "" + for range 5 { + q := "?limit=1" + if cursor != "" { + q += "&cursor=" + cursor + } + page := e.listDeployments(t, token, "demo", q) + paged = append(paged, ids(page)...) + cursor = page.NextCursor + if cursor == "" { + break + } + } + if want := []string{d3.ID, d2.ID, d1.ID}; !equalStrings(paged, want) { + t.Errorf("paged = %v, want %v", paged, want) + } + + // A typo in the filter is an error, not an empty list that reads as "this + // project has no deployments". + status, body := e.do(t, http.MethodGet, api.PathDeployments("demo")+"?state=redy", token, nil) + if status != http.StatusBadRequest { + t.Fatalf("state=redy: status = %d; body: %s", status, body) + } + if code := errCode(t, body); code != api.CodeBadRequest { + t.Errorf("code = %q, want %q", code, api.CodeBadRequest) + } + status, body = e.do(t, http.MethodGet, api.PathDeployments("demo")+"?limit=9000", token, nil) + if status != http.StatusBadRequest { + t.Errorf("limit=9000: status = %d; body: %s", status, body) + } + + // Another project's key cannot read this project's deployments. + e.createProject(t, "other") + otherToken := e.mintProject(t, e.projectID(t, "other"), "other-ci") + status, body = e.do(t, http.MethodGet, api.PathDeployments("demo"), otherToken, nil) + if status != http.StatusForbidden { + t.Errorf("cross-project list: status = %d; body: %s", status, body) + } +} + +func TestGetDeployment(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + contents := map[string]string{ + "index.html": "

hello

", + "assets/app.js": "console.log(1)", + } + dep := e.readyDeployment(t, token, "demo", contents) + + status, body := e.do(t, http.MethodGet, api.PathDeployment("demo", dep.ID), token, nil) + var got api.Deployment + mustJSON(t, status, http.StatusOK, body, &got) + if got.ID != dep.ID || got.State != api.StateReady || got.FileCount != 2 { + t.Errorf("deployment = %+v", got) + } + if got.Files != nil { + t.Errorf("files = %v, want them withheld unless asked for", got.Files) + } + if got.URL != "" { + t.Errorf("url = %q, want none: this deployment is not the one being served", got.URL) + } + + status, body = e.do(t, http.MethodGet, api.PathDeployment("demo", dep.ID)+"?files=true", token, nil) + got = api.Deployment{} + mustJSON(t, status, http.StatusOK, body, &got) + if len(got.Files) != len(contents) { + t.Fatalf("files = %+v, want %d entries", got.Files, len(contents)) + } + for _, f := range got.Files { + content, ok := contents[f.Path] + if !ok { + t.Errorf("unexpected manifest path %q", f.Path) + continue + } + if f.Digest != cas.Sum([]byte(content)).String() || f.Size != int64(len(content)) { + t.Errorf("%s = %+v, want the digest and size of its content", f.Path, f) + } + } + if got.Files[0].Path != "assets/app.js" { + t.Errorf("files start at %q, want them ordered by path", got.Files[0].Path) + } + + status, body = e.do(t, http.MethodGet, api.PathDeployment("demo", "dpl_0000000000000000"), token, nil) + if status != http.StatusNotFound || errCode(t, body) != api.CodeNotFound { + t.Errorf("unknown id: status = %d; body: %s", status, body) + } + + // A deployment is looked up within its project, so naming it under another + // project is "no such deployment" — not a way to read across the boundary, + // and not a confirmation that the id exists somewhere. + e.createProject(t, "other") + status, body = e.do(t, http.MethodGet, api.PathDeployment("other", dep.ID), e.adminToken, nil) + if status != http.StatusNotFound || errCode(t, body) != api.CodeNotFound { + t.Errorf("cross-project read: status = %d; body: %s", status, body) + } +} + +func TestDeleteDeploymentEndpoint(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + pid := e.projectID(t, "demo") + token := e.mintProject(t, pid, "ci") + + keep := e.readyDeployment(t, token, "demo", oneFile("v1")) + spare := e.readyDeployment(t, token, "demo", oneFile("v2")) + e.activate(t, token, "demo", keep.ID) + + // The one being served is a conflict, and specifically not a 403: the + // caller is allowed to do this, just not yet. + status, body := e.do(t, http.MethodDelete, api.PathDeployment("demo", keep.ID), token, nil) + if status != http.StatusConflict { + t.Fatalf("delete active: status = %d; body: %s", status, body) + } + if code := errCode(t, body); code != api.CodeDeploymentActive { + t.Errorf("code = %q, want %q", code, api.CodeDeploymentActive) + } + + dir := deploy.DeploymentDir(e.deployDir, pid, spare.ID) + if _, err := os.Stat(dir); err != nil { + t.Fatalf("stat %s: %v", dir, err) + } + status, body = e.do(t, http.MethodDelete, api.PathDeployment("demo", spare.ID), token, nil) + if status != http.StatusNoContent || len(body) != 0 { + t.Fatalf("delete: status = %d; body: %s", status, body) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Errorf("stat %s after delete: err = %v, want it gone", dir, err) + } + + // Gone means gone, and deleting it again says so rather than reporting a + // success that did nothing. + status, body = e.do(t, http.MethodGet, api.PathDeployment("demo", spare.ID), token, nil) + if status != http.StatusNotFound { + t.Errorf("get deleted: status = %d; body: %s", status, body) + } + status, body = e.do(t, http.MethodDelete, api.PathDeployment("demo", spare.ID), token, nil) + if status != http.StatusNotFound || errCode(t, body) != api.CodeNotFound { + t.Errorf("second delete: status = %d; body: %s", status, body) + } + + // The site is still being served by the deployment that was left alone. + if sp, ok := e.sites.Lookup("demo"); !ok || sp.Active() == nil || sp.Active().ID != keep.ID { + t.Errorf("serving %+v, want %s untouched", sp, keep.ID) + } + + status, body = e.do(t, http.MethodDelete, api.PathDeployment("demo", keep.ID), token, + map[string]string{"unexpected": "body"}) + if status != http.StatusBadRequest { + t.Errorf("delete with a body: status = %d; body: %s", status, body) + } +} + +func TestGCEndpoint(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + pid := e.projectID(t, "demo") + token := e.mintProject(t, pid, "ci") + + // Keep one spare deployment and no grace, so retention has something to do + // within the lifetime of a test. The blob grace is what protects content a + // request may be about to open; an hour of it would outlast any test, so + // this pass is told to collect immediately. + zero, one := 0, 1 + status, body := e.do(t, http.MethodPatch, api.PathProject(p.Name), e.adminToken, + api.ProjectPatch{RetentionCount: &one, RetentionGrace: &zero}) + mustJSON(t, status, http.StatusOK, body, nil) + e.server.Deploy.BlobGrace = -time.Minute + + d1 := e.readyDeployment(t, token, "demo", oneFile("v1")) + d2 := e.readyDeployment(t, token, "demo", oneFile("v2")) + d3 := e.readyDeployment(t, token, "demo", oneFile("v3")) + e.activate(t, token, "demo", d1.ID) + + // Active is excluded outright, then the newest inactive one is the single + // deployment retention keeps — so d2 is what a pass would delete. + status, body = e.do(t, http.MethodPost, api.PathGC(), e.adminToken, api.GCRequest{DryRun: true}) + var dry api.GCStats + mustJSON(t, status, http.StatusOK, body, &dry) + if !dry.DryRun || dry.DeploymentsDeleted != 1 { + t.Errorf("dry run = %+v, want 1 deployment reported", dry) + } + status, _ = e.do(t, http.MethodGet, api.PathDeployment("demo", d2.ID), token, nil) + if status != http.StatusOK { + t.Errorf("a dry run deleted %s", d2.ID) + } + + status, body = e.do(t, http.MethodPost, api.PathGC(), e.adminToken, api.GCRequest{}) + var stats api.GCStats + mustJSON(t, status, http.StatusOK, body, &stats) + if stats.DryRun || stats.DeploymentsDeleted != 1 { + t.Fatalf("collect = %+v, want 1 deployment deleted", stats) + } + if stats.BlobsDeleted != 1 || stats.BytesFreed == 0 { + t.Errorf("collect = %+v, want the content only that deployment held", stats) + } + if e.hasContent(t, oneFile("v2")["index.html"]) { + t.Error("v2's content survived the deployment that referenced it") + } + for _, keep := range []string{"v1", "v3"} { + if !e.hasContent(t, oneFile(keep)["index.html"]) { + t.Errorf("%s's content was collected while a deployment still referenced it", keep) + } + } + + status, _ = e.do(t, http.MethodGet, api.PathDeployment("demo", d2.ID), token, nil) + if status != http.StatusNotFound { + t.Errorf("get collected: status = %d, want 404", status) + } + for _, d := range []api.Deployment{d1, d3} { + if status, _ := e.do(t, http.MethodGet, api.PathDeployment("demo", d.ID), token, nil); status != http.StatusOK { + t.Errorf("%s: status = %d, want it kept", d.ID, status) + } + } + + // A body is optional, and a second pass finds nothing left to do. + status, body = e.do(t, http.MethodPost, api.PathGC(), e.adminToken, nil) + stats = api.GCStats{} + mustJSON(t, status, http.StatusOK, body, &stats) + if stats.DeploymentsDeleted != 0 || stats.BlobsDeleted != 0 { + t.Errorf("second pass = %+v, want nothing", stats) + } + + // Collection is server-wide, so it is admin-only however trusted the + // project key is. + status, body = e.do(t, http.MethodPost, api.PathGC(), token, api.GCRequest{}) + if status != http.StatusForbidden || errCode(t, body) != api.CodeForbidden { + t.Errorf("project key: status = %d; body: %s", status, body) + } +} + +func TestFsckEndpoint(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + e.readyDeployment(t, token, "demo", map[string]string{ + "index.html": "

hello

", + "app.js": "console.log(1)", + }) + + // Triggers maintain the counts, so a server that has only been used through + // the API is clean by construction. Asserting that is the baseline the + // injected drift below is measured against. + status, body := e.do(t, http.MethodPost, api.PathFsck(), e.adminToken, api.FsckRequest{}) + var rep api.FsckReport + mustJSON(t, status, http.StatusOK, body, &rep) + if rep.Blobs != 2 || rep.DriftCount != 0 || len(rep.Drift) != 0 || rep.Repaired != 0 { + t.Fatalf("clean report = %+v", rep) + } + + digest := cas.Sum([]byte("console.log(1)")) + e.exec(t, `UPDATE blobs SET refcount = 7 WHERE digest = ?`, digest[:]) + + status, body = e.do(t, http.MethodPost, api.PathFsck(), e.adminToken, api.FsckRequest{}) + rep = api.FsckReport{} + mustJSON(t, status, http.StatusOK, body, &rep) + if rep.DriftCount != 1 || len(rep.Drift) != 1 { + t.Fatalf("report = %+v, want the one drifted blob", rep) + } + if d := rep.Drift[0]; d.Digest != digest.String() || d.Stored != 7 || d.Actual != 1 { + t.Errorf("drift = %+v, want stored 7 and actual 1 for %s", d, digest) + } + if rep.Repaired != 0 { + t.Errorf("repaired = %d without being asked to", rep.Repaired) + } + + status, body = e.do(t, http.MethodPost, api.PathFsck(), e.adminToken, api.FsckRequest{Repair: true}) + rep = api.FsckReport{} + mustJSON(t, status, http.StatusOK, body, &rep) + if rep.DriftCount != 1 || rep.Repaired != 1 { + t.Fatalf("repair = %+v", rep) + } + + status, body = e.do(t, http.MethodPost, api.PathFsck(), e.adminToken, nil) + rep = api.FsckReport{} + mustJSON(t, status, http.StatusOK, body, &rep) + if rep.DriftCount != 0 { + t.Errorf("after repair = %+v, want no drift", rep) + } + + status, body = e.do(t, http.MethodPost, api.PathFsck(), token, api.FsckRequest{}) + if status != http.StatusForbidden || errCode(t, body) != api.CodeForbidden { + t.Errorf("project key: status = %d; body: %s", status, body) + } +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/adminapi/projects.go b/internal/adminapi/projects.go new file mode 100644 index 0000000..965b167 --- /dev/null +++ b/internal/adminapi/projects.go @@ -0,0 +1,169 @@ +package adminapi + +import ( + "errors" + "net/http" + "strings" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/httpx" + "github.com/iceBear67/simplepages/internal/store" +) + +// createProject handles POST /api/v1/projects. +func (s *Server) createProject(w http.ResponseWriter, r *http.Request) error { + var req api.CreateProjectRequest + if err := httpx.DecodeJSON(w, r, s.maxJSON(), &req); err != nil { + return err + } + name := strings.TrimSpace(req.Name) + if err := checkProjectName(name); err != nil { + return err + } + + p := store.DefaultProject(name) + s.clampDefaults(p) + if err := s.applyPatch(p, req.Patch); err != nil { + return err + } + if err := s.DB.CreateProject(r.Context(), p); err != nil { + if errors.Is(err, store.ErrExists) { + return api.Errorf(api.CodeProjectExists, "project %q already exists", name) + } + return err + } + + httpx.LogAttr(r.Context(), "project", name) + if s.Hooks.ProjectChanged != nil { + s.Hooks.ProjectChanged(r.Context(), p) + } + w.Header().Set("Location", api.PathProject(name)) + httpx.WriteJSON(w, http.StatusCreated, s.projectOf(p)) + return nil +} + +// listProjects handles GET /api/v1/projects. +func (s *Server) listProjects(w http.ResponseWriter, r *http.Request) error { + limit, err := intQuery(r, "limit", 100, 1, 500) + if err != nil { + return err + } + ps, next, err := s.DB.ListProjects(r.Context(), limit, r.URL.Query().Get("cursor")) + if err != nil { + return err + } + out := api.ProjectList{Projects: make([]api.Project, 0, len(ps)), NextCursor: next} + for _, p := range ps { + out.Projects = append(out.Projects, s.projectOf(p)) + } + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} + +// getProject handles GET /api/v1/projects/{name}. +func (s *Server) getProject(w http.ResponseWriter, r *http.Request) error { + p, err := s.project(r) + if err != nil { + return err + } + httpx.WriteJSON(w, http.StatusOK, s.projectOf(p)) + return nil +} + +// patchProject handles PATCH /api/v1/projects/{name}. +func (s *Server) patchProject(w http.ResponseWriter, r *http.Request) error { + p, err := s.project(r) + if err != nil { + return err + } + var patch api.ProjectPatch + if err := httpx.DecodeJSON(w, r, s.maxJSON(), &patch); err != nil { + return err + } + + // Apply to a copy so a rejected field cannot leave the caller looking at a + // partly-updated project in the error response. + updated := *p + if err := s.applyPatch(&updated, &patch); err != nil { + return err + } + if err := s.DB.UpdateProject(r.Context(), &updated); err != nil { + if errors.Is(err, store.ErrNotFound) { + return api.Errorf(api.CodeNotFound, "no such project") + } + return err + } + + if s.Hooks.ProjectChanged != nil { + s.Hooks.ProjectChanged(r.Context(), &updated) + } + httpx.WriteJSON(w, http.StatusOK, s.projectOf(&updated)) + return nil +} + +// deleteProject handles DELETE /api/v1/projects/{name}. +// +// The cascade takes the project's keys, deployments and manifest rows with it; +// blob refcounts fall as the manifest rows go, so the content is reclaimed by +// the next GC pass rather than synchronously here. +func (s *Server) deleteProject(w http.ResponseWriter, r *http.Request) error { + if err := httpx.NoBody(r); err != nil { + return err + } + p, err := s.project(r) + if err != nil { + return err + } + if err := s.DB.DeleteProject(r.Context(), p.ID); err != nil { + if errors.Is(err, store.ErrNotFound) { + return api.Errorf(api.CodeNotFound, "no such project") + } + return err + } + + // The project's keys went with it. Nothing else invalidates the auth cache, + // so without this a deleted project's key would keep working for up to the + // cache TTL. + s.Auth.V.Invalidate() + if s.Hooks.ProjectDeleted != nil { + s.Hooks.ProjectDeleted(r.Context(), p) + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +// project loads the project named by the {name} wildcard. +// +// The ownership guard already resolved the same name; looking it up again costs +// one indexed read on a cold path and keeps the guard free of any obligation to +// hand state to the handler. +func (s *Server) project(r *http.Request) (*store.Project, error) { + name := r.PathValue("name") + if err := checkProjectName(name); err != nil { + return nil, err + } + p, err := s.DB.ProjectByName(r.Context(), name) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, api.Errorf(api.CodeNotFound, "no such project") + } + return nil, err + } + httpx.LogAttr(r.Context(), "project", p.Name) + return p, nil +} + +// clampDefaults lowers the schema's per-project defaults to the server-wide +// ceilings, so a new project on a server with tightened limits does not start +// out already over them. +func (s *Server) clampDefaults(p *store.Project) { + if n := s.Limits.MaxManifestFiles; n > 0 && p.MaxFiles > n { + p.MaxFiles = n + } + if n := s.Limits.MaxFileBytes; n > 0 && p.MaxFileBytes > n { + p.MaxFileBytes = n + } + if p.MaxTotalBytes < p.MaxFileBytes { + p.MaxTotalBytes = p.MaxFileBytes + } +} diff --git a/internal/adminapi/projects_test.go b/internal/adminapi/projects_test.go new file mode 100644 index 0000000..220a077 --- /dev/null +++ b/internal/adminapi/projects_test.go @@ -0,0 +1,439 @@ +package adminapi + +import ( + "net/http" + "strconv" + "strings" + "testing" + + "github.com/iceBear67/simplepages/api" +) + +func TestCreateProjectAppliesDefaults(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + if p.Name != "demo" { + t.Errorf("name = %q", p.Name) + } + if p.IndexFile != "index.html" { + t.Errorf("index_file = %q, want index.html", p.IndexFile) + } + if p.CacheControl == "" { + t.Error("cache_control is empty; a project must always have one") + } + if p.RetentionCount != 10 { + t.Errorf("retention_count = %d, want 10", p.RetentionCount) + } + if p.CreatedAt.IsZero() || p.UpdatedAt.IsZero() { + t.Errorf("timestamps not set: %+v", p) + } +} + +func TestCreateProjectSetsLocation(t *testing.T) { + e := newEnv(t) + resp := e.doResp(t, http.MethodPost, api.PathProjects(), e.adminToken, + api.CreateProjectRequest{Name: "demo"}) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("status = %d, want 201", resp.StatusCode) + } + if loc := resp.Header.Get("Location"); loc != api.PathProject("demo") { + t.Errorf("Location = %q, want %q", loc, api.PathProject("demo")) + } +} + +func TestCreateProjectRejectsBadNames(t *testing.T) { + e := newEnv(t) + // The name becomes a "~name" entry in $WEBROOT, so anything that could carry + // a separator or a traversal has to be impossible by construction. + names := []string{ + "", " ", "Demo", "demo/evil", "demo\\evil", "..", ".hidden", "-lead", + "_lead", "demo\x00", "demo project", "デモ", "demo\x1b", "~demo", + strings.Repeat("a", 64), + } + for i, name := range names { + t.Run(strconv.Itoa(i), func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathProjects(), e.adminToken, + api.CreateProjectRequest{Name: name}) + if status != http.StatusBadRequest { + t.Fatalf("%q: status = %d, want 400; body: %s", name, status, body) + } + if got := errCode(t, body); got != api.CodeInvalidProjectName { + t.Errorf("%q: code = %q, want %q", name, got, api.CodeInvalidProjectName) + } + }) + } + // The boundary the pattern actually allows. + e.createProject(t, strings.Repeat("a", 63)) +} + +// A name arriving from a shell pipeline often has a trailing newline. Trimming +// it is a convenience, and it is the only normalisation the name gets — the +// pattern decides everything else. +func TestCreateProjectTrimsSurroundingSpace(t *testing.T) { + e := newEnv(t) + status, body := e.do(t, http.MethodPost, api.PathProjects(), e.adminToken, + api.CreateProjectRequest{Name: " demo\n"}) + var p api.Project + mustJSON(t, status, http.StatusCreated, body, &p) + if p.Name != "demo" { + t.Fatalf("name = %q, want demo", p.Name) + } +} + +func TestCreateProjectDuplicate(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + status, body := e.do(t, http.MethodPost, api.PathProjects(), e.adminToken, + api.CreateProjectRequest{Name: "demo"}) + if status != http.StatusConflict { + t.Fatalf("status = %d, want 409; body: %s", status, body) + } + if got := errCode(t, body); got != api.CodeProjectExists { + t.Errorf("code = %q, want %q", got, api.CodeProjectExists) + } +} + +func TestCreateProjectWithConfig(t *testing.T) { + e := newEnv(t) + spa := true + index := "app.html" + notFound := "404.html" + retention := 3 + status, body := e.do(t, http.MethodPost, api.PathProjects(), e.adminToken, + api.CreateProjectRequest{ + Name: "demo", + Patch: &api.ProjectPatch{ + IndexFile: &index, + NotFoundFile: ¬Found, + SPAFallback: &spa, + RetentionCount: &retention, + }, + }) + var p api.Project + mustJSON(t, status, http.StatusCreated, body, &p) + if p.IndexFile != index || p.NotFoundFile != notFound || !p.SPAFallback || p.RetentionCount != 3 { + t.Errorf("config not applied: %+v", p) + } +} + +func TestCreateProjectRejectsUnknownFields(t *testing.T) { + e := newEnv(t) + req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost, + e.ts.URL+api.PathProjects(), strings.NewReader(`{"name":"demo","retention_count":5}`)) + req.Header.Set("Authorization", "Bearer "+e.adminToken) + resp, err := e.ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + // retention_count belongs under "config"; silently ignoring a misplaced key + // would let a deploy script think it had configured something it had not. + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", resp.StatusCode) + } +} + +// cache_control is written verbatim into a response header on every request the +// project serves. A CR or LF there is response splitting. +func TestPatchRejectsHeaderInjection(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + + for _, bad := range []string{ + "public\r\nX-Evil: 1", + "public\nX-Evil: 1", + "public\rX-Evil: 1", + "public\x00", + "public, max-age=0\x7f", + strings.Repeat("a", maxCacheControlLen+1), + } { + t.Run(strings.NewReplacer("\r", "CR", "\n", "LF", "\x00", "NUL").Replace(bad), func(t *testing.T) { + status, body := e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{CacheControl: &bad}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + }) + } + + // And the project is untouched. + status, body := e.do(t, http.MethodGet, api.PathProject("demo"), e.adminToken, nil) + var p api.Project + mustJSON(t, status, http.StatusOK, body, &p) + if strings.ContainsAny(p.CacheControl, "\r\n") { + t.Fatalf("cache_control was stored with a newline: %q", p.CacheControl) + } +} + +// Display names are printed to operator terminals and written into logs. +func TestPatchRejectsControlCharacters(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + bad := "demo\x1b[31m site" + status, body := e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{DisplayName: &bad}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } +} + +func TestPatchRejectsBadSitePaths(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + for i, bad := range []string{ + "", "/etc/passwd", "../secret", "a/../../b", ".", "a//b", "a/", "a\\b", "a\x00b", + strings.Repeat("a", maxSitePathLen+1), + } { + t.Run(strconv.Itoa(i), func(t *testing.T) { + v := bad + status, body := e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{IndexFile: &v}) + if status != http.StatusBadRequest { + t.Fatalf("%q: status = %d, want 400; body: %s", bad, status, body) + } + if got := errCode(t, body); got != api.CodeInvalidPath { + t.Errorf("%q: code = %q, want %q", bad, got, api.CodeInvalidPath) + } + }) + } +} + +// The empty string is the only way a client can clear a custom 404 document, +// which is why not_found_file gets a different rule from index_file. +func TestPatchClearsNotFoundFile(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + + set := "404.html" + status, body := e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{NotFoundFile: &set}) + var p api.Project + mustJSON(t, status, http.StatusOK, body, &p) + if p.NotFoundFile != set { + t.Fatalf("not_found_file = %q, want %q", p.NotFoundFile, set) + } + + // A fresh destination: not_found_file is omitempty, so decoding a cleared + // project over the previous value would silently keep it. + clear := "" + status, body = e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{NotFoundFile: &clear}) + var cleared api.Project + mustJSON(t, status, http.StatusOK, body, &cleared) + if cleared.NotFoundFile != "" { + t.Fatalf("not_found_file = %q, want it cleared", cleared.NotFoundFile) + } + + // It must be SQL NULL, not the empty string, so the schema's "NULL means no + // custom document" contract holds. + row, err := e.db.ProjectByName(t.Context(), "demo") + if err != nil { + t.Fatal(err) + } + if row.NotFoundFile != "" { + t.Errorf("store kept %q", row.NotFoundFile) + } +} + +// An omitted field means "leave alone"; only a present one changes anything. +func TestPatchLeavesOmittedFieldsAlone(t *testing.T) { + e := newEnv(t) + spa := true + index := "app.html" + e.do(t, http.MethodPost, api.PathProjects(), e.adminToken, api.CreateProjectRequest{ + Name: "demo", + Patch: &api.ProjectPatch{IndexFile: &index, SPAFallback: &spa}, + }) + + display := "Demo Site" + status, body := e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{DisplayName: &display}) + var p api.Project + mustJSON(t, status, http.StatusOK, body, &p) + if p.IndexFile != index || !p.SPAFallback { + t.Errorf("patch clobbered untouched fields: %+v", p) + } + if p.DisplayName != display { + t.Errorf("display_name = %q, want %q", p.DisplayName, display) + } +} + +func TestPatchEnforcesServerCeilings(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + ceiling := e.server.Limits.MaxFileBytes + + over := ceiling + 1 + status, body := e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{MaxFileBytes: &over}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400; body: %s", status, body) + } + + // Lowering below the server limit is always allowed. + under := int64(1024) + status, body = e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{MaxFileBytes: &under}) + var p api.Project + mustJSON(t, status, http.StatusOK, body, &p) + if p.MaxFileBytes != under { + t.Errorf("max_file_bytes = %d, want %d", p.MaxFileBytes, under) + } + + zero := 0 + status, _ = e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{RetentionCount: &zero}) + if status != http.StatusBadRequest { + t.Errorf("retention_count = 0 accepted (status %d); it would delete the active deployment", status) + } +} + +// A patch is applied to a copy, so a field rejected halfway through must leave +// nothing behind. +func TestRejectedPatchChangesNothing(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + + display := "Fine" + bad := "x\r\ny" + status, _ := e.do(t, http.MethodPatch, api.PathProject("demo"), e.adminToken, + api.ProjectPatch{DisplayName: &display, CacheControl: &bad}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", status) + } + row, err := e.db.ProjectByName(t.Context(), "demo") + if err != nil { + t.Fatal(err) + } + if row.DisplayName != "" { + t.Errorf("display_name = %q; the valid half of a rejected patch was applied", row.DisplayName) + } +} + +func TestListProjectsPaging(t *testing.T) { + e := newEnv(t) + for _, n := range []string{"a", "b", "c", "d", "e"} { + e.createProject(t, n) + } + + var seen []string + cursor := "" + for range 10 { + path := api.PathProjects() + "?limit=2" + if cursor != "" { + path += "&cursor=" + cursor + } + status, body := e.do(t, http.MethodGet, path, e.adminToken, nil) + var list api.ProjectList + mustJSON(t, status, http.StatusOK, body, &list) + for _, p := range list.Projects { + seen = append(seen, p.Name) + } + if list.NextCursor == "" { + break + } + cursor = list.NextCursor + } + if got := strings.Join(seen, ","); got != "a,b,c,d,e" { + t.Errorf("paged through %q, want a,b,c,d,e", got) + } +} + +func TestListProjectsRejectsBadLimit(t *testing.T) { + e := newEnv(t) + for _, q := range []string{"?limit=0", "?limit=501", "?limit=abc", "?limit=-1"} { + status, body := e.do(t, http.MethodGet, api.PathProjects()+q, e.adminToken, nil) + if status != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400; body: %s", q, status, body) + } + } +} + +func TestGetUnknownProject(t *testing.T) { + e := newEnv(t) + status, body := e.do(t, http.MethodGet, api.PathProject("nope"), e.adminToken, nil) + if status != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body: %s", status, body) + } +} + +func TestDeleteProjectRevokesItsKeysImmediately(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + // Warm the auth cache so the test proves invalidation, not a cold lookup. + if status, body := e.do(t, http.MethodGet, api.PathWhoAmI(), token, nil); status != http.StatusOK { + t.Fatalf("whoami before delete: %d %s", status, body) + } + + status, body := e.do(t, http.MethodDelete, api.PathProject("demo"), e.adminToken, nil) + if status != http.StatusNoContent { + t.Fatalf("delete: status = %d, want 204; body: %s", status, body) + } + if len(body) != 0 { + t.Errorf("204 carried a body: %s", body) + } + + // The key cascaded away with the project. Without the cache invalidation in + // the delete handler it would keep authenticating for the cache TTL. + if status, _ := e.do(t, http.MethodGet, api.PathWhoAmI(), token, nil); status != http.StatusUnauthorized { + t.Errorf("deleted project's key still works: status = %d", status) + } +} + +func TestProjectKeyCannotManageProjects(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + e.createProject(t, "other") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + t.Run("create", func(t *testing.T) { + status, body := e.do(t, http.MethodPost, api.PathProjects(), token, + api.CreateProjectRequest{Name: "sneaky"}) + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body: %s", status, body) + } + }) + t.Run("list", func(t *testing.T) { + if status, _ := e.do(t, http.MethodGet, api.PathProjects(), token, nil); status != http.StatusForbidden { + t.Errorf("status = %d, want 403", status) + } + }) + t.Run("patch own", func(t *testing.T) { + // Reconfiguring a project is an admin operation even for its own key: + // a deploy credential should not be able to raise its own limits. + display := "x" + status, _ := e.do(t, http.MethodPatch, api.PathProject("demo"), token, + api.ProjectPatch{DisplayName: &display}) + if status != http.StatusForbidden { + t.Errorf("status = %d, want 403", status) + } + }) + t.Run("delete own", func(t *testing.T) { + if status, _ := e.do(t, http.MethodDelete, api.PathProject("demo"), token, nil); status != http.StatusForbidden { + t.Errorf("status = %d, want 403", status) + } + }) + t.Run("read own", func(t *testing.T) { + status, body := e.do(t, http.MethodGet, api.PathProject("demo"), token, nil) + var p api.Project + mustJSON(t, status, http.StatusOK, body, &p) + if p.Name != "demo" { + t.Errorf("name = %q", p.Name) + } + }) + t.Run("read other", func(t *testing.T) { + if status, _ := e.do(t, http.MethodGet, api.PathProject("other"), token, nil); status != http.StatusForbidden { + t.Errorf("status = %d, want 403", status) + } + }) + // A project key must not be able to use 404-vs-403 to enumerate which + // project names exist. + t.Run("read nonexistent", func(t *testing.T) { + status, _ := e.do(t, http.MethodGet, api.PathProject("ghost"), token, nil) + if status != http.StatusForbidden { + t.Errorf("status = %d, want 403; an unknown name must look like someone else's", status) + } + }) +} diff --git a/internal/adminapi/router_test.go b/internal/adminapi/router_test.go new file mode 100644 index 0000000..2aaa995 --- /dev/null +++ b/internal/adminapi/router_test.go @@ -0,0 +1,141 @@ +package adminapi + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/auth" +) + +// The client builds URLs with api.Path* and the server registers wildcard +// patterns. Nothing in the type system connects the two, so assert that every +// path the client can build actually lands on the route it is meant to. +func TestPathBuildersMatchRoutes(t *testing.T) { + mux := http.NewServeMux() + (&Server{Auth: &auth.Middleware{}}).Register(mux) + + cases := []struct { + method, path, want string + }{ + {http.MethodGet, api.PathWhoAmI(), "GET " + patWhoAmI}, + {http.MethodGet, api.PathSystemInfo(), "GET " + patSystemInfo}, + {http.MethodPost, api.PathProjects(), "POST " + patProjects}, + {http.MethodGet, api.PathProjects(), "GET " + patProjects}, + {http.MethodGet, api.PathProject("demo"), "GET " + patProject}, + {http.MethodPatch, api.PathProject("demo"), "PATCH " + patProject}, + {http.MethodDelete, api.PathProject("demo"), "DELETE " + patProject}, + {http.MethodPost, api.PathProjectKeys("demo"), "POST " + patProjectKeys}, + {http.MethodGet, api.PathProjectKeys("demo"), "GET " + patProjectKeys}, + {http.MethodPost, api.PathKeys(), "POST " + patKeys}, + {http.MethodGet, api.PathKeys(), "GET " + patKeys}, + {http.MethodDelete, api.PathKey("abcdefghijklmnop"), "DELETE " + patKey}, + } + for _, tc := range cases { + t.Run(tc.method+" "+tc.path, func(t *testing.T) { + r := httptest.NewRequest(tc.method, tc.path, nil) + _, pattern := mux.Handler(r) + if pattern != tc.want { + t.Errorf("routed to %q, want %q", pattern, tc.want) + } + }) + } +} + +// A project name that needs escaping must not be able to reach a different +// route by smuggling a slash through the path builder. +func TestPathEscapingCannotCrossRoutes(t *testing.T) { + mux := http.NewServeMux() + (&Server{Auth: &auth.Middleware{}}).Register(mux) + + built := api.PathProject("demo/keys") + if strings.Contains(built, "demo/keys") { + t.Fatalf("PathProject did not escape the slash: %q", built) + } + r := httptest.NewRequest(http.MethodGet, built, nil) + _, pattern := mux.Handler(r) + if pattern != "GET "+patProject { + t.Errorf("routed to %q, want %q", pattern, "GET "+patProject) + } +} + +func TestEveryRouteRequiresAuthentication(t *testing.T) { + e := newEnv(t) + p := e.createProject(t, "demo") + + cases := []struct{ method, path string }{ + {http.MethodGet, api.PathWhoAmI()}, + {http.MethodGet, api.PathSystemInfo()}, + {http.MethodPost, api.PathProjects()}, + {http.MethodGet, api.PathProjects()}, + {http.MethodGet, api.PathProject(p.Name)}, + {http.MethodPatch, api.PathProject(p.Name)}, + {http.MethodDelete, api.PathProject(p.Name)}, + {http.MethodPost, api.PathProjectKeys(p.Name)}, + {http.MethodGet, api.PathProjectKeys(p.Name)}, + {http.MethodPost, api.PathKeys()}, + {http.MethodGet, api.PathKeys()}, + {http.MethodDelete, api.PathKey("abcdefghijklmnop")}, + {http.MethodGet, api.Version + "/does-not-exist"}, + } + for _, tc := range cases { + t.Run(tc.method+" "+tc.path, func(t *testing.T) { + status, body := e.do(t, tc.method, tc.path, "", nil) + if status != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401; body: %s", status, body) + } + if got := errCode(t, body); got != api.CodeUnauthorized { + t.Errorf("code = %q, want %q", got, api.CodeUnauthorized) + } + }) + } +} + +// An unknown endpoint must answer with the same envelope as everything else, so +// a client has exactly one error shape to parse. +func TestUnknownEndpointReturnsEnvelope(t *testing.T) { + e := newEnv(t) + status, body := e.do(t, http.MethodGet, api.Version+"/nope", e.adminToken, nil) + if status != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body: %s", status, body) + } + if got := errCode(t, body); got != api.CodeNotFound { + t.Errorf("code = %q, want %q", got, api.CodeNotFound) + } + // The path is not echoed back: there is no reason to reflect caller-supplied + // bytes into a response body. + if bytes := string(body); strings.Contains(bytes, "nope") { + t.Errorf("response echoes the request path: %s", bytes) + } +} + +// The catch-all matches every path under the API prefix, so without the +// method-agnostic fallbacks a wrong verb on a real endpoint would report 404 +// "no such endpoint" — which sends a client looking for a typo that is not +// there. +func TestWrongMethodIsRejectedWithAllow(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + + cases := []struct { + method, path, wantAllow string + }{ + {http.MethodPut, api.PathProjects(), "POST"}, + {http.MethodPost, api.PathProject("demo"), "PATCH"}, + {http.MethodPut, api.PathKey("abcdefghijklmnop"), "DELETE"}, + {http.MethodDelete, api.PathWhoAmI(), "GET"}, + } + for _, tc := range cases { + t.Run(tc.method+" "+tc.path, func(t *testing.T) { + resp := e.doResp(t, tc.method, tc.path, e.adminToken, nil) + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("status = %d, want 405", resp.StatusCode) + } + if allow := resp.Header.Get("Allow"); !strings.Contains(allow, tc.wantAllow) { + t.Errorf("Allow = %q, want it to mention %s", allow, tc.wantAllow) + } + }) + } +} diff --git a/internal/adminapi/server.go b/internal/adminapi/server.go new file mode 100644 index 0000000..dc1f176 --- /dev/null +++ b/internal/adminapi/server.go @@ -0,0 +1,265 @@ +// Package adminapi implements the management API: projects, API keys and the +// system endpoints. It owns the translation between storage and the wire +// format, so the store never constructs api.Error values and the api package +// never learns that SQLite exists. +// +// Every route in here is authenticated. Authorisation is expressed as +// per-route middleware rather than as checks inside the handlers, so a new +// endpoint that forgets its guard is visible in the route table instead of +// being hidden three screens down in a handler body. +package adminapi + +import ( + "context" + "log/slog" + "net/http" + "strconv" + "strings" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/auth" + "github.com/iceBear67/simplepages/internal/config" + "github.com/iceBear67/simplepages/internal/deploy" + "github.com/iceBear67/simplepages/internal/httpx" + "github.com/iceBear67/simplepages/internal/site" + "github.com/iceBear67/simplepages/internal/store" +) + +// Route patterns. They are spelled out here rather than reusing the builders in +// api/paths.go because those escape their arguments to produce a concrete URL, +// which is the opposite of what a wildcard pattern needs. TestPathsMatchRoutes +// asserts the two stay in agreement. +const ( + patProjects = api.Version + "/projects" + patProject = api.Version + "/projects/{name}" + patProjectKeys = api.Version + "/projects/{name}/keys" + patKeys = api.Version + "/keys" + patKey = api.Version + "/keys/{key_id}" + patWhoAmI = api.Version + "/whoami" + patSystemInfo = api.Version + "/system/info" + patDeployments = patProject + "/deployments" + patDeployment = patDeployments + "/{id}" + patManifest = patDeployment + "/manifest" + patFinalize = patDeployment + "/finalize" + patActivate = patDeployment + "/activate" + patBlob = api.Version + "/blobs/{digest}" + patGC = api.Version + "/gc" + patFsck = api.Version + "/fsck" + patCatchAll = api.Version + "/" +) + +// Hooks let later milestones react to changes without adminapi taking a +// dependency on the site registry, the webroot or the deployment service. +// Every hook is optional and runs after the database transaction has committed, +// so a hook failure can never leave the store and the caller disagreeing about +// whether the change happened. +type Hooks struct { + // ProjectChanged fires after a project is created or reconfigured. + ProjectChanged func(ctx context.Context, p *store.Project) + // ProjectDeleted fires after a project and its cascade are gone. + ProjectDeleted func(ctx context.Context, p *store.Project) +} + +// Server holds the collaborators the management handlers need. +type Server struct { + DB *store.DB + Auth *auth.Middleware + Deploy *deploy.Service + Log *slog.Logger + + // Limits are the server-wide ceilings a per-project setting may not exceed. + Limits config.Limits + + // Resolver maps a project name to its row id for the ownership guard. When + // nil the database is consulted directly; the server substitutes the site + // registry so the hot deployment endpoints do not pay a query for it. + Resolver auth.ProjectResolver + + // Sites is the serving registry, consulted to report what a project is + // currently serving without a query per project. Nil in tests that do not + // wire up a serving layer, in which case active_deployment is omitted. + Sites *site.Registry + + // BaseURL is the public origin serving site content, without a trailing + // slash. Empty until the operator configures one, in which case responses + // simply omit the url field. + BaseURL string + + // LinkMode reports how deployment trees are assembled. Filled in from the + // CAS in M2. + LinkMode string + + Started time.Time + Hooks Hooks +} + +// Register mounts the management routes on mux. +// +// It takes a mux rather than returning a handler so the caller can put the +// health probes on the same listener without them inheriting authentication. +func (s *Server) Register(mux *http.ServeMux) { + admin := auth.RequireAdmin(s.Log) + + resolver := s.Resolver + if resolver == nil { + resolver = auth.ResolverFunc(s.resolveProject) + } + // owner admits admins and the project's own key. Note it reads the {name} + // wildcard, so it is only valid on patterns that declare one. + owner := auth.RequireProject("name", resolver, s.Log) + + s.route(mux, "GET "+patWhoAmI, s.whoami) + s.route(mux, "GET "+patSystemInfo, s.systemInfo, admin) + + s.route(mux, "POST "+patProjects, s.createProject, admin) + s.route(mux, "GET "+patProjects, s.listProjects, admin) + s.route(mux, "GET "+patProject, s.getProject, owner) + s.route(mux, "PATCH "+patProject, s.patchProject, admin) + s.route(mux, "DELETE "+patProject, s.deleteProject, admin) + + s.route(mux, "POST "+patKeys, s.createAdminKey, admin) + s.route(mux, "GET "+patKeys, s.listKeys, admin) + s.route(mux, "DELETE "+patKey, s.revokeKey) + s.route(mux, "POST "+patProjectKeys, s.createProjectKey, admin) + s.route(mux, "GET "+patProjectKeys, s.listProjectKeys, owner) + + s.route(mux, "POST "+patDeployments, s.createDeployment, owner) + s.route(mux, "GET "+patDeployments, s.listDeployments, owner) + s.route(mux, "GET "+patDeployment, s.getDeployment, owner) + s.route(mux, "DELETE "+patDeployment, s.deleteDeployment, owner) + s.route(mux, "POST "+patManifest, s.setManifest, owner) + s.route(mux, "POST "+patFinalize, s.finalize, owner) + s.route(mux, "POST "+patActivate, s.activate, owner) + + s.route(mux, "POST "+patGC, s.gc, admin) + s.route(mux, "POST "+patFsck, s.fsck, admin) + // Blob uploads carry no project in the path and so have no owner to check + // against. See putBlob for why authentication alone is the right guard. + s.route(mux, "PUT "+patBlob, s.putBlob) + + // A known path reached with an unregistered method must answer 405 and say + // what it does accept. ServeMux would do that by itself, but only when no + // pattern matches at all — and the catch-all below matches everything under + // the prefix, which would turn every method mismatch into a 404. These + // method-agnostic patterns are less specific than the ones above, so they + // only see requests the real routes rejected. + s.methods(mux, patProjects, http.MethodGet, http.MethodPost) + s.methods(mux, patProject, http.MethodGet, http.MethodPatch, http.MethodDelete) + s.methods(mux, patProjectKeys, http.MethodGet, http.MethodPost) + s.methods(mux, patKeys, http.MethodGet, http.MethodPost) + s.methods(mux, patKey, http.MethodDelete) + s.methods(mux, patWhoAmI, http.MethodGet) + s.methods(mux, patSystemInfo, http.MethodGet) + s.methods(mux, patDeployments, http.MethodGet, http.MethodPost) + s.methods(mux, patDeployment, http.MethodGet, http.MethodDelete) + s.methods(mux, patManifest, http.MethodPost) + s.methods(mux, patFinalize, http.MethodPost) + s.methods(mux, patActivate, http.MethodPost) + s.methods(mux, patBlob, http.MethodPut) + s.methods(mux, patGC, http.MethodPost) + s.methods(mux, patFsck, http.MethodPost) + + // Unknown paths under the API prefix answer with the same envelope as every + // other failure instead of net/http's plain-text 404, so a client has one + // shape of error to parse. It sits behind authentication too: an + // unauthenticated caller learns nothing about which endpoints exist. + s.route(mux, patCatchAll, func(w http.ResponseWriter, r *http.Request) error { + return api.Errorf(api.CodeNotFound, "no such endpoint") + }) +} + +// methods registers the 405 fallback for a path that exists under other verbs. +func (s *Server) methods(mux *http.ServeMux, pattern string, allow ...string) { + list := strings.Join(allow, ", ") + s.route(mux, pattern, func(w http.ResponseWriter, r *http.Request) error { + w.Header().Set("Allow", list) + return api.Errorf(api.CodeMethodNotAllowed, "method not allowed; this path accepts %s", list) + }) +} + +// handlerFunc is an http.HandlerFunc that may fail. Returning the error instead +// of writing it means a handler cannot accidentally answer twice, and the +// envelope is rendered in exactly one place. +type handlerFunc func(w http.ResponseWriter, r *http.Request) error + +func (s *Server) wrap(h handlerFunc) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := h(w, r); err != nil { + httpx.WriteError(w, r, s.Log, err) + } + }) +} + +// route registers pattern behind authentication plus the given guards, which +// run outermost first. +func (s *Server) route(mux *http.ServeMux, pattern string, h handlerFunc, guards ...func(http.Handler) http.Handler) { + var wrapped http.Handler = s.wrap(h) + for i := len(guards) - 1; i >= 0; i-- { + wrapped = guards[i](wrapped) + } + mux.Handle(pattern, s.Auth.Authenticate(wrapped)) +} + +// resolveProject is the fallback ProjectResolver used until the site registry +// exists. +func (s *Server) resolveProject(ctx context.Context, name string) (int64, error) { + p, err := s.DB.ProjectByName(ctx, name) + if err != nil { + return 0, err + } + return p.ID, nil +} + +// identity returns the caller. Authenticate guarantees one is present, so its +// absence is a routing bug rather than a client error. +func (s *Server) identity(r *http.Request) (*auth.Identity, error) { + id, ok := auth.IdentityFrom(r.Context()) + if !ok { + return nil, api.Errorf(api.CodeInternal, "handler reached without authentication") + } + return id, nil +} + +func (s *Server) maxJSON() int64 { + if s.Limits.MaxJSONBytes > 0 { + return s.Limits.MaxJSONBytes + } + return 1 << 20 +} + +// projectNames maps row ids to names for the key listings, which store an id +// but report a name. One query beats one lookup per key. +func (s *Server) projectNames(ctx context.Context) (map[int64]string, error) { + ps, err := s.DB.AllProjects(ctx) + if err != nil { + return nil, err + } + m := make(map[int64]string, len(ps)) + for _, p := range ps { + m[p.ID] = p.Name + } + return m, nil +} + +// intQuery reads a bounded integer query parameter. +func intQuery(r *http.Request, name string, def, min, max int) (int, error) { + raw := r.URL.Query().Get(name) + if raw == "" { + return def, nil + } + v, err := strconv.Atoi(raw) + if err != nil { + return 0, api.Errorf(api.CodeBadRequest, "%s must be an integer", name) + } + if v < min || v > max { + return 0, api.Errorf(api.CodeBadRequest, "%s must be between %d and %d", name, min, max) + } + return v, nil +} + +// noStore marks a response that must not be written to any cache. Used for the +// one response in the API that carries a credential. +func noStore(w http.ResponseWriter) { + w.Header().Set("Cache-Control", "no-store") +} diff --git a/internal/adminapi/system.go b/internal/adminapi/system.go new file mode 100644 index 0000000..2c13b22 --- /dev/null +++ b/internal/adminapi/system.go @@ -0,0 +1,125 @@ +package adminapi + +import ( + "net/http" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/httpx" + "github.com/iceBear67/simplepages/internal/version" +) + +// whoami handles GET /api/v1/whoami. Any valid key may call it; a CI job uses +// it to check that the token it was handed is the one it expected. +func (s *Server) whoami(w http.ResponseWriter, r *http.Request) error { + ident, err := s.identity(r) + if err != nil { + return err + } + out := api.WhoAmI{ + KeyID: ident.KeyID, + Scope: string(ident.Scope), + Name: ident.Name, + ExpiresAt: copyTime(ident.ExpiresAt), + } + if ident.ProjectID != nil { + // The identity carries the row id, not the name; the name is what a + // human wants to see. A missing row here would mean the project was + // deleted between authentication and now, in which case reporting an + // empty name is more honest than failing the request. + if p, err := s.DB.ProjectByID(r.Context(), *ident.ProjectID); err == nil { + out.Project = p.Name + } + } + noStore(w) + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} + +// systemInfo handles GET /api/v1/system/info, admin only. +func (s *Server) systemInfo(w http.ResponseWriter, r *http.Request) error { + counts, err := s.DB.Counts(r.Context()) + if err != nil { + return err + } + schema, err := s.DB.SchemaVersion(r.Context()) + if err != nil { + return err + } + linkMode := s.LinkMode + if linkMode == "" { + linkMode = "unknown" + } + uptime := int64(0) + if !s.Started.IsZero() { + uptime = int64(time.Since(s.Started).Seconds()) + } + httpx.WriteJSON(w, http.StatusOK, api.SystemInfo{ + Version: version.Short(), + UptimeS: uptime, + Projects: counts.Projects, + Deployments: counts.Deployments, + Blobs: counts.Blobs, + CASBytes: counts.CASBytes, + LinkMode: linkMode, + SchemaVer: schema, + }) + return nil +} + +// gc handles POST /api/v1/gc, admin only. +// +// A collection pass also runs on a timer; this exists so an operator who needs +// the disk back does not have to wait for it, and so that a dry run can answer +// "what would you delete" before anything is deleted. +func (s *Server) gc(w http.ResponseWriter, r *http.Request) error { + var req api.GCRequest + if r.ContentLength != 0 { + if err := httpx.DecodeJSON(w, r, s.maxJSON(), &req); err != nil { + return err + } + } + stats, err := s.Deploy.Collect(r.Context(), req.DryRun) + if err != nil { + return err + } + httpx.WriteJSON(w, http.StatusOK, stats) + return nil +} + +// fsck handles POST /api/v1/fsck, admin only. +// +// Refcounts are maintained by triggers, so on a healthy server this always +// reports zero drift. It exists for the cases outside normal operation — a +// database restored from a backup, a schema touched by hand — because the +// collector trusts those counters, and a count that reads low is the one way +// this design can lose data. +func (s *Server) fsck(w http.ResponseWriter, r *http.Request) error { + var req api.FsckRequest + if r.ContentLength != 0 { + if err := httpx.DecodeJSON(w, r, s.maxJSON(), &req); err != nil { + return err + } + } + rep, err := s.DB.Fsck(r.Context(), req.Repair) + if err != nil { + return err + } + out := api.FsckReport{ + Blobs: rep.Blobs, + DriftCount: rep.DriftCount, + Repaired: rep.Repaired, + Drift: make([]api.BlobDrift, 0, len(rep.Drift)), + } + for _, d := range rep.Drift { + out.Drift = append(out.Drift, api.BlobDrift{ + Digest: d.Digest.String(), Stored: d.Stored, Actual: d.Actual, + }) + } + if rep.DriftCount > 0 { + s.Log.WarnContext(r.Context(), "blob reference counts disagree with the manifests", + "blobs", rep.Blobs, "drift", rep.DriftCount, "repaired", rep.Repaired) + } + httpx.WriteJSON(w, http.StatusOK, out) + return nil +} diff --git a/internal/adminapi/system_test.go b/internal/adminapi/system_test.go new file mode 100644 index 0000000..982078c --- /dev/null +++ b/internal/adminapi/system_test.go @@ -0,0 +1,101 @@ +package adminapi + +import ( + "net/http" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" +) + +func TestWhoAmIReportsTheCaller(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + future := time.Now().Add(time.Hour).Truncate(time.Second) + proj := e.createKey(t, api.PathProjectKeys("demo"), e.adminToken, + api.CreateKeyRequest{Name: "ci", ExpiresAt: &future}) + + t.Run("project scope", func(t *testing.T) { + status, body := e.do(t, http.MethodGet, api.PathWhoAmI(), proj.Token, nil) + var who api.WhoAmI + mustJSON(t, status, http.StatusOK, body, &who) + if who.KeyID != proj.Key.ID { + t.Errorf("key_id = %q, want %q", who.KeyID, proj.Key.ID) + } + if who.Scope != api.ScopeProject || who.Project != "demo" || who.Name != "ci" { + t.Errorf("whoami = %+v, want scope project on demo named ci", who) + } + if who.ExpiresAt == nil || !who.ExpiresAt.Equal(future) { + t.Errorf("expires_at = %v, want %v", who.ExpiresAt, future) + } + }) + + t.Run("admin scope", func(t *testing.T) { + status, body := e.do(t, http.MethodGet, api.PathWhoAmI(), e.adminToken, nil) + var who api.WhoAmI + mustJSON(t, status, http.StatusOK, body, &who) + if who.Scope != api.ScopeAdmin { + t.Errorf("scope = %q, want admin", who.Scope) + } + // An admin key belongs to no project, and saying otherwise would suggest + // the caller is confined to one. + if who.Project != "" { + t.Errorf("project = %q, want empty for an admin key", who.Project) + } + }) + + // whoami names a credential, so it must not be cached by anything between + // the CLI and the server. + t.Run("no-store", func(t *testing.T) { + resp := e.doResp(t, http.MethodGet, api.PathWhoAmI(), proj.Token, nil) + if cc := resp.Header.Get("Cache-Control"); !strings.Contains(cc, "no-store") { + t.Errorf("Cache-Control = %q, want it to contain no-store", cc) + } + }) +} + +func TestSystemInfo(t *testing.T) { + e := newEnv(t) + e.createProject(t, "a") + e.createProject(t, "b") + + status, body := e.do(t, http.MethodGet, api.PathSystemInfo(), e.adminToken, nil) + var info api.SystemInfo + mustJSON(t, status, http.StatusOK, body, &info) + + if info.Projects != 2 { + t.Errorf("projects = %d, want 2", info.Projects) + } + if info.Deployments != 0 || info.Blobs != 0 || info.CASBytes != 0 { + t.Errorf("expected an empty CAS, got %+v", info) + } + if info.SchemaVer < 1 { + t.Errorf("schema_version = %d, want at least 1", info.SchemaVer) + } + if info.Version == "" { + t.Error("version is empty") + } + // The link mode is not known until the CAS is opened in M2; reporting an + // empty string would read as "no linking" rather than "not determined". + if info.LinkMode != "unknown" { + t.Errorf("link_mode = %q, want unknown", info.LinkMode) + } + if info.UptimeS < 0 { + t.Errorf("uptime_s = %d", info.UptimeS) + } +} + +func TestSystemInfoIsAdminOnly(t *testing.T) { + e := newEnv(t) + e.createProject(t, "demo") + token := e.mintProject(t, e.projectID(t, "demo"), "ci") + + status, body := e.do(t, http.MethodGet, api.PathSystemInfo(), token, nil) + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403; body: %s", status, body) + } + if got := errCode(t, body); got != api.CodeForbidden { + t.Errorf("code = %q, want %q", got, api.CodeForbidden) + } +} diff --git a/internal/adminapi/validate.go b/internal/adminapi/validate.go new file mode 100644 index 0000000..cf0f1f4 --- /dev/null +++ b/internal/adminapi/validate.go @@ -0,0 +1,107 @@ +package adminapi + +import ( + "io/fs" + "strings" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/config" +) + +// Field length caps. They exist to stop a client filling the database with a +// megabyte of display name, not to express any semantic limit. +const ( + maxDisplayNameLen = 200 + maxKeyNameLen = 200 + maxSitePathLen = 1024 + maxCacheControlLen = 256 +) + +// checkProjectName validates a name before it becomes a row and, later, a +// "~name" entry inside $WEBROOT. The pattern is the only thing standing between +// a project name and the filesystem, so this is a hard reject rather than a +// normalisation. +func checkProjectName(name string) error { + if name == "" { + return api.Errorf(api.CodeInvalidProjectName, "project name is required") + } + if !config.ProjectNamePattern.MatchString(name) { + return api.Errorf(api.CodeInvalidProjectName, + "project name must match %s", config.ProjectNamePattern.String()) + } + return nil +} + +// checkText rejects control characters in free-text fields. +// +// These strings are printed to operator terminals by "pages project list" and +// written into structured logs. An embedded ESC would let whoever set the field +// move the cursor, recolour the output or rewrite the line the operator is +// reading; a newline would forge a second log record. +func checkText(field, v string, max int) error { + if len(v) > max { + return api.Errorf(api.CodeBadRequest, "%s must be at most %d bytes", field, max) + } + for i := 0; i < len(v); i++ { + if c := v[i]; c < 0x20 || c == 0x7f { + return api.Errorf(api.CodeBadRequest, "%s must not contain control characters", field) + } + } + return nil +} + +// checkSitePath validates a path that names a document inside a deployment, +// such as index_file or not_found_file. +// +// It is checked here as well as at serve time because a value that cannot +// possibly resolve is a configuration mistake worth reporting at the moment it +// is made, rather than as a silent 404 on every request afterwards. +func checkSitePath(field, v string) error { + if v == "" { + return api.Errorf(api.CodeInvalidPath, "%s must not be empty", field) + } + if len(v) > maxSitePathLen { + return api.Errorf(api.CodeInvalidPath, "%s must be at most %d bytes", field, maxSitePathLen) + } + // fs.ValidPath rejects "..", absolute paths, empty segments and a trailing + // slash. It accepts ".", which is a directory rather than a document. + if !fs.ValidPath(v) || v == "." { + return api.Errorf(api.CodeInvalidPath, + "%s must be a relative slash-separated path with no . or .. segments", field) + } + if strings.ContainsRune(v, '\\') { + return api.Errorf(api.CodeInvalidPath, "%s must use forward slashes", field) + } + for i := 0; i < len(v); i++ { + if c := v[i]; c < 0x20 || c == 0x7f { + return api.Errorf(api.CodeInvalidPath, "%s must not contain control characters", field) + } + } + return nil +} + +// checkHeaderValue validates a string that is emitted verbatim as a response +// header value. +// +// A CR or LF here would be response splitting: the project could append headers +// of its own choosing to every response it serves, and under path routing those +// responses share an origin with every other project. net/http replaces newlines +// with spaces on write, so this is defence in depth — but it is the layer that +// keeps the bad value out of the database in the first place, and it is the one +// that tells the operator they typed something wrong. +func checkHeaderValue(field, v string) error { + if len(v) > maxCacheControlLen { + return api.Errorf(api.CodeBadRequest, "%s must be at most %d bytes", field, maxCacheControlLen) + } + for i := 0; i < len(v); i++ { + c := v[i] + if c == '\t' { + continue + } + if c < 0x20 || c > 0x7e { + return api.Errorf(api.CodeBadRequest, + "%s must contain only printable ASCII (no newlines)", field) + } + } + return nil +} diff --git a/internal/adminapi/validate_test.go b/internal/adminapi/validate_test.go new file mode 100644 index 0000000..f21e72d --- /dev/null +++ b/internal/adminapi/validate_test.go @@ -0,0 +1,114 @@ +package adminapi + +import ( + "strings" + "testing" + + "github.com/iceBear67/simplepages/api" +) + +func TestCheckProjectName(t *testing.T) { + valid := []string{ + "a", "0", "demo", "demo-site", "demo_site", "demo.site", "a1.b-c_d", + strings.Repeat("z", 63), + } + for _, name := range valid { + if err := checkProjectName(name); err != nil { + t.Errorf("checkProjectName(%q) = %v, want nil", name, err) + } + } + + // Everything that could become a path separator, a traversal or a hidden + // file once the name is turned into a "~name" entry in $WEBROOT. + invalid := []string{ + "", ".", "..", "./x", "../x", "/demo", "demo/", "a/b", `a\b`, + ".hidden", "-lead", "_lead", "Demo", "DEMO", "démo", "demo ", + " demo", "de mo", "demo\t", "demo\n", "demo\x00", "~demo", "demo%2f", + strings.Repeat("z", 64), + } + for _, name := range invalid { + err := checkProjectName(name) + if err == nil { + t.Errorf("checkProjectName(%q) = nil, want an error", name) + continue + } + if got := api.CodeOf(err); got != api.CodeInvalidProjectName { + t.Errorf("checkProjectName(%q) code = %q, want %q", name, got, api.CodeInvalidProjectName) + } + } +} + +func TestCheckSitePath(t *testing.T) { + valid := []string{ + "index.html", "404.html", "a/b/c.html", "a.b/c", "_next/index.html", + "страница.html", strings.Repeat("a", maxSitePathLen), + } + for _, p := range valid { + if err := checkSitePath("index_file", p); err != nil { + t.Errorf("checkSitePath(%q) = %v, want nil", p, err) + } + } + + invalid := []string{ + "", ".", "..", "/index.html", "index.html/", "a//b", "a/./b", "a/../b", + "../../etc/passwd", `a\b`, "a\x00b", "a\nb", "a\x7fb", + strings.Repeat("a", maxSitePathLen+1), + } + for _, p := range invalid { + err := checkSitePath("index_file", p) + if err == nil { + t.Errorf("checkSitePath(%q) = nil, want an error", p) + continue + } + if got := api.CodeOf(err); got != api.CodeInvalidPath { + t.Errorf("checkSitePath(%q) code = %q, want %q", p, got, api.CodeInvalidPath) + } + } +} + +func TestCheckHeaderValue(t *testing.T) { + valid := []string{ + "", "public, max-age=0, must-revalidate", "no-store", + "public,\tmax-age=31536000, immutable", strings.Repeat("a", maxCacheControlLen), + } + for _, v := range valid { + if err := checkHeaderValue("cache_control", v); err != nil { + t.Errorf("checkHeaderValue(%q) = %v, want nil", v, err) + } + } + + // A CR or LF here would let a project append headers of its own to every + // response it serves, on an origin it shares with every other project. + invalid := []string{ + "public\r\nX-Evil: 1", "public\nX-Evil: 1", "public\r", "public\x00", + "public\x7f", "public é", strings.Repeat("a", maxCacheControlLen+1), + } + for _, v := range invalid { + if err := checkHeaderValue("cache_control", v); err == nil { + t.Errorf("checkHeaderValue(%q) = nil, want an error", v) + } + } +} + +func TestCheckText(t *testing.T) { + // Free text is allowed to be anything printable, including non-ASCII: it is + // a display name, not a header value. + valid := []string{"", "Demo Site", "デモ", "a — b", strings.Repeat("x", maxDisplayNameLen)} + for _, v := range valid { + if err := checkText("display_name", v, maxDisplayNameLen); err != nil { + t.Errorf("checkText(%q) = %v, want nil", v, err) + } + } + + // Control characters are not: these strings are printed to operator + // terminals and written into structured logs. + invalid := []string{ + "a\x1b[31mred", "line\nbreak", "car\rriage", "nul\x00", "del\x7f", "tab\there", + strings.Repeat("x", maxDisplayNameLen+1), + } + for _, v := range invalid { + if err := checkText("display_name", v, maxDisplayNameLen); err == nil { + t.Errorf("checkText(%q) = nil, want an error", v) + } + } +} diff --git a/internal/auth/bootstrap.go b/internal/auth/bootstrap.go new file mode 100644 index 0000000..fe56bea --- /dev/null +++ b/internal/auth/bootstrap.go @@ -0,0 +1,110 @@ +package auth + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + + "github.com/iceBear67/simplepages/internal/store" +) + +// BootstrapKeyName is the name given to the first-run admin key, so an operator +// listing keys can tell it apart from ones they minted themselves. +const BootstrapKeyName = "bootstrap" + +// EnsureAdminKey mints an admin key and writes its token to tokenPath when the +// server has no usable admin key at all, and reports whether it did. +// +// This is the only place the server ever writes a token to disk, and it exists +// because a fresh install would otherwise have no way to authenticate the call +// that creates the first key. The file is 0600 and the log line tells the +// operator to delete it once they have copied the token out. +// +// "Usable" excludes revoked and expired keys, so an installation whose only +// admin key was revoked recovers by restarting rather than by hand-editing the +// database. +func EnsureAdminKey(ctx context.Context, db *store.DB, tokenPath string, log *slog.Logger) (bool, error) { + n, err := db.CountUsableAdminKeys(ctx) + if err != nil { + return false, fmt.Errorf("count admin keys: %w", err) + } + if n > 0 { + // Nothing to mint. If a token file is still lying around from an earlier + // bootstrap, say so: it is a live credential until that key is revoked. + if _, err := os.Lstat(tokenPath); err == nil && log != nil { + log.Warn("bootstrap token file still present; delete it once the token is stored elsewhere", + "path", tokenPath) + } + return false, nil + } + + token, keyID, hash, err := Mint() + if err != nil { + return false, err + } + if err := writeTokenFile(tokenPath, token); err != nil { + return false, err + } + k := &store.APIKey{ + ID: keyID, + SecretHash: hash[:], + Scope: store.ScopeAdmin, + Name: BootstrapKeyName, + } + if err := db.CreateKey(ctx, k); err != nil { + // The file names a key that does not exist. Remove it rather than leave + // an operator holding a token that will never authenticate. + _ = os.Remove(tokenPath) + return false, fmt.Errorf("create bootstrap key: %w", err) + } + + if log != nil { + // The key id is the public half of the token and is safe to log; the + // token itself is not, which is the whole reason for the file. + log.Warn("no admin key found; minted a bootstrap admin key", + "key_id", keyID, "path", tokenPath, + "action", "read the token, then delete this file") + } + return true, nil +} + +// writeTokenFile writes the token with 0600 permissions. +// +// It writes to a temporary file in the same directory and renames it into +// place. That is not for atomicity — nothing reads this concurrently — but +// because rename replaces the destination without following it. Opening the +// path directly would follow a symlink someone had planted there and write a +// live credential wherever it pointed. +func writeTokenFile(path, token string) error { + dir := filepath.Dir(path) + f, err := os.CreateTemp(dir, ".bootstrap-token-*") + if err != nil { + return fmt.Errorf("create bootstrap token file: %w", err) + } + tmp := f.Name() + defer func() { + f.Close() + os.Remove(tmp) // no-op once the rename has succeeded + }() + + // CreateTemp already makes the file 0600, but say so explicitly: this is the + // property that matters and it should not depend on a documented default. + if err := f.Chmod(0o600); err != nil { + return fmt.Errorf("chmod bootstrap token file: %w", err) + } + if _, err := f.WriteString(token + "\n"); err != nil { + return fmt.Errorf("write bootstrap token: %w", err) + } + if err := f.Sync(); err != nil { + return fmt.Errorf("sync bootstrap token: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("close bootstrap token: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("install bootstrap token: %w", err) + } + return nil +} diff --git a/internal/auth/bootstrap_test.go b/internal/auth/bootstrap_test.go new file mode 100644 index 0000000..e072019 --- /dev/null +++ b/internal/auth/bootstrap_test.go @@ -0,0 +1,285 @@ +package auth + +import ( + "bytes" + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/internal/store" +) + +// bootstrapEnv is a database, a token path in a directory of its own, and a log +// buffer to assert against. +type bootstrapEnv struct { + db *store.DB + dir string + log *slog.Logger + buf *bytes.Buffer +} + +func newBootstrapEnv(t *testing.T) *bootstrapEnv { + t.Helper() + var buf bytes.Buffer + return &bootstrapEnv{ + db: testStore(t), + dir: t.TempDir(), + log: slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})), + buf: &buf, + } +} + +func (e *bootstrapEnv) path() string { return filepath.Join(e.dir, "bootstrap-token") } + +func (e *bootstrapEnv) token(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile(e.path()) + if err != nil { + t.Fatalf("read token file: %v", err) + } + return strings.TrimSpace(string(raw)) +} + +func TestEnsureAdminKeyMintsOnEmptyDatabase(t *testing.T) { + ctx := context.Background() + e := newBootstrapEnv(t) + + minted, err := EnsureAdminKey(ctx, e.db, e.path(), e.log) + if err != nil { + t.Fatalf("EnsureAdminKey: %v", err) + } + if !minted { + t.Fatal("minted = false on an empty database") + } + + // The token in the file is the credential: it must actually authenticate. + v := newVerifier(t, e.db) + id, err := v.Verify(ctx, e.token(t)) + if err != nil { + t.Fatalf("the bootstrap token does not authenticate: %v", err) + } + if !id.IsAdmin() { + t.Errorf("scope = %q, want admin", id.Scope) + } + if id.Name != BootstrapKeyName { + t.Errorf("name = %q, want %q", id.Name, BootstrapKeyName) + } + + // The operator is told where the file is and that it must be removed, and + // the token itself never reaches the log — the file exists precisely so it + // does not have to. + logged := e.buf.String() + if !strings.Contains(logged, e.path()) { + t.Errorf("log does not name the token file: %s", logged) + } + if strings.Contains(logged, e.token(t)) { + t.Fatal("the bootstrap token was written to the log") + } + if !strings.Contains(logged, id.KeyID) { + t.Errorf("log does not name the key id, which is the safe half: %s", logged) + } +} + +// The file holds a live admin credential. Anything wider than 0600 hands it to +// every account on the host. +func TestBootstrapTokenFileIsPrivate(t *testing.T) { + ctx := context.Background() + e := newBootstrapEnv(t) + if _, err := EnsureAdminKey(ctx, e.db, e.path(), e.log); err != nil { + t.Fatal(err) + } + fi, err := os.Lstat(e.path()) + if err != nil { + t.Fatal(err) + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("mode = %#o, want 0600", perm) + } + if fi.Mode()&os.ModeSymlink != 0 { + t.Error("the token path is a symlink") + } +} + +func TestEnsureAdminKeyIsIdempotent(t *testing.T) { + ctx := context.Background() + e := newBootstrapEnv(t) + + if _, err := EnsureAdminKey(ctx, e.db, e.path(), e.log); err != nil { + t.Fatal(err) + } + first := e.token(t) + + // A restart must not mint a second admin key, and must not overwrite the + // token the operator has not yet collected. + minted, err := EnsureAdminKey(ctx, e.db, e.path(), e.log) + if err != nil { + t.Fatal(err) + } + if minted { + t.Error("minted a second bootstrap key") + } + if got := e.token(t); got != first { + t.Error("the token file was rewritten on the second run") + } + + n, err := e.db.CountUsableAdminKeys(ctx) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("usable admin keys = %d, want 1", n) + } + + // A leftover file is a live credential; the operator gets told about it. + if !strings.Contains(e.buf.String(), "still present") { + t.Errorf("no warning about the leftover token file: %s", e.buf.String()) + } +} + +// A revoked or expired admin key leaves the installation locked out, so +// restarting must mint a fresh one rather than counting the dead key. +func TestEnsureAdminKeyRecoversFromUnusableKeys(t *testing.T) { + ctx := context.Background() + + t.Run("revoked", func(t *testing.T) { + e := newBootstrapEnv(t) + _, keyID := mintInto(t, e.db, store.ScopeAdmin, nil) + if err := e.db.RevokeKey(ctx, keyID); err != nil { + t.Fatal(err) + } + minted, err := EnsureAdminKey(ctx, e.db, e.path(), e.log) + if err != nil { + t.Fatal(err) + } + if !minted { + t.Fatal("minted = false with only a revoked admin key") + } + }) + + t.Run("expired", func(t *testing.T) { + e := newBootstrapEnv(t) + token, _, hash, err := Mint() + if err != nil { + t.Fatal(err) + } + past := time.Now().Add(-time.Hour) + k := &store.APIKey{ID: keyIDOf(t, token), SecretHash: hash[:], + Scope: store.ScopeAdmin, Name: "old", ExpiresAt: &past} + if err := e.db.CreateKey(ctx, k); err != nil { + t.Fatal(err) + } + minted, err := EnsureAdminKey(ctx, e.db, e.path(), e.log) + if err != nil { + t.Fatal(err) + } + if !minted { + t.Fatal("minted = false with only an expired admin key") + } + }) + + t.Run("project keys do not count", func(t *testing.T) { + e := newBootstrapEnv(t) + id := createProjectRow(t, e.db, "demo") + mintInto(t, e.db, store.ScopeProject, &id) + minted, err := EnsureAdminKey(ctx, e.db, e.path(), e.log) + if err != nil { + t.Fatal(err) + } + if !minted { + t.Fatal("minted = false with only a project key") + } + }) +} + +// The token path is attacker-controllable on a host where the data directory is +// created before the server runs. Writing through a planted symlink would put a +// live admin credential wherever it pointed — /etc/cron.d, another user's +// ~/.ssh, a world-readable log. The rename-into-place replaces the link instead +// of following it. +func TestBootstrapTokenDoesNotFollowASymlink(t *testing.T) { + ctx := context.Background() + e := newBootstrapEnv(t) + + target := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(target, []byte("original\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, e.path()); err != nil { + t.Fatal(err) + } + + if _, err := EnsureAdminKey(ctx, e.db, e.path(), e.log); err != nil { + t.Fatalf("EnsureAdminKey: %v", err) + } + + victim, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if string(victim) != "original\n" { + t.Fatalf("the symlink target was overwritten with %q", victim) + } + fi, err := os.Lstat(e.path()) + if err != nil { + t.Fatal(err) + } + if fi.Mode()&os.ModeSymlink != 0 { + t.Fatal("the token path is still a symlink; the token went somewhere else") + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("mode = %#o, want 0600", perm) + } + if !strings.HasPrefix(e.token(t), Prefix+"_") { + t.Errorf("the token file does not hold a token") + } +} + +// A directory in the way must fail loudly rather than leave the server running +// with an admin key nobody can use. +func TestBootstrapTokenReportsAnUnwritablePath(t *testing.T) { + ctx := context.Background() + e := newBootstrapEnv(t) + blocked := filepath.Join(e.dir, "blocked") + if err := os.Mkdir(blocked, 0o755); err != nil { + t.Fatal(err) + } + + if _, err := EnsureAdminKey(ctx, e.db, blocked, e.log); err == nil { + t.Fatal("EnsureAdminKey = nil, want an error when the token cannot be stored") + } + // And no key was created: a key whose token was never delivered is just a + // row nobody can authenticate with. + n, err := e.db.CountUsableAdminKeys(ctx) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("usable admin keys = %d, want 0", n) + } +} + +// keyIDOf recovers the public half of a token, which is all a test needs to +// store the row. +func keyIDOf(t *testing.T, token string) string { + t.Helper() + keyID, _, err := Parse(token) + if err != nil { + t.Fatalf("Parse: %v", err) + } + return keyID +} + +// createProjectRow inserts a project so a project-scoped key has something to +// point at, and returns its id. +func createProjectRow(t *testing.T, db *store.DB, name string) int64 { + t.Helper() + p := store.DefaultProject(name) + if err := db.CreateProject(context.Background(), p); err != nil { + t.Fatalf("CreateProject: %v", err) + } + return p.ID +} diff --git a/internal/auth/middleware.go b/internal/auth/middleware.go new file mode 100644 index 0000000..aad80ef --- /dev/null +++ b/internal/auth/middleware.go @@ -0,0 +1,216 @@ +package auth + +import ( + "context" + "errors" + "log/slog" + "net/http" + "net/netip" + "strconv" + "strings" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/httpx" + "github.com/iceBear67/simplepages/internal/store" +) + +type ctxKey int + +const identityKey ctxKey = iota + +// IdentityFrom returns the identity established by Authenticate. +// +// A handler mounted behind Authenticate can treat a false result as a +// programming error: the middleware answers 401 itself and never calls through +// without an identity. +func IdentityFrom(ctx context.Context) (*Identity, bool) { + id, ok := ctx.Value(identityKey).(*Identity) + return id, ok +} + +// ContextWithIdentity is used by tests and by handlers that authenticate out of +// band; ordinary request handling gets its identity from Authenticate. +func ContextWithIdentity(ctx context.Context, id *Identity) context.Context { + return context.WithValue(ctx, identityKey, id) +} + +// errUnauthorized is the single response every authentication failure produces. +// Distinguishing "no such key" from "wrong secret" from "revoked" would tell an +// unauthenticated caller which key ids are real. +func errUnauthorized() *api.Error { + return api.Errorf(api.CodeUnauthorized, "missing or invalid API token") +} + +// Middleware carries the collaborators the auth handlers need. +type Middleware struct { + V *Verifier + Limiter *Limiter + Trusted []netip.Prefix + Log *slog.Logger +} + +// Authenticate requires a valid bearer token and puts the identity in the +// request context. +// +// The token is read only from the Authorization header, never from a query +// parameter: query strings land in proxy access logs, browser history and +// Referer headers, and a credential that ends up there is a credential leaked. +func (m *Middleware) Authenticate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token, ok := bearerToken(r) + if !ok { + m.reject(w, r, errors.New("auth: no bearer token")) + return + } + + client := m.clientKey(r) + if !m.Limiter.Allow(client) { + if d := m.Limiter.RetryAfter(client); d > 0 { + w.Header().Set("Retry-After", strconv.Itoa(int(d.Seconds()))) + } + httpx.WriteError(w, r, m.Log, api.Errorf(api.CodeRateLimited, + "too many failed authentication attempts; slow down")) + return + } + + id, err := m.V.Verify(r.Context(), token) + if err != nil { + m.Limiter.Fail(client) + m.reject(w, r, err) + return + } + + // The key id is the public half of the token and is safe to log; the + // secret never leaves this function. + httpx.LogAttr(r.Context(), "key_id", id.KeyID) + httpx.LogAttr(r.Context(), "scope", string(id.Scope)) + next.ServeHTTP(w, r.WithContext(ContextWithIdentity(r.Context(), id))) + }) +} + +// reject logs why authentication failed and tells the client only that it did. +func (m *Middleware) reject(w http.ResponseWriter, r *http.Request, cause error) { + if m.Log != nil { + // cause is one of this package's sentinels or a store error. None of + // them embed the token, which is what makes it safe to log at all. + m.Log.Debug("authentication failed", + "reason", cause, + "req_id", httpx.RequestIDFrom(r.Context()), + "path", r.URL.Path) + } + w.Header().Set("WWW-Authenticate", `Bearer realm="pages"`) + httpx.WriteError(w, r, m.Log, errUnauthorized()) +} + +// clientKey identifies the caller for rate limiting. +func (m *Middleware) clientKey(r *http.Request) string { + if addr, ok := httpx.ClientIP(r, m.Trusted); ok { + return addr.String() + } + return r.RemoteAddr +} + +// bearerToken extracts the credential from the Authorization header. The scheme +// comparison is case-insensitive per RFC 7235; the token itself is not touched. +func bearerToken(r *http.Request) (string, bool) { + h := r.Header.Get("Authorization") + if h == "" { + return "", false + } + scheme, token, ok := strings.Cut(h, " ") + if !ok || !strings.EqualFold(scheme, "Bearer") { + return "", false + } + token = strings.TrimSpace(token) + if token == "" { + return "", false + } + return token, true +} + +// RequireAdmin rejects identities that are not admin-scoped. +func RequireAdmin(log *slog.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id, ok := IdentityFrom(r.Context()) + if !ok { + httpx.WriteError(w, r, log, errUnauthorized()) + return + } + if !id.IsAdmin() { + httpx.WriteError(w, r, log, api.Errorf(api.CodeForbidden, + "this operation requires an admin key")) + return + } + next.ServeHTTP(w, r) + }) + } +} + +// ProjectResolver maps a project name from the URL to its row id. +// +// It is an interface rather than a concrete type so this package does not +// depend on the site registry, which does not exist until the serving layer is +// wired up, and so tests can supply a two-line fake. +type ProjectResolver interface { + ResolveProject(ctx context.Context, name string) (int64, error) +} + +// ResolverFunc adapts a function to ProjectResolver. +type ResolverFunc func(ctx context.Context, name string) (int64, error) + +func (f ResolverFunc) ResolveProject(ctx context.Context, name string) (int64, error) { + return f(ctx, name) +} + +// RequireProject allows admins through and otherwise requires the caller's key +// to belong to the project named by the {pathValue} URL wildcard. +// +// The comparison is on resolved row ids, never on the name string. Comparing +// names would make the trust boundary depend on every handler normalising the +// same way, and would break the moment two names can resolve to one project. +func RequireProject(pathValue string, r ProjectResolver, log *slog.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + id, ok := IdentityFrom(req.Context()) + if !ok { + httpx.WriteError(w, req, log, errUnauthorized()) + return + } + name := req.PathValue(pathValue) + if name == "" { + httpx.WriteError(w, req, log, api.Errorf(api.CodeBadRequest, "missing project name")) + return + } + + projectID, err := r.ResolveProject(req.Context(), name) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + // A project-scoped key must not be able to probe which + // project names exist, so an unknown name looks the same as + // someone else's project. + if !id.IsAdmin() { + httpx.WriteError(w, req, log, forbiddenProject()) + return + } + httpx.WriteError(w, req, log, + api.Errorf(api.CodeNotFound, "no such project: %s", name)) + return + } + httpx.WriteError(w, req, log, err) + return + } + + if !id.Owns(projectID) { + httpx.WriteError(w, req, log, forbiddenProject()) + return + } + httpx.LogAttr(req.Context(), "project", name) + next.ServeHTTP(w, req) + }) + } +} + +func forbiddenProject() *api.Error { + return api.Errorf(api.CodeForbidden, "this key does not have access to that project") +} diff --git a/internal/auth/middleware_test.go b/internal/auth/middleware_test.go new file mode 100644 index 0000000..6a049d0 --- /dev/null +++ b/internal/auth/middleware_test.go @@ -0,0 +1,407 @@ +package auth + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/httpx" + "github.com/iceBear67/simplepages/internal/store" +) + +// okHandler records that the request got past the middleware. +func okHandler(reached *bool) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if reached != nil { + *reached = true + } + w.WriteHeader(http.StatusNoContent) + }) +} + +func errorCode(t *testing.T, body []byte) api.Code { + t.Helper() + var env api.ErrorEnvelope + if err := json.Unmarshal(body, &env); err != nil { + t.Fatalf("response is not an error envelope: %v (%s)", err, body) + } + if env.Error.Code == "" { + t.Fatalf("envelope has no error code: %s", body) + } + return env.Error.Code +} + +func newMiddleware(t *testing.T, db *store.DB, logTo *bytes.Buffer) *Middleware { + t.Helper() + var h slog.Handler = slog.NewTextHandler(logTo, &slog.HandlerOptions{Level: slog.LevelDebug}) + log := slog.New(h) + return &Middleware{ + V: NewVerifier(db, log, DefaultCacheTTL), + Limiter: NewLimiter(5, time.Minute, 128), + Trusted: []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}, + Log: log, + } +} + +func TestAuthenticateAcceptsValidToken(t *testing.T) { + db := testStore(t) + m := newMiddleware(t, db, &bytes.Buffer{}) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + + var gotID *Identity + h := m.Authenticate(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id, ok := IdentityFrom(r.Context()) + if !ok { + t.Error("no identity in context behind Authenticate") + } + gotID = id + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/whoami", nil) + req.Header.Set("Authorization", "Bearer "+token) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, want 204: %s", rec.Code, rec.Body) + } + if gotID == nil || gotID.KeyID != keyID { + t.Errorf("identity = %+v, want key %s", gotID, keyID) + } +} + +func TestAuthenticateRejects(t *testing.T) { + db := testStore(t) + m := newMiddleware(t, db, &bytes.Buffer{}) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + _, secret, err := Parse(token) + if err != nil { + t.Fatal(err) + } + unknown, _, _, err := Mint() + if err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + header string + }{ + {"no header", ""}, + {"empty bearer", "Bearer "}, + {"wrong scheme", "Basic " + token}, + {"token without scheme", token}, + {"malformed token", "Bearer not-a-token"}, + {"unknown key", "Bearer " + unknown}, + {"truncated secret", "Bearer " + Prefix + "_" + keyID + "_" + secret[:42]}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reached := false + h := m.Authenticate(okHandler(&reached)) + req := httptest.NewRequest(http.MethodGet, "/api/v1/whoami", nil) + if tc.header != "" { + req.Header.Set("Authorization", tc.header) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if reached { + t.Error("request reached the handler") + } + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401: %s", rec.Code, rec.Body) + } + if got := errorCode(t, rec.Body.Bytes()); got != api.CodeUnauthorized { + t.Errorf("code = %q, want %q", got, api.CodeUnauthorized) + } + if got := rec.Header().Get("WWW-Authenticate"); !strings.Contains(got, "Bearer") { + t.Errorf("WWW-Authenticate = %q", got) + } + // The client must not learn which check failed. + body := rec.Body.String() + for _, leak := range []string{"revoked", "expired", "unknown key", "secret mismatch"} { + if strings.Contains(strings.ToLower(body), leak) { + t.Errorf("response distinguishes the failure reason (%q): %s", leak, body) + } + } + }) + } +} + +// The scheme is case-insensitive per RFC 7235, and some CI clients send "bearer". +func TestAuthenticateAcceptsAnyCaseScheme(t *testing.T) { + db := testStore(t) + m := newMiddleware(t, db, &bytes.Buffer{}) + token, _ := mintInto(t, db, store.ScopeAdmin, nil) + + for _, scheme := range []string{"Bearer", "bearer", "BEARER", "BeArEr"} { + reached := false + h := m.Authenticate(okHandler(&reached)) + req := httptest.NewRequest(http.MethodGet, "/x", nil) + req.Header.Set("Authorization", scheme+" "+token) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if !reached { + t.Errorf("scheme %q rejected: %d %s", scheme, rec.Code, rec.Body) + } + } +} + +// A token in the query string ends up in proxy logs and browser history, so it +// must never be accepted as a credential. +func TestTokenInQueryStringIsNotAccepted(t *testing.T) { + db := testStore(t) + m := newMiddleware(t, db, &bytes.Buffer{}) + token, _ := mintInto(t, db, store.ScopeAdmin, nil) + + reached := false + h := m.Authenticate(okHandler(&reached)) + req := httptest.NewRequest(http.MethodGet, "/api/v1/whoami?token="+token+"&access_token="+token, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if reached { + t.Fatal("a query-string token authenticated the request") + } + if rec.Code != http.StatusUnauthorized { + t.Errorf("status = %d, want 401", rec.Code) + } +} + +// The load-bearing one: nothing this middleware logs may contain the secret. +func TestAuthLogsNeverContainCredentials(t *testing.T) { + db := testStore(t) + var logBuf bytes.Buffer + m := newMiddleware(t, db, &logBuf) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + _, secret, err := Parse(token) + if err != nil { + t.Fatal(err) + } + + handler := httpx.Chain( + m.Authenticate(okHandler(nil)), + httpx.WithRequestID(m.Trusted), + httpx.AccessLog(m.Log, m.Trusted), + httpx.Recover(m.Log), + ) + + // A successful request, a wrong-secret request, and a garbage request: the + // three paths that each touch the token. + for _, hdr := range []string{ + "Bearer " + token, + "Bearer " + Prefix + "_" + keyID + "_" + strings.Repeat("z", 43), + "Bearer " + token + "trailing", + } { + req := httptest.NewRequest(http.MethodGet, "/api/v1/whoami", nil) + req.Header.Set("Authorization", hdr) + req.RemoteAddr = "10.1.2.3:5555" + handler.ServeHTTP(httptest.NewRecorder(), req) + } + + out := logBuf.String() + if out == "" { + t.Fatal("nothing was logged; the test would pass vacuously") + } + for _, forbidden := range []string{secret, token, "Bearer", "Authorization"} { + if strings.Contains(out, forbidden) { + t.Errorf("log contains %q:\n%s", forbidden, out) + } + } + // The public half is supposed to be there — otherwise an operator cannot + // tell which key made a request. + if !strings.Contains(out, keyID) { + t.Errorf("log does not record the key id:\n%s", out) + } +} + +func TestAuthenticateRateLimitsFailures(t *testing.T) { + db := testStore(t) + m := newMiddleware(t, db, &bytes.Buffer{}) + m.Limiter = NewLimiter(3, time.Minute, 32) + bad, _, _, err := Mint() + if err != nil { + t.Fatal(err) + } + good, _ := mintInto(t, db, store.ScopeAdmin, nil) + + send := func(token, remote string) *httptest.ResponseRecorder { + h := m.Authenticate(okHandler(nil)) + req := httptest.NewRequest(http.MethodGet, "/api/v1/whoami", nil) + req.Header.Set("Authorization", "Bearer "+token) + req.RemoteAddr = remote + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec + } + + for i := 0; i < 3; i++ { + if got := send(bad, "10.1.2.3:5555").Code; got != http.StatusUnauthorized { + t.Fatalf("attempt %d: status = %d, want 401", i, got) + } + } + rec := send(bad, "10.1.2.3:5555") + if rec.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429: %s", rec.Code, rec.Body) + } + if got := errorCode(t, rec.Body.Bytes()); got != api.CodeRateLimited { + t.Errorf("code = %q, want %q", got, api.CodeRateLimited) + } + if rec.Header().Get("Retry-After") == "" { + t.Error("429 without a Retry-After header") + } + + // A different address is unaffected... + if got := send(bad, "10.9.9.9:5555").Code; got != http.StatusUnauthorized { + t.Errorf("unrelated client got %d, want 401", got) + } + // ...and a client that had never failed can still authenticate. + if got := send(good, "10.8.8.8:5555").Code; got != http.StatusNoContent { + t.Errorf("valid token from a clean client got %d, want 204", got) + } +} + +func TestRequireAdmin(t *testing.T) { + db := testStore(t) + log := slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil)) + p := store.DefaultProject("demo") + if err := db.CreateProject(context.Background(), p); err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + identity *Identity + wantStatus int + }{ + {"admin", &Identity{KeyID: "a", Scope: store.ScopeAdmin}, http.StatusNoContent}, + {"project", &Identity{KeyID: "b", Scope: store.ScopeProject, ProjectID: &p.ID}, http.StatusForbidden}, + {"none", nil, http.StatusUnauthorized}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reached := false + h := RequireAdmin(log)(okHandler(&reached)) + req := httptest.NewRequest(http.MethodGet, "/api/v1/projects", nil) + if tc.identity != nil { + req = req.WithContext(ContextWithIdentity(req.Context(), tc.identity)) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != tc.wantStatus { + t.Errorf("status = %d, want %d: %s", rec.Code, tc.wantStatus, rec.Body) + } + if reached != (tc.wantStatus == http.StatusNoContent) { + t.Errorf("handler reached = %v", reached) + } + }) + } +} + +func TestRequireProject(t *testing.T) { + db := testStore(t) + ctx := context.Background() + log := slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil)) + + mine := store.DefaultProject("mine") + theirs := store.DefaultProject("theirs") + if err := db.CreateProject(ctx, mine); err != nil { + t.Fatal(err) + } + if err := db.CreateProject(ctx, theirs); err != nil { + t.Fatal(err) + } + + resolver := ResolverFunc(func(ctx context.Context, name string) (int64, error) { + p, err := db.ProjectByName(ctx, name) + if err != nil { + return 0, err + } + return p.ID, nil + }) + + admin := &Identity{KeyID: "admin00000000000", Scope: store.ScopeAdmin} + owner := &Identity{KeyID: "owner00000000000", Scope: store.ScopeProject, ProjectID: &mine.ID} + + cases := []struct { + name string + identity *Identity + project string + wantStatus int + }{ + {"owner on own project", owner, "mine", http.StatusNoContent}, + {"owner on another project", owner, "theirs", http.StatusForbidden}, + {"admin on any project", admin, "theirs", http.StatusNoContent}, + {"admin on unknown project", admin, "ghost", http.StatusNotFound}, + // An unknown name must look exactly like someone else's project to a + // project-scoped key, or the API becomes a project-name oracle. + {"owner on unknown project", owner, "ghost", http.StatusForbidden}, + {"no identity", nil, "mine", http.StatusUnauthorized}, + {"missing name", owner, "", http.StatusBadRequest}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reached := false + mux := http.NewServeMux() + mux.Handle("GET /api/v1/projects/{name}", RequireProject("name", resolver, log)(okHandler(&reached))) + // The "missing name" case cannot be produced through the mux, so it + // exercises the handler directly. + var h http.Handler = mux + target := "/api/v1/projects/" + tc.project + if tc.project == "" { + h = RequireProject("name", resolver, log)(okHandler(&reached)) + target = "/api/v1/projects/" + } + + req := httptest.NewRequest(http.MethodGet, target, nil) + if tc.identity != nil { + req = req.WithContext(ContextWithIdentity(req.Context(), tc.identity)) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != tc.wantStatus { + t.Errorf("status = %d, want %d: %s", rec.Code, tc.wantStatus, rec.Body) + } + if reached != (tc.wantStatus == http.StatusNoContent) { + t.Errorf("handler reached = %v", reached) + } + }) + } +} + +// Ownership is decided on row ids. A project key must not gain access to a +// project just because a name resolves to it. +func TestOwnsComparesIDsNotNames(t *testing.T) { + one := int64(1) + two := int64(2) + cases := []struct { + name string + id *Identity + ask int64 + want bool + }{ + {"admin owns anything", &Identity{Scope: store.ScopeAdmin}, 42, true}, + {"project owns itself", &Identity{Scope: store.ScopeProject, ProjectID: &one}, 1, true}, + {"project does not own another", &Identity{Scope: store.ScopeProject, ProjectID: &two}, 1, false}, + {"project key without a project owns nothing", &Identity{Scope: store.ScopeProject}, 1, false}, + {"nil identity owns nothing", nil, 1, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.id.Owns(tc.ask); got != tc.want { + t.Errorf("Owns(%d) = %v, want %v", tc.ask, got, tc.want) + } + }) + } +} diff --git a/internal/auth/ratelimit.go b/internal/auth/ratelimit.go new file mode 100644 index 0000000..674e651 --- /dev/null +++ b/internal/auth/ratelimit.go @@ -0,0 +1,118 @@ +package auth + +import ( + "sync" + "time" + + "github.com/iceBear67/simplepages/internal/cache" +) + +// Limiter throttles clients that keep presenting bad credentials. +// +// It exists for CPU, not for guessing. An 80-bit key id plus a 256-bit secret +// cannot be brute forced, so the realistic attack is not "eventually guess a +// token" but "make the server parse, look up and hash forever". Only failed +// attempts consume budget, so a busy CI runner pushing hundreds of valid +// requests a second is never touched. +// +// The bucket map is an LRU with a hard cap, because its keys are whatever +// addresses show up: an unbounded map keyed by attacker-chosen input would turn +// the defence into a memory exhaustion vector of its own. +type Limiter struct { + buckets *cache.Cache[string, *bucket] + burst float64 + refill float64 // tokens per second + + now func() time.Time // swapped in tests +} + +type bucket struct { + mu sync.Mutex + tokens float64 + last time.Time +} + +// NewLimiter allows burst consecutive failures per client, refilling to full +// over period, and tracks at most maxClients addresses. +func NewLimiter(burst int, period time.Duration, maxClients int) *Limiter { + if burst < 1 { + burst = 1 + } + if period <= 0 { + period = time.Minute + } + // Idle buckets are dropped after twice the refill period: by then a bucket + // has refilled completely, so forgetting it and recreating it full are the + // same thing. + return &Limiter{ + buckets: cache.New[string, *bucket](maxClients, 2*period), + burst: float64(burst), + refill: float64(burst) / period.Seconds(), + now: time.Now, + } +} + +// Allow reports whether client has any budget left. It does not consume any: +// a request that turns out to authenticate correctly should cost nothing. +func (l *Limiter) Allow(client string) bool { + if l == nil { + return true + } + b := l.bucket(client) + b.mu.Lock() + defer b.mu.Unlock() + l.refillLocked(b) + return b.tokens >= 1 +} + +// Fail records one failed attempt for client. +func (l *Limiter) Fail(client string) { + if l == nil { + return + } + b := l.bucket(client) + b.mu.Lock() + defer b.mu.Unlock() + l.refillLocked(b) + if b.tokens >= 1 { + b.tokens-- + } else { + b.tokens = 0 + } +} + +// RetryAfter estimates how long client must wait for one token, for the +// Retry-After header. It rounds up to whole seconds, and never returns zero +// while the client is actually throttled. +func (l *Limiter) RetryAfter(client string) time.Duration { + if l == nil { + return 0 + } + b := l.bucket(client) + b.mu.Lock() + defer b.mu.Unlock() + l.refillLocked(b) + if b.tokens >= 1 { + return 0 + } + need := 1 - b.tokens + d := time.Duration(need / l.refill * float64(time.Second)) + return d.Round(time.Second) + time.Second +} + +func (l *Limiter) bucket(client string) *bucket { + return l.buckets.GetOrCreate(client, func() *bucket { + return &bucket{tokens: l.burst, last: l.now()} + }) +} + +func (l *Limiter) refillLocked(b *bucket) { + now := l.now() + if elapsed := now.Sub(b.last); elapsed > 0 { + b.tokens += elapsed.Seconds() * l.refill + if b.tokens > l.burst { + b.tokens = l.burst + } + b.last = now + } +} diff --git a/internal/auth/ratelimit_test.go b/internal/auth/ratelimit_test.go new file mode 100644 index 0000000..bcc59e8 --- /dev/null +++ b/internal/auth/ratelimit_test.go @@ -0,0 +1,176 @@ +package auth + +import ( + "fmt" + "sync" + "testing" + "time" +) + +// fakeClock drives the limiter's refill without sleeping. +type fakeClock struct { + mu sync.Mutex + t time.Time +} + +func (c *fakeClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *fakeClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +func testLimiter(t *testing.T, burst int, period time.Duration, max int) (*Limiter, *fakeClock) { + t.Helper() + l := NewLimiter(burst, period, max) + clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} + l.now = clk.now + return l, clk +} + +// Successful requests must cost nothing: a CI runner pushing hundreds of valid +// deploys a second is not the thing this limiter is defending against. +func TestAllowDoesNotConsume(t *testing.T) { + l, _ := testLimiter(t, 3, time.Minute, 100) + for i := 0; i < 1000; i++ { + if !l.Allow("10.0.0.1") { + t.Fatalf("Allow denied a caller that never failed (i=%d)", i) + } + } +} + +func TestFailExhaustsBudget(t *testing.T) { + l, _ := testLimiter(t, 3, time.Minute, 100) + const ip = "10.0.0.1" + for i := 0; i < 3; i++ { + if !l.Allow(ip) { + t.Fatalf("denied before the burst was spent (i=%d)", i) + } + l.Fail(ip) + } + if l.Allow(ip) { + t.Error("burst exhausted but the caller is still allowed") + } + // Failing while already throttled must not push the balance negative, or + // the client would take proportionally longer to recover the more it tried. + for i := 0; i < 100; i++ { + l.Fail(ip) + } + if d := l.RetryAfter(ip); d > 2*time.Minute { + t.Errorf("RetryAfter = %v; over-failing drove the bucket negative", d) + } +} + +func TestBudgetRefills(t *testing.T) { + l, clk := testLimiter(t, 4, time.Minute, 100) + const ip = "10.0.0.1" + for i := 0; i < 4; i++ { + l.Fail(ip) + } + if l.Allow(ip) { + t.Fatal("expected to be throttled") + } + + // One quarter of the period restores one of four tokens. + clk.advance(15 * time.Second) + if !l.Allow(ip) { + t.Error("no token after a quarter period") + } + + clk.advance(time.Hour) + for i := 0; i < 4; i++ { + if !l.Allow(ip) { + t.Fatalf("bucket did not refill to full (i=%d)", i) + } + l.Fail(ip) + } + if l.Allow(ip) { + t.Error("bucket refilled past its burst") + } +} + +func TestClientsAreIndependent(t *testing.T) { + l, _ := testLimiter(t, 2, time.Minute, 100) + for i := 0; i < 2; i++ { + l.Fail("10.0.0.1") + } + if l.Allow("10.0.0.1") { + t.Error("attacker not throttled") + } + if !l.Allow("10.0.0.2") { + t.Error("one bad client throttled an unrelated one") + } +} + +func TestRetryAfter(t *testing.T) { + l, clk := testLimiter(t, 2, time.Minute, 100) + const ip = "10.0.0.1" + if d := l.RetryAfter(ip); d != 0 { + t.Errorf("RetryAfter with budget left = %v, want 0", d) + } + l.Fail(ip) + l.Fail(ip) + + d := l.RetryAfter(ip) + if d <= 0 { + t.Fatal("a throttled client must be told to wait a positive time") + } + if d > 2*time.Minute { + t.Errorf("RetryAfter = %v, unreasonably long for a 1m period", d) + } + // Waiting the advertised time must actually be enough. + clk.advance(d) + if !l.Allow(ip) { + t.Errorf("still throttled after waiting the advertised %v", d) + } +} + +// The bucket map is keyed by attacker-chosen input, so its bound is load +// bearing: without it the rate limiter becomes the memory exhaustion vector. +func TestBucketMapIsBounded(t *testing.T) { + l, _ := testLimiter(t, 2, time.Minute, 64) + for i := 0; i < 10000; i++ { + l.Fail(fmt.Sprintf("10.%d.%d.%d", i>>16&0xff, i>>8&0xff, i&0xff)) + } + if got := l.buckets.Len(); got > 64 { + t.Errorf("tracking %d clients, cap is 64", got) + } +} + +// A nil limiter is the "rate limiting disabled" configuration and must not +// panic in the request path. +func TestNilLimiterAllowsEverything(t *testing.T) { + var l *Limiter + if !l.Allow("10.0.0.1") { + t.Error("nil limiter denied a request") + } + l.Fail("10.0.0.1") + if d := l.RetryAfter("10.0.0.1"); d != 0 { + t.Errorf("RetryAfter = %v, want 0", d) + } +} + +func TestLimiterConcurrentUse(t *testing.T) { + l := NewLimiter(50, time.Minute, 256) + var wg sync.WaitGroup + for g := 0; g < 16; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < 200; i++ { + ip := fmt.Sprintf("10.0.0.%d", i%8) + l.Allow(ip) + if i%3 == 0 { + l.Fail(ip) + } + l.RetryAfter(ip) + } + }(g) + } + wg.Wait() +} diff --git a/internal/auth/token.go b/internal/auth/token.go new file mode 100644 index 0000000..d7465db --- /dev/null +++ b/internal/auth/token.go @@ -0,0 +1,134 @@ +// Package auth mints and verifies API tokens. +// +// A token looks like pgs__. The key id is the public half: it is +// stored in the clear, indexed, printed by `pages key list` and safe to log. +// The secret is 256 bits of crypto/rand, shown to the operator exactly once at +// creation and never stored — only its SHA-256. +// +// Splitting the two is what keeps verification a single indexed lookup instead +// of a table scan comparing every hash, and it gives the CLI something +// displayable that reveals nothing. +package auth + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base32" + "encoding/base64" + "errors" + "strings" +) + +const ( + // Prefix marks a pages token. Its main job is to be greppable: secret + // scanners and humans can both spot a leaked credential by shape alone. + Prefix = "pgs" + + keyIDBytes = 10 // 80 bits -> exactly 16 base32 characters, no padding + secretBytes = 32 // 256 bits -> exactly 43 base64url characters, no padding + + // KeyIDLen and secretLen are the encoded lengths. Parse checks them exactly + // so a truncated or padded token is rejected before any lookup happens. + KeyIDLen = 16 + secretLen = 43 +) + +// keyIDEncoding is lowercase base32 so a key id can be typed, double-clicked +// and pasted without case confusion. It is not standard base32; do not swap it +// for base32.StdEncoding without a migration, because existing ids would stop +// decoding. +var keyIDEncoding = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567").WithPadding(base32.NoPadding) + +// ErrMalformedToken is returned for anything that is not shaped like a token. +// +// It deliberately carries no detail about which check failed and never embeds +// the offending token: these errors reach logs, and a log line quoting a +// near-miss credential is a credential leak. +var ErrMalformedToken = errors.New("auth: malformed token") + +// Mint generates a new token. The caller stores keyID and secretHash and hands +// token to the operator; there is no way to recover token afterwards. +func Mint() (token, keyID string, secretHash [32]byte, err error) { + idRaw := make([]byte, keyIDBytes) + if _, err := rand.Read(idRaw); err != nil { + return "", "", [32]byte{}, err + } + secretRaw := make([]byte, secretBytes) + if _, err := rand.Read(secretRaw); err != nil { + return "", "", [32]byte{}, err + } + + keyID = keyIDEncoding.EncodeToString(idRaw) + secret := base64.RawURLEncoding.EncodeToString(secretRaw) + return Prefix + "_" + keyID + "_" + secret, keyID, sha256.Sum256([]byte(secret)), nil +} + +// Parse splits a token into its two halves, validating shape and alphabet. +// +// This runs before any database work, so it is also the cheap filter that keeps +// junk from reaching the store: an unauthenticated flood of garbage tokens +// costs a few string comparisons each, not a query. +func Parse(token string) (keyID, secret string, err error) { + rest, ok := strings.CutPrefix(token, Prefix+"_") + if !ok { + return "", "", ErrMalformedToken + } + keyID, secret, ok = strings.Cut(rest, "_") + if !ok { + return "", "", ErrMalformedToken + } + if len(keyID) != KeyIDLen || len(secret) != secretLen { + return "", "", ErrMalformedToken + } + if !validKeyID(keyID) || !validSecret(secret) { + return "", "", ErrMalformedToken + } + return keyID, secret, nil +} + +// ValidKeyID reports whether s could be a key id. Handlers that take a key id +// from the URL use it to reject junk before querying. +func ValidKeyID(s string) bool { return len(s) == KeyIDLen && validKeyID(s) } + +func validKeyID(s string) bool { + for i := 0; i < len(s); i++ { + c := s[i] + if (c >= 'a' && c <= 'z') || (c >= '2' && c <= '7') { + continue + } + return false + } + return true +} + +func validSecret(s string) bool { + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '-', c == '_': + continue + } + return false + } + return true +} + +// HashSecret returns the value stored in api_keys.secret_hash. +// +// A plain SHA-256, not bcrypt or argon2, and that is deliberate. Password +// hashing exists to make low-entropy human-chosen secrets expensive to guess +// offline; this secret is 256 uniformly random bits, so there is no dictionary +// and no brute force to slow down. Running a KDF per request would instead add +// 50-200ms to every API call and hand an unauthenticated client a CPU +// exhaustion attack: each wrong token would force a full key derivation. +func HashSecret(secret string) [32]byte { + return sha256.Sum256([]byte(secret)) +} + +// SecretMatches compares a presented secret against a stored hash in constant +// time, so a caller cannot learn the hash byte by byte from response timing. +func SecretMatches(secret string, storedHash []byte) bool { + got := HashSecret(secret) + return subtle.ConstantTimeCompare(got[:], storedHash) == 1 +} diff --git a/internal/auth/token_test.go b/internal/auth/token_test.go new file mode 100644 index 0000000..2d24771 --- /dev/null +++ b/internal/auth/token_test.go @@ -0,0 +1,135 @@ +package auth + +import ( + "strings" + "testing" +) + +func TestMintParseRoundTrip(t *testing.T) { + seen := map[string]bool{} + for i := 0; i < 100; i++ { + token, keyID, hash, err := Mint() + if err != nil { + t.Fatalf("Mint: %v", err) + } + if seen[keyID] { + t.Fatalf("Mint reused key id %q", keyID) + } + seen[keyID] = true + + if !strings.HasPrefix(token, Prefix+"_") { + t.Errorf("token %q lacks the %q prefix", token, Prefix) + } + gotID, secret, err := Parse(token) + if err != nil { + t.Fatalf("Parse(%q): %v", token, err) + } + if gotID != keyID { + t.Errorf("Parse key id = %q, want %q", gotID, keyID) + } + if len(gotID) != KeyIDLen { + t.Errorf("key id length = %d, want %d", len(gotID), KeyIDLen) + } + if !SecretMatches(secret, hash[:]) { + t.Error("minted secret does not match its own hash") + } + if strings.Contains(token, keyID+"_"+keyID) { + t.Error("secret must not repeat the key id") + } + if !ValidKeyID(keyID) { + t.Errorf("ValidKeyID rejected a minted id %q", keyID) + } + } +} + +func TestParseRejectsMalformed(t *testing.T) { + good, keyID, _, err := Mint() + if err != nil { + t.Fatal(err) + } + _, secret, err := Parse(good) + if err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + token string + }{ + {"empty", ""}, + {"no prefix", keyID + "_" + secret}, + {"wrong prefix", "xyz_" + keyID + "_" + secret}, + {"prefix only", "pgs_"}, + {"no separator", "pgs_" + keyID + secret}, + {"short key id", "pgs_" + keyID[:15] + "_" + secret}, + {"long key id", "pgs_" + keyID + "a_" + secret}, + {"short secret", "pgs_" + keyID + "_" + secret[:42]}, + {"long secret", "pgs_" + keyID + "_" + secret + "a"}, + {"uppercase key id", "pgs_" + strings.ToUpper(keyID) + "_" + secret}, + {"key id with 0 (not in base32)", "pgs_0" + keyID[1:] + "_" + secret}, + {"key id with 1 (not in base32)", "pgs_1" + keyID[1:] + "_" + secret}, + {"secret with padding", "pgs_" + keyID + "_" + secret[:42] + "="}, + {"secret with slash", "pgs_" + keyID + "_" + secret[:42] + "/"}, + {"secret with plus", "pgs_" + keyID + "_" + secret[:42] + "+"}, + {"embedded NUL", "pgs_" + keyID + "_" + secret[:42] + "\x00"}, + {"leading space", " " + good}, + {"newline", good + "\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, _, err := Parse(tc.token); err == nil { + t.Errorf("Parse(%q) accepted a malformed token", tc.token) + } + }) + } + + // The parse error must never quote the input: these errors reach logs. + if _, _, err := Parse(good[:len(good)-1] + "x"); err != nil { + if strings.Contains(err.Error(), secret[:20]) { + t.Error("parse error leaks part of the presented secret") + } + } +} + +func TestSecretMatches(t *testing.T) { + hash := HashSecret("correct horse battery staple") + if !SecretMatches("correct horse battery staple", hash[:]) { + t.Error("matching secret rejected") + } + if SecretMatches("correct horse battery stapl", hash[:]) { + t.Error("truncated secret accepted") + } + if SecretMatches("", hash[:]) { + t.Error("empty secret accepted") + } + if SecretMatches("correct horse battery staple", nil) { + t.Error("nil stored hash accepted") + } + if SecretMatches("correct horse battery staple", hash[:16]) { + t.Error("truncated stored hash accepted") + } +} + +func TestValidKeyID(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {"abcdefghijklmnop", true}, + {"234567234567abcd", true}, + {"", false}, + {"abcdefghijklmno", false}, // 15 + {"abcdefghijklmnopq", false}, // 17 + {"ABCDEFGHIJKLMNOP", false}, + {"abcdefghijklmno0", false}, + {"abcdefghijklmno1", false}, + {"abcdefghijklmno8", false}, + {"abcdefghijklmno-", false}, + {"abcdefghijklmn/p", false}, + } + for _, tc := range cases { + if got := ValidKeyID(tc.in); got != tc.want { + t.Errorf("ValidKeyID(%q) = %v, want %v", tc.in, got, tc.want) + } + } +} diff --git a/internal/auth/verify.go b/internal/auth/verify.go new file mode 100644 index 0000000..12b2ec4 --- /dev/null +++ b/internal/auth/verify.go @@ -0,0 +1,247 @@ +package auth + +import ( + "context" + "errors" + "log/slog" + "sync" + "sync/atomic" + "time" + + "github.com/iceBear67/simplepages/internal/store" +) + +// Identity is what a verified token proves. It is immutable once returned and +// is shared by every request using that key, so callers must not modify it. +type Identity struct { + KeyID string + Scope store.Scope + ProjectID *int64 // nil for admin keys + Name string + ExpiresAt *time.Time +} + +// IsAdmin reports whether the identity may act on every project. +func (i *Identity) IsAdmin() bool { return i != nil && i.Scope == store.ScopeAdmin } + +// Owns reports whether the identity may act on the project with this row id. +// Admins own everything. +// +// Callers must pass a resolved row id, never a name from the URL: comparing +// names would make the boundary depend on string handling in every handler. +func (i *Identity) Owns(projectID int64) bool { + if i == nil { + return false + } + if i.Scope == store.ScopeAdmin { + return true + } + return i.ProjectID != nil && *i.ProjectID == projectID +} + +// Failure reasons. All of them are reported to the client as one indistinct +// 401: telling an unauthenticated caller whether a key exists, is revoked or +// merely expired is free reconnaissance. +var ( + ErrUnknownKey = errors.New("auth: unknown key id") + ErrBadSecret = errors.New("auth: secret mismatch") + ErrRevoked = errors.New("auth: key revoked") + ErrExpired = errors.New("auth: key expired") +) + +// DefaultCacheTTL bounds how long a revocation can take to become visible if +// the process that revoked it is not this one. Within one process, Invalidate +// makes revocation immediate. +const DefaultCacheTTL = 60 * time.Second + +// Verifier turns a bearer token into an Identity. +// +// Verified keys are cached, because otherwise every deploy request would pay a +// database round trip before doing any work. The cache stores only positive +// results: caching unknown key ids would let anyone grow the map without bound +// by presenting random tokens. An unknown id costs one indexed lookup on a +// WITHOUT ROWID table, and the rate limiter covers the flood case. +// +// sync.Map fits this exactly — the key set is small and stable, entries are +// written once and read many times, and different goroutines mostly touch +// different keys. +type Verifier struct { + db *store.DB + log *slog.Logger + ttl time.Duration + + cache sync.Map // keyID -> *cacheEntry + gen atomic.Uint64 + + // Pending last-use timestamps, flushed in batches. Writing last_used_at per + // request would funnel every authenticated read through the single write + // connection, which is the contention the two-pool design exists to avoid. + mu sync.Mutex + touch map[string]time.Time + + now func() time.Time // swapped in tests +} + +type cacheEntry struct { + ident *Identity + hash []byte + gen uint64 + exp time.Time +} + +// NewVerifier returns a verifier reading from db. A ttl of zero means +// DefaultCacheTTL. +func NewVerifier(db *store.DB, log *slog.Logger, ttl time.Duration) *Verifier { + if ttl <= 0 { + ttl = DefaultCacheTTL + } + return &Verifier{ + db: db, + log: log, + ttl: ttl, + touch: make(map[string]time.Time), + now: time.Now, + } +} + +// Verify authenticates a bearer token. +// +// On success it also records the key as used; the timestamp is written to the +// database later, in a batch, so it is approximate by design. +func (v *Verifier) Verify(ctx context.Context, token string) (*Identity, error) { + keyID, secret, err := Parse(token) + if err != nil { + return nil, err + } + + now := v.now() + gen := v.gen.Load() + + entry, ok := v.lookupCache(keyID, gen, now) + if !ok { + key, err := v.db.KeyByID(ctx, keyID) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, ErrUnknownKey + } + return nil, err + } + entry = &cacheEntry{ + ident: identityOf(key), + hash: key.SecretHash, + gen: gen, + exp: now.Add(v.ttl), + } + // Revoked keys are never cached: the entry would only ever produce a + // rejection, and keeping it lets a caller pin memory with a dead key. + if key.RevokedAt != nil { + return nil, ErrRevoked + } + v.cache.Store(keyID, entry) + } + + // The comparison happens on every request, cache hit or not. The cache + // saves the database round trip; it must never save the check itself. + if !SecretMatches(secret, entry.hash) { + return nil, ErrBadSecret + } + if entry.ident.ExpiresAt != nil && !now.Before(*entry.ident.ExpiresAt) { + return nil, ErrExpired + } + + v.recordUse(keyID, now) + return entry.ident, nil +} + +func (v *Verifier) lookupCache(keyID string, gen uint64, now time.Time) (*cacheEntry, bool) { + raw, ok := v.cache.Load(keyID) + if !ok { + return nil, false + } + e := raw.(*cacheEntry) + if e.gen != gen || !now.Before(e.exp) { + v.cache.Delete(keyID) + return nil, false + } + return e, true +} + +// Invalidate discards every cached identity. +// +// Called after any key or project change. Bumping a generation counter rather +// than deleting individual entries is deliberate: a caller that forgets which +// ids a change touched cannot leave a stale entry behind, and the cost is one +// atomic load per verification. +func (v *Verifier) Invalidate() { v.gen.Add(1) } + +func (v *Verifier) recordUse(keyID string, at time.Time) { + v.mu.Lock() + defer v.mu.Unlock() + if prev, ok := v.touch[keyID]; !ok || at.After(prev) { + v.touch[keyID] = at + } +} + +// FlushTouches writes the accumulated last-use timestamps. +// +// The pending set is taken before the write and not restored on failure: a lost +// last_used_at is a cosmetic loss, and retrying would let a persistently +// failing write grow the map without bound. +func (v *Verifier) FlushTouches(ctx context.Context) error { + v.mu.Lock() + pending := v.touch + v.touch = make(map[string]time.Time) + v.mu.Unlock() + + if len(pending) == 0 { + return nil + } + return v.db.TouchKeys(ctx, pending) +} + +// RunFlusher writes pending last-use timestamps every interval until ctx is +// done, then flushes once more so a clean shutdown does not drop them. +func (v *Verifier) RunFlusher(ctx context.Context, interval time.Duration) { + if interval <= 0 { + interval = time.Minute + } + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + // ctx is already cancelled, so the final flush needs its own + // deadline or TouchKeys would return immediately. + final, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if err := v.FlushTouches(final); err != nil && v.log != nil { + v.log.Warn("final last_used_at flush failed", "error", err) + } + return + case <-t.C: + if err := v.FlushTouches(ctx); err != nil && v.log != nil { + v.log.Warn("last_used_at flush failed", "error", err) + } + } + } +} + +func identityOf(k *store.APIKey) *Identity { + // Everything reachable from a cached Identity is copied: the value is shared + // by every concurrent request using that key, so it must not alias a struct + // the store still owns. + id := &Identity{ + KeyID: k.ID, + Scope: k.Scope, + Name: k.Name, + } + if k.ProjectID != nil { + pid := *k.ProjectID + id.ProjectID = &pid + } + if k.ExpiresAt != nil { + exp := *k.ExpiresAt + id.ExpiresAt = &exp + } + return id +} diff --git a/internal/auth/verify_test.go b/internal/auth/verify_test.go new file mode 100644 index 0000000..2ac7dd8 --- /dev/null +++ b/internal/auth/verify_test.go @@ -0,0 +1,407 @@ +package auth + +import ( + "context" + "errors" + "io" + "log/slog" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/iceBear67/simplepages/internal/store" +) + +func testStore(t *testing.T) *store.DB { + t.Helper() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + db, err := store.Open(context.Background(), filepath.Join(t.TempDir(), "pages.db"), log) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +// mintInto creates a real key in the database and returns its token. +func mintInto(t *testing.T, db *store.DB, scope store.Scope, projectID *int64) (token, keyID string) { + t.Helper() + token, keyID, hash, err := Mint() + if err != nil { + t.Fatal(err) + } + k := &store.APIKey{ + ID: keyID, + SecretHash: hash[:], + Scope: scope, + ProjectID: projectID, + Name: "test", + } + if err := db.CreateKey(context.Background(), k); err != nil { + t.Fatalf("CreateKey: %v", err) + } + return token, keyID +} + +func newVerifier(t *testing.T, db *store.DB) *Verifier { + t.Helper() + return NewVerifier(db, slog.New(slog.NewTextHandler(io.Discard, nil)), DefaultCacheTTL) +} + +func TestVerifyAdminKey(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + + id, err := v.Verify(ctx, token) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if id.KeyID != keyID { + t.Errorf("KeyID = %q, want %q", id.KeyID, keyID) + } + if !id.IsAdmin() { + t.Error("admin key did not produce an admin identity") + } + if !id.Owns(1) || !id.Owns(999) { + t.Error("an admin must own every project") + } + if id.ProjectID != nil { + t.Errorf("admin identity carries project %v", *id.ProjectID) + } +} + +func TestVerifyProjectKey(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + + p := store.DefaultProject("demo") + if err := db.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + token, _ := mintInto(t, db, store.ScopeProject, &p.ID) + + id, err := v.Verify(ctx, token) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if id.IsAdmin() { + t.Error("project key produced an admin identity") + } + if !id.Owns(p.ID) { + t.Error("project key does not own its own project") + } + if id.Owns(p.ID + 1) { + t.Error("project key owns someone else's project") + } +} + +func TestVerifyRejects(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + _, secret, err := Parse(token) + if err != nil { + t.Fatal(err) + } + + // A well-formed token for a key that was never created. + otherToken, _, _, err := Mint() + if err != nil { + t.Fatal(err) + } + // The right key id with someone else's secret. + strangerToken, _, _, err := Mint() + if err != nil { + t.Fatal(err) + } + _, wrong, err := Parse(strangerToken) + if err != nil { + t.Fatal(err) + } + + cases := []struct { + name string + token string + want error + }{ + {"garbage", "not-a-token", ErrMalformedToken}, + {"unknown key", otherToken, ErrUnknownKey}, + {"wrong secret", Prefix + "_" + keyID + "_" + wrong, ErrBadSecret}, + {"empty", "", ErrMalformedToken}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := v.Verify(ctx, tc.token); !errors.Is(err, tc.want) { + t.Errorf("Verify: got %v, want %v", err, tc.want) + } + }) + } + + // Sanity: the real token still works after all those failures. + if _, err := v.Verify(ctx, Prefix+"_"+keyID+"_"+secret); err != nil { + t.Errorf("valid token rejected after failed attempts: %v", err) + } +} + +func TestVerifyRejectsExpiredKey(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + + token, keyID, hash, err := Mint() + if err != nil { + t.Fatal(err) + } + past := time.Now().Add(-time.Hour) + if err := db.CreateKey(ctx, &store.APIKey{ + ID: keyID, SecretHash: hash[:], Scope: store.ScopeAdmin, ExpiresAt: &past, + }); err != nil { + t.Fatal(err) + } + if _, err := v.Verify(ctx, token); !errors.Is(err, ErrExpired) { + t.Errorf("got %v, want ErrExpired", err) + } +} + +// Expiry must be re-evaluated on every call, not frozen into the cache entry, +// or a key cached one second before it expires would stay valid for the whole +// cache TTL. +func TestCachedKeyStillExpires(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + + now := time.Now() + fake := now + var mu sync.Mutex + v.now = func() time.Time { + mu.Lock() + defer mu.Unlock() + return fake + } + + token, keyID, hash, err := Mint() + if err != nil { + t.Fatal(err) + } + exp := now.Add(30 * time.Second) + if err := db.CreateKey(ctx, &store.APIKey{ + ID: keyID, SecretHash: hash[:], Scope: store.ScopeAdmin, ExpiresAt: &exp, + }); err != nil { + t.Fatal(err) + } + + if _, err := v.Verify(ctx, token); err != nil { + t.Fatalf("key should be valid before expiry: %v", err) + } + mu.Lock() + fake = now.Add(31 * time.Second) // still inside the 60s cache TTL + mu.Unlock() + if _, err := v.Verify(ctx, token); !errors.Is(err, ErrExpired) { + t.Errorf("expired key still accepted from cache: %v", err) + } +} + +func TestVerifyRejectsRevokedKey(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + + if err := db.RevokeKey(ctx, keyID); err != nil { + t.Fatal(err) + } + if _, err := v.Verify(ctx, token); !errors.Is(err, ErrRevoked) { + t.Errorf("got %v, want ErrRevoked", err) + } +} + +// Revocation has to take effect immediately in the process that performed it; +// this both proves that and demonstrates the cache is really being consulted. +func TestInvalidateMakesRevocationImmediate(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + + if _, err := v.Verify(ctx, token); err != nil { + t.Fatal(err) + } + + // Revoke behind the verifier's back. The cached entry is still live, so the + // key keeps working — which is exactly what makes the next assertion mean + // something. + if err := db.RevokeKey(ctx, keyID); err != nil { + t.Fatal(err) + } + if _, err := v.Verify(ctx, token); err != nil { + t.Fatalf("cache was not consulted (or the test is not measuring it): %v", err) + } + + v.Invalidate() + if _, err := v.Verify(ctx, token); !errors.Is(err, ErrRevoked) { + t.Errorf("after Invalidate: got %v, want ErrRevoked", err) + } +} + +// An unknown key id must not create a cache entry: the id space is +// attacker-chosen, so caching misses would be an unbounded memory sink. +func TestUnknownKeysAreNotCached(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + + for i := 0; i < 200; i++ { + token, _, _, err := Mint() + if err != nil { + t.Fatal(err) + } + if _, err := v.Verify(ctx, token); !errors.Is(err, ErrUnknownKey) { + t.Fatalf("got %v, want ErrUnknownKey", err) + } + } + entries := 0 + v.cache.Range(func(any, any) bool { entries++; return true }) + if entries != 0 { + t.Errorf("%d unknown key ids were cached", entries) + } +} + +// Same idea for revoked keys: a dead credential must not pin memory. +func TestRevokedKeysAreNotCached(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + if err := db.RevokeKey(ctx, keyID); err != nil { + t.Fatal(err) + } + for i := 0; i < 10; i++ { + if _, err := v.Verify(ctx, token); !errors.Is(err, ErrRevoked) { + t.Fatal(err) + } + } + entries := 0 + v.cache.Range(func(any, any) bool { entries++; return true }) + if entries != 0 { + t.Errorf("%d revoked keys were cached", entries) + } +} + +func TestFlushTouches(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + + if err := v.FlushTouches(ctx); err != nil { + t.Errorf("flushing an empty batch: %v", err) + } + before, err := db.KeyByID(ctx, keyID) + if err != nil { + t.Fatal(err) + } + if before.LastUsedAt != nil { + t.Error("last_used_at set before any use") + } + + for i := 0; i < 5; i++ { + if _, err := v.Verify(ctx, token); err != nil { + t.Fatal(err) + } + } + // Nothing is written until the batch is flushed; that is the whole point of + // keeping the write connection out of the request path. + mid, err := db.KeyByID(ctx, keyID) + if err != nil { + t.Fatal(err) + } + if mid.LastUsedAt != nil { + t.Error("last_used_at written per request instead of in a batch") + } + + if err := v.FlushTouches(ctx); err != nil { + t.Fatalf("FlushTouches: %v", err) + } + after, err := db.KeyByID(ctx, keyID) + if err != nil { + t.Fatal(err) + } + if after.LastUsedAt == nil { + t.Fatal("last_used_at still unset after flush") + } + + // A flush drains the pending set, so a second one has nothing to do. + v.mu.Lock() + pending := len(v.touch) + v.mu.Unlock() + if pending != 0 { + t.Errorf("%d pending touches survived the flush", pending) + } +} + +func TestRunFlusherFlushesOnShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + db := testStore(t) + v := newVerifier(t, db) + token, keyID := mintInto(t, db, store.ScopeAdmin, nil) + if _, err := v.Verify(ctx, token); err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + go func() { + defer close(done) + v.RunFlusher(ctx, time.Hour) // never ticks; only the shutdown path runs + }() + cancel() + <-done + + k, err := db.KeyByID(context.Background(), keyID) + if err != nil { + t.Fatal(err) + } + if k.LastUsedAt == nil { + t.Error("shutdown flush dropped the pending timestamps") + } +} + +func TestVerifyIsConcurrencySafe(t *testing.T) { + ctx := context.Background() + db := testStore(t) + v := newVerifier(t, db) + token, _ := mintInto(t, db, store.ScopeAdmin, nil) + bad, _, _, err := Mint() + if err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + for g := 0; g < 16; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < 100; i++ { + if g%4 == 0 { + if _, err := v.Verify(ctx, bad); err == nil { + t.Error("bad token accepted") + } + continue + } + if _, err := v.Verify(ctx, token); err != nil { + t.Errorf("good token rejected: %v", err) + return + } + if i%25 == 0 { + v.Invalidate() + } + } + }(g) + } + wg.Wait() +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..a4edcd2 --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,179 @@ +// Package cache provides a bounded, TTL-expiring LRU map. +// +// It exists for the one place that genuinely needs eviction: the failed-auth +// rate limiter, whose key space is attacker-controlled and must not be allowed +// to grow without bound. The other caches in this server deliberately do not +// use it — the project registry and deployment manifests are copy-on-write +// snapshots read without any lock at all, and adding an LRU there would put a +// contended mutex on the hot serving path to solve a problem that does not +// exist. +package cache + +import ( + "sync" + "time" +) + +type node[K comparable, V any] struct { + key K + val V + expires time.Time + prev, next *node[K, V] +} + +// Cache maps K to V with a size cap and a per-entry TTL. It is safe for +// concurrent use. Entries are evicted when the cap is exceeded (least recently +// used first) or when their TTL passes, whichever comes first. +// +// The zero value is not usable; call New. +type Cache[K comparable, V any] struct { + mu sync.Mutex + m map[K]*node[K, V] + head *node[K, V] // most recently used + tail *node[K, V] // least recently used + max int + ttl time.Duration + + // now is swapped out by tests. Production always uses time.Now. + now func() time.Time +} + +// New returns a cache holding at most max entries for at most ttl each. +func New[K comparable, V any](max int, ttl time.Duration) *Cache[K, V] { + if max < 1 { + max = 1 + } + return &Cache[K, V]{ + m: make(map[K]*node[K, V]), + max: max, + ttl: ttl, + now: time.Now, + } +} + +// Get returns the value for k, refreshing its recency. A value whose TTL has +// passed is reported as absent and dropped. +func (c *Cache[K, V]) Get(k K) (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + n, ok := c.m[k] + if !ok { + var zero V + return zero, false + } + if !c.now().Before(n.expires) { + c.remove(n) + var zero V + return zero, false + } + c.moveToFront(n) + return n.val, true +} + +// Put inserts or replaces the value for k and resets its TTL. +func (c *Cache[K, V]) Put(k K, v V) { + c.mu.Lock() + defer c.mu.Unlock() + c.put(k, v) +} + +// GetOrCreate returns the existing value for k, or stores and returns the one +// newVal produces. +// +// The whole operation happens under the lock, which is what makes it usable for +// the rate limiter: two concurrent requests from the same address must share +// one token bucket, and a Get-then-Put pair would hand each of them its own. +// newVal must not call back into the cache. +func (c *Cache[K, V]) GetOrCreate(k K, newVal func() V) V { + c.mu.Lock() + defer c.mu.Unlock() + + if n, ok := c.m[k]; ok { + if c.now().Before(n.expires) { + c.moveToFront(n) + return n.val + } + c.remove(n) + } + v := newVal() + c.put(k, v) + return v +} + +// Delete drops k if present. +func (c *Cache[K, V]) Delete(k K) { + c.mu.Lock() + defer c.mu.Unlock() + if n, ok := c.m[k]; ok { + c.remove(n) + } +} + +// Len reports the number of entries, including any whose TTL has passed but +// that have not been touched since. It is meant for tests and diagnostics. +func (c *Cache[K, V]) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.m) +} + +// ---------------------------------------------------------------- internals +// All of these require c.mu. + +func (c *Cache[K, V]) put(k K, v V) { + if n, ok := c.m[k]; ok { + n.val = v + n.expires = c.now().Add(c.ttl) + c.moveToFront(n) + return + } + n := &node[K, V]{key: k, val: v, expires: c.now().Add(c.ttl)} + c.m[k] = n + c.pushFront(n) + for len(c.m) > c.max { + c.remove(c.tail) + } +} + +func (c *Cache[K, V]) pushFront(n *node[K, V]) { + n.prev = nil + n.next = c.head + if c.head != nil { + c.head.prev = n + } + c.head = n + if c.tail == nil { + c.tail = n + } +} + +func (c *Cache[K, V]) moveToFront(n *node[K, V]) { + if c.head == n { + return + } + c.unlink(n) + c.pushFront(n) +} + +func (c *Cache[K, V]) remove(n *node[K, V]) { + if n == nil { + return + } + c.unlink(n) + delete(c.m, n.key) +} + +func (c *Cache[K, V]) unlink(n *node[K, V]) { + if n.prev != nil { + n.prev.next = n.next + } else if c.head == n { + c.head = n.next + } + if n.next != nil { + n.next.prev = n.prev + } else if c.tail == n { + c.tail = n.prev + } + n.prev, n.next = nil, nil +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go new file mode 100644 index 0000000..d08e58c --- /dev/null +++ b/internal/cache/cache_test.go @@ -0,0 +1,223 @@ +package cache + +import ( + "fmt" + "sync" + "testing" + "time" +) + +// clock lets the TTL tests run without sleeping. +type clock struct { + mu sync.Mutex + t time.Time +} + +func (c *clock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *clock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +func newTestCache[K comparable, V any](t *testing.T, max int, ttl time.Duration) (*Cache[K, V], *clock) { + t.Helper() + c := New[K, V](max, ttl) + clk := &clock{t: time.Unix(1_700_000_000, 0)} + c.now = clk.now + return c, clk +} + +func TestGetPut(t *testing.T) { + c, _ := newTestCache[string, int](t, 4, time.Minute) + if _, ok := c.Get("missing"); ok { + t.Error("empty cache returned a hit") + } + c.Put("a", 1) + if v, ok := c.Get("a"); !ok || v != 1 { + t.Errorf("Get(a) = %v, %v", v, ok) + } + c.Put("a", 2) + if v, _ := c.Get("a"); v != 2 { + t.Errorf("Put did not overwrite: %v", v) + } + if c.Len() != 1 { + t.Errorf("Len = %d, want 1", c.Len()) + } + c.Delete("a") + if _, ok := c.Get("a"); ok { + t.Error("deleted key still present") + } +} + +func TestTTLExpiry(t *testing.T) { + c, clk := newTestCache[string, int](t, 4, time.Minute) + c.Put("a", 1) + + clk.advance(59 * time.Second) + if _, ok := c.Get("a"); !ok { + t.Error("entry expired early") + } + clk.advance(time.Second) + if _, ok := c.Get("a"); ok { + t.Error("entry outlived its TTL") + } + if c.Len() != 0 { + t.Errorf("expired entry not dropped: Len = %d", c.Len()) + } +} + +// The bound is the whole point: the rate limiter's key space is whatever +// addresses show up, so an unbounded map would be a memory DoS. +func TestEvictsLeastRecentlyUsed(t *testing.T) { + c, _ := newTestCache[string, int](t, 3, time.Minute) + c.Put("a", 1) + c.Put("b", 2) + c.Put("c", 3) + + // Touching "a" makes "b" the least recently used. + if _, ok := c.Get("a"); !ok { + t.Fatal("a missing") + } + c.Put("d", 4) + + if c.Len() != 3 { + t.Errorf("Len = %d, want 3", c.Len()) + } + if _, ok := c.Get("b"); ok { + t.Error("b should have been evicted") + } + for _, k := range []string{"a", "c", "d"} { + if _, ok := c.Get(k); !ok { + t.Errorf("%s should have survived", k) + } + } +} + +func TestEvictionKeepsListConsistent(t *testing.T) { + c, _ := newTestCache[int, int](t, 8, time.Minute) + for i := 0; i < 1000; i++ { + c.Put(i, i) + if i%3 == 0 { + c.Get(i / 2) + } + if i%7 == 0 { + c.Delete(i - 1) + } + if c.Len() > 8 { + t.Fatalf("cap exceeded at i=%d: Len = %d", i, c.Len()) + } + } + // Every surviving node must be reachable from both ends, or a later + // eviction would corrupt the list rather than free anything. + c.mu.Lock() + defer c.mu.Unlock() + forward := 0 + for n := c.head; n != nil; n = n.next { + forward++ + if forward > 100 { + t.Fatal("forward walk did not terminate (cycle in the list)") + } + } + backward := 0 + for n := c.tail; n != nil; n = n.prev { + backward++ + if backward > 100 { + t.Fatal("backward walk did not terminate (cycle in the list)") + } + } + if forward != len(c.m) || backward != len(c.m) { + t.Errorf("list holds %d/%d nodes, map holds %d", forward, backward, len(c.m)) + } +} + +func TestGetOrCreate(t *testing.T) { + c, clk := newTestCache[string, *int](t, 4, time.Minute) + calls := 0 + newVal := func() *int { + calls++ + v := calls + return &v + } + + first := c.GetOrCreate("a", newVal) + second := c.GetOrCreate("a", newVal) + if first != second { + t.Error("GetOrCreate must return the same value for a live entry") + } + if calls != 1 { + t.Errorf("newVal called %d times, want 1", calls) + } + + clk.advance(2 * time.Minute) + third := c.GetOrCreate("a", newVal) + if third == first { + t.Error("an expired entry must be replaced") + } + if calls != 2 { + t.Errorf("newVal called %d times, want 2", calls) + } +} + +// Two callers racing on the same key must end up sharing one value — the rate +// limiter relies on this to avoid handing every request its own token bucket. +func TestGetOrCreateIsAtomic(t *testing.T) { + c := New[string, *int](64, time.Minute) + const goroutines = 32 + var wg sync.WaitGroup + got := make([]*int, goroutines) + start := make(chan struct{}) + for i := 0; i < goroutines; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + got[i] = c.GetOrCreate("shared", func() *int { v := 0; return &v }) + }(i) + } + close(start) + wg.Wait() + for i := 1; i < goroutines; i++ { + if got[i] != got[0] { + t.Fatalf("goroutine %d got a different value", i) + } + } +} + +func TestConcurrentUse(t *testing.T) { + c := New[string, int](16, time.Minute) + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < 500; i++ { + k := fmt.Sprintf("k%d", i%40) + c.Put(k, i) + c.Get(k) + if i%10 == 0 { + c.Delete(k) + } + c.GetOrCreate(k, func() int { return g }) + } + }(g) + } + wg.Wait() + if c.Len() > 16 { + t.Errorf("Len = %d, want <= 16", c.Len()) + } +} + +func TestNewClampsMax(t *testing.T) { + c := New[string, int](0, time.Minute) + c.Put("a", 1) + c.Put("b", 2) + if c.Len() != 1 { + t.Errorf("Len = %d, want 1", c.Len()) + } +} diff --git a/internal/cas/cas.go b/internal/cas/cas.go new file mode 100644 index 0000000..adbc965 --- /dev/null +++ b/internal/cas/cas.go @@ -0,0 +1,414 @@ +package cas + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path" + "path/filepath" +) + +// LinkMode is how an assembled deployment tree gets at a blob's content. +type LinkMode string + +const ( + // LinkHard hardlinks. A deployment tree then costs directory entries and + // nothing else, however many deployments share the same files. + LinkHard LinkMode = "hardlink" + // LinkCopy copies. Correct everywhere, at the cost of disk proportional to + // deployed content rather than to unique content. + LinkCopy LinkMode = "copy" +) + +// Failures a caller distinguishes. Everything else is an I/O error and is +// returned as it came back from the operating system. +var ( + ErrDigestMismatch = errors.New("cas: content does not hash to the declared digest") + ErrSizeMismatch = errors.New("cas: content length does not match the declared size") + ErrTooLarge = errors.New("cas: content exceeds the maximum file size") + ErrNotFound = errors.New("cas: no such blob") +) + +const ( + // tmpDir holds uploads in progress. Anything in it is unreferenced by + // definition: a blob only becomes reachable by being renamed out of here. + tmpDir = "tmp" + + // blobMode is read-only for everyone, and that is load-bearing rather than + // tidy. An assembled deployment file is usually a hardlink to the blob — + // the same inode — so anything that writes through the copy in $WEBROOT + // corrupts the blob itself, and with it every project that shares that + // content. Read-only is the cheap barrier; docs/operations.md carries the + // warning that $WEBROOT is read-only to outside consumers. + blobMode = 0o444 + dirMode = 0o755 + tmpMode = 0o600 +) + +// Options configure Open. +type Options struct { + // Mode forces a link mode. The zero value probes and falls back to copying. + Mode LinkMode + // ProbeDir is where the probe tries to place a link. It must be the + // directory deployment trees are assembled in: hardlinks cannot cross + // filesystems, so probing anywhere else answers a different question. + ProbeDir string + Log *slog.Logger +} + +// Store is the on-disk blob store. +type Store struct { + dir string + root *os.Root + linkMode LinkMode + log *slog.Logger +} + +// Open prepares the store at dir, creating it if necessary. +func Open(dir string, opt Options) (*Store, error) { + if dir == "" { + return nil, errors.New("cas: directory is required") + } + abs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(abs, tmpDir), dirMode); err != nil { + return nil, err + } + log := opt.Log + if log == nil { + log = slog.New(slog.DiscardHandler) + } + + // Every name handed to root below is derived from a digest, so traversal is + // not the threat being defended against here. What os.Root buys is that a + // symlink planted inside the store — by a restore from a bad backup, by a + // misdirected rsync — cannot make a write land outside it. + root, err := os.OpenRoot(abs) + if err != nil { + return nil, err + } + s := &Store{dir: abs, root: root, log: log} + + switch opt.Mode { + case LinkCopy: + s.linkMode = LinkCopy + case LinkHard: + if err := probeLink(abs, opt.ProbeDir); err != nil { + root.Close() + return nil, fmt.Errorf("cas: hardlinks were requested but %s cannot be hardlinked into %s: %w", abs, opt.ProbeDir, err) + } + s.linkMode = LinkHard + case "": + if err := probeLink(abs, opt.ProbeDir); err != nil { + s.linkMode = LinkCopy + log.Warn("hardlinks unavailable, deployment trees will be copies; disk use will be proportional to deployed content rather than to unique content", + "cas_dir", abs, "deployments_dir", opt.ProbeDir, "err", err) + } else { + s.linkMode = LinkHard + } + default: + root.Close() + return nil, fmt.Errorf("cas: unknown link mode %q", opt.Mode) + } + return s, nil +} + +// Dir is the store's absolute root directory. +func (s *Store) Dir() string { return s.dir } + +// LinkMode reports how LinkInto will place content. +func (s *Store) LinkMode() LinkMode { return s.linkMode } + +// Close releases the store's directory handle. +func (s *Store) Close() error { return s.root.Close() } + +// Path is where a blob lives on disk. Only for hardlinking, which needs a name +// the kernel resolves from the process's own root; every other operation goes +// through the store's directory handle. +func (s *Store) Path(d Digest) string { + return filepath.Join(s.dir, filepath.FromSlash(d.Rel())) +} + +// Put stores r's bytes under want, if and only if they really hash to want. +// +// The verification is the reason cross-project deduplication is safe at all. +// Blobs are shared: if the server took the caller's word for the digest, a +// client could declare the digest of another project's index.html, upload +// whatever it liked, and every project referencing that content would start +// serving the attacker's bytes. So the hash is recomputed over the stream as it +// arrives and the upload is discarded unless it matches — the claimed digest is +// only ever a claim. +// +// declaredSize may be -1 when the caller genuinely does not know the length; any +// other value must match what arrives. maxBytes is a hard ceiling and is +// enforced by reading one byte past it rather than by trusting Content-Length. +// +// Returns the number of bytes stored. A blob that is already present is left +// alone and the upload discarded, which is safe precisely because both are known +// to hash to want. +func (s *Store) Put(ctx context.Context, want Digest, declaredSize, maxBytes int64, r io.Reader) (int64, error) { + if maxBytes <= 0 { + return 0, fmt.Errorf("cas: maxBytes must be positive, got %d", maxBytes) + } + if declaredSize > maxBytes { + return 0, fmt.Errorf("%w: declared %d bytes, limit is %d", ErrTooLarge, declaredSize, maxBytes) + } + + tmpRel, f, err := s.newTemp() + if err != nil { + return 0, err + } + // Every path out of this function that is not a successful rename has to + // remove the temporary file, including the ones that are not our error — a + // client that hung up mid-upload, a cancelled context. One defer covers all + // of them. + committed := false + defer func() { + f.Close() // no-op if already closed below + if !committed { + if err := s.root.Remove(tmpRel); err != nil && !errors.Is(err, fs.ErrNotExist) { + s.log.Warn("removing abandoned upload", "path", tmpRel, "err", err) + } + } + }() + + h := sha256.New() + n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(&ctxReader{ctx: ctx, r: r}, maxBytes+1)) + if err != nil { + return 0, err + } + if n > maxBytes { + return 0, fmt.Errorf("%w: limit is %d bytes", ErrTooLarge, maxBytes) + } + if declaredSize >= 0 && n != declaredSize { + return 0, fmt.Errorf("%w: declared %d bytes, received %d", ErrSizeMismatch, declaredSize, n) + } + if got := Digest(h.Sum(nil)); got != want { + return 0, fmt.Errorf("%w: declared %s, computed %s", ErrDigestMismatch, want, got) + } + + // Durability before visibility. Losing the content while keeping the + // directory entry would be corruption — the database would say present=1 + // and the file would be a hole — whereas losing the rename is merely a blob + // that has to be uploaded again, which deploy.Recover already handles by + // setting present=0 for digests whose file is missing. So the file is + // fsynced and the containing directory deliberately is not. + if err := f.Sync(); err != nil { + return 0, err + } + if err := f.Chmod(blobMode); err != nil { + return 0, err + } + if err := f.Close(); err != nil { + return 0, err + } + + rel := want.Rel() + if err := s.root.MkdirAll(path.Dir(rel), dirMode); err != nil { + return 0, err + } + if _, err := s.root.Stat(rel); err == nil { + // Two clients uploading identical content at once is a benign race, not + // a conflict: keep what is there and drop ours. Losing this race to a + // writer that renames between the Stat and here is equally benign, + // since rename is atomic and both files hold the same bytes. + return n, nil + } else if !errors.Is(err, fs.ErrNotExist) { + return 0, err + } + if err := s.root.Rename(tmpRel, rel); err != nil { + return 0, err + } + committed = true + return n, nil +} + +// Has reports whether the blob's bytes are on disk. +func (s *Store) Has(d Digest) (bool, error) { + if _, err := s.root.Stat(d.Rel()); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err + } + return true, nil +} + +// Open returns the blob for reading; the caller closes it. +// +// This is the read path of every request the server serves, and it is +// deliberately the only filesystem call on it. The name comes from Rel(), which +// is derived from 32 bytes that came out of an in-memory map, so no byte of +// user input reaches the filesystem here. Traversal on the read path is not so +// much prevented as inexpressible. +func (s *Store) Open(d Digest) (*os.File, error) { + f, err := s.root.Open(d.Rel()) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("%w: %s", ErrNotFound, d) + } + return nil, err + } + return f, nil +} + +// LinkInto places the blob's content at destDir/relPath. +// +// destDir is a staging directory the caller has just created and relPath has +// passed pathutil.Validate, so the join cannot leave destDir and there is no +// pre-existing symlink for it to follow. +func (s *Store) LinkInto(d Digest, destDir, relPath string) error { + dest := filepath.Join(destDir, filepath.FromSlash(relPath)) + if s.linkMode == LinkHard { + err := os.Link(s.Path(d), dest) + switch { + case err == nil: + return nil + case errors.Is(err, fs.ErrNotExist): + // The blob is gone, or a parent directory was never created. Either + // is a bug here, not a filesystem limitation, and copying would only + // fail again with a less informative error. + return fmt.Errorf("%w: %s: %w", ErrNotFound, d, err) + case errors.Is(err, fs.ErrExist): + // pathutil.Set rejects duplicate paths within a manifest, so two + // files claiming one name means the manifest was not checked. + return fmt.Errorf("cas: %s already exists in the deployment tree: %w", relPath, err) + } + // Anything else — most often EMLINK, since ext4 caps a file at 65,000 + // links and a blob shared by enough deployments does reach that — is a + // per-file limitation rather than a reason to fail the deployment. + s.log.Debug("hardlink failed, copying this file instead", "digest", d, "path", relPath, "err", err) + } + return s.copyInto(d, dest) +} + +func (s *Store) copyInto(d Digest, dest string) error { + src, err := s.Open(d) + if err != nil { + return err + } + defer src.Close() + + // O_EXCL because a collision means two manifest entries claimed one path, + // which pathutil.Set is supposed to have made impossible. + dst, err := os.OpenFile(dest, os.O_CREATE|os.O_EXCL|os.O_WRONLY, blobMode) + if err != nil { + return err + } + defer dst.Close() + if _, err := io.Copy(dst, src); err != nil { + return err + } + // A copy is the only content in the deployment tree that is not already + // durable — a hardlink shares the inode that Put fsynced. + if err := dst.Sync(); err != nil { + return err + } + return dst.Close() +} + +// Remove deletes a blob. Missing is success, so GC can be re-run after a crash. +func (s *Store) Remove(d Digest) error { + if err := s.root.Remove(d.Rel()); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil +} + +// PurgeTemp deletes every upload in progress and reports how many. Startup +// calls it: an interrupted upload is unreferenced garbage by construction. +func (s *Store) PurgeTemp() (int, error) { + entries, err := os.ReadDir(filepath.Join(s.dir, tmpDir)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return 0, nil + } + return 0, err + } + n := 0 + for _, e := range entries { + if err := s.root.RemoveAll(tmpDir + "/" + e.Name()); err != nil { + return n, err + } + n++ + } + return n, nil +} + +// newTemp creates a uniquely named file under tmp/ and returns its store-relative +// name. 128 bits of randomness make a collision impossible; the retry loop costs +// nothing and means a hypothetical one is not a failed upload. +func (s *Store) newTemp() (string, *os.File, error) { + var buf [16]byte + for attempt := 0; attempt < 3; attempt++ { + if _, err := rand.Read(buf[:]); err != nil { + return "", nil, err + } + name := tmpDir + "/" + hex.EncodeToString(buf[:]) + f, err := s.root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, tmpMode) + if err == nil { + return name, f, nil + } + if !errors.Is(err, fs.ErrExist) { + return "", nil, err + } + } + return "", nil, errors.New("cas: could not create a temporary file") +} + +// probeLink answers whether a hardlink from the blob store into the deployment +// tree actually works here. +// +// It has to be a real attempt rather than a check of the filesystem type. +// overlayfs — Docker's default, and this machine's /home — can fail or silently +// copy up across layers; a bind mount or a separate volume for the deployment +// tree puts the two directories on different devices and link(2) returns EXDEV; +// some hardened mounts refuse link(2) outright. The only reliable answer is to +// try it against the directory that will actually be used. +func probeLink(casDir, probeDir string) error { + if probeDir == "" { + return errors.New("no deployments directory to probe against") + } + if err := os.MkdirAll(probeDir, dirMode); err != nil { + return err + } + var buf [8]byte + if _, err := rand.Read(buf[:]); err != nil { + return err + } + suffix := hex.EncodeToString(buf[:]) + src := filepath.Join(casDir, tmpDir, ".link-probe-"+suffix) + dst := filepath.Join(probeDir, ".link-probe-"+suffix) + if err := os.WriteFile(src, []byte("probe"), tmpMode); err != nil { + return err + } + defer os.Remove(src) + if err := os.Link(src, dst); err != nil { + return err + } + return os.Remove(dst) +} + +// ctxReader makes a long upload cancellable. The HTTP server closes the body +// when a client disappears, but Put also runs against local readers on recovery +// paths where nothing else would notice a shutdown. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c *ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} diff --git a/internal/cas/cas_test.go b/internal/cas/cas_test.go new file mode 100644 index 0000000..dbe883c --- /dev/null +++ b/internal/cas/cas_test.go @@ -0,0 +1,498 @@ +package cas + +import ( + "bytes" + "context" + "errors" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +func newStore(t *testing.T, mode LinkMode) (*Store, string) { + t.Helper() + base := t.TempDir() + deployDir := filepath.Join(base, "deployments") + s, err := Open(filepath.Join(base, "cas"), Options{ + Mode: mode, + ProbeDir: deployDir, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + if mode == LinkHard { + // Not every filesystem supports link(2), which is the whole reason + // Open probes. Skipping is honest here; TestOpenAutoFallsBackToCopy + // covers what happens when it is unavailable. + t.Skipf("hardlinks unavailable under %s: %v", base, err) + } + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s, deployDir +} + +// tempFiles is how many uploads are sitting in tmp/. Every test that exercises a +// failure path checks this: a leaked temporary file is a slow disk leak that no +// other test would notice. +func tempFiles(t *testing.T, s *Store) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(s.Dir(), tmpDir)) + if err != nil { + t.Fatal(err) + } + return len(entries) +} + +func put(t *testing.T, s *Store, content string) Digest { + t.Helper() + d := Sum([]byte(content)) + n, err := s.Put(context.Background(), d, int64(len(content)), 1<<20, strings.NewReader(content)) + if err != nil { + t.Fatalf("Put(%q): %v", content, err) + } + if n != int64(len(content)) { + t.Fatalf("Put returned %d bytes, want %d", n, len(content)) + } + return d +} + +func TestParseDigest(t *testing.T) { + d := Sum([]byte("hello")) + hexForm := d.String() + + got, err := ParseDigest(hexForm) + if err != nil { + t.Fatalf("ParseDigest: %v", err) + } + if got != d { + t.Errorf("round trip changed the digest") + } + + bad := []string{ + "", + hexForm[:HexLen-1], + hexForm + "0", + strings.ToUpper(hexForm), // uppercase is a second spelling; see ParseDigest + strings.Repeat("g", HexLen), + hexForm[:HexLen-2] + "!!", + "../../../etc/passwd", + } + for _, s := range bad { + if _, err := ParseDigest(s); !errors.Is(err, ErrBadDigest) { + t.Errorf("ParseDigest(%q) = %v, want ErrBadDigest", s, err) + } + } +} + +func TestDigestRelIsSharded(t *testing.T) { + d := Sum([]byte("hello")) + rel := d.Rel() + hexForm := d.String() + want := hexForm[0:2] + "/" + hexForm[2:4] + "/" + hexForm + if rel != want { + t.Errorf("Rel = %q, want %q", rel, want) + } + if !fs.ValidPath(rel) { + t.Errorf("Rel = %q is not a valid path", rel) + } +} + +func TestFromBytes(t *testing.T) { + d := Sum([]byte("hello")) + // database/sql reuses its scan buffers, so a digest must not alias one. + buf := append([]byte(nil), d.Bytes()...) + got, err := FromBytes(buf) + if err != nil { + t.Fatal(err) + } + for i := range buf { + buf[i] = 0 + } + if got != d { + t.Error("FromBytes aliased its argument instead of copying") + } + if _, err := FromBytes(buf[:8]); err == nil { + t.Error("a short digest must be rejected") + } +} + +func TestPutAndOpen(t *testing.T) { + s, _ := newStore(t, LinkCopy) + const content = "

hello

" + d := put(t, s, content) + + ok, err := s.Has(d) + if err != nil || !ok { + t.Fatalf("Has = %v, %v", ok, err) + } + f, err := s.Open(d) + if err != nil { + t.Fatal(err) + } + defer f.Close() + got, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if string(got) != content { + t.Errorf("read back %q, want %q", got, content) + } + if tempFiles(t, s) != 0 { + t.Error("a successful Put left a temporary file behind") + } +} + +func TestPutIsIdempotent(t *testing.T) { + s, _ := newStore(t, LinkCopy) + const content = "same bytes" + d := put(t, s, content) + + before, err := os.Stat(s.Path(d)) + if err != nil { + t.Fatal(err) + } + put(t, s, content) + after, err := os.Stat(s.Path(d)) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(before, after) { + t.Error("re-uploading identical content replaced the blob instead of keeping it") + } + if tempFiles(t, s) != 0 { + t.Error("a redundant Put left a temporary file behind") + } +} + +func TestOpenMissingBlob(t *testing.T) { + s, _ := newStore(t, LinkCopy) + d := Sum([]byte("never uploaded")) + if ok, err := s.Has(d); err != nil || ok { + t.Fatalf("Has = %v, %v", ok, err) + } + if _, err := s.Open(d); !errors.Is(err, ErrNotFound) { + t.Errorf("Open = %v, want ErrNotFound", err) + } +} + +// The heart of the store: a caller's digest is a claim, never a fact. +func TestPutRejectsAClaimedDigest(t *testing.T) { + s, _ := newStore(t, LinkCopy) + ctx := context.Background() + + // Stand in for another project's file, already deduplicated into the store. + victim := "the real index.html" + victimDigest := put(t, s, victim) + + attack := "" + _, err := s.Put(ctx, victimDigest, int64(len(attack)), 1<<20, strings.NewReader(attack)) + if !errors.Is(err, ErrDigestMismatch) { + t.Fatalf("Put = %v, want ErrDigestMismatch", err) + } + + f, err := s.Open(victimDigest) + if err != nil { + t.Fatal(err) + } + defer f.Close() + got, _ := io.ReadAll(f) + if string(got) != victim { + t.Fatalf("the existing blob was overwritten: %q", got) + } + if tempFiles(t, s) != 0 { + t.Error("the rejected upload left a temporary file behind") + } +} + +type errReader struct{ err error } + +func (e errReader) Read([]byte) (int, error) { return 0, e.err } + +// Every way Put can fail has to clean up after itself, or a store slowly fills +// with the debris of clients that hung up. +func TestPutCleansUpOnEveryFailure(t *testing.T) { + s, _ := newStore(t, LinkCopy) + const content = "some content" + d := Sum([]byte(content)) + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + + cases := []struct { + name string + ctx context.Context + want Digest + size int64 + max int64 + body io.Reader + wantErr error + }{ + {"digest mismatch", context.Background(), Sum([]byte("other")), int64(len(content)), 1 << 20, strings.NewReader(content), ErrDigestMismatch}, + {"size mismatch", context.Background(), d, int64(len(content)) + 1, 1 << 20, strings.NewReader(content), ErrSizeMismatch}, + {"over the limit", context.Background(), d, -1, 4, strings.NewReader(content), ErrTooLarge}, + {"declared over the limit", context.Background(), d, 1 << 30, 4, strings.NewReader(content), ErrTooLarge}, + {"reader failed", context.Background(), d, -1, 1 << 20, errReader{errors.New("connection reset")}, nil}, + {"context cancelled", cancelled, d, -1, 1 << 20, strings.NewReader(content), context.Canceled}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := s.Put(tc.ctx, tc.want, tc.size, tc.max, tc.body) + if err == nil { + t.Fatal("Put should have failed") + } + if tc.wantErr != nil && !errors.Is(err, tc.wantErr) { + t.Errorf("err = %v, want %v", err, tc.wantErr) + } + if n := tempFiles(t, s); n != 0 { + t.Errorf("%d temporary file(s) left behind", n) + } + if ok, _ := s.Has(tc.want); ok { + t.Error("a failed Put made a blob visible") + } + }) + } +} + +// A client that lies in Content-Length must not get past the ceiling; the limit +// is enforced against the bytes that arrive, not against the declaration. +func TestPutEnforcesTheLimitAgainstActualBytes(t *testing.T) { + s, _ := newStore(t, LinkCopy) + body := strings.Repeat("x", 100) + d := Sum([]byte(body)) + // Declares that it fits, then sends more. + _, err := s.Put(context.Background(), d, 4, 8, strings.NewReader(body)) + if err == nil { + t.Fatal("Put should have failed") + } + if tempFiles(t, s) != 0 { + t.Error("temporary file left behind") + } +} + +func TestPutExactlyAtTheLimit(t *testing.T) { + s, _ := newStore(t, LinkCopy) + body := strings.Repeat("x", 64) + d := Sum([]byte(body)) + if _, err := s.Put(context.Background(), d, 64, 64, strings.NewReader(body)); err != nil { + t.Errorf("a file exactly at the limit must be accepted: %v", err) + } +} + +func TestPutEmptyBlob(t *testing.T) { + s, _ := newStore(t, LinkCopy) + d := put(t, s, "") + f, err := s.Open(d) + if err != nil { + t.Fatal(err) + } + defer f.Close() + b, _ := io.ReadAll(f) + if len(b) != 0 { + t.Errorf("read %d bytes from the empty blob", len(b)) + } +} + +// Concurrent uploads of identical content are expected — two CI jobs deploying +// the same vendored asset — and all of them must succeed with one file left. +func TestConcurrentIdenticalPut(t *testing.T) { + s, _ := newStore(t, LinkCopy) + const content = "shared asset" + d := Sum([]byte(content)) + + const n = 16 + errs := make([]error, n) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range n { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, errs[i] = s.Put(context.Background(), d, int64(len(content)), 1<<20, strings.NewReader(content)) + }() + } + close(start) + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Errorf("goroutine %d: %v", i, err) + } + } + if tempFiles(t, s) != 0 { + t.Error("temporary files left behind") + } + f, err := s.Open(d) + if err != nil { + t.Fatal(err) + } + defer f.Close() + got, _ := io.ReadAll(f) + if string(got) != content { + t.Errorf("blob = %q, want %q", got, content) + } +} + +func TestPurgeTemp(t *testing.T) { + s, _ := newStore(t, LinkCopy) + d := put(t, s, "keep me") + + for _, name := range []string{"aaaa", "bbbb"} { + if err := os.WriteFile(filepath.Join(s.Dir(), tmpDir, name), []byte("interrupted"), 0o600); err != nil { + t.Fatal(err) + } + } + n, err := s.PurgeTemp() + if err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("purged %d, want 2", n) + } + if tempFiles(t, s) != 0 { + t.Error("tmp is not empty") + } + if ok, _ := s.Has(d); !ok { + t.Error("PurgeTemp removed a committed blob") + } + if n, err := s.PurgeTemp(); err != nil || n != 0 { + t.Errorf("second PurgeTemp = %d, %v", n, err) + } +} + +func TestRemoveIsIdempotent(t *testing.T) { + s, _ := newStore(t, LinkCopy) + d := put(t, s, "temporary") + if err := s.Remove(d); err != nil { + t.Fatal(err) + } + if ok, _ := s.Has(d); ok { + t.Error("blob still present after Remove") + } + // GC re-running after a crash must not fail on what it already deleted. + if err := s.Remove(d); err != nil { + t.Errorf("second Remove: %v", err) + } +} + +func TestLinkIntoCopyMode(t *testing.T) { + s, deployDir := newStore(t, LinkCopy) + const content = "console.log(1)\n" + d := put(t, s, content) + + dest := filepath.Join(deployDir, "assets") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + if err := s.LinkInto(d, dest, "app.js"); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filepath.Join(dest, "app.js")) + if err != nil { + t.Fatal(err) + } + if string(got) != content { + t.Errorf("content = %q, want %q", got, content) + } +} + +func TestLinkIntoRejectsAMissingBlob(t *testing.T) { + for _, mode := range []LinkMode{LinkCopy, LinkHard} { + t.Run(string(mode), func(t *testing.T) { + s, deployDir := newStore(t, mode) + if err := os.MkdirAll(deployDir, 0o755); err != nil { + t.Fatal(err) + } + d := Sum([]byte("never uploaded")) + if err := s.LinkInto(d, deployDir, "x.html"); !errors.Is(err, ErrNotFound) { + t.Errorf("LinkInto = %v, want ErrNotFound", err) + } + }) + } +} + +func TestLinkIntoRejectsADuplicatePath(t *testing.T) { + for _, mode := range []LinkMode{LinkCopy, LinkHard} { + t.Run(string(mode), func(t *testing.T) { + s, deployDir := newStore(t, mode) + if err := os.MkdirAll(deployDir, 0o755); err != nil { + t.Fatal(err) + } + d := put(t, s, "x") + if err := s.LinkInto(d, deployDir, "x.html"); err != nil { + t.Fatal(err) + } + if err := s.LinkInto(d, deployDir, "x.html"); err == nil { + t.Error("writing twice to one path must fail rather than overwrite") + } + }) + } +} + +func TestOpenRejectsAnUnknownMode(t *testing.T) { + if _, err := Open(t.TempDir(), Options{Mode: "symlink"}); err == nil { + t.Error("an unknown link mode must be refused at startup, not at deploy time") + } +} + +// Forcing hardlinks has to fail loudly when they do not work, or an operator who +// asked for them silently gets copies and a full disk. +func TestOpenHardlinkModeFailsWithoutAProbeTarget(t *testing.T) { + if _, err := Open(filepath.Join(t.TempDir(), "cas"), Options{Mode: LinkHard}); err == nil { + t.Error("hardlink mode with nothing to probe against must fail") + } +} + +func TestOpenAutoFallsBackToCopy(t *testing.T) { + // No ProbeDir, so the probe cannot succeed and auto must degrade rather than + // refuse to start. + s, err := Open(filepath.Join(t.TempDir(), "cas"), Options{}) + if err != nil { + t.Fatalf("auto mode must start anyway: %v", err) + } + defer s.Close() + if s.LinkMode() != LinkCopy { + t.Errorf("LinkMode = %q, want %q", s.LinkMode(), LinkCopy) + } +} + +func TestBlobsAreReadOnly(t *testing.T) { + s, _ := newStore(t, LinkCopy) + d := put(t, s, "immutable") + fi, err := os.Stat(s.Path(d)) + if err != nil { + t.Fatal(err) + } + // A writable blob is a writable inode shared by every project referencing + // that content; see blobMode. + if perm := fi.Mode().Perm(); perm != blobMode { + t.Errorf("blob mode = %04o, want %04o", perm, blobMode) + } +} + +func TestPutLargeStream(t *testing.T) { + s, _ := newStore(t, LinkCopy) + body := bytes.Repeat([]byte("0123456789abcdef"), 1<<16) // 1 MiB + d := Sum(body) + n, err := s.Put(context.Background(), d, int64(len(body)), 4<<20, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if n != int64(len(body)) { + t.Fatalf("stored %d bytes, want %d", n, len(body)) + } + f, err := s.Open(d) + if err != nil { + t.Fatal(err) + } + defer f.Close() + got, _ := io.ReadAll(f) + if !bytes.Equal(got, body) { + t.Error("stored content differs from what was written") + } +} diff --git a/internal/cas/cas_unix_test.go b/internal/cas/cas_unix_test.go new file mode 100644 index 0000000..ae45358 --- /dev/null +++ b/internal/cas/cas_unix_test.go @@ -0,0 +1,115 @@ +//go:build unix + +package cas + +import ( + "os" + "path/filepath" + "syscall" + "testing" +) + +func nlink(t *testing.T, path string) uint64 { + t.Helper() + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + t.Skip("no stat_t on this platform") + } + return uint64(st.Nlink) +} + +// The deduplication claim, stated as a filesystem fact: a deployed file and its +// blob are the same inode, so a second deployment of the same content costs a +// directory entry and nothing more. +func TestLinkIntoSharesTheInode(t *testing.T) { + s, deployDir := newStore(t, LinkHard) + const content = "shared across deployments\n" + d := put(t, s, content) + + if got := nlink(t, s.Path(d)); got != 1 { + t.Fatalf("a fresh blob has %d links, want 1", got) + } + + first := filepath.Join(deployDir, "dpl_one") + second := filepath.Join(deployDir, "dpl_two") + for _, dir := range []string{first, second} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := s.LinkInto(d, dir, "index.html"); err != nil { + t.Fatal(err) + } + } + + if got := nlink(t, s.Path(d)); got != 3 { + t.Errorf("blob has %d links after two deployments, want 3 (blob + 2)", got) + } + blob, err := os.Stat(s.Path(d)) + if err != nil { + t.Fatal(err) + } + for _, dir := range []string{first, second} { + deployed, err := os.Stat(filepath.Join(dir, "index.html")) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(blob, deployed) { + t.Errorf("%s is a copy, not a hardlink to the blob", dir) + } + if got, err := os.ReadFile(filepath.Join(dir, "index.html")); err != nil || string(got) != content { + t.Errorf("%s: content = %q, %v", dir, got, err) + } + } + + // Removing a deployment tree must not take the blob with it: the other + // deployment still references it, and so may other projects. + if err := os.RemoveAll(first); err != nil { + t.Fatal(err) + } + if got := nlink(t, s.Path(d)); got != 2 { + t.Errorf("blob has %d links after one tree was removed, want 2", got) + } +} + +// The mode is what stops a deployed file from being written through into the +// blob every other project shares. +func TestDeployedFilesAreReadOnly(t *testing.T) { + for _, mode := range []LinkMode{LinkHard, LinkCopy} { + t.Run(string(mode), func(t *testing.T) { + s, deployDir := newStore(t, mode) + d := put(t, s, "content") + if err := os.MkdirAll(deployDir, 0o755); err != nil { + t.Fatal(err) + } + if err := s.LinkInto(d, deployDir, "index.html"); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(filepath.Join(deployDir, "index.html")) + if err != nil { + t.Fatal(err) + } + if perm := fi.Mode().Perm(); perm != blobMode { + t.Errorf("deployed file mode = %04o, want %04o", perm, blobMode) + } + }) + } +} + +// A copy must be a copy: writing through it may not reach the blob. +func TestCopyModeDoesNotShareTheInode(t *testing.T) { + s, deployDir := newStore(t, LinkCopy) + d := put(t, s, "content") + if err := os.MkdirAll(deployDir, 0o755); err != nil { + t.Fatal(err) + } + if err := s.LinkInto(d, deployDir, "index.html"); err != nil { + t.Fatal(err) + } + if got := nlink(t, s.Path(d)); got != 1 { + t.Errorf("blob has %d links in copy mode, want 1", got) + } +} diff --git a/internal/cas/digest.go b/internal/cas/digest.go new file mode 100644 index 0000000..5b287ea --- /dev/null +++ b/internal/cas/digest.go @@ -0,0 +1,88 @@ +// Package cas is the content-addressed blob store. Every file any deployment +// contains is stored once, named by the SHA-256 of its bytes, and shared by +// every deployment and every project that references that content. +// +// Sharing is what makes redeploying a mostly-unchanged site nearly free, and it +// is only safe because the store never takes a caller's word for a digest: see +// Store.Put. +package cas + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" +) + +const ( + // Size is a digest's length in bytes. + Size = sha256.Size + // HexLen is a digest's length in its hex form. + HexLen = 2 * Size +) + +// Digest is the SHA-256 of a blob's contents. +// +// An array rather than a slice, so it can be a map key, compared with ==, and +// copied out of a database buffer instead of aliasing one — database/sql reuses +// the byte slices it scans into. +type Digest [Size]byte + +// ErrBadDigest rejects anything that is not a digest in the canonical form. +var ErrBadDigest = errors.New("cas: not a 64-character lowercase hex sha-256") + +// ParseDigest decodes the hex form used on the wire and in URLs. +// +// Strict about case rather than normalising, because a digest is simultaneously +// a primary key in the blobs table and a component of a filesystem path. +// Accepting two spellings of one value would mean two rows, two files, and +// deduplication that quietly stops deduplicating. +func ParseDigest(s string) (Digest, error) { + var d Digest + if len(s) != HexLen { + return d, ErrBadDigest + } + for i := 0; i < len(s); i++ { + // encoding/hex accepts uppercase; this loop is what makes lowercase the + // only accepted spelling. + c := s[i] + if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') { + continue + } + return d, ErrBadDigest + } + if _, err := hex.Decode(d[:], []byte(s)); err != nil { + return Digest{}, ErrBadDigest + } + return d, nil +} + +// FromBytes converts the raw 32 bytes stored in the database. It copies, so the +// result does not alias the scan buffer it came from. +func FromBytes(b []byte) (Digest, error) { + var d Digest + if len(b) != Size { + return d, fmt.Errorf("cas: digest is %d bytes, want %d", len(b), Size) + } + copy(d[:], b) + return d, nil +} + +// String is the canonical hex form. +func (d Digest) String() string { return hex.EncodeToString(d[:]) } + +// Bytes is the raw form stored in the database. The slice belongs to the +// caller's copy of the digest, so writing to it cannot affect anything else. +func (d Digest) Bytes() []byte { return d[:] } + +// Rel is the blob's path inside the store, sharded two levels deep: +// "ab/cd/abcd…". Two hex characters per level gives 65,536 leaf directories, so +// even a store with millions of blobs keeps every directory small enough that +// readdir and lookup stay fast on ext4 and xfs alike. +func (d Digest) Rel() string { + s := d.String() + return s[0:2] + "/" + s[2:4] + "/" + s +} + +// Sum digests b. +func Sum(b []byte) Digest { return sha256.Sum256(b) } diff --git a/internal/clicmd/config.go b/internal/clicmd/config.go new file mode 100644 index 0000000..5c36a86 --- /dev/null +++ b/internal/clicmd/config.go @@ -0,0 +1,118 @@ +package clicmd + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// ConfigFile is the CLI's on-disk configuration. +// +// JSON rather than TOML: encoding/json is already linked into the binary for +// the API, and adding a TOML parser to read four fields would cost more than +// the file saves. CI never reads it at all — there, everything comes from +// PAGES_* environment variables. +type ConfigFile struct { + Server string `json:"server,omitempty"` + Token string `json:"token,omitempty"` + Project string `json:"project,omitempty"` + Output string `json:"output,omitempty"` +} + +// DefaultConfigPath is $PAGES_CONFIG, else $XDG_CONFIG_HOME/pages/config.json, +// else ~/.config/pages/config.json. It returns "" when no home directory can be +// determined, which simply means there is no config file. +func DefaultConfigPath() string { + if p := os.Getenv("PAGES_CONFIG"); p != "" { + return p + } + dir, err := os.UserConfigDir() + if err != nil { + return "" + } + return filepath.Join(dir, "pages", "config.json") +} + +// LoadConfig reads path. A missing file is not an error — it is the normal +// state on a CI runner. warn receives a note if the file is readable by anyone +// but its owner, because it may hold a token. +func LoadConfig(path string, warn io.Writer) (ConfigFile, error) { + var f ConfigFile + if path == "" { + return f, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return f, nil + } + if err != nil { + return f, fmt.Errorf("read config %s: %w", path, err) + } + if fi, err := os.Stat(path); err == nil && fi.Mode().Perm()&0o077 != 0 && warn != nil { + fmt.Fprintf(warn, "warning: %s is mode %#o and may contain a token; run: chmod 600 %s\n", + path, fi.Mode().Perm(), path) + } + if err := json.Unmarshal(raw, &f); err != nil { + return ConfigFile{}, fmt.Errorf("parse config %s: %w", path, err) + } + return f, nil +} + +// SaveConfig writes f to path with mode 0600, replacing any existing file +// atomically so a crash cannot leave a truncated config behind. +func SaveConfig(path string, f ConfigFile) error { + if path == "" { + return errors.New("no config path: set PAGES_CONFIG or pass --config") + } + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("create config directory: %w", err) + } + raw, err := json.MarshalIndent(f, "", " ") + if err != nil { + return err + } + raw = append(raw, '\n') + + // Written in the destination directory so the rename stays within one + // filesystem, and created 0600 from the start so the token is never briefly + // world-readable. + tmp, err := os.CreateTemp(dir, ".config-*.tmp") + if err != nil { + return fmt.Errorf("create temporary config: %w", err) + } + defer os.Remove(tmp.Name()) + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return err + } + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmp.Name(), path); err != nil { + return fmt.Errorf("install config: %w", err) + } + return nil +} + +// redactToken shows enough of a token to recognise which one it is and nothing +// that could be used with it. The key id is the public half by construction +// (pgs__), so it is safe to print in full. +func redactToken(token string) string { + if token == "" { + return "" + } + parts := strings.SplitN(token, "_", 3) + if len(parts) == 3 { + return parts[0] + "_" + parts[1] + "_…" + } + return "…" +} diff --git a/internal/clicmd/config_test.go b/internal/clicmd/config_test.go new file mode 100644 index 0000000..ca06db2 --- /dev/null +++ b/internal/clicmd/config_test.go @@ -0,0 +1,136 @@ +package clicmd + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestSaveConfigIsOwnerOnlyAndAtomic(t *testing.T) { + dir := filepath.Join(t.TempDir(), "nested") + path := filepath.Join(dir, "config.json") + + want := ConfigFile{Server: "https://p.example.com", Token: "pgs_abcdefghijklmnop_secret", Output: "json"} + if err := SaveConfig(path, want); err != nil { + t.Fatalf("SaveConfig: %v", err) + } + + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("config mode = %#o, want 0600 — it holds a token", perm) + } + if di, err := os.Stat(dir); err == nil { + if perm := di.Mode().Perm(); perm&0o077 != 0 { + t.Errorf("config directory mode = %#o, want no group or other bits", perm) + } + } + + got, err := LoadConfig(path, io.Discard) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + if got != want { + t.Errorf("round trip = %+v, want %+v", got, want) + } + + // The temporary file is written in the destination directory; leaving one + // behind would leave a mode-0600 copy of the token lying around. + ents, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(ents) != 1 || ents[0].Name() != "config.json" { + names := make([]string, len(ents)) + for i, e := range ents { + names[i] = e.Name() + } + t.Errorf("directory contains %q, want only config.json", names) + } +} + +func TestSaveConfigReplacesInPlace(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := SaveConfig(path, ConfigFile{Server: "https://one.example.com"}); err != nil { + t.Fatal(err) + } + if err := SaveConfig(path, ConfigFile{Server: "https://two.example.com"}); err != nil { + t.Fatal(err) + } + got, err := LoadConfig(path, io.Discard) + if err != nil { + t.Fatal(err) + } + if got.Server != "https://two.example.com" { + t.Errorf("server = %q, want the second write", got.Server) + } +} + +func TestLoadConfig(t *testing.T) { + t.Run("a missing file is the normal state on CI", func(t *testing.T) { + got, err := LoadConfig(filepath.Join(t.TempDir(), "absent.json"), io.Discard) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if got != (ConfigFile{}) { + t.Errorf("got %+v, want the zero value", got) + } + }) + + t.Run("an empty path means there is no config file", func(t *testing.T) { + if _, err := LoadConfig("", io.Discard); err != nil { + t.Fatalf("err = %v, want nil", err) + } + }) + + t.Run("malformed JSON is reported, not ignored", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(path, []byte("{not json"), 0o600) + if _, err := LoadConfig(path, io.Discard); err == nil { + t.Fatal("expected an error") + } + }) + + t.Run("a readable-by-others config warns", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + os.WriteFile(path, []byte(`{"token":"pgs_abcdefghijklmnop_secret"}`), 0o644) + var warn strings.Builder + if _, err := LoadConfig(path, &warn); err != nil { + t.Fatal(err) + } + if !strings.Contains(warn.String(), "chmod 600") { + t.Errorf("warning = %q, want it to say how to fix the mode", warn.String()) + } + if strings.Contains(warn.String(), "secret") { + t.Error("the warning printed the token it was warning about") + } + }) +} + +func TestRedactToken(t *testing.T) { + cases := []struct{ in, want string }{ + {"pgs_abcdefghijklmnop_thesecrethalf", "pgs_abcdefghijklmnop_…"}, + {"", ""}, + {"garbage", "…"}, + {"pgs_onlytwo", "…"}, + // The secret half is base64url, so it can itself contain underscores; + // SplitN with n=3 keeps them in the part that gets dropped. + {"pgs_abcdefghijklmnop_a_b_c", "pgs_abcdefghijklmnop_…"}, + } + for _, tc := range cases { + if got := redactToken(tc.in); got != tc.want { + t.Errorf("redactToken(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestDefaultConfigPathPrefersEnv(t *testing.T) { + t.Setenv("PAGES_CONFIG", "/tmp/explicit.json") + if got := DefaultConfigPath(); got != "/tmp/explicit.json" { + t.Errorf("DefaultConfigPath = %q", got) + } +} diff --git a/internal/clicmd/deploy.go b/internal/clicmd/deploy.go new file mode 100644 index 0000000..d242711 --- /dev/null +++ b/internal/clicmd/deploy.go @@ -0,0 +1,220 @@ +package clicmd + +import ( + "context" + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/iceBear67/simplepages/internal/client" + "github.com/iceBear67/simplepages/internal/cliutil" +) + +// stringList is a flag that may be given more than once. +type stringList []string + +func (s *stringList) String() string { return strings.Join(*s, ",") } + +func (s *stringList) Set(v string) error { + *s = append(*s, v) + return nil +} + +func deployCmd(g *Globals) *cliutil.Command { + var ( + activate = true + meta stringList + include stringList + exclude stringList + concurrency int + retries int + follow bool + dryRun bool + ) + return &cliutil.Command{ + Name: "deploy", + Args: "", + Short: "Upload a directory and switch the site to it", + Long: "Hashes the directory, uploads only the files the server does not\n" + + "already have, then switches the project over in one step — a visitor\n" + + "sees the old site or the new one, never a mixture.\n\n" + + "Interrupting a deploy is safe: the blobs that made it are kept, so\n" + + "running the same command again uploads only what is still missing.\n\n" + + "Commit metadata is read from the CI environment (GitHub Actions,\n" + + "GitLab CI) and can be set or overridden with --meta.", + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&activate, "activate", true, "switch the project to this deployment once it is uploaded") + fs.Var(&meta, "meta", "`key=value` recorded with the deployment; repeatable") + fs.Var(&include, "include", "only upload files matching `glob`; repeatable") + fs.Var(&exclude, "exclude", "skip files and directories matching `glob`; repeatable") + fs.IntVar(&concurrency, "concurrency", 8, "`number` of blobs uploaded at once") + fs.IntVar(&retries, "retries", 4, "`number` of extra attempts per blob on a transient failure") + fs.BoolVar(&follow, "follow-symlinks", false, "upload what symlinks point at instead of refusing them") + fs.BoolVar(&dryRun, "dry-run", false, "scan and report without contacting the server") + }, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 1, "one directory"); err != nil { + return err + } + dir := args[0] + + metaMap, err := parseMeta(meta) + if err != nil { + return err + } + + p, err := g.Printer() + if err != nil { + return err + } + + src, err := client.Scan(ctx, dir, client.ScanOptions{ + Include: include, + Exclude: exclude, + FollowSymlinks: follow, + }) + if err != nil { + return err + } + defer src.Close() + + if dryRun { + // Deliberately offline: negotiating the manifest to find out what is + // missing would create a deployment on the server, which is exactly + // what --dry-run promises not to do. + return p.Print(dryRunResult{ + Dir: src.Dir, + FileCount: len(src.Files), + TotalBytes: src.TotalBytes, + UniqueBlobs: src.UniqueBlobs(), + Files: src.Files, + }, func() *cliutil.Table { + t := cliutil.NewTable("PATH", "SIZE", "DIGEST") + for _, f := range src.Files { + t.Row(f.Path, cliutil.Bytes(f.Size), f.Digest[:12]) + } + t.Row("", "", "") + t.Row(fmt.Sprintf("%d files", len(src.Files)), + cliutil.Bytes(src.TotalBytes), + fmt.Sprintf("%d unique", src.UniqueBlobs())) + return t + }) + } + + project, err := g.ProjectName() + if err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + + res, err := c.Deploy(ctx, client.DeployOptions{ + Project: project, + Source: src, + Meta: metaMap, + Activate: activate, + Concurrency: concurrency, + Retries: retries, + // Progress goes to stderr so it stays out of a piped -o json + // document, and is shown even without --verbose: the deduplication + // win is the reason this tool exists, and a CI log should record it. + Progress: func(msg string) { fmt.Fprintln(g.Err, msg) }, + }) + if err != nil { + return err + } + + return p.Print(res, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("deployment", res.Deployment.ID) + t.Row("project", project) + t.Row("state", res.Deployment.State) + t.Row("files", strconv.Itoa(res.FileCount)) + t.Row("total_bytes", cliutil.Bytes(res.TotalBytes)) + t.Row("uploaded", fmt.Sprintf("%s, %s", + cliutil.Plural(res.Uploaded, "blob"), cliutil.Bytes(res.UploadedBytes))) + t.Row("reused", strconv.Itoa(res.Deduplicated)) + t.Row("activated", cliutil.Bool(res.Activated)) + t.Row("url", cliutil.Str(res.URL)) + return t + }) + }, + } +} + +// dryRunResult is what --dry-run reports: everything decided locally, and +// nothing that would need the server. +type dryRunResult struct { + Dir string `json:"dir"` + FileCount int `json:"file_count"` + TotalBytes int64 `json:"total_bytes"` + UniqueBlobs int `json:"unique_blobs"` + Files []client.LocalFile `json:"files"` +} + +// parseMeta folds --meta over whatever the CI environment reveals, so an +// explicit flag always wins over a guessed value. +func parseMeta(pairs []string) (map[string]string, error) { + out := detectCIMeta() + for _, p := range pairs { + k, v, ok := strings.Cut(p, "=") + if !ok { + return nil, cliutil.UsageErrorf("--meta %q: expected key=value", p) + } + k = strings.TrimSpace(k) + if k == "" { + return nil, cliutil.UsageErrorf("--meta %q: empty key", p) + } + if out == nil { + out = make(map[string]string, len(pairs)) + } + out[k] = v + } + return out, nil +} + +// detectCIMeta reads the commit metadata the common CI systems export, so a +// deployment can be traced back to what produced it without every pipeline +// having to spell out the same four --meta flags. +func detectCIMeta() map[string]string { + out := make(map[string]string, 4) + set := func(key string, envs ...string) { + for _, e := range envs { + if v := strings.TrimSpace(os.Getenv(e)); v != "" { + out[key] = v + return + } + } + } + set("git_sha", "GITHUB_SHA", "CI_COMMIT_SHA", "GIT_COMMIT") + set("git_ref", "GITHUB_REF_NAME", "CI_COMMIT_REF_NAME", "GIT_BRANCH") + set("ci_run", "GITHUB_RUN_ID", "CI_PIPELINE_ID", "BUILD_NUMBER") + set("actor", "GITHUB_ACTOR", "GITLAB_USER_LOGIN") + + if url := ciRunURL(); url != "" { + out["ci_url"] = url + } + if len(out) == 0 { + return nil + } + return out +} + +// ciRunURL reconstructs a link back to the job. GitHub does not export one +// directly; GitLab does. +func ciRunURL() string { + if v := strings.TrimSpace(os.Getenv("CI_PIPELINE_URL")); v != "" { + return v + } + server := strings.TrimRight(os.Getenv("GITHUB_SERVER_URL"), "/") + repo := os.Getenv("GITHUB_REPOSITORY") + run := os.Getenv("GITHUB_RUN_ID") + if server != "" && repo != "" && run != "" { + return server + "/" + repo + "/actions/runs/" + run + } + return "" +} diff --git a/internal/clicmd/deployment.go b/internal/clicmd/deployment.go new file mode 100644 index 0000000..ea30a2d --- /dev/null +++ b/internal/clicmd/deployment.go @@ -0,0 +1,357 @@ +package clicmd + +import ( + "context" + "flag" + "fmt" + "maps" + "slices" + "strconv" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/client" + "github.com/iceBear67/simplepages/internal/cliutil" +) + +func deploymentCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "deployment", + Short: "Inspect, roll back and remove deployments", + Long: "Every deployment a project has ever finished is kept until retention\n" + + "drops it, which is what makes a rollback one command rather than a\n" + + "rebuild. A project key may manage only its own project's deployments.", + Sub: []*cliutil.Command{ + deploymentListCmd(g), + deploymentShowCmd(g), + deploymentActivateCmd(g), + deploymentDeleteCmd(g), + }, + } +} + +func deploymentListCmd(g *Globals) *cliutil.Command { + var ( + state string + limit int + ) + return &cliutil.Command{ + Name: "list", + Short: "List a project's deployments", + Long: "Newest first. The one marked active is what the site is serving, and\n" + + "is the one to roll back from; any other ready deployment can be rolled\n" + + "back to with \"pages deployment activate\".", + Flags: func(fs *flag.FlagSet) { + fs.StringVar(&state, "state", "", "show only deployments in this `state`: pending, uploading, ready, failed or deleting") + fs.IntVar(&limit, "limit", 0, "stop after this many deployments; 0 lists them all") + }, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + project, err := g.ProjectName() + if err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + deps, err := listDeployments(ctx, c, project, state, limit) + if err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + // The API's own shape, so -o json is the same document whether it came + // from here or from curl, and an empty listing prints as [] not null. + out := api.DeploymentList{Deployments: deps} + if out.Deployments == nil { + out.Deployments = []api.Deployment{} + } + return p.Print(out, func() *cliutil.Table { + t := cliutil.NewTable("ID", "STATE", "ACTIVE", "FILES", "SIZE", "CREATED", "COMMIT") + for _, d := range deps { + t.Row(d.ID, d.State, cliutil.Bool(d.Active), + strconv.Itoa(d.FileCount), cliutil.Bytes(d.TotalBytes), + cliutil.Time(d.CreatedAt), + cliutil.Str(cliutil.Truncate(d.Meta["git_sha"], 12))) + } + return t + }) + }, + } +} + +// listDeployments follows the cursor, stopping at limit when one was given. +// Paging on the caller's behalf matters here for the same reason it does for +// projects: a listing that silently showed the first page would be a lie, and +// this one is read to decide which deployment to roll back to. +func listDeployments(ctx context.Context, c *client.Client, project, state string, limit int) ([]api.Deployment, error) { + opts := client.DeploymentListOptions{State: state} + opts.Limit = 500 + if limit > 0 && limit < opts.Limit { + opts.Limit = limit + } + var all []api.Deployment + for { + page, err := c.ListDeployments(ctx, project, opts) + if err != nil { + return nil, err + } + all = append(all, page.Deployments...) + if limit > 0 && len(all) >= limit { + return all[:limit], nil + } + if page.NextCursor == "" || len(page.Deployments) == 0 { + return all, nil + } + opts.Cursor = page.NextCursor + } +} + +func deploymentShowCmd(g *Globals) *cliutil.Command { + var files bool + return &cliutil.Command{ + Name: "show", + Args: "", + Short: "Show one deployment", + Long: "With --files, also lists the manifest: every path, its size and the\n" + + "digest of its content. That is one line per file, so it is a lot of\n" + + "output for a large site.", + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&files, "files", false, "also list the deployment's files") + }, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 1, "one deployment id"); err != nil { + return err + } + project, err := g.ProjectName() + if err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + d, err := c.GetDeployment(ctx, project, args[0], files) + if err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(d, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("id", d.ID) + t.Row("project", d.Project) + t.Row("state", d.State) + t.Row("active", cliutil.Bool(d.Active)) + t.Row("files", strconv.Itoa(d.FileCount)) + t.Row("total_bytes", cliutil.Bytes(d.TotalBytes)) + t.Row("created_at", cliutil.Time(d.CreatedAt)) + t.Row("finalized_at", cliutil.TimePtr(d.FinalizedAt)) + t.Row("activated_at", cliutil.TimePtr(d.ActivatedAt)) + if d.URL != "" { + t.Row("url", d.URL) + } + if d.Error != "" { + t.Row("error", d.Error) + } + for _, k := range sortedKeys(d.Meta) { + t.Row("meta."+k, d.Meta[k]) + } + for _, f := range d.Files { + t.Row(f.Path, fmt.Sprintf("%s %s", + cliutil.Bytes(f.Size), cliutil.Truncate(f.Digest, 12))) + } + return t + }) + }, + } +} + +// sortedKeys gives map-backed output a stable order, so two runs of the same +// command produce the same lines and a diff of them means something. +func sortedKeys(m map[string]string) []string { + return slices.Sorted(maps.Keys(m)) +} + +func deploymentActivateCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "activate", + Args: "", + Short: "Switch the site to a deployment", + Long: "This is how a rollback is done: name an older deployment and the\n" + + "project serves it again. The switch is atomic and costs nothing —\n" + + "the content is still on disk — so it takes effect immediately.", + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 1, "one deployment id"); err != nil { + return err + } + project, err := g.ProjectName() + if err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + d, err := c.Activate(ctx, project, args[0]) + if err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(d, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("deployment", d.ID) + t.Row("project", project) + t.Row("state", d.State) + t.Row("active", cliutil.Bool(d.Active)) + t.Row("files", strconv.Itoa(d.FileCount)) + t.Row("url", cliutil.Str(d.URL)) + return t + }) + }, + } +} + +func deploymentDeleteCmd(g *Globals) *cliutil.Command { + var yes bool + return &cliutil.Command{ + Name: "delete", + Args: "", + Short: "Delete a deployment", + Long: "The deployment the project is serving cannot be deleted; activate\n" + + "another one first. Content no other deployment references is\n" + + "reclaimed by the next garbage collection, not immediately.", + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&yes, "yes", false, "do not ask for confirmation") + }, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 1, "one deployment id"); err != nil { + return err + } + id := args[0] + project, err := g.ProjectName() + if err != nil { + return err + } + if !yes { + if err := cliutil.Confirm(g.In, g.Err, + fmt.Sprintf("Delete deployment %s of project %q?", id, project)); err != nil { + return err + } + } + c, err := g.Client() + if err != nil { + return err + } + if err := c.DeleteDeployment(ctx, project, id); err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + p.Printf("deleted deployment %s\n", id) + return nil + }, + } +} + +// ------------------------------------------------------------------ upkeep +// +// These two are server-wide rather than per-project, which is why they sit +// under "pages system" next to "system info" and not under "deployment". + +func systemGCCmd(g *Globals) *cliutil.Command { + var dryRun bool + return &cliutil.Command{ + Name: "gc", + Short: "Run a garbage collection pass now", + Long: "The server collects on a timer anyway; this is for an operator who\n" + + "wants the disk back sooner. --dry-run reports what would be deleted\n" + + "without deleting it, though it cannot count the content the listed\n" + + "deployments hold — nothing was deleted, so it is all still in use.\n" + + "Requires an admin key.", + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&dryRun, "dry-run", false, "report what would be deleted without deleting it") + }, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + stats, err := c.Collect(ctx, dryRun) + if err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(stats, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("dry_run", cliutil.Bool(stats.DryRun)) + t.Row("deployments_deleted", strconv.Itoa(stats.DeploymentsDeleted)) + t.Row("blobs_deleted", strconv.Itoa(stats.BlobsDeleted)) + t.Row("bytes_freed", cliutil.Bytes(stats.BytesFreed)) + return t + }) + }, + } +} + +func systemFsckCmd(g *Globals) *cliutil.Command { + var repair bool + return &cliutil.Command{ + Name: "fsck", + Short: "Check the stored reference counts against the manifests", + Long: "On a healthy server this always reports no drift: the counts are\n" + + "maintained by database triggers. It is for the cases outside normal\n" + + "operation — a restored backup, a database edited by hand — because a\n" + + "count that reads low is content the collector will delete while a\n" + + "deployment still needs it. --repair rewrites the counts from the\n" + + "manifests. Requires an admin key.", + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&repair, "repair", false, "correct the counts that disagree") + }, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + rep, err := c.Fsck(ctx, repair) + if err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(rep, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("blobs", strconv.FormatInt(rep.Blobs, 10)) + t.Row("drift", strconv.Itoa(rep.DriftCount)) + t.Row("repaired", strconv.Itoa(rep.Repaired)) + for _, d := range rep.Drift { + t.Row(cliutil.Truncate(d.Digest, 12), + fmt.Sprintf("stored %d, actual %d", d.Stored, d.Actual)) + } + return t + }) + }, + } +} diff --git a/internal/clicmd/deployment_test.go b/internal/clicmd/deployment_test.go new file mode 100644 index 0000000..942cd94 --- /dev/null +++ b/internal/clicmd/deployment_test.go @@ -0,0 +1,326 @@ +package clicmd + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cliutil" +) + +// cliToken is syntactically plausible and otherwise meaningless: none of the +// fake servers below look at it, but Globals refuses to build a client without +// one. +const cliToken = "pgs_abcdefghijklmnop_secret" + +// runCLI drives the real command tree the way main does and returns what the +// user would have seen. Going through Root rather than calling a command's Exec +// directly is the point: it covers the flag registration and the dispatch that +// a hand-built call would skip. +func runCLI(t *testing.T, server, stdin string, args ...string) (stdout, stderr string, err error) { + t.Helper() + clearEnv(t) + t.Setenv("PAGES_TOKEN", cliToken) + var out, errOut strings.Builder + g := &Globals{ + In: strings.NewReader(stdin), + Out: &out, + Err: &errOut, + Config: filepath.Join(t.TempDir(), "absent.json"), + Server: server, + } + err = cliutil.Run(context.Background(), Root(g), args, &errOut, g.Register) + return out.String(), errOut.String(), err +} + +func writeJSON(t *testing.T, w http.ResponseWriter, v any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(v); err != nil { + t.Error(err) + } +} + +func fakeDeployment(id string) api.Deployment { + return api.Deployment{ + ID: id, Project: "demo", State: "ready", + FileCount: 2, TotalBytes: 4096, + CreatedAt: time.Unix(1700000000, 0).UTC(), + } +} + +// indexOf is strings.Index with a failure message, used to assert ordering. +func indexOf(t *testing.T, haystack, needle string) int { + t.Helper() + i := strings.Index(haystack, needle) + if i < 0 { + t.Fatalf("output does not mention %q:\n%s", needle, haystack) + } + return i +} + +// TestDeploymentListFollowsTheCursor: the listing is read to decide which +// deployment to roll back to, so one that silently showed the first page would +// be worse than one that failed. +func TestDeploymentListFollowsTheCursor(t *testing.T) { + var queries []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries = append(queries, r.URL.RawQuery) + if r.URL.Query().Get("cursor") == "dpl_2" { + writeJSON(t, w, api.DeploymentList{ + Deployments: []api.Deployment{fakeDeployment("dpl_1")}, + }) + return + } + writeJSON(t, w, api.DeploymentList{ + Deployments: []api.Deployment{fakeDeployment("dpl_3"), fakeDeployment("dpl_2")}, + NextCursor: "dpl_2", + }) + })) + defer srv.Close() + + out, _, err := runCLI(t, srv.URL, "", "deployment", "list", "--project", "demo") + if err != nil { + t.Fatalf("deployment list: %v", err) + } + // Newest first, as the server returned them: the order is what tells the + // reader which one is the previous release. + first := indexOf(t, out, "dpl_3") + second := indexOf(t, out, "dpl_2") + third := indexOf(t, out, "dpl_1") + if !(first < second && second < third) { + t.Errorf("rows out of order:\n%s", out) + } + if len(queries) != 2 { + t.Fatalf("made %d requests (%q), want 2", len(queries), queries) + } + if !strings.Contains(queries[1], "cursor=dpl_2") { + t.Errorf("second request query = %q, want the cursor from the first page", queries[1]) + } +} + +// TestDeploymentListStopsAtTheLimit — the server here always offers another +// page, so a --limit that was not honoured would page until the test timed out. +func TestDeploymentListStopsAtTheLimit(t *testing.T) { + var requests int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if requests > 4 { + t.Errorf("still paging after %d requests; --limit was ignored", requests) + writeJSON(t, w, api.DeploymentList{}) + return + } + writeJSON(t, w, api.DeploymentList{ + Deployments: []api.Deployment{fakeDeployment("dpl_3"), fakeDeployment("dpl_2")}, + NextCursor: "dpl_2", + }) + })) + defer srv.Close() + + out, _, err := runCLI(t, srv.URL, "", "deployment", "list", "--project", "demo", "--limit", "2") + if err != nil { + t.Fatalf("deployment list: %v", err) + } + if requests != 1 { + t.Errorf("made %d requests, want 1: two rows already satisfy --limit 2", requests) + } + if !strings.Contains(out, "dpl_3") || !strings.Contains(out, "dpl_2") { + t.Errorf("output is missing a row:\n%s", out) + } +} + +// TestDeploymentListEmptyIsAnEmptyArray: -o json is what a CI step parses, and +// jq treats null and [] very differently. +func TestDeploymentListEmptyIsAnEmptyArray(t *testing.T) { + var query string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query = r.URL.RawQuery + writeJSON(t, w, api.DeploymentList{}) + })) + defer srv.Close() + + out, _, err := runCLI(t, srv.URL, "", + "deployment", "list", "--project", "demo", "--state", "failed", "-o", "json") + if err != nil { + t.Fatalf("deployment list: %v", err) + } + if !strings.Contains(query, "state=failed") { + t.Errorf("query = %q, want the state filter", query) + } + if !strings.Contains(out, `"deployments": []`) { + t.Errorf("output = %s, want an empty array", out) + } + if strings.Contains(out, "null") { + t.Errorf("output = %s, want no null", out) + } +} + +// TestDeploymentShowAsksForFilesOnlyWhenTold — the manifest is one line per +// file, so a large site's would drown the rest of the output. +func TestDeploymentShowAsksForFilesOnlyWhenTold(t *testing.T) { + d := fakeDeployment("dpl_1") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + out := d + if r.URL.Query().Get("files") == "true" { + out.Files = []api.FileEntry{ + {Path: "assets/app.js", Digest: strings.Repeat("ab", 32), Size: 15}, + {Path: "index.html", Digest: strings.Repeat("cd", 32), Size: 42}, + } + } + writeJSON(t, w, out) + })) + defer srv.Close() + + plain, _, err := runCLI(t, srv.URL, "", "deployment", "show", "dpl_1", "--project", "demo") + if err != nil { + t.Fatalf("deployment show: %v", err) + } + if strings.Contains(plain, "assets/app.js") { + t.Errorf("the manifest was printed without --files:\n%s", plain) + } + + full, _, err := runCLI(t, srv.URL, "", "deployment", "show", "dpl_1", "--project", "demo", "--files") + if err != nil { + t.Fatalf("deployment show --files: %v", err) + } + if !strings.Contains(full, "assets/app.js") || !strings.Contains(full, "index.html") { + t.Errorf("--files did not list the manifest:\n%s", full) + } +} + +// TestDeploymentDeleteAsksFirst. Deleting the wrong deployment is not +// recoverable from the CLI, so the prompt is the safety net and --yes is the +// documented way past it. +func TestDeploymentDeleteAsksFirst(t *testing.T) { + newServer := func(seen *[]string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *seen = append(*seen, r.Method+" "+r.URL.Path) + w.WriteHeader(http.StatusNoContent) + })) + } + + t.Run("declining deletes nothing", func(t *testing.T) { + var seen []string + srv := newServer(&seen) + defer srv.Close() + + _, errOut, err := runCLI(t, srv.URL, "n\n", "deployment", "delete", "dpl_1", "--project", "demo") + if !errors.Is(err, cliutil.ErrAborted) { + t.Fatalf("err = %v, want it to report the abort", err) + } + if len(seen) != 0 { + t.Errorf("requests = %q, want none", seen) + } + if !strings.Contains(errOut, "dpl_1") { + t.Errorf("prompt = %q, want it to name the deployment", errOut) + } + }) + + t.Run("confirming deletes", func(t *testing.T) { + var seen []string + srv := newServer(&seen) + defer srv.Close() + + out, _, err := runCLI(t, srv.URL, "y\n", "deployment", "delete", "dpl_1", "--project", "demo") + if err != nil { + t.Fatalf("deployment delete: %v", err) + } + want := "DELETE " + api.PathDeployment("demo", "dpl_1") + if len(seen) != 1 || seen[0] != want { + t.Errorf("requests = %q, want [%q]", seen, want) + } + if !strings.Contains(out, "dpl_1") { + t.Errorf("output = %q, want it to confirm what was deleted", out) + } + }) + + t.Run("--yes does not prompt", func(t *testing.T) { + var seen []string + srv := newServer(&seen) + defer srv.Close() + + // Empty stdin: without --yes this would abort rather than delete. + _, errOut, err := runCLI(t, srv.URL, "", "deployment", "delete", "dpl_1", "--project", "demo", "--yes") + if err != nil { + t.Fatalf("deployment delete --yes: %v", err) + } + if len(seen) != 1 { + t.Errorf("requests = %q, want one delete", seen) + } + if strings.Contains(errOut, "[y/N]") { + t.Errorf("stderr = %q, want no prompt", errOut) + } + }) +} + +// TestSystemGCPostsTheDryRunFlag: a dry run that silently ran for real is the +// worst bug this command could have, so the flag's trip to the wire is checked +// rather than assumed. +func TestSystemGCPostsTheDryRunFlag(t *testing.T) { + var gotPath, gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := make([]byte, 256) + n, _ := r.Body.Read(body) + gotPath, gotBody = r.Method+" "+r.URL.Path, string(body[:n]) + writeJSON(t, w, api.GCStats{DryRun: true, DeploymentsDeleted: 3, BlobsDeleted: 4, BytesFreed: 5120}) + })) + defer srv.Close() + + out, _, err := runCLI(t, srv.URL, "", "system", "gc", "--dry-run") + if err != nil { + t.Fatalf("system gc: %v", err) + } + if want := "POST " + api.PathGC(); gotPath != want { + t.Errorf("request = %q, want %q", gotPath, want) + } + if !strings.Contains(gotBody, `"dry_run":true`) { + t.Errorf("body = %q, want dry_run set", gotBody) + } + for _, want := range []string{"dry_run", "yes", "deployments_deleted", "3", "blobs_deleted", "4"} { + if !strings.Contains(out, want) { + t.Errorf("output is missing %q:\n%s", want, out) + } + } +} + +// TestSystemFsckReportsDrift — the report exists for the case where the counts +// are wrong, so the drifting digests have to reach the operator's screen. +func TestSystemFsckReportsDrift(t *testing.T) { + digest := strings.Repeat("ab", 32) + var gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := make([]byte, 256) + n, _ := r.Body.Read(body) + gotBody = string(body[:n]) + writeJSON(t, w, api.FsckReport{ + Blobs: 9, + DriftCount: 1, + Repaired: 1, + Drift: []api.BlobDrift{{Digest: digest, Stored: 7, Actual: 1}}, + }) + })) + defer srv.Close() + + out, _, err := runCLI(t, srv.URL, "", "system", "fsck", "--repair") + if err != nil { + t.Fatalf("system fsck: %v", err) + } + if !strings.Contains(gotBody, `"repair":true`) { + t.Errorf("body = %q, want repair set", gotBody) + } + // Truncate spends its last column on the ellipsis, so twelve columns of + // digest are eleven characters and a marker that there is more. + if !strings.Contains(out, digest[:11]+"…") { + t.Errorf("output does not name the drifting blob:\n%s", out) + } + if !strings.Contains(out, "stored 7, actual 1") { + t.Errorf("output does not give the counts:\n%s", out) + } +} diff --git a/internal/clicmd/globals.go b/internal/clicmd/globals.go new file mode 100644 index 0000000..605aaa7 --- /dev/null +++ b/internal/clicmd/globals.go @@ -0,0 +1,216 @@ +// Package clicmd implements the pages subcommands. +package clicmd + +import ( + "errors" + "flag" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/iceBear67/simplepages/internal/client" + "github.com/iceBear67/simplepages/internal/cliutil" +) + +// Globals are the settings every command shares. +// +// Precedence is flag > environment > config file > default. Flags are parsed +// into these fields first, so Resolve fills in only what is still empty — which +// is why a flag given as the empty string counts as absent. +type Globals struct { + Server string + Token string + TokenFile string + Project string + Config string + Output string + Timeout time.Duration + Verbose bool + + In io.Reader + Out io.Writer + Err io.Writer + + file ConfigFile + resolved bool +} + +// NewGlobals returns Globals wired to the process's standard streams. +func NewGlobals() *Globals { + return &Globals{In: os.Stdin, Out: os.Stdout, Err: os.Stderr} +} + +// Register adds the global flags to fs. +// +// Every flag's default is the field's current value, so registering on a +// subcommand's FlagSet preserves what an outer parse already set. Passing a +// fixed default here would make "pages --server X project list" lose --server +// the moment the subcommand registered its own copy. +func (g *Globals) Register(fs *flag.FlagSet) { + fs.StringVar(&g.Server, "server", g.Server, "management API base `URL` (env PAGES_SERVER)") + fs.StringVar(&g.Token, "token", g.Token, + "API `token`; avoid it — argv is world-readable via /proc on shared runners, so prefer --token-file or PAGES_TOKEN") + fs.StringVar(&g.TokenFile, "token-file", g.TokenFile, "read the API token from `path` (env PAGES_TOKEN_FILE)") + fs.StringVar(&g.Project, "project", g.Project, "project `name` to operate on (env PAGES_PROJECT)") + fs.StringVar(&g.Config, "config", g.Config, "config file `path` (env PAGES_CONFIG)") + fs.StringVar(&g.Output, "o", g.Output, "output `format`: table or json (env PAGES_OUTPUT)") + fs.StringVar(&g.Output, "output", g.Output, "output `format`: table or json") + fs.DurationVar(&g.Timeout, "timeout", g.Timeout, "per-request `timeout`") + fs.BoolVar(&g.Verbose, "v", g.Verbose, "verbose progress on stderr") + fs.BoolVar(&g.Verbose, "verbose", g.Verbose, "verbose progress on stderr") +} + +// Resolve applies the environment, then the config file, to anything the flags +// did not set. It runs once; later calls are no-ops. +func (g *Globals) Resolve() error { + if g.resolved { + return nil + } + g.resolved = true + + if g.Config == "" { + g.Config = DefaultConfigPath() + } + f, err := LoadConfig(g.Config, g.Err) + if err != nil { + return err + } + g.file = f + + g.Server = firstNonEmpty(g.Server, os.Getenv("PAGES_SERVER"), f.Server) + g.Project = firstNonEmpty(g.Project, os.Getenv("PAGES_PROJECT"), f.Project) + + if err := g.resolveToken(); err != nil { + return err + } + + g.Output = firstNonEmpty(g.Output, os.Getenv("PAGES_OUTPUT"), f.Output, cliutil.FormatTable) + if !cliutil.ValidFormat(g.Output) { + return fmt.Errorf("unknown output format %q: use table or json", g.Output) + } + + if g.Timeout == 0 { + if s := os.Getenv("PAGES_TIMEOUT"); s != "" { + d, err := cliutil.ParseDuration(s) + if err != nil { + return fmt.Errorf("PAGES_TIMEOUT: %w", err) + } + g.Timeout = d + } + } + if g.Timeout <= 0 { + g.Timeout = client.DefaultTimeout + } + return nil +} + +// resolveToken picks the token from the most specific source that has one: +// --token, --token-file, $PAGES_TOKEN, $PAGES_TOKEN_FILE, config file. +func (g *Globals) resolveToken() error { + if g.Token != "" { + // Only a flag can have set this: the environment and the config file are + // read below. Say so once — on a shared runner every other user can read + // this token out of /proc//cmdline for as long as the process lives. + fmt.Fprintln(g.Err, "warning: --token puts the token in the process list; prefer --token-file or PAGES_TOKEN") + return nil + } + if g.TokenFile != "" { + t, err := readTokenFile(g.TokenFile, g.Err) + if err != nil { + return err + } + g.Token = t + return nil + } + if t := os.Getenv("PAGES_TOKEN"); t != "" { + g.Token = strings.TrimSpace(t) + return nil + } + if p := os.Getenv("PAGES_TOKEN_FILE"); p != "" { + t, err := readTokenFile(p, g.Err) + if err != nil { + return err + } + g.Token = t + return nil + } + g.Token = g.file.Token + return nil +} + +// readTokenFile reads a token, tolerating the trailing newline that every text +// editor and `pages key create ... > token` adds. +func readTokenFile(path string, warn io.Writer) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read token file: %w", err) + } + if fi, err := os.Stat(path); err == nil && fi.Mode().Perm()&0o077 != 0 && warn != nil { + fmt.Fprintf(warn, "warning: token file %s is mode %#o; run: chmod 600 %s\n", + path, fi.Mode().Perm(), path) + } + t := strings.TrimSpace(string(raw)) + if t == "" { + return "", fmt.Errorf("token file %s is empty", path) + } + return t, nil +} + +// Client builds an API client from the resolved settings. +func (g *Globals) Client() (*client.Client, error) { + if err := g.Resolve(); err != nil { + return nil, err + } + return client.New(client.Config{ + BaseURL: g.Server, + Token: g.Token, + Timeout: g.Timeout, + }) +} + +// Printer renders command output in the requested format. +func (g *Globals) Printer() (*cliutil.Printer, error) { + if err := g.Resolve(); err != nil { + return nil, err + } + return &cliutil.Printer{Out: g.Out, Format: g.Output}, nil +} + +// ProjectName returns the project to operate on, or an explanation of how to +// name one. +func (g *Globals) ProjectName() (string, error) { + if err := g.Resolve(); err != nil { + return "", err + } + if g.Project == "" { + return "", errors.New("no project: pass --project or set PAGES_PROJECT") + } + return g.Project, nil +} + +// logf writes a progress note to stderr under --verbose. +func (g *Globals) logf(format string, args ...any) { + if g.Verbose { + fmt.Fprintf(g.Err, format+"\n", args...) + } +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} + +// exactArgs is the argument-count check every leaf command starts with. It +// reports a usage error, so the command's own usage text follows the message. +func exactArgs(args []string, n int, want string) error { + if len(args) != n { + return cliutil.UsageErrorf("expected %s, got %d argument(s)", want, len(args)) + } + return nil +} diff --git a/internal/clicmd/globals_test.go b/internal/clicmd/globals_test.go new file mode 100644 index 0000000..5d2f57b --- /dev/null +++ b/internal/clicmd/globals_test.go @@ -0,0 +1,341 @@ +package clicmd + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/iceBear67/simplepages/internal/cliutil" +) + +// clearEnv makes a test independent of whatever the developer has exported. +func clearEnv(t *testing.T) { + t.Helper() + for _, k := range []string{ + "PAGES_SERVER", "PAGES_TOKEN", "PAGES_TOKEN_FILE", + "PAGES_PROJECT", "PAGES_OUTPUT", "PAGES_TIMEOUT", "PAGES_CONFIG", + } { + t.Setenv(k, "") + } +} + +// writeConfig returns the path to a config file holding f. +func writeConfig(t *testing.T, f ConfigFile) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.json") + if err := SaveConfig(path, f); err != nil { + t.Fatal(err) + } + return path +} + +// newTestGlobals is what NewGlobals would give a command, with the streams +// captured and the config file pinned so no real one can interfere. +func newTestGlobals(config string) (*Globals, *strings.Builder, *strings.Builder) { + var out, errOut strings.Builder + return &Globals{In: strings.NewReader(""), Out: &out, Err: &errOut, Config: config}, &out, &errOut +} + +// TestPrecedence is the rule stated in the help text: flag, then environment, +// then config file, then default. It is easy to get subtly wrong, and wrong +// here means a CI job deploying to the wrong server. +func TestPrecedence(t *testing.T) { + file := ConfigFile{Server: "https://file.example.com", Project: "fileproj", Output: "json"} + + t.Run("flag wins", func(t *testing.T) { + clearEnv(t) + t.Setenv("PAGES_SERVER", "https://env.example.com") + t.Setenv("PAGES_PROJECT", "envproj") + g, _, _ := newTestGlobals(writeConfig(t, file)) + g.Server, g.Project = "https://flag.example.com", "flagproj" + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Server != "https://flag.example.com" || g.Project != "flagproj" { + t.Errorf("server=%q project=%q, want the flag values", g.Server, g.Project) + } + }) + + t.Run("environment beats the config file", func(t *testing.T) { + clearEnv(t) + t.Setenv("PAGES_SERVER", "https://env.example.com") + t.Setenv("PAGES_PROJECT", "envproj") + g, _, _ := newTestGlobals(writeConfig(t, file)) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Server != "https://env.example.com" || g.Project != "envproj" { + t.Errorf("server=%q project=%q, want the environment values", g.Server, g.Project) + } + }) + + t.Run("the config file is the last word before defaults", func(t *testing.T) { + clearEnv(t) + g, _, _ := newTestGlobals(writeConfig(t, file)) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Server != "https://file.example.com" || g.Project != "fileproj" || g.Output != "json" { + t.Errorf("server=%q project=%q output=%q, want the file values", g.Server, g.Project, g.Output) + } + }) + + t.Run("defaults fill the rest", func(t *testing.T) { + clearEnv(t) + g, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Output != cliutil.FormatTable { + t.Errorf("output = %q, want %q", g.Output, cliutil.FormatTable) + } + if g.Timeout <= 0 { + t.Errorf("timeout = %v, want a positive default", g.Timeout) + } + }) +} + +// TestFlagsSurviveTheSubcommandParse covers the trap that makes this design +// work: every global flag is re-registered on each subcommand's FlagSet, and +// registering with a fixed default would wipe a value given before the +// subcommand name. +func TestFlagsSurviveTheSubcommandParse(t *testing.T) { + clearEnv(t) + cfg := writeConfig(t, ConfigFile{}) + + for _, args := range [][]string{ + {"--server", "https://flag.example.com", "--config", cfg, "probe", "demo"}, + {"probe", "--server", "https://flag.example.com", "--config", cfg, "demo"}, + {"--server", "https://flag.example.com", "probe", "demo", "--config", cfg}, + } { + t.Run(strings.Join(args, " "), func(t *testing.T) { + g, _, _ := newTestGlobals("") + var gotProject string + root := &cliutil.Command{ + Name: "pages", + Sub: []*cliutil.Command{{ + Name: "probe", + Exec: func(ctx context.Context, args []string) error { + if err := g.Resolve(); err != nil { + return err + } + if len(args) == 1 { + gotProject = args[0] + } + return nil + }, + }}, + } + if err := cliutil.Run(context.Background(), root, args, io.Discard, g.Register); err != nil { + t.Fatalf("Run: %v", err) + } + if g.Server != "https://flag.example.com" { + t.Errorf("server = %q, want the flag to survive dispatch", g.Server) + } + if gotProject != "demo" { + t.Errorf("positional = %q, want demo", gotProject) + } + }) + } +} + +func TestTokenPrecedence(t *testing.T) { + const ( + flagTok = "pgs_flagflagflagfl_secret" + flagFileTok = "pgs_flagfileflagfi_secret" + envTok = "pgs_envenvenvenven_secret" + envFileTok = "pgs_envfileenvfile_secret" + fileTok = "pgs_fileconfigfile_secret" + ) + tokenFile := func(t *testing.T, content string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p + } + + t.Run("--token wins but warns", func(t *testing.T) { + clearEnv(t) + t.Setenv("PAGES_TOKEN", envTok) + g, _, errOut := newTestGlobals(writeConfig(t, ConfigFile{Token: fileTok})) + g.Token = flagTok + g.TokenFile = tokenFile(t, flagFileTok) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Token != flagTok { + t.Errorf("token = %q, want the flag", g.Token) + } + // argv is world-readable through /proc on a shared runner, so this + // warning is the whole reason --token is documented as a last resort. + if !strings.Contains(errOut.String(), "process list") { + t.Errorf("stderr = %q, want a warning about the process list", errOut.String()) + } + if strings.Contains(errOut.String(), "secret") { + t.Error("the warning printed the token") + } + }) + + t.Run("--token-file beats the environment", func(t *testing.T) { + clearEnv(t) + t.Setenv("PAGES_TOKEN", envTok) + g, _, _ := newTestGlobals(writeConfig(t, ConfigFile{Token: fileTok})) + g.TokenFile = tokenFile(t, flagFileTok) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Token != flagFileTok { + t.Errorf("token = %q, want the one from --token-file", g.Token) + } + }) + + t.Run("PAGES_TOKEN beats PAGES_TOKEN_FILE", func(t *testing.T) { + clearEnv(t) + t.Setenv("PAGES_TOKEN", envTok) + t.Setenv("PAGES_TOKEN_FILE", tokenFile(t, envFileTok)) + g, _, _ := newTestGlobals(writeConfig(t, ConfigFile{Token: fileTok})) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Token != envTok { + t.Errorf("token = %q, want PAGES_TOKEN", g.Token) + } + }) + + t.Run("PAGES_TOKEN_FILE beats the config file", func(t *testing.T) { + clearEnv(t) + t.Setenv("PAGES_TOKEN_FILE", tokenFile(t, envFileTok)) + g, _, _ := newTestGlobals(writeConfig(t, ConfigFile{Token: fileTok})) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Token != envFileTok { + t.Errorf("token = %q, want PAGES_TOKEN_FILE", g.Token) + } + }) + + t.Run("the config file is the fallback", func(t *testing.T) { + clearEnv(t) + g, _, _ := newTestGlobals(writeConfig(t, ConfigFile{Token: fileTok})) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Token != fileTok { + t.Errorf("token = %q, want the config file's", g.Token) + } + }) +} + +// TestTokenFileTrimsTrailingNewline: `pages key create ... > token` and every +// text editor add one. +func TestTokenFileTrimsTrailingNewline(t *testing.T) { + clearEnv(t) + p := filepath.Join(t.TempDir(), "token") + os.WriteFile(p, []byte("pgs_abcdefghijklmnop_secret\n"), 0o600) + + g, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + g.TokenFile = p + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Token != "pgs_abcdefghijklmnop_secret" { + t.Errorf("token = %q, want the newline trimmed", g.Token) + } +} + +func TestTokenFileProblemsAreReported(t *testing.T) { + t.Run("missing", func(t *testing.T) { + clearEnv(t) + g, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + g.TokenFile = filepath.Join(t.TempDir(), "nope") + if err := g.Resolve(); err == nil { + t.Fatal("expected an error") + } + }) + + t.Run("empty", func(t *testing.T) { + clearEnv(t) + p := filepath.Join(t.TempDir(), "token") + os.WriteFile(p, []byte("\n\n"), 0o600) + g, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + g.TokenFile = p + err := g.Resolve() + if err == nil || !strings.Contains(err.Error(), "empty") { + t.Fatalf("err = %v, want it to say the file is empty", err) + } + }) + + t.Run("world readable warns", func(t *testing.T) { + clearEnv(t) + p := filepath.Join(t.TempDir(), "token") + os.WriteFile(p, []byte("pgs_abcdefghijklmnop_secret"), 0o644) + g, _, errOut := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + g.TokenFile = p + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if !strings.Contains(errOut.String(), "chmod 600") { + t.Errorf("stderr = %q, want a mode warning", errOut.String()) + } + }) +} + +func TestResolveRejectsUnknownOutputFormat(t *testing.T) { + clearEnv(t) + g, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + g.Output = "yaml" + err := g.Resolve() + if err == nil || !strings.Contains(err.Error(), "table or json") { + t.Fatalf("err = %v, want it to list the formats", err) + } +} + +func TestResolveParsesFriendlyTimeouts(t *testing.T) { + clearEnv(t) + t.Setenv("PAGES_TIMEOUT", "2m") + g, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + if err := g.Resolve(); err != nil { + t.Fatal(err) + } + if g.Timeout.Minutes() != 2 { + t.Errorf("timeout = %v, want 2m", g.Timeout) + } + + clearEnv(t) + t.Setenv("PAGES_TIMEOUT", "later") + g2, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + err := g2.Resolve() + if err == nil || !strings.Contains(err.Error(), "PAGES_TIMEOUT") { + t.Fatalf("err = %v, want it to name the variable", err) + } +} + +func TestProjectNameExplainsHowToSetIt(t *testing.T) { + clearEnv(t) + g, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + _, err := g.ProjectName() + if err == nil || !strings.Contains(err.Error(), "PAGES_PROJECT") { + t.Fatalf("err = %v, want it to name the flag and the variable", err) + } +} + +// TestClientNeedsServerAndToken: the two settings a fresh CI job forgets. +func TestClientNeedsServerAndToken(t *testing.T) { + clearEnv(t) + g, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + if _, err := g.Client(); err == nil || !strings.Contains(err.Error(), "PAGES_SERVER") { + t.Fatalf("err = %v, want it to name PAGES_SERVER", err) + } + + clearEnv(t) + g2, _, _ := newTestGlobals(filepath.Join(t.TempDir(), "absent.json")) + g2.Server = "https://p.example.com" + if _, err := g2.Client(); err == nil || !strings.Contains(err.Error(), "PAGES_TOKEN") { + t.Fatalf("err = %v, want it to name PAGES_TOKEN", err) + } +} diff --git a/internal/clicmd/key.go b/internal/clicmd/key.go new file mode 100644 index 0000000..2ecd1fa --- /dev/null +++ b/internal/clicmd/key.go @@ -0,0 +1,208 @@ +package clicmd + +import ( + "context" + "errors" + "flag" + "fmt" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cliutil" +) + +func keyCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "key", + Short: "Manage API keys", + Sub: []*cliutil.Command{ + keyCreateCmd(g), + keyListCmd(g), + keyRevokeCmd(g), + }, + } +} + +func keyCreateCmd(g *Globals) *cliutil.Command { + var ( + name string + admin bool + expires string + ) + return &cliutil.Command{ + Name: "create", + Short: "Mint a key", + Long: "The token is printed once and cannot be retrieved again — the server\n" + + "stores only its hash. Redirect it straight into a file or a secret\n" + + "store; do not let it reach a CI log.\n\n" + + "Without --admin the key is scoped to one project and can do nothing\n" + + "outside it. Requires an admin key either way.", + Flags: func(fs *flag.FlagSet) { + fs.StringVar(&name, "name", "", "`label` recorded with the key, e.g. github-actions") + fs.BoolVar(&admin, "admin", false, "mint an admin key instead of a project key") + fs.StringVar(&expires, "expires", "", "expire after this `duration`, e.g. 90d; default never") + }, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + req := api.CreateKeyRequest{Name: name} + if expires != "" { + d, err := cliutil.ParseDuration(expires) + if err != nil { + return err + } + if d <= 0 { + return errors.New("--expires must be positive") + } + // Computed here, sent absolute: a clock difference between this + // machine and the server then shifts nothing. + t := time.Now().Add(d).UTC().Truncate(time.Second) + req.ExpiresAt = &t + } + + c, err := g.Client() + if err != nil { + return err + } + var out api.CreateKeyResponse + if admin { + out, err = c.CreateAdminKey(ctx, req) + } else { + var project string + if project, err = g.ProjectName(); err != nil { + return fmt.Errorf("%w, or pass --admin for a server-wide key", err) + } + out, err = c.CreateProjectKey(ctx, project, req) + } + if err != nil { + return err + } + + p, err := g.Printer() + if err != nil { + return err + } + if p.Format == cliutil.FormatJSON { + return p.JSON(out) + } + // Table format prints the token on a line of its own so it survives + // a copy-paste and so `pages key create | tail -1` is not tempting. + if err := keyTable(out.Key).Write(g.Out); err != nil { + return err + } + fmt.Fprintf(g.Out, "\n%s\n", out.Token) + fmt.Fprintln(g.Err, "this token is shown once and cannot be recovered; store it now") + return nil + }, + } +} + +func keyListCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "list", + Short: "List keys", + Long: "With --project, lists that project's keys; a project key may list its\n" + + "own. Without it, lists every key on the server and requires an admin\n" + + "key. Secrets are never listed — only key ids.", + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + if err := g.Resolve(); err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + var list api.KeyList + if g.Project != "" { + list, err = c.ListProjectKeys(ctx, g.Project) + } else { + list, err = c.ListKeys(ctx) + } + if err != nil { + return err + } + if list.Keys == nil { + list.Keys = []api.Key{} + } + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(list, func() *cliutil.Table { + t := cliutil.NewTable("ID", "SCOPE", "PROJECT", "NAME", "CREATED", "EXPIRES", "LAST USED", "STATE") + for _, k := range list.Keys { + t.Row(k.ID, k.Scope, cliutil.Str(k.Project), cliutil.Str(k.Name), + cliutil.Time(k.CreatedAt), cliutil.TimePtr(k.ExpiresAt), + cliutil.TimePtr(k.LastUsed), keyState(k)) + } + return t + }) + }, + } +} + +func keyRevokeCmd(g *Globals) *cliutil.Command { + var yes bool + return &cliutil.Command{ + Name: "revoke", + Args: "", + Short: "Revoke a key", + Long: "Takes effect immediately across the server. The key id is the middle\n" + + "segment of a token (pgs__) and is what `key list`\n" + + "shows. Revoking an already-revoked key succeeds.", + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&yes, "yes", false, "do not ask for confirmation") + }, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 1, "one key id"); err != nil { + return err + } + id := args[0] + if !yes { + if err := cliutil.Confirm(g.In, g.Err, fmt.Sprintf("Revoke key %s?", id)); err != nil { + return err + } + } + c, err := g.Client() + if err != nil { + return err + } + if err := c.RevokeKey(ctx, id); err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + p.Printf("revoked key %s\n", id) + return nil + }, + } +} + +// keyState collapses the two timestamps that decide whether a key still works. +func keyState(k api.Key) string { + switch { + case k.Revoked(): + return "revoked" + case k.ExpiresAt != nil && k.ExpiresAt.Before(time.Now()): + return "expired" + default: + return "active" + } +} + +func keyTable(k api.Key) *cliutil.Table { + t := cliutil.NewTable() + t.Row("id", k.ID) + t.Row("scope", k.Scope) + t.Row("project", cliutil.Str(k.Project)) + t.Row("name", cliutil.Str(k.Name)) + t.Row("created_at", cliutil.Time(k.CreatedAt)) + t.Row("expires_at", cliutil.TimePtr(k.ExpiresAt)) + return t +} diff --git a/internal/clicmd/project.go b/internal/clicmd/project.go new file mode 100644 index 0000000..127d87e --- /dev/null +++ b/internal/clicmd/project.go @@ -0,0 +1,278 @@ +package clicmd + +import ( + "context" + "flag" + "fmt" + "strconv" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cliutil" +) + +// projectFlags are the settings shared by "project create" and "project +// update". They map one-to-one onto api.ProjectPatch, whose fields are pointers +// so an absent flag and an explicitly emptied one stay distinguishable. +type projectFlags struct { + displayName cliutil.OptString + indexFile cliutil.OptString + notFoundFile cliutil.OptString + spa cliutil.OptBool + cacheControl cliutil.OptString + retention cliutil.OptInt + grace cliutil.OptDuration + maxFiles cliutil.OptInt + maxFileBytes cliutil.OptBytes + maxTotalBytes cliutil.OptBytes +} + +func (p *projectFlags) register(fs *flag.FlagSet) { + fs.Var(&p.displayName, "display-name", "human-readable `name` shown in listings") + fs.Var(&p.indexFile, "index-file", "document served for a directory, e.g. `index.html`") + fs.Var(&p.notFoundFile, "not-found-file", "document served with 404, e.g. `404.html`; empty clears it") + fs.Var(&p.spa, "spa", "serve the index document for unknown paths that accept HTML") + fs.Var(&p.cacheControl, "cache-control", "Cache-Control `header` sent with every file") + fs.Var(&p.retention, "retention", "`count` of finished deployments to keep per project") + fs.Var(&p.grace, "retention-grace", "`duration` a deployment stays after being replaced, e.g. 1h") + fs.Var(&p.maxFiles, "max-files", "`count` of files allowed in one deployment") + fs.Var(&p.maxFileBytes, "max-file-bytes", "largest single file, e.g. `256MiB`") + fs.Var(&p.maxTotalBytes, "max-total-bytes", "largest total deployment, e.g. `2GiB`") +} + +func (p *projectFlags) patch() api.ProjectPatch { + return api.ProjectPatch{ + DisplayName: p.displayName.Ptr(), + IndexFile: p.indexFile.Ptr(), + NotFoundFile: p.notFoundFile.Ptr(), + SPAFallback: p.spa.Ptr(), + CacheControl: p.cacheControl.Ptr(), + RetentionCount: p.retention.Ptr(), + RetentionGrace: p.grace.SecondsPtr(), + MaxFiles: p.maxFiles.Ptr(), + MaxFileBytes: p.maxFileBytes.Ptr(), + MaxTotalBytes: p.maxTotalBytes.Ptr(), + } +} + +func projectCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "project", + Short: "Manage projects", + Sub: []*cliutil.Command{ + projectCreateCmd(g), + projectListCmd(g), + projectShowCmd(g), + projectUpdateCmd(g), + projectDeleteCmd(g), + }, + } +} + +func projectCreateCmd(g *Globals) *cliutil.Command { + var pf projectFlags + return &cliutil.Command{ + Name: "create", + Args: "", + Short: "Create a project", + Long: "The name becomes the URL prefix, so it is restricted to lowercase\n" + + "letters, digits, dot, dash and underscore, and cannot be changed later.\n" + + "Settings left unset take the server's defaults. Requires an admin key.", + Flags: pf.register, + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 1, "one project name"); err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + patch := pf.patch() + p, err := c.CreateProject(ctx, api.CreateProjectRequest{Name: args[0], Patch: &patch}) + if err != nil { + return err + } + return g.printProject(p) + }, + } +} + +func projectListCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "list", + Short: "List projects", + Long: "Requires an admin key. Follows paging to the end.", + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + ps, err := c.ListAllProjects(ctx) + if err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + // api.ProjectList rather than the bare slice, so -o json produces the + // same shape the API returns and a nil slice still prints as []. + out := api.ProjectList{Projects: ps} + if out.Projects == nil { + out.Projects = []api.Project{} + } + return p.Print(out, func() *cliutil.Table { + t := cliutil.NewTable("NAME", "DISPLAY NAME", "INDEX", "SPA", "KEEP", "UPDATED") + for _, pr := range ps { + t.Row(pr.Name, cliutil.Str(cliutil.Truncate(pr.DisplayName, 32)), + pr.IndexFile, cliutil.Bool(pr.SPAFallback), + strconv.Itoa(pr.RetentionCount), cliutil.Time(pr.UpdatedAt)) + } + return t + }) + }, + } +} + +func projectShowCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "show", + Args: "[name]", + Short: "Show one project", + Long: "Defaults to --project. A project key may read only its own project.", + Exec: func(ctx context.Context, args []string) error { + name, err := g.oneProject(args) + if err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + p, err := c.GetProject(ctx, name) + if err != nil { + return err + } + return g.printProject(p) + }, + } +} + +func projectUpdateCmd(g *Globals) *cliutil.Command { + var pf projectFlags + return &cliutil.Command{ + Name: "update", + Args: "[name]", + Short: "Change a project's settings", + Long: "Only the settings named by flags are changed. Requires an admin key.\n" + + "--not-found-file= with an empty value clears the custom 404 document.", + Flags: pf.register, + Exec: func(ctx context.Context, args []string) error { + name, err := g.oneProject(args) + if err != nil { + return err + } + patch := pf.patch() + if patch == (api.ProjectPatch{}) { + return fmt.Errorf("nothing to change: pass at least one setting flag") + } + c, err := g.Client() + if err != nil { + return err + } + p, err := c.PatchProject(ctx, name, patch) + if err != nil { + return err + } + return g.printProject(p) + }, + } +} + +func projectDeleteCmd(g *Globals) *cliutil.Command { + var yes bool + return &cliutil.Command{ + Name: "delete", + Args: "[name]", + Short: "Delete a project and everything in it", + Long: "Removes the project's keys and deployments and unpublishes the site.\n" + + "The uploaded content is reclaimed by the next garbage collection.\n" + + "Requires an admin key.", + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&yes, "yes", false, "do not ask for confirmation") + }, + Exec: func(ctx context.Context, args []string) error { + name, err := g.oneProject(args) + if err != nil { + return err + } + if !yes { + if err := cliutil.Confirm(g.In, g.Err, + fmt.Sprintf("Delete project %q, its keys and all its deployments?", name)); err != nil { + return err + } + } + c, err := g.Client() + if err != nil { + return err + } + if err := c.DeleteProject(ctx, name); err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + p.Printf("deleted project %s\n", name) + return nil + }, + } +} + +// oneProject takes the project from the positional argument, falling back to +// --project. Both are accepted because "pages project show demo" reads better +// at a prompt while "--project" is what a CI job already has set. +func (g *Globals) oneProject(args []string) (string, error) { + switch len(args) { + case 0: + return g.ProjectName() + case 1: + return args[0], nil + default: + return "", fmt.Errorf("expected at most one project name") + } +} + +func (g *Globals) printProject(pr api.Project) error { + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(pr, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("name", pr.Name) + t.Row("display_name", cliutil.Str(pr.DisplayName)) + t.Row("url", cliutil.Str(pr.URL)) + t.Row("index_file", pr.IndexFile) + t.Row("not_found_file", cliutil.Str(pr.NotFoundFile)) + t.Row("spa_fallback", cliutil.Bool(pr.SPAFallback)) + t.Row("cache_control", pr.CacheControl) + t.Row("retention_count", strconv.Itoa(pr.RetentionCount)) + t.Row("retention_grace", (time.Duration(pr.RetentionGrace) * time.Second).String()) + t.Row("max_files", strconv.Itoa(pr.MaxFiles)) + t.Row("max_file_bytes", cliutil.Bytes(pr.MaxFileBytes)) + t.Row("max_total_bytes", cliutil.Bytes(pr.MaxTotalBytes)) + t.Row("created_at", cliutil.Time(pr.CreatedAt)) + t.Row("updated_at", cliutil.Time(pr.UpdatedAt)) + if d := pr.ActiveDeployment; d != nil { + t.Row("active_deployment", d.ID) + t.Row("active_files", strconv.Itoa(d.FileCount)) + t.Row("active_bytes", cliutil.Bytes(d.TotalBytes)) + t.Row("activated_at", cliutil.TimePtr(d.ActivatedAt)) + } + return t + }) +} diff --git a/internal/clicmd/root.go b/internal/clicmd/root.go new file mode 100644 index 0000000..9c65dfe --- /dev/null +++ b/internal/clicmd/root.go @@ -0,0 +1,272 @@ +package clicmd + +import ( + "context" + "flag" + "fmt" + "strconv" + "time" + + "github.com/iceBear67/simplepages/internal/cliutil" + "github.com/iceBear67/simplepages/internal/version" +) + +// Root builds the command tree. +func Root(g *Globals) *cliutil.Command { + var showVersion bool + return &cliutil.Command{ + Name: "pages", + Short: "Deploy static sites atomically", + Long: "Settings are taken from flags first, then PAGES_* environment\n" + + "variables, then the config file, then built-in defaults.\n\n" + + " PAGES_SERVER management API base URL\n" + + " PAGES_TOKEN API token\n" + + " PAGES_TOKEN_FILE file holding the API token\n" + + " PAGES_PROJECT default project\n" + + " PAGES_OUTPUT table or json\n" + + " PAGES_TIMEOUT per-request timeout\n" + + " PAGES_CONFIG config file path", + Flags: func(fs *flag.FlagSet) { + fs.BoolVar(&showVersion, "version", false, "print the version and exit") + }, + Exec: func(ctx context.Context, args []string) error { + if showVersion { + fmt.Fprintln(g.Out, version.String()) + return nil + } + return cliutil.ErrUsage + }, + Sub: []*cliutil.Command{ + deployCmd(g), + projectCmd(g), + deploymentCmd(g), + keyCmd(g), + whoamiCmd(g), + systemCmd(g), + configCmd(g), + versionCmd(g), + }, + } +} + +func versionCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "version", + Short: "Print the version", + Long: "Reports the client build only; it does not contact the server.", + Exec: func(ctx context.Context, args []string) error { + fmt.Fprintln(g.Out, version.String()) + return nil + }, + } +} + +func whoamiCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "whoami", + Short: "Show which key is being used", + Long: "Useful for confirming a CI runner picked up the credential you meant.", + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + who, err := c.WhoAmI(ctx) + if err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(who, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("key_id", who.KeyID) + t.Row("scope", who.Scope) + t.Row("project", cliutil.Str(who.Project)) + t.Row("name", cliutil.Str(who.Name)) + t.Row("expires_at", cliutil.TimePtr(who.ExpiresAt)) + t.Row("server", c.BaseURL()) + return t + }) + }, + } +} + +func systemCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "system", + Short: "Inspect and maintain the server", + Sub: []*cliutil.Command{systemInfoCmd(g), systemGCCmd(g), systemFsckCmd(g)}, + } +} + +func systemInfoCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "info", + Short: "Show server version and storage counters", + Long: "Requires an admin key.", + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + c, err := g.Client() + if err != nil { + return err + } + info, err := c.SystemInfo(ctx) + if err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(info, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("version", info.Version) + t.Row("uptime", (time.Duration(info.UptimeS) * time.Second).String()) + t.Row("schema_version", strconv.Itoa(info.SchemaVer)) + t.Row("projects", strconv.FormatInt(info.Projects, 10)) + t.Row("deployments", strconv.FormatInt(info.Deployments, 10)) + t.Row("blobs", strconv.FormatInt(info.Blobs, 10)) + t.Row("cas_bytes", cliutil.Bytes(info.CASBytes)) + // How deployment trees are built on this host: hardlink shares + // inodes with the store, copy does not, and the difference shows + // up as disk usage. + t.Row("link_mode", info.LinkMode) + return t + }) + }, + } +} + +// -------------------------------------------------------------------- config + +func configCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "config", + Short: "Read and write the CLI config file", + Long: "The config file is JSON, mode 0600, at $PAGES_CONFIG or\n" + + "$XDG_CONFIG_HOME/pages/config.json. It is a convenience for a\n" + + "workstation; CI should use PAGES_* environment variables instead.", + Sub: []*cliutil.Command{ + configShowCmd(g), + configSetCmd(g), + configPathCmd(g), + }, + } +} + +// configView is what "config show" prints: the file's contents with the token +// reduced to its public half. +type configView struct { + Path string `json:"path"` + Server string `json:"server,omitempty"` + Token string `json:"token,omitempty"` + Project string `json:"project,omitempty"` + Output string `json:"output,omitempty"` +} + +func configShowCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "show", + Short: "Show the config file, with the token redacted", + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + if err := g.Resolve(); err != nil { + return err + } + // From the file, not from the resolved settings: this command answers + // "what is stored here", and printing an environment-supplied token + // back at the user would be actively misleading. + view := configView{ + Path: g.Config, + Server: g.file.Server, + Token: redactToken(g.file.Token), + Project: g.file.Project, + Output: g.file.Output, + } + p, err := g.Printer() + if err != nil { + return err + } + return p.Print(view, func() *cliutil.Table { + t := cliutil.NewTable() + t.Row("path", cliutil.Str(view.Path)) + t.Row("server", cliutil.Str(view.Server)) + t.Row("token", cliutil.Str(view.Token)) + t.Row("project", cliutil.Str(view.Project)) + t.Row("output", cliutil.Str(view.Output)) + return t + }) + }, + } +} + +func configSetCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "set", + Args: " ", + Short: "Set one config value", + Long: "An empty value removes the setting. The file is created mode 0600.\n" + + "Reading a token from a file avoids putting it in your shell history:\n" + + " pages config set token \"$(cat token.txt)\"", + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 2, "a key and a value"); err != nil { + return err + } + if err := g.Resolve(); err != nil { + return err + } + key, value := args[0], args[1] + f := g.file + switch key { + case "server": + f.Server = value + case "token": + f.Token = value + case "project": + f.Project = value + case "output": + if value != "" && !cliutil.ValidFormat(value) { + return fmt.Errorf("unknown output format %q: use table or json", value) + } + f.Output = value + default: + return cliutil.UsageErrorf("unknown config key %q", key) + } + if err := SaveConfig(g.Config, f); err != nil { + return err + } + p, err := g.Printer() + if err != nil { + return err + } + p.Printf("set %s in %s\n", key, g.Config) + return nil + }, + } +} + +func configPathCmd(g *Globals) *cliutil.Command { + return &cliutil.Command{ + Name: "path", + Short: "Print the config file path", + Exec: func(ctx context.Context, args []string) error { + if err := exactArgs(args, 0, "no arguments"); err != nil { + return err + } + if err := g.Resolve(); err != nil { + return err + } + fmt.Fprintln(g.Out, g.Config) + return nil + }, + } +} diff --git a/internal/client/calls.go b/internal/client/calls.go new file mode 100644 index 0000000..1a653f7 --- /dev/null +++ b/internal/client/calls.go @@ -0,0 +1,210 @@ +package client + +import ( + "context" + "net/http" + "net/url" + "strconv" + + "github.com/iceBear67/simplepages/api" +) + +// ---------------------------------------------------------------- identity + +// WhoAmI describes the credential the client is using. +func (c *Client) WhoAmI(ctx context.Context) (api.WhoAmI, error) { + var out api.WhoAmI + err := c.do(ctx, http.MethodGet, api.PathWhoAmI(), nil, &out) + return out, err +} + +// SystemInfo returns server-wide counters. Admin scope only. +func (c *Client) SystemInfo(ctx context.Context) (api.SystemInfo, error) { + var out api.SystemInfo + err := c.do(ctx, http.MethodGet, api.PathSystemInfo(), nil, &out) + return out, err +} + +// ---------------------------------------------------------------- projects + +// CreateProject creates a project. Admin scope only. +func (c *Client) CreateProject(ctx context.Context, req api.CreateProjectRequest) (api.Project, error) { + var out api.Project + err := c.do(ctx, http.MethodPost, api.PathProjects(), req, &out) + return out, err +} + +// ListOptions pages a listing endpoint. A zero Limit takes the server default. +type ListOptions struct { + Limit int + Cursor string +} + +func (o ListOptions) query() string { + q := url.Values{} + if o.Limit > 0 { + q.Set("limit", strconv.Itoa(o.Limit)) + } + if o.Cursor != "" { + q.Set("cursor", o.Cursor) + } + if len(q) == 0 { + return "" + } + return "?" + q.Encode() +} + +// ListProjects returns one page of projects. Admin scope only. +func (c *Client) ListProjects(ctx context.Context, opts ListOptions) (api.ProjectList, error) { + var out api.ProjectList + err := c.do(ctx, http.MethodGet, api.PathProjects()+opts.query(), nil, &out) + return out, err +} + +// ListAllProjects follows the cursor to the end. +// +// The CLI pages on the user's behalf because "pages project list" that silently +// showed the first hundred of three hundred projects would be a lie; a caller +// that wants one page asks for ListProjects. +func (c *Client) ListAllProjects(ctx context.Context) ([]api.Project, error) { + var all []api.Project + opts := ListOptions{Limit: 500} + for { + page, err := c.ListProjects(ctx, opts) + if err != nil { + return nil, err + } + all = append(all, page.Projects...) + if page.NextCursor == "" || len(page.Projects) == 0 { + return all, nil + } + opts.Cursor = page.NextCursor + } +} + +// GetProject reads one project. A project-scoped key may read only its own. +func (c *Client) GetProject(ctx context.Context, name string) (api.Project, error) { + var out api.Project + err := c.do(ctx, http.MethodGet, api.PathProject(name), nil, &out) + return out, err +} + +// PatchProject applies a partial update. Admin scope only. +func (c *Client) PatchProject(ctx context.Context, name string, patch api.ProjectPatch) (api.Project, error) { + var out api.Project + err := c.do(ctx, http.MethodPatch, api.PathProject(name), patch, &out) + return out, err +} + +// DeleteProject removes a project and everything under it. Admin scope only. +func (c *Client) DeleteProject(ctx context.Context, name string) error { + return c.do(ctx, http.MethodDelete, api.PathProject(name), nil, nil) +} + +// -------------------------------------------------------------------- keys + +// CreateAdminKey mints an admin key. Admin scope only. +// +// The returned token is the only copy that will ever exist; the caller shows it +// once and must not log it. +func (c *Client) CreateAdminKey(ctx context.Context, req api.CreateKeyRequest) (api.CreateKeyResponse, error) { + var out api.CreateKeyResponse + err := c.do(ctx, http.MethodPost, api.PathKeys(), req, &out) + return out, err +} + +// CreateProjectKey mints a key scoped to one project. Admin scope only. +func (c *Client) CreateProjectKey(ctx context.Context, project string, req api.CreateKeyRequest) (api.CreateKeyResponse, error) { + var out api.CreateKeyResponse + err := c.do(ctx, http.MethodPost, api.PathProjectKeys(project), req, &out) + return out, err +} + +// ListKeys returns every key on the server. Admin scope only. +func (c *Client) ListKeys(ctx context.Context) (api.KeyList, error) { + var out api.KeyList + err := c.do(ctx, http.MethodGet, api.PathKeys(), nil, &out) + return out, err +} + +// ListProjectKeys returns the keys of one project. +func (c *Client) ListProjectKeys(ctx context.Context, project string) (api.KeyList, error) { + var out api.KeyList + err := c.do(ctx, http.MethodGet, api.PathProjectKeys(project), nil, &out) + return out, err +} + +// RevokeKey revokes a key by id. It takes effect immediately, and is +// idempotent: revoking an already-revoked key succeeds. +func (c *Client) RevokeKey(ctx context.Context, keyID string) error { + return c.do(ctx, http.MethodDelete, api.PathKey(keyID), nil, nil) +} + +// ------------------------------------------------------------- deployments + +// DeploymentListOptions selects and pages a project's deployments. A zero State +// lists every state. +type DeploymentListOptions struct { + ListOptions + State string +} + +func (o DeploymentListOptions) query() string { + q := url.Values{} + if o.Limit > 0 { + q.Set("limit", strconv.Itoa(o.Limit)) + } + if o.Cursor != "" { + q.Set("cursor", o.Cursor) + } + if o.State != "" { + q.Set("state", o.State) + } + if len(q) == 0 { + return "" + } + return "?" + q.Encode() +} + +// ListDeployments returns one page of a project's deployments, newest first. +func (c *Client) ListDeployments(ctx context.Context, project string, opts DeploymentListOptions) (api.DeploymentList, error) { + var out api.DeploymentList + err := c.do(ctx, http.MethodGet, api.PathDeployments(project)+opts.query(), nil, &out) + return out, err +} + +// GetDeployment reads one deployment. withFiles asks for its manifest too, +// which for a large site is a great deal more response than the rest of it. +func (c *Client) GetDeployment(ctx context.Context, project, id string, withFiles bool) (api.Deployment, error) { + var out api.Deployment + path := api.PathDeployment(project, id) + if withFiles { + path += "?files=true" + } + err := c.do(ctx, http.MethodGet, path, nil, &out) + return out, err +} + +// DeleteDeployment removes a deployment and its files. The active one cannot be +// deleted; the server answers deployment_active until something else is +// activated. +func (c *Client) DeleteDeployment(ctx context.Context, project, id string) error { + return c.do(ctx, http.MethodDelete, api.PathDeployment(project, id), nil, nil) +} + +// ---------------------------------------------------------------- upkeep + +// Collect runs a garbage collection pass. Admin scope only. +func (c *Client) Collect(ctx context.Context, dryRun bool) (api.GCStats, error) { + var out api.GCStats + err := c.do(ctx, http.MethodPost, api.PathGC(), api.GCRequest{DryRun: dryRun}, &out) + return out, err +} + +// Fsck checks the blob reference counts against the manifests, optionally +// correcting them. Admin scope only. +func (c *Client) Fsck(ctx context.Context, repair bool) (api.FsckReport, error) { + var out api.FsckReport + err := c.do(ctx, http.MethodPost, api.PathFsck(), api.FsckRequest{Repair: repair}, &out) + return out, err +} diff --git a/internal/client/client.go b/internal/client/client.go new file mode 100644 index 0000000..39d3fba --- /dev/null +++ b/internal/client/client.go @@ -0,0 +1,224 @@ +// Package client is the HTTP client for the pages management API. +// +// It depends on the standard library and github.com/iceBear67/simplepages/api +// only — see cmd/pages/deps_test.go, which fails the build if a server-side +// package ever reaches the CLI through here. +package client + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/version" +) + +// DefaultTimeout bounds a single management request. +const DefaultTimeout = 30 * time.Second + +// maxErrorBody caps how much of a non-2xx body is read before giving up on +// finding an error envelope in it. A misrouted request can land on something +// that answers with a megabyte of HTML. +const maxErrorBody = 64 << 10 + +// Config configures a Client. +type Config struct { + // BaseURL is the management API root, e.g. "https://pages.example.com". + // Any trailing slash is trimmed. + BaseURL string + // Token is the bearer credential. It is sent in the Authorization header + // and must never appear in a URL, a log line or an error message. + Token string + // Timeout bounds each request. Zero means DefaultTimeout. + Timeout time.Duration + // HTTP overrides the underlying client. Its Timeout field is ignored: + // deadlines come from the request context so the deploy path can give a + // large blob upload longer than a management call. + HTTP *http.Client + // UserAgent overrides the default "pages/". + UserAgent string +} + +// Client talks to the management API. +type Client struct { + base string + token string + timeout time.Duration + http *http.Client + agent string +} + +// New validates cfg and returns a client. +func New(cfg Config) (*Client, error) { + raw := strings.TrimSpace(cfg.BaseURL) + if raw == "" { + return nil, errors.New("no server URL: pass --server or set PAGES_SERVER") + } + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("invalid server URL %q: %w", raw, err) + } + switch u.Scheme { + case "http", "https": + case "": + return nil, fmt.Errorf("invalid server URL %q: missing scheme, try https://%s", raw, raw) + default: + return nil, fmt.Errorf("invalid server URL %q: scheme must be http or https", raw) + } + if u.Host == "" { + return nil, fmt.Errorf("invalid server URL %q: missing host", raw) + } + if cfg.Token == "" { + return nil, errors.New("no token: pass --token-file or set PAGES_TOKEN") + } + + hc := cfg.HTTP + if hc == nil { + tr := http.DefaultTransport.(*http.Transport).Clone() + // The deploy path uploads blobs concurrently to one host. + tr.MaxIdleConnsPerHost = 32 + hc = &http.Client{Transport: tr} + } + // A redirect is a misconfigured --server, and following it silently would + // send the bearer token somewhere the operator did not name. Report it and + // let them fix the URL. (Go strips Authorization across hosts anyway, which + // would turn the redirect into a confusing 401 instead.) + hc.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return fmt.Errorf("server redirected to %s — use that as --server", req.URL.Redacted()) + } + + agent := cfg.UserAgent + if agent == "" { + agent = "pages/" + version.Version + } + timeout := cfg.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + + return &Client{ + base: strings.TrimRight(u.String(), "/"), + token: cfg.Token, + timeout: timeout, + http: hc, + agent: agent, + }, nil +} + +// BaseURL returns the server root the client was configured with. +func (c *Client) BaseURL() string { return c.base } + +// do performs a request with a JSON body and decodes a JSON response. +// +// body may be nil for a bodyless request; out may be nil to discard the +// response (204, or a response the caller does not need). +func (c *Client) do(ctx context.Context, method, path string, body, out any) error { + var rdr io.Reader + if body != nil { + buf, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("encode request: %w", err) + } + rdr = bytes.NewReader(buf) + } + + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", c.agent) + if rdr != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := c.http.Do(req) + if err != nil { + return requestError(method, path, err) + } + defer func() { + // Drain a little so the connection can be reused; a huge unread body is + // not worth keeping the connection for. + io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) + resp.Body.Close() + }() + + if resp.StatusCode >= 400 { + return responseError(resp) + } + if out == nil || resp.StatusCode == http.StatusNoContent { + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("decode %s %s response: %w", method, path, err) + } + return nil +} + +// requestError describes a transport-level failure. The URL is included; the +// token is not, and cannot be — it never leaves the Authorization header. +func requestError(method, path string, err error) error { + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("%s %s: timed out; raise --timeout if the server is slow", method, path) + } + return fmt.Errorf("%s %s: %w", method, path, err) +} + +// responseError turns a non-2xx response into an *api.Error, so callers can +// switch on api.CodeOf and the CLI can print "project_exists: ...". +func responseError(resp *http.Response) error { + raw, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBody)) + + var env api.ErrorEnvelope + if err := json.Unmarshal(raw, &env); err == nil && env.Error.Code != "" { + return &env.Error + } + + // Not an envelope: a proxy, a wrong --server, or a bug. Say what arrived + // instead of pretending to a code we did not receive. + msg := strings.TrimSpace(string(raw)) + if len(msg) > 200 { + msg = msg[:200] + "…" + } + if msg == "" { + msg = http.StatusText(resp.StatusCode) + } + return &api.Error{Code: codeForStatus(resp.StatusCode), Message: msg} +} + +func codeForStatus(status int) api.Code { + switch status { + case http.StatusBadRequest: + return api.CodeBadRequest + case http.StatusUnauthorized: + return api.CodeUnauthorized + case http.StatusForbidden: + return api.CodeForbidden + case http.StatusNotFound: + return api.CodeNotFound + case http.StatusMethodNotAllowed: + return api.CodeMethodNotAllowed + case http.StatusConflict: + return api.CodeConflict + case http.StatusRequestEntityTooLarge: + return api.CodePayloadTooLarge + case http.StatusTooManyRequests: + return api.CodeRateLimited + case http.StatusServiceUnavailable: + return api.CodeUnavailable + default: + return api.CodeInternal + } +} diff --git a/internal/client/client_test.go b/internal/client/client_test.go new file mode 100644 index 0000000..322f8a8 --- /dev/null +++ b/internal/client/client_test.go @@ -0,0 +1,363 @@ +package client + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" +) + +const testToken = "pgs_abcdefghijklmnop_ThisIsTheSecretHalfAndMustNotLeakAnywhere" + +func TestNewValidatesConfig(t *testing.T) { + cases := []struct { + desc string + cfg Config + wantErr string + }{ + {"no server", Config{Token: testToken}, "no server URL"}, + {"no token", Config{BaseURL: "https://p.example.com"}, "no token"}, + {"missing scheme", Config{BaseURL: "p.example.com", Token: testToken}, "missing scheme"}, + {"wrong scheme", Config{BaseURL: "ftp://p.example.com", Token: testToken}, "scheme must be http or https"}, + {"missing host", Config{BaseURL: "https://", Token: testToken}, "missing host"}, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + _, err := New(tc.cfg) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want one containing %q", err, tc.wantErr) + } + // A rejected URL is echoed back so the operator can see the typo, but + // the token must never turn up in a message they might paste anywhere. + if err != nil && strings.Contains(err.Error(), testToken) { + t.Error("error message contains the token") + } + }) + } +} + +func TestNewTrimsTrailingSlash(t *testing.T) { + c, err := New(Config{BaseURL: "https://p.example.com/", Token: testToken}) + if err != nil { + t.Fatal(err) + } + if got := c.BaseURL(); got != "https://p.example.com" { + t.Errorf("BaseURL = %q, want no trailing slash", got) + } +} + +// TestRequestCarriesTokenInHeaderOnly is the wire half of the rule that a +// token never reaches a proxy access log: it belongs in Authorization, and +// nowhere near the URL. +func TestRequestCarriesTokenInHeaderOnly(t *testing.T) { + var gotAuth, gotURL, gotAccept, gotAgent string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth, gotURL = r.Header.Get("Authorization"), r.URL.String() + gotAccept, gotAgent = r.Header.Get("Accept"), r.Header.Get("User-Agent") + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"key_id":"abcdefghijklmnop","scope":"admin"}`)) + })) + defer srv.Close() + + c, err := New(Config{BaseURL: srv.URL, Token: testToken}) + if err != nil { + t.Fatal(err) + } + who, err := c.WhoAmI(context.Background()) + if err != nil { + t.Fatal(err) + } + if who.KeyID != "abcdefghijklmnop" || who.Scope != "admin" { + t.Errorf("decoded %+v", who) + } + if gotAuth != "Bearer "+testToken { + t.Errorf("Authorization = %q", gotAuth) + } + if strings.Contains(gotURL, "pgs_") { + t.Errorf("token reached the URL: %q", gotURL) + } + if gotAccept != "application/json" { + t.Errorf("Accept = %q", gotAccept) + } + if !strings.HasPrefix(gotAgent, "pages/") { + t.Errorf("User-Agent = %q", gotAgent) + } +} + +// TestRedirectIsRefused: following a redirect would send the bearer token to a +// host the operator never named. +func TestRedirectIsRefused(t *testing.T) { + elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("request followed the redirect and reached %s", r.Host) + })) + defer elsewhere.Close() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, elsewhere.URL+r.URL.Path, http.StatusFound) + })) + defer srv.Close() + + c, _ := New(Config{BaseURL: srv.URL, Token: testToken}) + _, err := c.WhoAmI(context.Background()) + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), "redirected") { + t.Errorf("err = %v, want it to explain the redirect", err) + } +} + +func TestErrorResponses(t *testing.T) { + cases := []struct { + desc string + status int + body string + contentType string + wantCode api.Code + wantMsg string + }{ + { + desc: "server envelope is used verbatim", + status: http.StatusConflict, + body: `{"error":{"code":"project_exists","message":"project demo already exists"}}`, + wantCode: "project_exists", + wantMsg: "project demo already exists", + }, + { + desc: "a bare proxy error still gets a code from the status", + status: http.StatusForbidden, + body: "403 Forbidden", + wantCode: api.CodeForbidden, + wantMsg: "403 Forbidden", + }, + { + desc: "an empty body falls back to the status text", + status: http.StatusBadGateway, + body: "", + wantCode: api.CodeInternal, + wantMsg: "Bad Gateway", + }, + { + desc: "an envelope without a code is not an envelope", + status: http.StatusNotFound, + body: `{"error":{"message":"nope"}}`, + wantCode: api.CodeNotFound, + wantMsg: `{"error":{"message":"nope"}}`, + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.status) + w.Write([]byte(tc.body)) + })) + defer srv.Close() + + c, _ := New(Config{BaseURL: srv.URL, Token: testToken}) + _, err := c.WhoAmI(context.Background()) + + var apiErr *api.Error + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v (%T), want *api.Error", err, err) + } + if apiErr.Code != tc.wantCode { + t.Errorf("code = %q, want %q", apiErr.Code, tc.wantCode) + } + if apiErr.Message != tc.wantMsg { + t.Errorf("message = %q, want %q", apiErr.Message, tc.wantMsg) + } + }) + } +} + +// TestHugeErrorBodyIsTruncated: a wrong --server can point at something that +// answers every request with a megabyte of HTML. +func TestHugeErrorBodyIsTruncated(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(strings.Repeat("x", 1<<20))) + })) + defer srv.Close() + + c, _ := New(Config{BaseURL: srv.URL, Token: testToken}) + _, err := c.WhoAmI(context.Background()) + if err == nil { + t.Fatal("expected an error") + } + if len(err.Error()) > 400 { + t.Errorf("error message is %d bytes; it should be truncated", len(err.Error())) + } +} + +func TestNoContentNeedsNoBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Errorf("method = %s", r.Method) + } + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + c, _ := New(Config{BaseURL: srv.URL, Token: testToken}) + if err := c.DeleteProject(context.Background(), "demo"); err != nil { + t.Fatalf("DeleteProject: %v", err) + } +} + +// TestTimeoutSaysWhichFlagToRaise — the timeout is the one failure a user can +// fix from the message alone, so the message has to name the flag. +func TestTimeoutSaysWhichFlagToRaise(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + defer func() { close(release); srv.Close() }() + + c, _ := New(Config{BaseURL: srv.URL, Token: testToken, Timeout: 50 * time.Millisecond}) + _, err := c.WhoAmI(context.Background()) + if err == nil { + t.Fatal("expected a timeout") + } + if !strings.Contains(err.Error(), "--timeout") { + t.Errorf("err = %v, want it to mention --timeout", err) + } +} + +// TestDeploymentQueriesTravelInTheURL. State, limit and cursor are the three +// knobs of a deployment listing and all three ride the query string; one that +// went missing would leave a listing that quietly answers a different question. +func TestDeploymentQueriesTravelInTheURL(t *testing.T) { + var got string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.URL.RequestURI() + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"deployments":[]}`)) + })) + defer srv.Close() + + c, _ := New(Config{BaseURL: srv.URL, Token: testToken}) + ctx := context.Background() + list := api.PathDeployments("demo") + one := api.PathDeployment("demo", "dpl_1") + + cases := []struct { + desc string + call func() error + want string + }{ + { + "no options, no query", + func() error { _, err := c.ListDeployments(ctx, "demo", DeploymentListOptions{}); return err }, + list, + }, + { + "every option set", + func() error { + opts := DeploymentListOptions{State: "ready"} + opts.Limit, opts.Cursor = 50, "dpl_9" + _, err := c.ListDeployments(ctx, "demo", opts) + return err + }, + list + "?cursor=dpl_9&limit=50&state=ready", + }, + { + "a manifest is not fetched by default", + func() error { _, err := c.GetDeployment(ctx, "demo", "dpl_1", false); return err }, + one, + }, + { + "asking for the manifest", + func() error { _, err := c.GetDeployment(ctx, "demo", "dpl_1", true); return err }, + one + "?files=true", + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + if err := tc.call(); err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("requested %q, want %q", got, tc.want) + } + }) + } +} + +// TestUpkeepPostsItsFlag: a --dry-run that silently ran for real, or a fsck +// that repaired without being asked, are the two ways these calls can be +// dangerous, and both live in the request body. +func TestUpkeepPostsItsFlag(t *testing.T) { + var gotMethod, gotPath, gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + gotMethod, gotPath, gotBody = r.Method, r.URL.Path, string(body) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{}`)) + })) + defer srv.Close() + + c, _ := New(Config{BaseURL: srv.URL, Token: testToken}) + ctx := context.Background() + + if _, err := c.Collect(ctx, true); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPost || gotPath != api.PathGC() { + t.Errorf("collect sent %s %s", gotMethod, gotPath) + } + if !strings.Contains(gotBody, `"dry_run":true`) { + t.Errorf("collect body = %q, want dry_run set", gotBody) + } + + if _, err := c.Fsck(ctx, true); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPost || gotPath != api.PathFsck() { + t.Errorf("fsck sent %s %s", gotMethod, gotPath) + } + if !strings.Contains(gotBody, `"repair":true`) { + t.Errorf("fsck body = %q, want repair set", gotBody) + } +} + +// TestListAllProjectsFollowsTheCursor: paging is invisible to the CLI, so the +// only place it can break is here. +func TestListAllProjectsFollowsTheCursor(t *testing.T) { + var seen []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cursor := r.URL.Query().Get("cursor") + seen = append(seen, cursor) + w.Header().Set("Content-Type", "application/json") + switch cursor { + case "": + w.Write([]byte(`{"projects":[{"name":"a"},{"name":"b"}],"next_cursor":"b"}`)) + case "b": + w.Write([]byte(`{"projects":[{"name":"c"}]}`)) + default: + t.Errorf("unexpected cursor %q", cursor) + } + })) + defer srv.Close() + + c, _ := New(Config{BaseURL: srv.URL, Token: testToken}) + got, err := c.ListAllProjects(context.Background()) + if err != nil { + t.Fatal(err) + } + var names []string + for _, p := range got { + names = append(names, p.Name) + } + if strings.Join(names, ",") != "a,b,c" { + t.Errorf("projects = %v, want a,b,c", names) + } + if len(seen) != 2 || seen[0] != "" || seen[1] != "b" { + t.Errorf("cursors requested = %q", seen) + } +} diff --git a/internal/client/deploy.go b/internal/client/deploy.go new file mode 100644 index 0000000..ef00553 --- /dev/null +++ b/internal/client/deploy.go @@ -0,0 +1,492 @@ +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/iceBear67/simplepages/api" +) + +// ------------------------------------------------------------ single calls + +// CreateDeployment opens a deployment. Nothing is served until it is finalized +// and activated. +func (c *Client) CreateDeployment(ctx context.Context, project string, meta map[string]string) (api.Deployment, error) { + var out api.Deployment + req := api.CreateDeploymentRequest{Meta: meta} + err := c.do(ctx, http.MethodPost, api.PathDeployments(project), req, &out) + return out, err +} + +// SetManifest declares the deployment's complete file list and returns the +// digests the server does not have yet. +// +// A large manifest is both slow to send and slow to insert, so it gets a +// deadline scaled to its size rather than the management timeout. +func (c *Client) SetManifest(ctx context.Context, project, id string, files []api.FileEntry) (api.ManifestResponse, error) { + var out api.ManifestResponse + req := api.ManifestRequest{Files: files} + // Roughly a millisecond per file on top of the base timeout: a 50,000-file + // manifest gets a minute of headroom, a small one gets no extra. + extra := time.Duration(len(files)) * time.Millisecond + err := c.doWithTimeout(ctx, c.timeout+extra, http.MethodPost, + api.PathManifest(project, id), req, &out) + return out, err +} + +// Finalize verifies every blob arrived and assembles the deployment. +// +// It returns an *api.Error with code blobs_missing when uploads are outstanding; +// the digests are in details["missing"]. +func (c *Client) Finalize(ctx context.Context, project, id string, fileCount int) (api.Deployment, error) { + var out api.Deployment + // Assembly hardlinks or copies every file, so this scales with the file + // count in the same way the manifest insert does. + extra := time.Duration(fileCount) * time.Millisecond + err := c.doWithTimeout(ctx, c.timeout+extra, http.MethodPost, + api.PathFinalize(project, id), nil, &out) + return out, err +} + +// Activate switches the project to this deployment. Passing an older id is how +// a rollback is performed. +func (c *Client) Activate(ctx context.Context, project, id string) (api.Deployment, error) { + var out api.Deployment + err := c.do(ctx, http.MethodPost, api.PathActivate(project, id), nil, &out) + return out, err +} + +// PutBlob uploads one blob's contents. +// +// It bypasses do, which is JSON-only: the body is raw bytes, the length is +// declared up front so the server can refuse an oversized file before reading +// it, and the deadline has to accommodate a file rather than an API call. +// +// A digest the server already holds is a fast 200 — that is what makes a +// re-run of a failed deploy cheap. +func (c *Client) PutBlob(ctx context.Context, digest string, size int64, body io.Reader) (api.BlobResponse, error) { + var out api.BlobResponse + path := api.PathBlob(digest) + + ctx, cancel := context.WithTimeout(ctx, uploadTimeout(c.timeout, size)) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.base+path, body) + if err != nil { + return out, err + } + req.ContentLength = size + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", c.agent) + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := c.http.Do(req) + if err != nil { + return out, requestError(http.MethodPut, path, err) + } + defer func() { + io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) + resp.Body.Close() + }() + + if resp.StatusCode >= 400 { + return out, wrapRetryable(resp, responseError(resp)) + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return out, fmt.Errorf("decode PUT %s response: %w", path, err) + } + return out, nil +} + +// uploadTimeout gives a transfer the base timeout plus enough time to move its +// bytes over a slow link. The floor is deliberately pessimistic — a CI runner +// on a hotel connection should finish, not time out halfway and start over. +func uploadTimeout(base time.Duration, size int64) time.Duration { + const bytesPerSecond = 128 << 10 + return base + time.Duration(size/bytesPerSecond)*time.Second +} + +// doWithTimeout is do with an explicit deadline instead of the client's. +// +// do reads c.timeout, and one Client is shared by every upload goroutine, so +// the field cannot be swapped in place. The struct is a string, a duration and +// two pointers; copying it is cheaper than the synchronisation would be. +func (c *Client) doWithTimeout(ctx context.Context, timeout time.Duration, method, path string, body, out any) error { + tmp := *c + tmp.timeout = timeout + return tmp.do(ctx, method, path, body, out) +} + +// ------------------------------------------------------------------ retries + +// throttled marks an error the deploy loop should retry, and carries the +// server's Retry-After when it sent one. +// +// The delay rides on a wrapper rather than on api.Error because api.Error is +// the wire type: a field that never appears in JSON does not belong in it. +// api.CodeOf unwraps, so callers still see the underlying code. +type throttled struct { + err error + after time.Duration +} + +func (t *throttled) Error() string { return t.err.Error() } +func (t *throttled) Unwrap() error { return t.err } + +// wrapRetryable tags the responses that are worth trying again: rate limiting, +// and anything the server reports as a transient failure of its own. +func wrapRetryable(resp *http.Response, err error) error { + switch { + case resp.StatusCode == http.StatusTooManyRequests, + resp.StatusCode >= 500: + return &throttled{err: err, after: retryAfter(resp)} + } + return err +} + +// retryAfter reads the header in its delay-seconds form. The HTTP-date form is +// ignored on purpose: honouring it means trusting the server's clock against +// ours, and the backoff below is a perfectly good fallback. +func retryAfter(resp *http.Response) time.Duration { + v := strings.TrimSpace(resp.Header.Get("Retry-After")) + if v == "" { + return 0 + } + secs, err := strconv.Atoi(v) + if err != nil || secs < 0 { + return 0 + } + const maxWait = 60 * time.Second + d := time.Duration(secs) * time.Second + return min(d, maxWait) +} + +// retryable reports whether err is worth another attempt, and how long to wait +// before it if the server asked for a specific delay. +// +// Transport errors are retried because a dropped connection mid-upload is the +// single most common failure on a CI runner. A 4xx other than 429 is not: the +// request is wrong and repeating it will not fix it. +func retryable(err error) (time.Duration, bool) { + if err == nil { + return 0, false + } + var t *throttled + if errors.As(err, &t) { + return t.after, true + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // A deadline that came from the caller's context means give up; one from + // our own per-upload timeout is indistinguishable here, so treat both as + // fatal rather than risk a retry storm against a wedged server. + return 0, false + } + var apiErr *api.Error + if errors.As(err, &apiErr) { + // An envelope arrived, so the server is reachable and answered on + // purpose. Only rate limiting is worth repeating. + return 0, apiErr.Code == api.CodeRateLimited + } + // Anything else is a transport failure. + return 0, true +} + +// backoff is exponential with jitter, capped. The jitter matters when a CI +// fleet retries in lockstep after a server restart. +func backoff(attempt int, seed uint64) time.Duration { + const ( + base = 250 * time.Millisecond + longest = 15 * time.Second + ) + d := min(base< 0 { + up, bytes, err := c.upload(ctx, src, man.Missing, opts, report) + res.Uploaded, res.UploadedBytes = up, bytes + if err != nil { + return nil, err + } + } + + fin, err := c.Finalize(ctx, opts.Project, dep.ID, len(src.Files)) + if err != nil { + // The server may have lost a blob between our upload and the finalize — + // a GC race, or a restart mid-write. It tells us exactly which, so send + // those again and finalize once more rather than failing the build. + missing, ok := missingFrom(err) + if !ok { + return nil, err + } + report("server is missing %s after upload; resending", plural(len(missing), "blob")) + up, bytes, uerr := c.upload(ctx, src, missing, opts, report) + res.Uploaded += up + res.UploadedBytes += bytes + if uerr != nil { + return nil, uerr + } + fin, err = c.Finalize(ctx, opts.Project, dep.ID, len(src.Files)) + if err != nil { + return nil, err + } + } + res.Deployment = fin + res.URL = fin.URL + report("finalized %s (%d files, %s)", fin.ID, fin.FileCount, humanBytes(fin.TotalBytes)) + + if opts.Activate { + act, err := c.Activate(ctx, opts.Project, dep.ID) + if err != nil { + return nil, err + } + res.Deployment = act + res.Activated = true + if act.URL != "" { + res.URL = act.URL + } + report("activated %s", act.ID) + } + return res, nil +} + +// upload sends the named digests, at most Concurrency at a time. +func (c *Client) upload(ctx context.Context, src *Source, digests []string, + opts DeployOptions, report func(string, ...any)) (int, int64, error) { + + // A digest may back several paths; any one of them has the bytes. + byDigest := make(map[string]LocalFile, len(src.Files)) + for _, f := range src.Files { + if _, ok := byDigest[f.Digest]; !ok { + byDigest[f.Digest] = f + } + } + + var ( + mu sync.Mutex + count int + sent int64 + ) + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(opts.Concurrency) + for _, digest := range digests { + f, ok := byDigest[digest] + if !ok { + // The server asked for something we never offered. Failing here beats + // finalizing into a deployment that can never become ready. + return count, sent, fmt.Errorf("server reported digest %s as missing, "+ + "but it is not in the manifest", digest) + } + g.Go(func() error { + if err := c.putRetrying(ctx, src, f, opts.Retries, report); err != nil { + return err + } + mu.Lock() + count++ + sent += f.Size + mu.Unlock() + return nil + }) + } + err := g.Wait() + if err == nil { + report("uploaded %s (%s)", plural(count, "blob"), humanBytes(sent)) + } + return count, sent, err +} + +// putRetrying uploads one blob, retrying transient failures. +// +// The body is reopened for every attempt: an io.Reader that has already been +// partly consumed cannot be replayed, and a retry that sent the tail of a file +// would be rejected as a digest mismatch — correctly, but confusingly. +func (c *Client) putRetrying(ctx context.Context, src *Source, f LocalFile, + retries int, report func(string, ...any)) error { + + var last error + for attempt := 0; attempt <= retries; attempt++ { + if attempt > 0 { + wait, _ := retryable(last) + if wait == 0 { + wait = backoff(attempt-1, seedOf(f.Digest)) + } + report("retrying %s in %s (%v)", shortDigest(f.Digest), wait.Round(time.Millisecond), last) + t := time.NewTimer(wait) + select { + case <-ctx.Done(): + t.Stop() + return ctx.Err() + case <-t.C: + } + } + + body, err := src.Open(f.Path) + if err != nil { + return fmt.Errorf("%s: %w", f.Path, err) + } + _, err = c.PutBlob(ctx, f.Digest, f.Size, body) + body.Close() + if err == nil { + return nil + } + last = err + if _, ok := retryable(err); !ok { + return fmt.Errorf("upload %s: %w", f.Path, err) + } + } + return fmt.Errorf("upload %s: giving up after %d attempts: %w", f.Path, retries+1, last) +} + +// missingFrom extracts the digest list from a blobs_missing error. +func missingFrom(err error) ([]string, bool) { + var apiErr *api.Error + if !errors.As(err, &apiErr) || apiErr.Code != api.CodeBlobsMissing { + return nil, false + } + raw, ok := apiErr.Details["missing"].([]any) + if !ok || len(raw) == 0 { + return nil, false + } + out := make([]string, 0, len(raw)) + for _, v := range raw { + s, ok := v.(string) + if !ok { + return nil, false + } + out = append(out, s) + } + return out, true +} + +// seedOf derives a per-blob jitter seed from its digest, so retries of +// different blobs spread out without a shared random source. +func seedOf(digest string) uint64 { + var h uint64 = 1469598103934665603 // FNV-1a offset basis + for i := 0; i < len(digest); i++ { + h ^= uint64(digest[i]) + h *= 1099511628211 + } + return h +} + +func shortDigest(d string) string { + if len(d) > 12 { + return d[:12] + } + return d +} + +func plural(n int, noun string) string { + if n == 1 { + return "1 " + noun + } + return strconv.Itoa(n) + " " + noun + "s" +} + +// humanBytes renders a size the way a person reads it. It lives here rather +// than in cliutil because the progress messages are produced by this package. +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return strconv.FormatInt(n, 10) + " B" + } + div, exp := int64(unit), 0 + for v := n / unit; v >= unit && exp < 4; v /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTP"[exp]) +} diff --git a/internal/client/scan.go b/internal/client/scan.go new file mode 100644 index 0000000..5d17953 --- /dev/null +++ b/internal/client/scan.go @@ -0,0 +1,323 @@ +package client + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + "slices" + + "golang.org/x/sync/errgroup" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/pathutil" +) + +// skipDirs are never walked into. A build output directory should not contain +// them at all, but "pages deploy ." on a repository root is a mistake people +// make once, and uploading a .git directory publishes the whole history. +var skipDirs = []string{".git", ".hg", ".svn"} + +// LocalFile is one file of a scanned directory, already hashed. +type LocalFile struct { + // Path is site-relative and slash-separated: it is what the URL will be. + Path string `json:"path"` + // Digest is the lowercase hex SHA-256 of the contents. + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// ScanOptions filters and paces a scan. +type ScanOptions struct { + // Include, when non-empty, keeps only files matching at least one pattern. + // Exclude drops files matching any pattern, and prunes whole directories. + // A pattern is path.Match syntax, tried against the site-relative path and + // against the base name, so both "assets/*.map" and "*.map" work. + Include []string + Exclude []string + + // FollowSymlinks reads through symbolic links instead of refusing them. + // Links are still resolved inside the scanned directory, so one pointing at + // /etc/passwd fails rather than publishing it. + FollowSymlinks bool + + // Concurrency bounds the hashing goroutines. Zero means GOMAXPROCS. + Concurrency int +} + +// Source is a scanned directory: the manifest it produced, plus the handle the +// upload path reads the contents back through. +// +// Files are read through an os.Root rather than by path, so a symlink swapped +// in between the scan and the upload still cannot reach outside the directory +// the user named. +type Source struct { + Dir string + Files []LocalFile + TotalBytes int64 + + root *os.Root +} + +// Scan walks dir, hashes what it finds, and returns the result. The caller must +// Close the Source. +func Scan(ctx context.Context, dir string, opts ScanOptions) (*Source, error) { + if err := checkPatterns("include", opts.Include); err != nil { + return nil, err + } + if err := checkPatterns("exclude", opts.Exclude); err != nil { + return nil, err + } + + abs, err := filepath.Abs(dir) + if err != nil { + return nil, fmt.Errorf("%s: %w", dir, err) + } + fi, err := os.Stat(abs) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fmt.Errorf("%s is not a directory", dir) + } + root, err := os.OpenRoot(abs) + if err != nil { + return nil, err + } + + s := &Source{Dir: abs, root: root} + if err := s.walk(ctx, opts); err != nil { + root.Close() + return nil, err + } + if len(s.Files) == 0 { + root.Close() + return nil, fmt.Errorf("%s contains no files to deploy", dir) + } + if err := s.hash(ctx, opts.Concurrency); err != nil { + root.Close() + return nil, err + } + return s, nil +} + +// Close releases the directory handle. +func (s *Source) Close() error { + if s == nil || s.root == nil { + return nil + } + return s.root.Close() +} + +// Open reads one of the scanned files. +func (s *Source) Open(p string) (*os.File, error) { + return s.root.Open(filepath.FromSlash(p)) +} + +// Manifest renders the scan as the wire form the server expects. +func (s *Source) Manifest() []api.FileEntry { + out := make([]api.FileEntry, len(s.Files)) + for i, f := range s.Files { + out[i] = api.FileEntry{Path: f.Path, Digest: f.Digest, Size: f.Size} + } + return out +} + +// UniqueBlobs counts distinct digests, which is what the deduplicating upload +// actually has to deal with. +func (s *Source) UniqueBlobs() int { + seen := make(map[string]struct{}, len(s.Files)) + for _, f := range s.Files { + seen[f.Digest] = struct{}{} + } + return len(seen) +} + +// walk collects the paths and sizes. Hashing is a separate pass so it can run +// concurrently over a list that is already known to be valid: finding out on +// file 40,000 that file 3 has an unusable name would waste the whole scan. +func (s *Source) walk(ctx context.Context, opts ScanOptions) error { + set := pathutil.NewSet(0) + return filepath.WalkDir(s.Dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + rel, err := filepath.Rel(s.Dir, p) + if err != nil { + return err + } + if rel == "." { + return nil + } + name := filepath.ToSlash(rel) + + if d.IsDir() { + if slices.Contains(skipDirs, d.Name()) || matchAny(opts.Exclude, name) { + return fs.SkipDir + } + return nil + } + + // WalkDir reports entry types from Lstat, so a symlink arrives as a + // symlink and is never silently followed. + var size int64 + switch { + case d.Type()&fs.ModeSymlink != 0: + if !opts.FollowSymlinks { + return fmt.Errorf("%s is a symbolic link; a deployment holds regular files only "+ + "(pass --follow-symlinks to upload what it points at)", name) + } + fi, err := s.root.Stat(filepath.FromSlash(name)) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + if !fi.Mode().IsRegular() { + return fmt.Errorf("%s points at a %s, not a regular file", name, kindOf(fi.Mode())) + } + size = fi.Size() + case d.Type().IsRegular(): + fi, err := d.Info() + if err != nil { + return err + } + size = fi.Size() + default: + return fmt.Errorf("%s is a %s; a deployment holds regular files only", + name, kindOf(d.Type())) + } + + if !keep(opts, name) { + return nil + } + // The same checks the server runs, so a name that could never be stored + // is reported here — with the local path in hand — instead of as a + // rejected manifest after the walk. + if err := set.Add(name); err != nil { + return fmt.Errorf("%s: %w", name, err) + } + s.Files = append(s.Files, LocalFile{Path: name, Size: size}) + s.TotalBytes += size + return nil + }) +} + +// hash fills in every digest. It is CPU-bound on small files and IO-bound on +// large ones, so it runs at GOMAXPROCS by default. +func (s *Source) hash(ctx context.Context, concurrency int) error { + if concurrency <= 0 { + concurrency = runtime.GOMAXPROCS(0) + } + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) + for i := range s.Files { + g.Go(func() error { + if err := ctx.Err(); err != nil { + return err + } + f := &s.Files[i] + digest, size, err := s.digest(f.Path) + if err != nil { + return fmt.Errorf("%s: %w", f.Path, err) + } + // The file may have been rewritten between the walk and now. The + // digest and the size have to describe the same bytes, so take both + // from the read that produced the digest. + f.Digest, f.Size = digest, size + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + // Sorted output makes "pages deploy --dry-run" diffable between runs. + slices.SortFunc(s.Files, func(a, b LocalFile) int { + if a.Path < b.Path { + return -1 + } + if a.Path > b.Path { + return 1 + } + return 0 + }) + s.TotalBytes = 0 + for _, f := range s.Files { + s.TotalBytes += f.Size + } + return nil +} + +func (s *Source) digest(p string) (string, int64, error) { + f, err := s.Open(p) + if err != nil { + return "", 0, err + } + defer f.Close() + h := sha256.New() + n, err := io.Copy(h, f) + if err != nil { + return "", 0, err + } + return hex.EncodeToString(h.Sum(nil)), n, nil +} + +// keep applies the include/exclude filters to a file. +func keep(opts ScanOptions, name string) bool { + if len(opts.Include) > 0 && !matchAny(opts.Include, name) { + return false + } + return !matchAny(opts.Exclude, name) +} + +// matchAny reports whether name matches a pattern, either as a whole path or by +// its base name. Matching the base name too is what makes "--exclude '*.map'" +// behave the way everyone expects, since path.Match's "*" does not cross "/". +func matchAny(patterns []string, name string) bool { + base := path.Base(name) + for _, pat := range patterns { + if ok, _ := path.Match(pat, name); ok { + return true + } + if ok, _ := path.Match(pat, base); ok { + return true + } + } + return false +} + +// checkPatterns rejects malformed globs up front. path.Match reports a bad +// pattern only when it is tried, so an unchecked one would silently match +// nothing and quietly deploy the wrong file set. +func checkPatterns(flag string, patterns []string) error { + for _, pat := range patterns { + if _, err := path.Match(pat, "x"); err != nil { + return fmt.Errorf("--%s %q: %w", flag, pat, err) + } + } + return nil +} + +func kindOf(m fs.FileMode) string { + switch { + case m&fs.ModeDir != 0: + return "directory" + case m&fs.ModeSymlink != 0: + return "symbolic link" + case m&fs.ModeDevice != 0: + return "device file" + case m&fs.ModeNamedPipe != 0: + return "named pipe" + case m&fs.ModeSocket != 0: + return "socket" + default: + return "special file" + } +} diff --git a/internal/cliutil/command.go b/internal/cliutil/command.go new file mode 100644 index 0000000..7b96d37 --- /dev/null +++ b/internal/cliutil/command.go @@ -0,0 +1,237 @@ +// Package cliutil is the CLI's plumbing: a small subcommand tree over the +// standard flag package, and table/JSON rendering. +// +// It is deliberately not a CLI framework, but not for the reason usually +// given. Measured on this machine, net/http and crypto/tls alone put the floor +// for any Go HTTP client at 5.4 MB stripped; the pages binary is 6.0 MB, and +// the same program written with cobra came out at 5.9 MB. Half a megabyte +// either way is noise next to the TLS stack, so "cobra is too big" would be a +// claim the numbers do not support. +// +// The actual reason is dependency surface. This tool has a fixed set of about +// a dozen commands whose flags are plain strings, bools and ints; it needs no +// shell completion, no generated man pages, no dynamic command registration. +// The tree below is the part of a framework it would use, and it is small +// enough to read in one sitting — which matters more for a binary that CI jobs +// download and run with a deployment credential in the environment. +package cliutil + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "sort" + "strings" + "text/tabwriter" +) + +// ErrUsage is returned when the command line does not name something runnable. +// The caller prints the usage text that accompanies it and exits non-zero. +var ErrUsage = errors.New("usage") + +// Command is one node of the command tree. A node either runs (Exec) or has +// children (Sub), never both. +type Command struct { + // Name is the single word that selects this command. + Name string + // Args describes the positional arguments for the usage line, e.g. "". + Args string + // Short is the one-line summary listed by the parent. + Short string + // Long is optional additional prose printed by --help. + Long string + + // Flags registers this command's flags. It runs once per invocation, before + // parsing, so the variables it binds are the ones Exec reads. + Flags func(fs *flag.FlagSet) + + // Exec runs the command with the positional arguments left after parsing. + Exec func(ctx context.Context, args []string) error + + // Sub are the child commands, if any. + Sub []*Command + + // parent is filled in during dispatch so usage text can print the full path. + parent *Command +} + +// Persistent registers flags that every command in the tree accepts. +// +// The values it binds must already hold whatever an outer parse produced, and +// it must register them with those values as the defaults — see Globals.Register +// in internal/clicmd. Registering with a fixed default instead would reset a +// flag given before the subcommand name ("pages --server X project list"), +// because flag.StringVar assigns the default at registration time. +type Persistent func(fs *flag.FlagSet) + +// Run parses args against c and dispatches. out receives usage and help text. +func Run(ctx context.Context, c *Command, args []string, out io.Writer, persistent Persistent) error { + fs := flag.NewFlagSet(c.path(), flag.ContinueOnError) + fs.SetOutput(out) + fs.Usage = func() { c.printUsage(out, fs) } + if persistent != nil { + persistent(fs) + } + if c.Flags != nil { + c.Flags(fs) + } + if len(c.Sub) == 0 { + args = permute(fs, args) + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + // flag has already printed the usage via fs.Usage. + return nil + } + // flag printed the message; adding our own would double it. + return ErrUsage + } + rest := fs.Args() + + if len(c.Sub) > 0 && len(rest) > 0 { + for _, sub := range c.Sub { + if sub.Name == rest[0] { + sub.parent = c + return Run(ctx, sub, rest[1:], out, persistent) + } + } + fmt.Fprintf(out, "unknown command %q\n\n", rest[0]) + c.printUsage(out, fs) + return ErrUsage + } + + // A leaf, or a parent invoked without naming a child. The latter still gets + // its Exec so the root can answer --version before falling back to usage. + if c.Exec == nil { + c.printUsage(out, fs) + return ErrUsage + } + err := c.Exec(ctx, rest) + if errors.Is(err, ErrUsage) { + c.printUsage(out, fs) + } + return err +} + +// permute moves flags ahead of positional arguments, so that +// +// pages project create demo --spa +// +// works as well as the order the flag package wants: +// +// pages project create --spa demo +// +// Stopping at the first non-flag argument is right for a command that has +// children — the first positional there names the child, and its flags belong +// to it, not to us. A leaf has no such ambiguity, so it gets the permuting +// parse people expect from every other CLI they use. +// +// Two tokens are left exactly where they are: "--" terminates flags for good, +// and an unrecognised flag is passed through alone so that flag.Parse reports +// it rather than this function silently swallowing the argument after it. +func permute(fs *flag.FlagSet, args []string) []string { + known := map[string]*flag.Flag{} + fs.VisitAll(func(f *flag.Flag) { known[f.Name] = f }) + + var flags, rest []string + for i := 0; i < len(args); i++ { + a := args[i] + if a == "--" { + rest = append(rest, args[i+1:]...) + break + } + // "-" alone is the conventional name for stdin, not a flag. + if len(a) < 2 || a[0] != '-' { + rest = append(rest, a) + continue + } + flags = append(flags, a) + name := strings.TrimLeft(a, "-") + if strings.ContainsRune(name, '=') { + continue // --name=value carries its own argument + } + // A non-boolean flag takes the next token with it. Booleans must not + // swallow anything: "--spa demo" is a flag and a positional. + if f, ok := known[name]; ok && !isBoolFlag(f) && i+1 < len(args) { + i++ + flags = append(flags, args[i]) + } + } + // The separator makes the tail positional even if a value there looks like + // a flag, which matters for paths and metadata values. + return append(flags, append([]string{"--"}, rest...)...) +} + +func isBoolFlag(f *flag.Flag) bool { + b, ok := f.Value.(interface{ IsBoolFlag() bool }) + return ok && b.IsBoolFlag() +} + +// UsageErrorf returns an error that makes Run print the command's usage after +// the caller reports the message. Use it for "wrong number of arguments" and +// friends, where the fix is visible in the usage text. +func UsageErrorf(format string, args ...any) error { + return &usageError{msg: fmt.Sprintf(format, args...)} +} + +type usageError struct{ msg string } + +func (e *usageError) Error() string { return e.msg } + +// Is makes errors.Is(err, ErrUsage) true for any usageError. +func (e *usageError) Is(target error) bool { return target == ErrUsage } + +// path is the space-separated command path, used in usage text and as the +// FlagSet name so flag's own error messages name the right command. +func (c *Command) path() string { + if c.parent == nil { + return c.Name + } + return c.parent.path() + " " + c.Name +} + +func (c *Command) printUsage(out io.Writer, fs *flag.FlagSet) { + if c.Short != "" { + fmt.Fprintf(out, "%s — %s\n\n", c.path(), c.Short) + } + + fmt.Fprintf(out, "Usage:\n %s", c.path()) + if len(c.Sub) > 0 { + fmt.Fprint(out, " ") + } + if hasFlags(fs) { + fmt.Fprint(out, " [flags]") + } + if c.Args != "" { + fmt.Fprintf(out, " %s", c.Args) + } + fmt.Fprint(out, "\n") + + if len(c.Sub) > 0 { + fmt.Fprint(out, "\nCommands:\n") + tw := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) + subs := append([]*Command(nil), c.Sub...) + sort.Slice(subs, func(i, j int) bool { return subs[i].Name < subs[j].Name }) + for _, sub := range subs { + fmt.Fprintf(tw, " %s\t%s\n", sub.Name, sub.Short) + } + tw.Flush() + } + + if hasFlags(fs) { + fmt.Fprint(out, "\nFlags:\n") + fs.PrintDefaults() + } + + if c.Long != "" { + fmt.Fprintf(out, "\n%s\n", strings.TrimSpace(c.Long)) + } +} + +func hasFlags(fs *flag.FlagSet) bool { + n := 0 + fs.VisitAll(func(*flag.Flag) { n++ }) + return n > 0 +} diff --git a/internal/cliutil/command_test.go b/internal/cliutil/command_test.go new file mode 100644 index 0000000..45080c5 --- /dev/null +++ b/internal/cliutil/command_test.go @@ -0,0 +1,197 @@ +package cliutil + +import ( + "context" + "errors" + "flag" + "io" + "reflect" + "strings" + "testing" +) + +// optBool is the minimal stand-in for clicmd's optional flags: a flag.Value +// that announces it takes no argument. +type optBool struct{ v bool } + +func (o *optBool) String() string { return "" } +func (o *optBool) Set(string) error { o.v = true; return nil } +func (o *optBool) IsBoolFlag() bool { return true } + +func TestPermute(t *testing.T) { + newFS := func() *flag.FlagSet { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + fs.String("name", "", "") + fs.Bool("yes", false, "") + fs.Var(&optBool{}, "spa", "") + return fs + } + + cases := []struct { + desc string + in []string + want []string + }{ + { + desc: "flags after the positional, which is what people type", + in: []string{"demo", "--name", "Demo", "--spa"}, + want: []string{"--name", "Demo", "--spa", "--", "demo"}, + }, + { + desc: "already in flag package order, unchanged apart from the separator", + in: []string{"--name", "Demo", "demo"}, + want: []string{"--name", "Demo", "--", "demo"}, + }, + { + desc: "a bool flag must not swallow the positional that follows it", + in: []string{"--yes", "demo"}, + want: []string{"--yes", "--", "demo"}, + }, + { + desc: "single-dash spelling is the same flag", + in: []string{"demo", "-name", "Demo"}, + want: []string{"-name", "Demo", "--", "demo"}, + }, + { + desc: "--flag=value carries its own argument", + in: []string{"demo", "--name=Demo", "other"}, + want: []string{"--name=Demo", "--", "demo", "other"}, + }, + { + desc: "a value that looks like a flag is still the flag's value", + in: []string{"demo", "--name", "-weird"}, + want: []string{"--name", "-weird", "--", "demo"}, + }, + { + desc: "everything after -- stays positional", + in: []string{"--yes", "--", "--name", "demo"}, + want: []string{"--yes", "--", "--name", "demo"}, + }, + { + desc: "a lone dash is a positional, not a flag", + in: []string{"-"}, + want: []string{"--", "-"}, + }, + { + desc: "an unknown flag is passed through alone so flag.Parse reports it", + in: []string{"--nope", "demo"}, + want: []string{"--nope", "--", "demo"}, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + got := permute(newFS(), tc.in) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("permute(%q)\n got %q\nwant %q", tc.in, got, tc.want) + } + }) + } +} + +// TestPermutedFlagsReachExec is the property the permutation exists for: the +// command sees the same flags and the same positionals either way round. +func TestPermutedFlagsReachExec(t *testing.T) { + for _, args := range [][]string{ + {"create", "demo", "--name", "Demo", "--spa"}, + {"create", "--name", "Demo", "--spa", "demo"}, + {"create", "--name", "Demo", "demo", "--spa"}, + } { + t.Run(strings.Join(args, " "), func(t *testing.T) { + var ( + name string + spa optBool + rest []string + ) + root := &Command{ + Name: "test", + Sub: []*Command{{ + Name: "create", + Flags: func(fs *flag.FlagSet) { + fs.StringVar(&name, "name", "", "") + fs.Var(&spa, "spa", "") + }, + Exec: func(ctx context.Context, args []string) error { + rest = args + return nil + }, + }}, + } + if err := Run(context.Background(), root, args, io.Discard, nil); err != nil { + t.Fatalf("Run: %v", err) + } + if name != "Demo" || !spa.v { + t.Errorf("flags: name=%q spa=%v, want Demo/true", name, spa.v) + } + if !reflect.DeepEqual(rest, []string{"demo"}) { + t.Errorf("positionals: %q, want [demo]", rest) + } + }) + } +} + +// TestParentDoesNotPermute guards the other half of the rule: a parent must +// leave a child's flags alone, or "pages project create --spa" would try to +// parse --spa against the project command and fail. +func TestParentDoesNotPermute(t *testing.T) { + var spa bool + root := &Command{ + Name: "test", + Sub: []*Command{{ + Name: "project", + Sub: []*Command{{ + Name: "create", + Flags: func(fs *flag.FlagSet) { fs.BoolVar(&spa, "spa", false, "") }, + Exec: func(context.Context, []string) error { return nil }, + }}, + }}, + } + if err := Run(context.Background(), root, []string{"project", "create", "--spa"}, io.Discard, nil); err != nil { + t.Fatalf("Run: %v", err) + } + if !spa { + t.Error("--spa did not reach the leaf command") + } +} + +func TestUsageErrors(t *testing.T) { + root := &Command{ + Name: "test", + Sub: []*Command{{ + Name: "leaf", + Exec: func(context.Context, []string) error { return UsageErrorf("expected %d args", 1) }, + }}, + } + + t.Run("unknown command", func(t *testing.T) { + var out strings.Builder + err := Run(context.Background(), root, []string{"nope"}, &out, nil) + if !errors.Is(err, ErrUsage) { + t.Fatalf("err = %v, want ErrUsage", err) + } + if !strings.Contains(out.String(), `unknown command "nope"`) { + t.Errorf("output did not name the unknown command:\n%s", out.String()) + } + }) + + t.Run("a parent with no child named is usage, not a crash", func(t *testing.T) { + err := Run(context.Background(), root, nil, io.Discard, nil) + if !errors.Is(err, ErrUsage) { + t.Fatalf("err = %v, want ErrUsage", err) + } + }) + + t.Run("UsageErrorf prints the command's usage and keeps its message", func(t *testing.T) { + var out strings.Builder + err := Run(context.Background(), root, []string{"leaf"}, &out, nil) + if !errors.Is(err, ErrUsage) { + t.Fatalf("err = %v, want ErrUsage", err) + } + if err.Error() != "expected 1 args" { + t.Errorf("message = %q", err.Error()) + } + if !strings.Contains(out.String(), "Usage:\n test leaf") { + t.Errorf("usage text missing or misnamed:\n%s", out.String()) + } + }) +} diff --git a/internal/cliutil/input.go b/internal/cliutil/input.go new file mode 100644 index 0000000..3aee85a --- /dev/null +++ b/internal/cliutil/input.go @@ -0,0 +1,131 @@ +package cliutil + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" +) + +// ParseDuration accepts Go's duration syntax plus the day, week and year +// suffixes an operator actually types for a key lifetime ("90d", "1y"). +// +// A year is 365 days and a day is 24 hours: no calendar arithmetic, no time +// zones, no leap seconds. The value becomes an absolute expiry timestamp on the +// client precisely so a difference of a few hours cannot matter. +func ParseDuration(s string) (time.Duration, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, errors.New("empty duration") + } + var mult time.Duration + switch s[len(s)-1] { + case 'd': + mult = 24 * time.Hour + case 'w': + mult = 7 * 24 * time.Hour + case 'y': + mult = 365 * 24 * time.Hour + default: + d, err := time.ParseDuration(s) + if err != nil { + return 0, fmt.Errorf("invalid duration %q: use 30s, 5m, 12h, 90d, 2w or 1y", s) + } + return d, nil + } + n, err := strconv.ParseFloat(s[:len(s)-1], 64) + if err != nil { + return 0, fmt.Errorf("invalid duration %q: use 30s, 5m, 12h, 90d, 2w or 1y", s) + } + return time.Duration(n * float64(mult)), nil +} + +// ParseBytes accepts a plain byte count or one with a unit suffix: "1048576", +// "1MiB", "256M", "1.5G". +// +// K, M, G and T all mean the binary multiple, whether or not the suffix is +// spelled with an "i". Disk-quota flags are set to round binary numbers, and a +// tool that quietly read "256MB" as 256,000,000 would produce a limit its user +// did not ask for. +func ParseBytes(s string) (int64, error) { + t := strings.TrimSpace(s) + if t == "" { + return 0, errors.New("empty size") + } + digits := strings.TrimRight(t, "bBiIkKmMgGtT") + // Uppercase before trimming so "MiB", "MIB" and "mib" all reduce to "M". + unit := strings.ToUpper(t[len(digits):]) + unit = strings.TrimSuffix(unit, "B") + unit = strings.TrimSuffix(unit, "I") + + n, err := strconv.ParseFloat(strings.TrimSpace(digits), 64) + if err != nil || n < 0 { + return 0, fmt.Errorf("invalid size %q: use a byte count or 256MiB, 2GiB", s) + } + var mult float64 = 1 + switch unit { + case "": + case "K": + mult = 1 << 10 + case "M": + mult = 1 << 20 + case "G": + mult = 1 << 30 + case "T": + mult = 1 << 40 + default: + return 0, fmt.Errorf("invalid size %q: unknown unit %q", s, unit) + } + return int64(n * mult), nil +} + +// KeyValue splits a "k=v" argument. The value may contain further '=' signs. +func KeyValue(s string) (key, value string, err error) { + k, v, ok := strings.Cut(s, "=") + if !ok || k == "" { + return "", "", fmt.Errorf("expected key=value, got %q", s) + } + return k, v, nil +} + +// ErrAborted is returned when the user declines a confirmation prompt. +var ErrAborted = errors.New("aborted") + +// Confirm asks for a yes before a destructive operation. +// +// It refuses rather than prompts when stdin is not a terminal: a pipeline that +// blocks forever on a prompt nobody can see is worse than one that fails and +// tells the operator to pass --yes. +func Confirm(in io.Reader, out io.Writer, prompt string) error { + if f, ok := in.(*os.File); ok && !isTerminal(f) { + return fmt.Errorf("%w: refusing to prompt for confirmation with no terminal; pass --yes", ErrAborted) + } + fmt.Fprintf(out, "%s [y/N]: ", prompt) + line, err := bufio.NewReader(in).ReadString('\n') + if err != nil && line == "" { + return ErrAborted + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + return nil + } + return ErrAborted +} + +// isTerminal reports whether f is a character device. This is the cheap +// stdlib-only approximation of a tty check; it is used to decide whether to +// prompt and whether to draw progress, never for anything security-relevant. +func isTerminal(f *os.File) bool { + fi, err := f.Stat() + if err != nil { + return false + } + return fi.Mode()&os.ModeCharDevice != 0 +} + +// IsTerminal reports whether f is attached to a terminal. +func IsTerminal(f *os.File) bool { return isTerminal(f) } diff --git a/internal/cliutil/optflag.go b/internal/cliutil/optflag.go new file mode 100644 index 0000000..fb08fb5 --- /dev/null +++ b/internal/cliutil/optflag.go @@ -0,0 +1,169 @@ +package cliutil + +import ( + "strconv" + "time" +) + +// The Opt* types are flag.Value implementations that remember whether they were +// given on the command line. +// +// They exist for PATCH: api.ProjectPatch uses a pointer per field so the server +// can tell "leave this alone" from "set it to the zero value". A plain +// fs.StringVar cannot express that difference — an unset --not-found-file and +// an explicit --not-found-file="" both arrive as "". Each type's Ptr method +// produces exactly the pointer the patch field wants. + +// OptString is an optional string flag. +type OptString struct { + Val string + Present bool +} + +func (o *OptString) String() string { + if o == nil { + return "" + } + return o.Val +} + +func (o *OptString) Set(s string) error { + o.Val, o.Present = s, true + return nil +} + +// Ptr returns nil unless the flag was given. +func (o *OptString) Ptr() *string { + if !o.Present { + return nil + } + return &o.Val +} + +// OptBool is an optional boolean flag. It may be written as --flag as well as +// --flag=false. +type OptBool struct { + Val bool + Present bool +} + +func (o *OptBool) String() string { + if o == nil { + return "false" + } + return strconv.FormatBool(o.Val) +} + +func (o *OptBool) Set(s string) error { + v, err := strconv.ParseBool(s) + if err != nil { + return err + } + o.Val, o.Present = v, true + return nil +} + +// IsBoolFlag lets flag accept "--spa" without an explicit value. +func (o *OptBool) IsBoolFlag() bool { return true } + +// Ptr returns nil unless the flag was given. +func (o *OptBool) Ptr() *bool { + if !o.Present { + return nil + } + return &o.Val +} + +// OptInt is an optional integer flag. +type OptInt struct { + Val int + Present bool +} + +func (o *OptInt) String() string { + if o == nil { + return "" + } + return strconv.Itoa(o.Val) +} + +func (o *OptInt) Set(s string) error { + v, err := strconv.Atoi(s) + if err != nil { + return err + } + o.Val, o.Present = v, true + return nil +} + +// Ptr returns nil unless the flag was given. +func (o *OptInt) Ptr() *int { + if !o.Present { + return nil + } + return &o.Val +} + +// OptBytes is an optional byte count, written as a plain number or with a unit +// suffix ("256MiB"). +type OptBytes struct { + Val int64 + Present bool +} + +func (o *OptBytes) String() string { + if o == nil { + return "" + } + return strconv.FormatInt(o.Val, 10) +} + +func (o *OptBytes) Set(s string) error { + v, err := ParseBytes(s) + if err != nil { + return err + } + o.Val, o.Present = v, true + return nil +} + +// Ptr returns nil unless the flag was given. +func (o *OptBytes) Ptr() *int64 { + if !o.Present { + return nil + } + return &o.Val +} + +// OptDuration is an optional duration, accepting the same suffixes as +// ParseDuration. +type OptDuration struct { + Val time.Duration + Present bool +} + +func (o *OptDuration) String() string { + if o == nil { + return "" + } + return o.Val.String() +} + +func (o *OptDuration) Set(s string) error { + v, err := ParseDuration(s) + if err != nil { + return err + } + o.Val, o.Present = v, true + return nil +} + +// SecondsPtr returns the duration in whole seconds, or nil if the flag was not +// given. The API expresses grace periods as an integer number of seconds. +func (o *OptDuration) SecondsPtr() *int { + if !o.Present { + return nil + } + s := int(o.Val / time.Second) + return &s +} diff --git a/internal/cliutil/output.go b/internal/cliutil/output.go new file mode 100644 index 0000000..9704372 --- /dev/null +++ b/internal/cliutil/output.go @@ -0,0 +1,159 @@ +package cliutil + +import ( + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "text/tabwriter" + "time" +) + +// Output formats. +const ( + FormatTable = "table" + FormatJSON = "json" +) + +// ValidFormat reports whether s names an output format. +func ValidFormat(s string) bool { return s == FormatTable || s == FormatJSON } + +// Printer writes command output in the format the user asked for. +type Printer struct { + Out io.Writer + Format string +} + +// Print renders v. In table format it calls table to build the rendering; a nil +// table means the value has no tabular form and JSON is used regardless. +func (p *Printer) Print(v any, table func() *Table) error { + if p.Format == FormatJSON || table == nil { + return p.JSON(v) + } + return table().Write(p.Out) +} + +// JSON writes v as indented JSON with a trailing newline. +func (p *Printer) JSON(v any) error { + enc := json.NewEncoder(p.Out) + enc.SetIndent("", " ") + // The output is read by humans and by jq, neither of which wants &, < and > + // spelled as & and friends. + enc.SetEscapeHTML(false) + return enc.Encode(v) +} + +// Printf writes a human-readable line, and nothing at all in JSON format — +// progress chatter must never end up in a stream something is parsing. +func (p *Printer) Printf(format string, args ...any) { + if p.Format == FormatJSON { + return + } + fmt.Fprintf(p.Out, format, args...) +} + +// Table is a column-aligned rendering built up row by row. +type Table struct { + header []string + rows [][]string +} + +// NewTable starts a table with the given column headings. +func NewTable(header ...string) *Table { return &Table{header: header} } + +// Row appends a row. Cells beyond the header count are kept: a ragged table is +// a formatting annoyance, not a reason to drop data. +func (t *Table) Row(cells ...string) { t.rows = append(t.rows, cells) } + +// Len reports how many rows have been added. +func (t *Table) Len() int { return len(t.rows) } + +// Write renders the table. An empty table prints its header only, so a caller +// can tell "no rows" from "the command did nothing". +func (t *Table) Write(w io.Writer) error { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + if len(t.header) > 0 { + fmt.Fprintln(tw, strings.Join(t.header, "\t")) + } + for _, row := range t.rows { + fmt.Fprintln(tw, strings.Join(row, "\t")) + } + return tw.Flush() +} + +// ------------------------------------------------------------ cell helpers + +// Dash is what an empty cell shows, so a missing value is visibly missing +// rather than looking like a column-alignment mistake. +const Dash = "-" + +// Str renders a string cell, showing Dash when empty. +func Str(s string) string { + if s == "" { + return Dash + } + return s +} + +// Bool renders a boolean cell. +func Bool(b bool) string { + if b { + return "yes" + } + return "no" +} + +// Plural renders a count with its noun, adding "s" for anything but one. +// Only regular nouns are pluralized correctly, which is all this CLI has. +func Plural(n int, noun string) string { + if n == 1 { + return "1 " + noun + } + return strconv.Itoa(n) + " " + noun + "s" +} + +// Time renders an absolute local timestamp to the second. Absolute rather than +// relative: these values go into CI logs that are read days later, where "2 +// hours ago" has lost its reference point. +func Time(t time.Time) string { + if t.IsZero() { + return Dash + } + return t.Local().Format("2006-01-02 15:04:05") +} + +// TimePtr renders an optional timestamp. +func TimePtr(t *time.Time) string { + if t == nil { + return Dash + } + return Time(*t) +} + +// Bytes renders a byte count in binary units, as a human reads it. +func Bytes(n int64) string { + const unit = 1024 + if n < unit { + return strconv.FormatInt(n, 10) + " B" + } + div, exp := int64(unit), 0 + for n/div >= unit && exp < 4 { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTP"[exp]) +} + +// Truncate shortens s to at most max runes, marking the cut with an ellipsis so +// a clipped value is never mistaken for a complete one. +func Truncate(s string, max int) string { + if max <= 1 { + return s + } + r := []rune(s) + if len(r) <= max { + return s + } + return string(r[:max-1]) + "…" +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..b5aa87e --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,311 @@ +// Package config defines the pages-server configuration: its shape, defaults, +// and validation. Loading (and the flag > env > file > default precedence) lives +// in load.go. +package config + +import ( + "encoding" + "fmt" + "net" + "net/netip" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// ProjectNamePattern constrains project names. It is deliberately strict: the +// name becomes a "~name" entry inside $WEBROOT, so anything that could contain +// a path separator, a "..", or a leading dot must be impossible by construction. +var ProjectNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,62}$`) + +// AssembleMode selects how deployment directories are built from the CAS. +type AssembleMode string + +const ( + // AssembleAuto probes for hardlink support at startup and falls back to copy. + AssembleAuto AssembleMode = "auto" + // AssembleHardlink requires hardlinks and fails loudly if unavailable. + AssembleHardlink AssembleMode = "hardlink" + // AssembleCopy always copies. Correct but uses disk proportional to content. + AssembleCopy AssembleMode = "copy" + // AssembleNone skips on-disk assembly entirely; content is served straight + // from the CAS and $WEBROOT symlinks are not maintained. + AssembleNone AssembleMode = "none" +) + +func (m AssembleMode) valid() bool { + switch m { + case AssembleAuto, AssembleHardlink, AssembleCopy, AssembleNone: + return true + } + return false +} + +// Duration wraps time.Duration so it can be written as "15m" in TOML. +type Duration time.Duration + +var _ encoding.TextUnmarshaler = (*Duration)(nil) + +func (d *Duration) UnmarshalText(text []byte) error { + v, err := time.ParseDuration(string(text)) + if err != nil { + return err + } + *d = Duration(v) + return nil +} + +// Set implements flag.Value, so the same type backs both the TOML field and the +// command-line flag. +func (d *Duration) Set(s string) error { return d.UnmarshalText([]byte(s)) } + +func (d Duration) MarshalText() ([]byte, error) { return []byte(d.String()), nil } +func (d Duration) String() string { return time.Duration(d).String() } +func (d Duration) D() time.Duration { return time.Duration(d) } + +// Limits bounds what a single deployment may push. Per-project overrides live +// in the projects table; these are the server-wide ceilings. +type Limits struct { + MaxFileBytes int64 `toml:"max_file_bytes"` + MaxManifestFiles int `toml:"max_manifest_files"` + MaxConcurrentUploads int `toml:"max_concurrent_uploads"` + MaxManifestBytes int64 `toml:"max_manifest_bytes"` + MaxJSONBytes int64 `toml:"max_json_bytes"` +} + +// Config is the fully resolved server configuration. +type Config struct { + DataDir string `toml:"data_dir"` + Webroot string `toml:"webroot"` + + Listen string `toml:"listen"` // public static-content listener + APIListen string `toml:"api_listen"` // management API listener + + // SiteURL is the public origin the static listener is reachable at, as seen + // from outside — behind a reverse proxy that is a name the server itself + // never learns. It exists only so API responses can tell a CI job where its + // deployment landed; nothing about serving depends on it, so leaving it + // empty simply omits the url field. + SiteURL string `toml:"site_url"` + + TrustedProxyCIDRs []string `toml:"trusted_proxy_cidrs"` + AssembleMode AssembleMode `toml:"assemble_mode"` + + LogLevel string `toml:"log_level"` + LogFormat string `toml:"log_format"` + + GCInterval Duration `toml:"gc_interval"` + ReconcileInterval Duration `toml:"reconcile_interval"` + ShutdownGrace Duration `toml:"shutdown_grace"` + + // ReadHeaderTimeout and ReadTimeout guard both listeners. There is + // deliberately no WriteTimeout: a global write deadline on the static + // listener would kill legitimate large downloads over slow links. + ReadHeaderTimeout Duration `toml:"read_header_timeout"` + ReadTimeout Duration `toml:"read_timeout"` + IdleTimeout Duration `toml:"idle_timeout"` + + Limits Limits `toml:"limits"` + + // trustedNets is the parsed form of TrustedProxyCIDRs, filled by Validate. + trustedNets []netip.Prefix +} + +// Default returns the baseline configuration. Every other layer (file, env, +// flags) is applied on top of this. +func Default() Config { + return Config{ + DataDir: "/var/lib/pages-server", + Webroot: "/srv/www", + Listen: ":8080", + APIListen: "127.0.0.1:8081", + TrustedProxyCIDRs: []string{"127.0.0.1/32", "::1/128"}, + AssembleMode: AssembleAuto, + LogLevel: "info", + LogFormat: "json", + GCInterval: Duration(15 * time.Minute), + ReconcileInterval: Duration(5 * time.Minute), + ShutdownGrace: Duration(30 * time.Second), + ReadHeaderTimeout: Duration(10 * time.Second), + ReadTimeout: Duration(5 * time.Minute), + IdleTimeout: Duration(120 * time.Second), + Limits: Limits{ + MaxFileBytes: 256 << 20, // 256 MiB + MaxManifestFiles: 50000, + MaxConcurrentUploads: 32, + MaxManifestBytes: 64 << 20, // 64 MiB of manifest JSON + MaxJSONBytes: 1 << 20, // 1 MiB for ordinary API bodies + }, + } +} + +// DBPath is the SQLite database file. +func (c *Config) DBPath() string { return filepath.Join(c.DataDir, "pages.db") } + +// CASDir holds content-addressed blobs. +func (c *Config) CASDir() string { return filepath.Join(c.DataDir, "cas") } + +// DeploymentsDir holds assembled deployment trees. +func (c *Config) DeploymentsDir() string { return filepath.Join(c.DataDir, "deployments") } + +// BootstrapTokenPath is where the first-run admin token is written. +func (c *Config) BootstrapTokenPath() string { return filepath.Join(c.DataDir, "bootstrap-token") } + +// TrustedProxies returns the parsed trusted-proxy prefixes. Only requests whose +// direct peer falls inside one of these may have their X-Forwarded-For honoured. +func (c *Config) TrustedProxies() []netip.Prefix { return c.trustedNets } + +// Validate checks the configuration and normalises it in place. It performs no +// I/O beyond stat-ing the configured directories' parents, so it is safe to run +// from --check-config without binding ports. +func (c *Config) Validate() error { + if c.DataDir == "" { + return fmt.Errorf("data_dir is required") + } + abs, err := filepath.Abs(c.DataDir) + if err != nil { + return fmt.Errorf("data_dir: %w", err) + } + c.DataDir = filepath.Clean(abs) + + if c.AssembleMode != AssembleNone { + if c.Webroot == "" { + return fmt.Errorf("webroot is required unless assemble_mode is %q", AssembleNone) + } + abs, err = filepath.Abs(c.Webroot) + if err != nil { + return fmt.Errorf("webroot: %w", err) + } + c.Webroot = filepath.Clean(abs) + + // $WEBROOT must not sit inside $DATA_DIR (or vice versa): the reconciler + // removes stale "~name" symlinks from the webroot, and the GC removes + // trees from the data dir. Overlapping them makes each capable of + // deleting the other's state. + if c.Webroot == c.DataDir { + return fmt.Errorf("webroot and data_dir must differ (both %q)", c.DataDir) + } + if isUnder(c.Webroot, c.DataDir) || isUnder(c.DataDir, c.Webroot) { + return fmt.Errorf("webroot %q and data_dir %q must not be nested", c.Webroot, c.DataDir) + } + } + + if !c.AssembleMode.valid() { + return fmt.Errorf("assemble_mode: want auto|hardlink|copy|none, got %q", c.AssembleMode) + } + + for _, spec := range []struct{ name, addr string }{ + {"listen", c.Listen}, + {"api_listen", c.APIListen}, + } { + if spec.addr == "" { + return fmt.Errorf("%s is required", spec.name) + } + if _, _, err := net.SplitHostPort(spec.addr); err != nil { + return fmt.Errorf("%s %q: %w", spec.name, spec.addr, err) + } + } + if c.Listen == c.APIListen { + return fmt.Errorf("listen and api_listen must differ (both %q); "+ + "the management API must not share an origin with served content", c.Listen) + } + + if c.SiteURL != "" { + // Stored without the trailing slash so api.SiteURL can append one and + // get exactly one back. + c.SiteURL = strings.TrimRight(c.SiteURL, "/") + u, err := url.Parse(c.SiteURL) + if err != nil { + return fmt.Errorf("site_url %q: %w", c.SiteURL, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("site_url %q must be an absolute http:// or https:// URL", c.SiteURL) + } + if u.Host == "" { + return fmt.Errorf("site_url %q is missing a host", c.SiteURL) + } + if u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("site_url %q must not carry a query or fragment", c.SiteURL) + } + } + + c.trustedNets = c.trustedNets[:0] + for _, s := range c.TrustedProxyCIDRs { + p, err := netip.ParsePrefix(strings.TrimSpace(s)) + if err != nil { + return fmt.Errorf("trusted_proxy_cidrs %q: %w", s, err) + } + c.trustedNets = append(c.trustedNets, p) + } + + switch c.LogLevel { + case "debug", "info", "warn", "error": + default: + return fmt.Errorf("log_level: want debug|info|warn|error, got %q", c.LogLevel) + } + switch c.LogFormat { + case "json", "text": + default: + return fmt.Errorf("log_format: want json|text, got %q", c.LogFormat) + } + + for _, spec := range []struct { + name string + d Duration + }{ + {"gc_interval", c.GCInterval}, + {"reconcile_interval", c.ReconcileInterval}, + {"shutdown_grace", c.ShutdownGrace}, + {"read_header_timeout", c.ReadHeaderTimeout}, + {"read_timeout", c.ReadTimeout}, + {"idle_timeout", c.IdleTimeout}, + } { + if spec.d <= 0 { + return fmt.Errorf("%s must be positive, got %s", spec.name, spec.d) + } + } + + if c.Limits.MaxFileBytes <= 0 { + return fmt.Errorf("limits.max_file_bytes must be positive") + } + if c.Limits.MaxManifestFiles <= 0 { + return fmt.Errorf("limits.max_manifest_files must be positive") + } + if c.Limits.MaxConcurrentUploads <= 0 { + return fmt.Errorf("limits.max_concurrent_uploads must be positive") + } + if c.Limits.MaxManifestBytes <= 0 { + return fmt.Errorf("limits.max_manifest_bytes must be positive") + } + if c.Limits.MaxJSONBytes <= 0 { + return fmt.Errorf("limits.max_json_bytes must be positive") + } + return nil +} + +// EnsureDirs creates the data directory layout. Separated from Validate so +// --check-config stays read-only. +func (c *Config) EnsureDirs() error { + dirs := []string{c.DataDir, c.CASDir(), filepath.Join(c.CASDir(), "tmp"), c.DeploymentsDir()} + if c.AssembleMode != AssembleNone { + dirs = append(dirs, c.Webroot) + } + for _, d := range dirs { + if err := os.MkdirAll(d, 0o755); err != nil { + return fmt.Errorf("create %s: %w", d, err) + } + } + return nil +} + +// isUnder reports whether path is lexically inside base. +func isUnder(path, base string) bool { + rel, err := filepath.Rel(base, path) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..161e286 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,266 @@ +package config + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestDefaultIsValid(t *testing.T) { + cfg := Default() + if err := cfg.Validate(); err != nil { + t.Fatalf("the built-in default must validate: %v", err) + } + if len(cfg.TrustedProxies()) != 2 { + t.Errorf("trusted proxies = %v, want loopback v4 + v6", cfg.TrustedProxies()) + } +} + +func writeConfig(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +// Precedence is flag > env > file > default, and — the part that is easy to get +// wrong — a flag left unset must not shadow the file or the environment with its +// default value. +func TestLoadPrecedence(t *testing.T) { + path := writeConfig(t, ` +data_dir = "/srv/from-file" +webroot = "/var/www/from-file" +listen = ":9001" +log_level = "warn" +gc_interval = "9m" +`) + + t.Setenv("PAGES_WEBROOT", "/var/www/from-env") + t.Setenv("PAGES_LOG_LEVEL", "debug") + + opts, err := Load([]string{ + "-config", path, + "-log-level", "error", + "-gc-interval", "1m", + }, io.Discard) + if err != nil { + t.Fatalf("Load: %v", err) + } + cfg := opts.Config + + if cfg.LogLevel != "error" { + t.Errorf("log_level = %q, want the flag to win", cfg.LogLevel) + } + if cfg.Webroot != "/var/www/from-env" { + t.Errorf("webroot = %q, want the env to beat the file", cfg.Webroot) + } + if cfg.DataDir != "/srv/from-file" { + t.Errorf("data_dir = %q, want the file to beat the default", cfg.DataDir) + } + if cfg.Listen != ":9001" { + t.Errorf("listen = %q, want the file value (no flag, no env)", cfg.Listen) + } + if cfg.GCInterval.D() != time.Minute { + t.Errorf("gc_interval = %s, want the flag to win", cfg.GCInterval) + } + if cfg.APIListen != Default().APIListen { + t.Errorf("api_listen = %q, want the untouched default", cfg.APIListen) + } +} + +// An unset flag defaults to the same value as Default(), so a naive +// implementation silently overwrites whatever the file said with that default. +func TestUnsetFlagDoesNotShadowFile(t *testing.T) { + path := writeConfig(t, "log_format = \"text\"\napi_listen = \"127.0.0.1:9999\"\n") + opts, err := Load([]string{"-config", path}, io.Discard) + if err != nil { + t.Fatalf("Load: %v", err) + } + if opts.Config.LogFormat != "text" { + t.Errorf("log_format = %q, want text", opts.Config.LogFormat) + } + if opts.Config.APIListen != "127.0.0.1:9999" { + t.Errorf("api_listen = %q, want the file value", opts.Config.APIListen) + } +} + +func TestLoadRejectsUnknownKey(t *testing.T) { + path := writeConfig(t, "data_dir = \"/srv/x\"\nlog_levle = \"debug\"\n") + _, err := Load([]string{"-config", path}, io.Discard) + if err == nil { + t.Fatal("a typo in the config file must fail loudly") + } + if !strings.Contains(err.Error(), "log_levle") { + t.Errorf("error should name the offending key, got: %v", err) + } +} + +func TestLoadRejectsUnknownArgs(t *testing.T) { + if _, err := Load([]string{"serve"}, io.Discard); err == nil { + t.Fatal("positional arguments must be rejected") + } +} + +func TestLoadEnvDurationError(t *testing.T) { + t.Setenv("PAGES_GC_INTERVAL", "fifteen minutes") + _, err := Load(nil, io.Discard) + if err == nil || !strings.Contains(err.Error(), "PAGES_GC_INTERVAL") { + t.Fatalf("want an error naming the variable, got %v", err) + } +} + +func TestLoadTrustedProxiesFromEnv(t *testing.T) { + t.Setenv("PAGES_TRUSTED_PROXY_CIDRS", "10.0.0.0/8, 192.168.0.0/16 ,") + opts, err := Load(nil, io.Discard) + if err != nil { + t.Fatalf("Load: %v", err) + } + got := opts.Config.TrustedProxies() + if len(got) != 2 || got[0].String() != "10.0.0.0/8" || got[1].String() != "192.168.0.0/16" { + t.Errorf("trusted proxies = %v", got) + } +} + +func TestValidate(t *testing.T) { + cases := []struct { + name string + mutate func(*Config) + wantErr string + }{ + {"webroot inside data_dir", func(c *Config) { + c.DataDir = "/var/lib/pages" + c.Webroot = "/var/lib/pages/www" + }, "nested"}, + {"data_dir inside webroot", func(c *Config) { + c.DataDir = "/srv/www/state" + c.Webroot = "/srv/www" + }, "nested"}, + {"identical dirs", func(c *Config) { + c.DataDir = "/srv/www" + c.Webroot = "/srv/www" + }, "must differ"}, + {"listeners collide", func(c *Config) { + c.Listen = "127.0.0.1:8080" + c.APIListen = "127.0.0.1:8080" + }, "must differ"}, + {"listen without port", func(c *Config) { c.Listen = "8080" }, "listen"}, + {"bad cidr", func(c *Config) { c.TrustedProxyCIDRs = []string{"10.0.0.1"} }, "trusted_proxy_cidrs"}, + {"bad assemble mode", func(c *Config) { c.AssembleMode = "symlink" }, "assemble_mode"}, + {"bad log level", func(c *Config) { c.LogLevel = "verbose" }, "log_level"}, + {"zero interval", func(c *Config) { c.GCInterval = 0 }, "gc_interval"}, + {"negative limit", func(c *Config) { c.Limits.MaxFileBytes = -1 }, "max_file_bytes"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := Default() + tc.mutate(&cfg) + err := cfg.Validate() + if err == nil { + t.Fatalf("want an error mentioning %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("error = %v, want it to mention %q", err, tc.wantErr) + } + }) + } +} + +// assemble_mode = "none" means no webroot is maintained, so the webroot checks +// must not fire. +func TestValidateAllowsEmptyWebrootWhenAssemblyDisabled(t *testing.T) { + cfg := Default() + cfg.AssembleMode = AssembleNone + cfg.Webroot = "" + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } +} + +func TestValidateMakesPathsAbsolute(t *testing.T) { + cfg := Default() + cfg.DataDir = "state/./" + cfg.Webroot = "www" + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if !filepath.IsAbs(cfg.DataDir) || strings.HasSuffix(cfg.DataDir, "/") { + t.Errorf("data_dir = %q, want a cleaned absolute path", cfg.DataDir) + } + if !filepath.IsAbs(cfg.Webroot) { + t.Errorf("webroot = %q, want an absolute path", cfg.Webroot) + } +} + +func TestEnsureDirs(t *testing.T) { + base := t.TempDir() + cfg := Default() + cfg.DataDir = filepath.Join(base, "state") + cfg.Webroot = filepath.Join(base, "www") + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + if err := cfg.EnsureDirs(); err != nil { + t.Fatalf("EnsureDirs: %v", err) + } + for _, d := range []string{cfg.DataDir, cfg.CASDir(), filepath.Join(cfg.CASDir(), "tmp"), cfg.DeploymentsDir(), cfg.Webroot} { + fi, err := os.Stat(d) + if err != nil { + t.Errorf("missing %s: %v", d, err) + continue + } + if !fi.IsDir() { + t.Errorf("%s is not a directory", d) + } + } + // Idempotent: a restart must not fail on directories that already exist. + if err := cfg.EnsureDirs(); err != nil { + t.Fatalf("second EnsureDirs: %v", err) + } +} + +func TestProjectNamePattern(t *testing.T) { + valid := []string{"a", "demo", "my-site", "my.site", "my_site", "a1", strings.Repeat("x", 63)} + invalid := []string{ + "", "-lead", ".lead", "_lead", "UPPER", "has space", "has/slash", "..", + "a/../b", "a\\b", "tilde~", strings.Repeat("x", 64), "naïve", "a\x00b", + } + for _, s := range valid { + if !ProjectNamePattern.MatchString(s) { + t.Errorf("%q should be a valid project name", s) + } + } + for _, s := range invalid { + if ProjectNamePattern.MatchString(s) { + t.Errorf("%q must be rejected as a project name", s) + } + } +} + +func TestDurationRoundTrip(t *testing.T) { + var d Duration + if err := d.Set("15m30s"); err != nil { + t.Fatal(err) + } + if d.D() != 15*time.Minute+30*time.Second { + t.Errorf("d = %s", d) + } + b, err := d.MarshalText() + if err != nil { + t.Fatal(err) + } + var back Duration + if err := back.UnmarshalText(b); err != nil { + t.Fatal(err) + } + if back != d { + t.Errorf("round trip: %s != %s", back, d) + } + if err := d.Set("soon"); err == nil { + t.Error("Set must reject a non-duration") + } +} diff --git a/internal/config/load.go b/internal/config/load.go new file mode 100644 index 0000000..361f3c0 --- /dev/null +++ b/internal/config/load.go @@ -0,0 +1,200 @@ +package config + +import ( + "flag" + "fmt" + "io" + "log/slog" + "os" + "strconv" + "strings" + + "github.com/BurntSushi/toml" +) + +// Options is the result of parsing the server command line. +type Options struct { + Config Config + ConfigPath string + CheckOnly bool + ShowVersion bool +} + +// Load resolves the server configuration with precedence +// +// flag > PAGES_* env > config file > built-in default +// +// args excludes the program name. It returns flag.ErrHelp when -h was passed. +func Load(args []string, out io.Writer) (*Options, error) { + fs := flag.NewFlagSet("pages-server", flag.ContinueOnError) + fs.SetOutput(out) + + // flagCfg receives whatever the user typed; we later copy across only the + // fields whose flags were actually visited, so unset flags never shadow the + // file or the environment. + flagCfg := Default() + opts := &Options{} + + fs.StringVar(&opts.ConfigPath, "config", os.Getenv("PAGES_CONFIG"), "path to the TOML config file") + fs.BoolVar(&opts.CheckOnly, "check-config", false, "validate configuration and exit without binding ports") + fs.BoolVar(&opts.ShowVersion, "version", false, "print version and exit") + + fs.StringVar(&flagCfg.DataDir, "data-dir", flagCfg.DataDir, "directory for the database, CAS blobs and deployment trees") + fs.StringVar(&flagCfg.Webroot, "webroot", flagCfg.Webroot, "directory in which ~PROJECT symlinks are maintained") + fs.StringVar(&flagCfg.Listen, "listen", flagCfg.Listen, "address for the public static-content listener") + fs.StringVar(&flagCfg.APIListen, "api-listen", flagCfg.APIListen, "address for the management API listener") + fs.StringVar(&flagCfg.SiteURL, "site-url", flagCfg.SiteURL, "public base URL of the static listener, e.g. https://pages.example.com (reported in API responses)") + fs.StringVar((*string)(&flagCfg.AssembleMode), "assemble-mode", string(flagCfg.AssembleMode), "how deployment trees are built: auto|hardlink|copy|none") + fs.StringVar(&flagCfg.LogLevel, "log-level", flagCfg.LogLevel, "debug|info|warn|error") + fs.StringVar(&flagCfg.LogFormat, "log-format", flagCfg.LogFormat, "json|text") + fs.Var(&flagCfg.GCInterval, "gc-interval", "how often the retention/blob sweep runs") + fs.Var(&flagCfg.ShutdownGrace, "shutdown-grace", "how long in-flight requests may finish during shutdown") + + fs.Usage = func() { + fmt.Fprintf(out, "Usage: pages-server [flags]\n\n"+ + "Serves static sites deployed through the pages CLI, switching each\n"+ + "project's content atomically.\n\nFlags:\n") + fs.PrintDefaults() + } + + if err := fs.Parse(args); err != nil { + return nil, err + } + if opts.ShowVersion { + return opts, nil + } + if fs.NArg() > 0 { + return nil, fmt.Errorf("unexpected argument %q", fs.Arg(0)) + } + + set := make(map[string]bool, 16) + fs.Visit(func(f *flag.Flag) { set[f.Name] = true }) + + cfg := Default() + + if opts.ConfigPath != "" { + if err := applyFile(&cfg, opts.ConfigPath); err != nil { + return nil, err + } + } + if err := applyEnv(&cfg); err != nil { + return nil, err + } + + // Highest precedence: flags the user actually typed. + overrides := map[string]func(){ + "data-dir": func() { cfg.DataDir = flagCfg.DataDir }, + "webroot": func() { cfg.Webroot = flagCfg.Webroot }, + "listen": func() { cfg.Listen = flagCfg.Listen }, + "api-listen": func() { cfg.APIListen = flagCfg.APIListen }, + "site-url": func() { cfg.SiteURL = flagCfg.SiteURL }, + "assemble-mode": func() { cfg.AssembleMode = flagCfg.AssembleMode }, + "log-level": func() { cfg.LogLevel = flagCfg.LogLevel }, + "log-format": func() { cfg.LogFormat = flagCfg.LogFormat }, + "gc-interval": func() { cfg.GCInterval = flagCfg.GCInterval }, + "shutdown-grace": func() { cfg.ShutdownGrace = flagCfg.ShutdownGrace }, + } + for name, apply := range overrides { + if set[name] { + apply() + } + } + + if err := cfg.Validate(); err != nil { + return nil, err + } + opts.Config = cfg + return opts, nil +} + +// applyFile decodes the TOML file over cfg. Unknown keys are an error: a typo in +// a config file should fail loudly rather than silently leave a default in place. +func applyFile(cfg *Config, path string) error { + md, err := toml.DecodeFile(path, cfg) + if err != nil { + return fmt.Errorf("config %s: %w", path, err) + } + if undec := md.Undecoded(); len(undec) > 0 { + keys := make([]string, len(undec)) + for i, k := range undec { + keys[i] = k.String() + } + return fmt.Errorf("config %s: unknown key(s): %s", path, strings.Join(keys, ", ")) + } + return nil +} + +func applyEnv(cfg *Config) error { + str := func(key string, dst *string) { + if v, ok := os.LookupEnv(key); ok { + *dst = v + } + } + str("PAGES_DATA_DIR", &cfg.DataDir) + str("PAGES_WEBROOT", &cfg.Webroot) + str("PAGES_LISTEN", &cfg.Listen) + str("PAGES_API_LISTEN", &cfg.APIListen) + str("PAGES_SITE_URL", &cfg.SiteURL) + str("PAGES_LOG_LEVEL", &cfg.LogLevel) + str("PAGES_LOG_FORMAT", &cfg.LogFormat) + str("PAGES_ASSEMBLE_MODE", (*string)(&cfg.AssembleMode)) + + dur := func(key string, dst *Duration) error { + v, ok := os.LookupEnv(key) + if !ok { + return nil + } + if err := dst.UnmarshalText([]byte(v)); err != nil { + return fmt.Errorf("%s=%q: %w", key, v, err) + } + return nil + } + if err := dur("PAGES_GC_INTERVAL", &cfg.GCInterval); err != nil { + return err + } + if err := dur("PAGES_RECONCILE_INTERVAL", &cfg.ReconcileInterval); err != nil { + return err + } + if err := dur("PAGES_SHUTDOWN_GRACE", &cfg.ShutdownGrace); err != nil { + return err + } + + if v, ok := os.LookupEnv("PAGES_TRUSTED_PROXY_CIDRS"); ok { + parts := strings.Split(v, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + cfg.TrustedProxyCIDRs = out + } + if v, ok := os.LookupEnv("PAGES_MAX_FILE_BYTES"); ok { + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return fmt.Errorf("PAGES_MAX_FILE_BYTES=%q: %w", v, err) + } + cfg.Limits.MaxFileBytes = n + } + return nil +} + +// Logger builds the structured logger described by the configuration. +func (c *Config) Logger(w io.Writer) *slog.Logger { + var level slog.Level + switch c.LogLevel { + case "debug": + level = slog.LevelDebug + case "warn": + level = slog.LevelWarn + case "error": + level = slog.LevelError + default: + level = slog.LevelInfo + } + opts := &slog.HandlerOptions{Level: level} + if c.LogFormat == "text" { + return slog.New(slog.NewTextHandler(w, opts)) + } + return slog.New(slog.NewJSONHandler(w, opts)) +} diff --git a/internal/deploy/assemble.go b/internal/deploy/assemble.go new file mode 100644 index 0000000..dd11189 --- /dev/null +++ b/internal/deploy/assemble.go @@ -0,0 +1,147 @@ +// Package deploy owns the deployment lifecycle: creating one, negotiating its +// manifest, accepting blob uploads, assembling the directory tree, and (from M3) +// switching a project over to it. +// +// It is the only package that writes to $DATA_DIR/deployments. Everything it +// writes lands in a staging directory first and becomes visible with a single +// rename, which is the same discipline the CAS uses for blobs and the registry +// uses for the active pointer. +package deploy + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strconv" + + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/pathutil" + "github.com/iceBear67/simplepages/internal/store" +) + +const ( + dirMode = 0o755 + // stagingSuffix marks a tree that is still being built. Recovery deletes + // every directory carrying it, since by definition nothing references one. + stagingSuffix = ".staging" +) + +// DeploymentDir is where a deployment's assembled tree lives. +// +// The project id rather than its name: a project that is renamed keeps its +// deployments where they are, and no user-chosen string is ever a path segment +// under $DATA_DIR. +func DeploymentDir(root string, projectID int64, publicID string) string { + return filepath.Join(root, strconv.FormatInt(projectID, 10), publicID) +} + +// Assemble builds destDir from the CAS. +// +// The tree is built under destDir+".staging" and moved into place with one +// rename, so destDir either does not exist or is the complete deployment — +// there is no state in which a reader could walk a half-built tree. +// +// It is idempotent in the way finalize needs: an existing destDir is a finished +// tree (rename is atomic, so a partial one cannot survive a crash) and is left +// alone. +func Assemble(ctx context.Context, cs *cas.Store, files []store.FileRow, destDir string) error { + staging := destDir + stagingSuffix + if fi, err := os.Stat(destDir); err == nil { + if !fi.IsDir() { + return fmt.Errorf("deploy: %s exists and is not a directory", destDir) + } + return os.RemoveAll(staging) + } else if !os.IsNotExist(err) { + return err + } + + // Whatever an earlier attempt left behind is unreferenced by construction. + if err := os.RemoveAll(staging); err != nil { + return err + } + if err := os.MkdirAll(staging, dirMode); err != nil { + return err + } + // One cleanup for every failure path: a staging tree that outlives its + // attempt is wasted disk that only recovery would find. + ok := false + defer func() { + if !ok { + os.RemoveAll(staging) + } + }() + + // The set of directories that had to be created, so each can be fsynced + // once. Recorded per ancestor because MkdirAll creates parents silently. + dirs := map[string]bool{".": true} + for _, f := range files { + if err := ctx.Err(); err != nil { + return err + } + // The manifest was validated when it was accepted. Checking again here + // costs a few hundred nanoseconds per file and means the one place that + // turns stored strings into filesystem paths does not depend on a + // promise made by a different package at a different time. + if err := pathutil.Validate(f.Path); err != nil { + return fmt.Errorf("deploy: manifest path %q: %w", f.Path, err) + } + if dir := path.Dir(f.Path); !dirs[dir] { + if err := os.MkdirAll(filepath.Join(staging, filepath.FromSlash(dir)), dirMode); err != nil { + return err + } + for d := dir; !dirs[d]; d = path.Dir(d) { + dirs[d] = true + } + } + if err := cs.LinkInto(f.Digest, staging, f.Path); err != nil { + return fmt.Errorf("deploy: %s: %w", f.Path, err) + } + } + + // Blob content is already durable — Put fsynced it, and a hardlink shares + // that inode — but the directory entries pointing at it are not. Without + // this, a crash could leave a renamed-into-place tree with missing files, + // which is exactly the half-updated site the whole design exists to avoid. + if err := syncDirs(staging, dirs); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(destDir), dirMode); err != nil { + return err + } + if err := os.Rename(staging, destDir); err != nil { + return err + } + ok = true + return syncDir(filepath.Dir(destDir)) +} + +// syncDirs fsyncs every directory of the staged tree, deepest first, so a +// parent is only made durable once the entries it names are. +func syncDirs(staging string, dirs map[string]bool) error { + rel := make([]string, 0, len(dirs)) + for d := range dirs { + rel = append(rel, d) + } + sort.Sort(sort.Reverse(sort.StringSlice(rel))) + for _, d := range rel { + if err := syncDir(filepath.Join(staging, filepath.FromSlash(d))); err != nil { + return err + } + } + return nil +} + +func syncDir(dir string) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + if err := f.Sync(); err != nil { + return err + } + return f.Close() +} diff --git a/internal/deploy/assemble_test.go b/internal/deploy/assemble_test.go new file mode 100644 index 0000000..0713484 --- /dev/null +++ b/internal/deploy/assemble_test.go @@ -0,0 +1,228 @@ +package deploy + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/store" +) + +// fixture is a CAS holding some content plus the manifest that names it. +type fixture struct { + cas *cas.Store + dir string // deployments root + files []store.FileRow +} + +func newFixture(t *testing.T, contents map[string]string) *fixture { + t.Helper() + base := t.TempDir() + deployDir := filepath.Join(base, "deployments") + cs, err := cas.Open(filepath.Join(base, "cas"), cas.Options{ProbeDir: deployDir}) + if err != nil { + t.Fatalf("cas.Open: %v", err) + } + t.Cleanup(func() { cs.Close() }) + + f := &fixture{cas: cs, dir: deployDir} + for path, content := range contents { + d := cas.Sum([]byte(content)) + if _, err := cs.Put(t.Context(), d, int64(len(content)), 1<<20, strings.NewReader(content)); err != nil { + t.Fatalf("put %s: %v", path, err) + } + f.files = append(f.files, store.FileRow{Path: path, Digest: d, Size: int64(len(content))}) + } + return f +} + +// walk reads back an assembled tree as path -> content. +func walk(t *testing.T, dir string) map[string]string { + t.Helper() + out := map[string]string{} + err := filepath.WalkDir(dir, func(p string, e fs.DirEntry, err error) error { + if err != nil || e.IsDir() { + return err + } + b, err := os.ReadFile(p) + if err != nil { + return err + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + out[filepath.ToSlash(rel)] = string(b) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + return out +} + +func TestAssembleBuildsTheTree(t *testing.T) { + want := map[string]string{ + "index.html": "

hi

", + "assets/app.js": "console.log(1)", + "assets/css/app.css": "body{}", + "a/b/c/d/deep.txt": "deep", + "copy.html": "

hi

", // shares a blob with index.html + } + f := newFixture(t, want) + dest := DeploymentDir(f.dir, 7, "dpl_0123456789abcdef") + + if err := Assemble(t.Context(), f.cas, f.files, dest); err != nil { + t.Fatalf("Assemble: %v", err) + } + got := walk(t, dest) + if len(got) != len(want) { + t.Fatalf("assembled %d files, want %d: %v", len(got), len(want), got) + } + for p, content := range want { + if got[p] != content { + t.Errorf("%s = %q, want %q", p, got[p], content) + } + } + // The staging directory is gone: it became the tree by rename. + if _, err := os.Stat(dest + stagingSuffix); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("staging directory survived: %v", err) + } + // The project id is a path segment, so the tree is where the registry will + // later expect to find it. + if !strings.HasSuffix(filepath.Dir(dest), string(filepath.Separator)+"7") { + t.Errorf("deployment dir %q is not under its project id", dest) + } + + // Where the filesystem allows it, an assembled file is the blob rather than a + // copy of it. This is what keeps a hundred deployments of one site costing + // one site's worth of disk, so it is worth asserting rather than assuming. + if f.cas.LinkMode() != cas.LinkHard { + t.Skipf("link mode is %s on this filesystem; skipping the hardlink assertion", f.cas.LinkMode()) + } + for _, e := range f.files { + blob, err := os.Stat(f.cas.Path(e.Digest)) + if err != nil { + t.Fatal(err) + } + placed, err := os.Stat(filepath.Join(dest, e.Path)) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(blob, placed) { + t.Errorf("%s is a copy of its blob, not a link to it", e.Path) + } + } +} + +// Finalize can be retried, so assembling onto a finished tree must be a no-op +// rather than a rebuild — a rebuild would briefly unlink files that an +// external consumer of $WEBROOT is reading. +func TestAssembleIsIdempotent(t *testing.T) { + f := newFixture(t, map[string]string{"index.html": "one"}) + dest := DeploymentDir(f.dir, 1, "dpl_a") + if err := Assemble(t.Context(), f.cas, f.files, dest); err != nil { + t.Fatal(err) + } + before, err := os.Stat(filepath.Join(dest, "index.html")) + if err != nil { + t.Fatal(err) + } + if err := Assemble(t.Context(), f.cas, f.files, dest); err != nil { + t.Fatalf("second Assemble: %v", err) + } + after, err := os.Stat(filepath.Join(dest, "index.html")) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(before, after) { + t.Error("the second Assemble replaced a file that was already in place") + } +} + +// A crash leaves a staging tree behind. The next attempt must clear it rather +// than build on top of files it did not put there. +func TestAssembleDiscardsALeftoverStagingTree(t *testing.T) { + f := newFixture(t, map[string]string{"index.html": "real"}) + dest := DeploymentDir(f.dir, 1, "dpl_a") + staging := dest + stagingSuffix + if err := os.MkdirAll(staging, 0o755); err != nil { + t.Fatal(err) + } + // Same path as a manifest entry, so a build that did not clear this would + // fail on the exclusive create rather than silently serve the wrong bytes. + if err := os.WriteFile(filepath.Join(staging, "index.html"), []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(staging, "orphan.txt"), []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + + if err := Assemble(t.Context(), f.cas, f.files, dest); err != nil { + t.Fatalf("Assemble: %v", err) + } + got := walk(t, dest) + if len(got) != 1 || got["index.html"] != "real" { + t.Errorf("tree = %v, want just the real index.html", got) + } +} + +// A failure must leave nothing behind: no partial tree at the destination, and +// no staging directory quietly consuming disk until recovery notices it. +func TestAssembleLeavesNothingBehindOnFailure(t *testing.T) { + f := newFixture(t, map[string]string{"index.html": "real"}) + missing := store.FileRow{Path: "gone.txt", Digest: cas.Sum([]byte("never stored")), Size: 12} + dest := DeploymentDir(f.dir, 1, "dpl_a") + + err := Assemble(t.Context(), f.cas, append(f.files, missing), dest) + if !errors.Is(err, cas.ErrNotFound) { + t.Fatalf("err = %v, want cas.ErrNotFound", err) + } + for _, p := range []string{dest, dest + stagingSuffix} { + if _, err := os.Stat(p); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("%s still exists: %v", p, err) + } + } +} + +// Assembly is the one place that turns stored strings into filesystem paths, so +// it re-checks them even though the API validated the manifest on the way in. +// A row that got past the API — or into the table by some other route — must not +// be able to place a file outside the tree being built. +func TestAssembleRejectsAnEscapingPath(t *testing.T) { + f := newFixture(t, map[string]string{"index.html": "real"}) + dest := DeploymentDir(f.dir, 1, "dpl_a") + + for _, bad := range []string{"../../etc/passwd", "/etc/passwd", "a/../../b", "a//b", `a\..\b`, "a/./b", ""} { + row := store.FileRow{Path: bad, Digest: f.files[0].Digest, Size: f.files[0].Size} + if err := Assemble(t.Context(), f.cas, []store.FileRow{row}, dest); err == nil { + t.Errorf("Assemble accepted %q", bad) + } + // Nothing is left behind for the next attempt to inherit, and in + // particular no directory was created on the way to the rejection. + for _, p := range []string{dest, dest + stagingSuffix} { + if _, err := os.Stat(p); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("%q left %s behind: %v", bad, p, err) + } + } + } +} + +func TestAssembleHonoursCancellation(t *testing.T) { + f := newFixture(t, map[string]string{"index.html": "real"}) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + dest := DeploymentDir(f.dir, 1, "dpl_a") + if err := Assemble(ctx, f.cas, f.files, dest); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if _, err := os.Stat(dest + stagingSuffix); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("staging directory survived cancellation: %v", err) + } +} diff --git a/internal/deploy/atomicity_test.go b/internal/deploy/atomicity_test.go new file mode 100644 index 0000000..e3602db --- /dev/null +++ b/internal/deploy/atomicity_test.go @@ -0,0 +1,454 @@ +package deploy + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/site" + "github.com/iceBear67/simplepages/internal/store" + "github.com/iceBear67/simplepages/internal/webroot" +) + +// These are the tests the whole program exists for. +// +// The failure they are written against is the one rsync has: during a +// deployment a visitor sees the new HTML with the old JavaScript, or an asset +// that is not there yet. Everything else in this repository — the immutable +// snapshot, the single pointer store, the database-before-memory ordering — is +// a means to making that state unobservable, and unobservable is a claim about +// concurrent behaviour that only a concurrent test can support. +// +// What is actually asserted, and why it is the strongest true statement: +// +// - Every single response is internally consistent. big.bin is half a +// megabyte of one repeated byte, so it spans many writes and a switch +// landing mid-body would show up as a seam. Every byte of it equal to the +// same version is the claim "no request ever saw a half-updated site". +// - A reader's successive responses never go backwards. Requests within one +// reader are strictly ordered — the next is not sent until the previous has +// been read to completion — so the version it observes may only rise. +// +// It would be tempting to also demand that three separate GETs issued around +// the same time report the same version. That is not a property this or any +// design has: they are three requests, an activation may legitimately land +// between any two of them, and asserting otherwise would be asserting that the +// switch never happens. The per-response guarantee above is what "atomic +// deployment" means. + +const ( + // Large enough that a response spans many socket writes, so a switch has + // somewhere to land mid-body. + stormFileSize = 512 << 10 + // The in-flight tests need the server to still be blocked writing when the + // test does something underneath it, which means comfortably more than a + // loopback socket will buffer for a client that has stopped reading. + inflightFileSize = 8 << 20 +) + +// switchEnv is an env with the serving layer attached: a registry, a webroot +// and an HTTP server, which is the only configuration in which the switch is +// observable from the outside. +type switchEnv struct { + *env + reg *site.Registry + wrDir string + srv *httptest.Server + cl *http.Client +} + +func newSwitchEnv(t *testing.T) *switchEnv { + t.Helper() + e := newEnv(t) + log := slog.New(slog.DiscardHandler) + + reg := site.NewRegistry() + wrDir := t.TempDir() + wr, err := webroot.Open(wrDir, e.dir) + if err != nil { + t.Fatalf("webroot.Open: %v", err) + } + e.svc.Sites = reg + e.svc.Webroot = wr + reg.Put(e.p) + + srv := httptest.NewServer(&site.Handler{Registry: reg, CAS: e.cas, Log: log}) + t.Cleanup(srv.Close) + + // The default transport keeps two idle connections per host, which would + // turn 64 readers into a connection churn benchmark instead of a switching + // one. + cl := &http.Client{Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512}} + t.Cleanup(cl.CloseIdleConnections) + + return &switchEnv{env: e, reg: reg, wrDir: wrDir, srv: srv, cl: cl} +} + +// versionFiles is deployment n: three small files that name their version, and +// one large one filled with the single byte n so that any part of it identifies +// the whole. +func versionFiles(n, size int) map[string]string { + v := strconv.Itoa(n) + return map[string]string{ + "marker.txt": v, + "a.txt": v, + "b.txt": v, + "big.bin": string(bytes.Repeat([]byte{byte(n)}, size)), + } +} + +// publish takes version n all the way to ready without activating it. +func (e *switchEnv) publish(t *testing.T, n, size int) *store.Deployment { + t.Helper() + contents := versionFiles(n, size) + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { + t.Fatalf("SetManifest v%d: %v", n, err) + } + names := make([]string, 0, len(contents)) + for p := range contents { + names = append(names, p) + } + e.upload(t, contents, names...) + dep, err := e.svc.Finalize(t.Context(), e.p, dep) + if err != nil { + t.Fatalf("Finalize v%d: %v", n, err) + } + return dep +} + +func (e *switchEnv) activate(t *testing.T, dep *store.Deployment) *store.Deployment { + t.Helper() + out, err := e.svc.Activate(t.Context(), e.p, dep) + if err != nil { + t.Fatalf("Activate %s: %v", dep.PublicID, err) + } + return out +} + +// get fetches one file from the served site. It returns errors rather than +// failing the test, because most of its callers are goroutines. +func (e *switchEnv) get(name string) ([]byte, error) { + resp, err := e.cl.Get(e.srv.URL + "/~demo/" + name) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("GET %s: status %d", name, resp.StatusCode) + } + return io.ReadAll(resp.Body) +} + +// bigVersion is the assertion that matters: it reports which version a big.bin +// body came from, and fails if the body is not entirely from one version. +func bigVersion(b []byte, size int) (int, error) { + if len(b) != size { + return 0, fmt.Errorf("big.bin is %d bytes, want %d", len(b), size) + } + first := b[0] + for i, c := range b { + if c != first { + return 0, fmt.Errorf( + "big.bin mixes two deployments: byte 0 is from version %d but byte %d is from version %d", + first, i, c) + } + } + return int(first), nil +} + +func smallVersion(b []byte) (int, error) { + n, err := strconv.Atoi(string(b)) + if err != nil { + return 0, fmt.Errorf("file body %q is not a version number: %v", b, err) + } + return n, nil +} + +func TestActivationAtomicity(t *testing.T) { + const versions = 50 + const readers = 64 + + e := newSwitchEnv(t) + deps := make([]*store.Deployment, versions) + valid := make(map[string]bool, versions) + for i := range deps { + deps[i] = e.publish(t, i+1, stormFileSize) + valid[deps[i].PublicID] = true + } + e.activate(t, deps[0]) + + ctx, stop := context.WithCancel(t.Context()) + var wg sync.WaitGroup + + // Anti-vacuity: if every reader only ever saw the last version the + // invariants above hold trivially and prove nothing. + var mu sync.Mutex + lowest, highest := versions+1, 0 + record := func(n int) { + mu.Lock() + defer mu.Unlock() + lowest = min(lowest, n) + highest = max(highest, n) + } + + for range readers { + wg.Add(1) + go func() { + defer wg.Done() + seen := 0 + for ctx.Err() == nil { + for _, name := range []string{"a.txt", "b.txt", "big.bin"} { + body, err := e.get(name) + if err != nil { + if ctx.Err() == nil { + t.Errorf("%v", err) + } + return + } + var n int + if name == "big.bin" { + n, err = bigVersion(body, stormFileSize) + } else { + n, err = smallVersion(body) + } + if err != nil { + t.Errorf("%s: %v", name, err) + return + } + if n < seen { + t.Errorf("%s reported version %d after version %d had already been "+ + "served to this reader: the switch was observed running backwards", + name, n, seen) + return + } + seen = n + record(n) + } + } + }() + } + + // The symlink is not what serves the site, but an external reader — a + // reverse proxy, a backup job — follows it, and it must never be missing or + // dangling while the switch runs. + link := filepath.Join(e.wrDir, "~demo") + wg.Add(1) + go func() { + defer wg.Done() + for ctx.Err() == nil { + target, err := os.Readlink(link) + if err != nil { + t.Errorf("$WEBROOT/~demo: %v", err) + return + } + // Stat follows the link, so a dangling one fails here. + fi, err := os.Stat(target) + if err != nil { + t.Errorf("$WEBROOT/~demo -> %s: %v", target, err) + return + } + if !fi.IsDir() { + t.Errorf("$WEBROOT/~demo -> %s is not a directory", target) + return + } + if !valid[filepath.Base(target)] { + t.Errorf("$WEBROOT/~demo -> %s, which is not a deployment of this project", target) + return + } + } + }() + + for _, dep := range deps[1:] { + if _, err := e.svc.Activate(t.Context(), e.p, dep); err != nil { + t.Errorf("Activate %s: %v", dep.PublicID, err) + break + } + } + stop() + wg.Wait() + + if t.Failed() { + return + } + if lowest >= versions { + t.Fatalf("every observation was of version %d or later: the readers never "+ + "overlapped the switching, so this test proved nothing", lowest) + } + if lowest == highest { + t.Fatalf("every observation was of version %d: no switch was observed", lowest) + } + for _, name := range []string{"marker.txt", "a.txt", "b.txt"} { + body, err := e.get(name) + if err != nil { + t.Fatalf("%v", err) + } + if got, err := smallVersion(body); err != nil || got != versions { + t.Errorf("after the storm %s = %q (%v), want version %d", name, body, err, versions) + } + } +} + +// openBig starts a request for big.bin and reads only its first kilobyte, so +// the response is still open and the server is still blocked writing it. +func (e *switchEnv) openBig(t *testing.T) (*http.Response, []byte) { + t.Helper() + resp, err := e.cl.Get(e.srv.URL + "/~demo/big.bin") + if err != nil { + t.Fatalf("GET big.bin: %v", err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + t.Fatalf("GET big.bin: status %d", resp.StatusCode) + } + head := make([]byte, 1024) + if _, err := io.ReadFull(resp.Body, head); err != nil { + resp.Body.Close() + t.Fatalf("reading the start of big.bin: %v", err) + } + return resp, head +} + +// drain finishes a response opened by openBig and reports which version the +// whole body came from. +func drain(t *testing.T, resp *http.Response, head []byte) int { + t.Helper() + defer resp.Body.Close() + rest, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("reading the rest of big.bin: %v", err) + } + n, err := bigVersion(append(head, rest...), inflightFileSize) + if err != nil { + t.Fatalf("%v", err) + } + return n +} + +func TestAnInFlightRequestKeepsReadingTheDeploymentItStartedOn(t *testing.T) { + e := newSwitchEnv(t) + v1 := e.publish(t, 1, inflightFileSize) + v2 := e.publish(t, 2, inflightFileSize) + e.activate(t, v1) + + resp, head := e.openBig(t) + e.activate(t, v2) + + // This is the property a symlink rename cannot give you: the switch has + // already happened, and this response is still the one it started as. + if n := drain(t, resp, head); n != 1 { + t.Errorf("a request that started before the switch finished on version %d, want 1", n) + } + body, err := e.get("marker.txt") + if err != nil { + t.Fatalf("%v", err) + } + if n, _ := smallVersion(body); n != 2 { + t.Errorf("a request that started after the switch got version %d, want 2", n) + } +} + +func TestAnInFlightRequestSurvivesTheContentBeingCollected(t *testing.T) { + e := newSwitchEnv(t) + v1 := e.publish(t, 1, inflightFileSize) + v2 := e.publish(t, 2, inflightFileSize) + e.activate(t, v1) + + resp, head := e.openBig(t) + e.activate(t, v2) + + // Everything a collector could possibly remove, with no grace period at + // all: the assembled tree and the content it was hardlinked from. Both, + // because removing only one of them leaves the inode alive through the + // other and the test would prove nothing. + if err := os.RemoveAll(DeploymentDir(e.dir, e.p.ID, v1.PublicID)); err != nil { + t.Fatalf("removing the old tree: %v", err) + } + for _, c := range versionFiles(1, inflightFileSize) { + if err := e.cas.Remove(cas.Sum([]byte(c))); err != nil { + t.Fatalf("removing old content: %v", err) + } + } + + // The handler is holding an open descriptor, and POSIX keeps the inode + // alive until it closes. The grace period in the collector exists so this + // never has to be relied on, but relying on it has to work. + if n := drain(t, resp, head); n != 1 { + t.Errorf("a request whose content was deleted under it finished on version %d, want 1", n) + } + if _, err := e.get("marker.txt"); err != nil { + t.Errorf("the live deployment stopped serving after the old one was collected: %v", err) + } +} + +func TestAFailedActivationChangesNothing(t *testing.T) { + e := newSwitchEnv(t) + v1 := e.publish(t, 1, 64) + v2 := e.publish(t, 2, 64) + e.activate(t, v1) + + link := filepath.Join(e.wrDir, "~demo") + before, err := os.Readlink(link) + if err != nil { + t.Fatalf("$WEBROOT/~demo: %v", err) + } + + // Make v2's manifest unreadable, which fails Activate inside index() — + // before the transaction, before the pointer store, before the symlink. The + // bogus blobs row exists only to satisfy the foreign key; the point is the + // one-byte digest, which cas.FromBytes refuses. + err = e.db.Tx(t.Context(), func(tx *sql.Tx) error { + if _, err := tx.ExecContext(t.Context(), ` + INSERT INTO blobs (digest, size, present, created_at, last_ref_at) + VALUES (x'00', 1, 1, 0, 0) ON CONFLICT(digest) DO NOTHING`); err != nil { + return err + } + _, err := tx.ExecContext(t.Context(), + `UPDATE deployment_files SET digest = x'00' WHERE deployment_id = ?`, v2.ID) + return err + }) + if err != nil { + t.Fatalf("corrupting the manifest: %v", err) + } + + if _, err := e.svc.Activate(t.Context(), e.p, v2); err == nil { + t.Fatal("Activate succeeded on a deployment whose manifest cannot be read") + } + + // In memory. + body, err := e.get("marker.txt") + if err != nil { + t.Fatalf("%v", err) + } + if n, _ := smallVersion(body); n != 1 { + t.Errorf("after the failed activation the site serves version %d, want 1", n) + } + // In the database, which is what a restart would come back to. + active, err := e.db.ActiveDeployment(t.Context(), e.p.ID) + if err != nil { + t.Fatalf("ActiveDeployment: %v", err) + } + if active.PublicID != v1.PublicID { + t.Errorf("the database says %s is active, want %s", active.PublicID, v1.PublicID) + } + // And on disk. + after, err := os.Readlink(link) + if err != nil { + t.Fatalf("$WEBROOT/~demo: %v", err) + } + if after != before { + t.Errorf("$WEBROOT/~demo moved to %s, want it left at %s", after, before) + } +} diff --git a/internal/deploy/gc.go b/internal/deploy/gc.go new file mode 100644 index 0000000..d24776a --- /dev/null +++ b/internal/deploy/gc.go @@ -0,0 +1,259 @@ +package deploy + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/store" +) + +const ( + // defaultBlobGrace is how long a blob must have been unreferenced before + // its content is removed. + // + // It is what makes the read path safe without reference counting requests. + // A handler resolves a digest from its snapshot and then opens it; between + // those two instants the collector could in principle delete the file. An + // hour is an enormous margin for a gap that is measured in microseconds, + // and it costs only some disk that was going to be reclaimed anyway. + defaultBlobGrace = time.Hour + + // failedRetention is how long a failed deployment's row is kept. Its + // manifest is already gone, so this is purely so that an operator + // investigating a broken CI job can still see that it failed and why. + failedRetention = 24 * time.Hour +) + +// Collect runs one garbage collection pass: retention first, then the content +// nothing references any more. +// +// Two passes in that order, and not one, because the first is what makes work +// for the second. Deleting a deployment drops its manifest rows, the delete +// trigger takes each blob's refcount down, and only then can a blob be seen to +// be unreferenced. The second pass will not act on those blobs in this same +// run — the trigger sets last_ref_at to now and the grace period has not +// elapsed — which is deliberate: content that just became unreachable is +// exactly the content some in-flight request is most likely to still be +// reading. +// +// A dry run reports the deployments that would be deleted and the blobs that +// are collectable right now. It cannot report the blobs the deletions would +// free, because nothing has been deleted; the number is a floor, not an +// estimate, and it is honest about being one. +func (s *Service) Collect(ctx context.Context, dryRun bool) (api.GCStats, error) { + stats := api.GCStats{DryRun: dryRun} + + if !dryRun { + // Abandoned uploads first, so that whatever only they referenced is + // already unreferenced by the time the blob pass looks. + n, err := s.DB.ExpireStaleDeployments(ctx, time.Now().Add(-staleUploadAge), + "abandoned: no activity for "+staleUploadAge.String()) + if err != nil { + return stats, err + } + if n > 0 { + s.Log.InfoContext(ctx, "expired unfinished deployments", "count", n) + } + } + + projects, err := s.DB.AllProjects(ctx) + if err != nil { + return stats, err + } + for _, p := range projects { + n, err := s.collectProject(ctx, p, dryRun) + stats.DeploymentsDeleted += n + if err != nil { + return stats, err + } + } + + blobs, err := s.DB.UnreferencedBlobs(ctx, time.Now().Add(-s.blobGrace()), 0) + if err != nil { + return stats, err + } + for _, b := range blobs { + if dryRun { + stats.BlobsDeleted++ + stats.BytesFreed += b.Size + continue + } + deleted, err := s.DB.DeleteBlob(ctx, b.Digest, func() error { return s.CAS.Remove(b.Digest) }) + if err != nil { + return stats, err + } + // Not deleted means the blob was referenced again between the listing + // and the delete — a new deployment naming content that was about to be + // collected. Leaving it alone is the whole point of rechecking the + // refcount inside the transaction. + if deleted { + stats.BlobsDeleted++ + stats.BytesFreed += b.Size + } + } + return stats, nil +} + +// collectProject applies one project's retention policy. +// +// The active deployment is not considered at all: it is excluded by the query, +// so no counting mistake here can reach it. +func (s *Service) collectProject(ctx context.Context, p *store.Project, dryRun bool) (int, error) { + deps, err := s.DB.InactiveDeployments(ctx, p.ID) + if err != nil { + return 0, err + } + grace := time.Duration(p.RetentionGraceS) * time.Second + now := time.Now() + + var deleted, kept int + for _, dep := range deps { + switch dep.State { + case store.StateReady: + // Newest first, so the first RetentionCount of them are the ones + // worth keeping for a rollback. + if kept < p.RetentionCount { + kept++ + continue + } + if now.Sub(retiredAt(dep)) < grace { + continue + } + case store.StateFailed: + if now.Sub(dep.CreatedAt) < failedRetention { + continue + } + case store.StateDeleting: + // Already claimed by a sweep that did not finish. No grace applies: + // nothing may serve a tree that is half removed. + default: + // pending or uploading. Someone may still be uploading to it, and + // ExpireStaleDeployments is what decides when they are not. + continue + } + if dryRun { + deleted++ + continue + } + if err := s.claim(ctx, p, dep); err != nil { + switch { + case errors.Is(err, store.ErrNotFound): + // Deleted by someone else between the listing and now. + case errors.Is(err, store.ErrConflict): + // Activated between the listing and now — a rollback landed on + // a deployment retention had picked. Correct outcome: the claim + // is refused and the deployment stays. + s.Log.InfoContext(ctx, "skipped a deployment that was activated during collection", + "project", p.Name, "deployment", dep.PublicID) + default: + return deleted, err + } + continue + } + if err := s.removeDeployment(ctx, dep); err != nil { + // The row is in the deleting state and committed, so recovery or + // the next sweep will finish it. Nothing serves it in the meantime. + s.Log.WarnContext(ctx, "could not finish deleting a deployment", + "project", p.Name, "deployment", dep.PublicID, "err", err) + continue + } + deleted++ + } + return deleted, nil +} + +// retiredAt is when a deployment stopped being served, or when it was created +// if it never was. It is what the retention grace is measured from. +func retiredAt(dep *store.Deployment) time.Time { + if dep.DeactivatedAt != nil { + return *dep.DeactivatedAt + } + return dep.CreatedAt +} + +// claim marks a deployment as being deleted, which is what makes it safe to +// start removing files. +// +// The project lock is held for exactly this statement. Activation takes the +// same lock and re-reads the row under it, so the two orderings are the only +// possible ones: either activation commits first and this fails on the +// active = 0 condition, or this commits first and activation finds a deployment +// in the deleting state and refuses it. There is no interleaving in which a +// tree is removed from underneath a deployment that has just become active. +func (s *Service) claim(ctx context.Context, p *store.Project, dep *store.Deployment) error { + unlock, err := s.locks.lock(ctx, p.ID) + if err != nil { + return err + } + defer unlock() + return s.DB.MarkDeploymentDeleting(ctx, dep.ID) +} + +// Delete removes one deployment on request. The active one cannot be deleted: +// that is a conflict, not a permission problem, and the client is expected to +// activate something else first. +func (s *Service) Delete(ctx context.Context, p *store.Project, dep *store.Deployment) error { + if err := s.claim(ctx, p, dep); err != nil { + switch { + case errors.Is(err, store.ErrNotFound): + return api.Errorf(api.CodeNotFound, "no such deployment") + case errors.Is(err, store.ErrConflict): + return api.Errorf(api.CodeDeploymentActive, + "this deployment is the one the project is serving; activate another one first") + } + return err + } + if err := s.removeDeployment(ctx, dep); err != nil { + return err + } + s.Log.InfoContext(ctx, "deployment deleted", "project", p.Name, "deployment", dep.PublicID) + return nil +} + +// RemoveProjectTrees deletes what a project left on disk once its rows are +// gone. Its blobs are freed by the same cascade and collected on the next pass. +func (s *Service) RemoveProjectTrees(projectID int64) error { + if s.Dir == "" { + return nil + } + return os.RemoveAll(filepath.Join(s.Dir, strconv.FormatInt(projectID, 10))) +} + +// RunCollector collects on a timer until ctx is done. A failed pass is logged +// and the next one runs as scheduled: everything the collector does is +// idempotent, so there is nothing to unwind and no reason to stop. +func (s *Service) RunCollector(ctx context.Context, every time.Duration) { + if every <= 0 { + return + } + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + stats, err := s.Collect(ctx, false) + if err != nil { + s.Log.WarnContext(ctx, "garbage collection did not finish", "err", err) + } + if stats.DeploymentsDeleted > 0 || stats.BlobsDeleted > 0 { + s.Log.InfoContext(ctx, "collected", + "deployments", stats.DeploymentsDeleted, + "blobs", stats.BlobsDeleted, "bytes", stats.BytesFreed) + } + } + } +} + +func (s *Service) blobGrace() time.Duration { + if s.BlobGrace == 0 { + return defaultBlobGrace + } + return s.BlobGrace +} diff --git a/internal/deploy/gc_test.go b/internal/deploy/gc_test.go new file mode 100644 index 0000000..906a0f0 --- /dev/null +++ b/internal/deploy/gc_test.go @@ -0,0 +1,617 @@ +package deploy + +import ( + "database/sql" + "errors" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/store" +) + +// These tests set BlobGrace negative. The grace exists so that a request which +// has resolved a digest and is about to open it cannot lose the file underneath +// it, and the default hour is far longer than any test wants to wait. Timestamps +// are whole seconds, so a grace of zero would not collect a blob dereferenced in +// the same second either — negative is the only value that means "now". +const collectNow = -time.Minute + +// deployReady publishes a finished, inactive deployment holding contents. +func (e *env) deployReady(t *testing.T, contents map[string]string) *store.Deployment { + t.Helper() + dep := e.create(t) + files := manifest(contents) + if _, _, err := e.svc.SetManifest(t.Context(), dep, files); err != nil { + t.Fatalf("SetManifest: %v", err) + } + paths := make([]string, 0, len(contents)) + for p := range contents { + paths = append(paths, p) + } + e.upload(t, contents, paths...) + dep, err := e.svc.Finalize(t.Context(), e.p, dep) + if err != nil { + t.Fatalf("Finalize: %v", err) + } + return dep +} + +// version publishes a deployment whose single file identifies it, which is +// enough for retention tests: what matters is how many survive and which. +func (e *env) version(t *testing.T, n int) *store.Deployment { + t.Helper() + return e.deployReady(t, map[string]string{"index.html": "v" + strconv.Itoa(n)}) +} + +func (e *env) activate(t *testing.T, dep *store.Deployment) { + t.Helper() + if _, err := e.svc.Activate(t.Context(), e.p, dep); err != nil { + t.Fatalf("Activate %s: %v", dep.PublicID, err) + } +} + +// exec runs a statement the service has no method for. Retention tests need to +// backdate rows, because the alternative is a test that sleeps for an hour. +func (e *env) exec(t *testing.T, query string, args ...any) { + t.Helper() + err := e.db.Tx(t.Context(), func(tx *sql.Tx) error { + _, err := tx.ExecContext(t.Context(), query, args...) + return err + }) + if err != nil { + t.Fatalf("%s: %v", query, err) + } +} + +// setRetention rewrites the project's policy and re-reads it, because the +// collector reads the row and not the struct the test is holding. +func (e *env) setRetention(t *testing.T, count int, graceS int64) { + t.Helper() + e.exec(t, `UPDATE projects SET retention_count = ?, retention_grace_s = ? WHERE id = ?`, + count, graceS, e.p.ID) + p, err := e.db.ProjectByID(t.Context(), e.p.ID) + if err != nil { + t.Fatal(err) + } + e.p = p +} + +// alive reports whether a deployment still has a row and a tree. +func (e *env) alive(t *testing.T, dep *store.Deployment) (row, tree bool) { + t.Helper() + _, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID) + switch { + case err == nil: + row = true + case errors.Is(err, store.ErrNotFound): + default: + t.Fatalf("DeploymentByPublicID: %v", err) + } + _, err = os.Stat(DeploymentDir(e.dir, e.p.ID, dep.PublicID)) + switch { + case err == nil: + tree = true + case os.IsNotExist(err): + default: + t.Fatalf("stat deployment tree: %v", err) + } + return +} + +func (e *env) mustSurvive(t *testing.T, dep *store.Deployment, why string) { + t.Helper() + row, tree := e.alive(t, dep) + if !row || !tree { + t.Errorf("%s (%s) was collected: row=%v tree=%v", why, dep.PublicID, row, tree) + } +} + +func (e *env) mustBeGone(t *testing.T, dep *store.Deployment, why string) { + t.Helper() + row, tree := e.alive(t, dep) + if row || tree { + t.Errorf("%s (%s) survived: row=%v tree=%v", why, dep.PublicID, row, tree) + } +} + +// blobCount is how many blobs the database knows about, which is the number the +// milestone's manual check watches drop. +func blobCount(t *testing.T, e *env) int64 { + t.Helper() + counts, err := e.db.Counts(t.Context()) + if err != nil { + t.Fatal(err) + } + return counts.Blobs +} + +func hasContent(t *testing.T, e *env, content string) bool { + t.Helper() + ok, err := e.cas.Has(cas.Sum([]byte(content))) + if err != nil { + t.Fatalf("cas.Has: %v", err) + } + return ok +} + +// The headline retention rule from the milestone: deploy repeatedly, keep +// retention_count of them plus whichever one is being served, and watch the +// content of the rest go away. +func TestCollectKeepsRetentionCountPlusTheActiveOne(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 10, 0) + + var deps []*store.Deployment + for i := 1; i <= 15; i++ { + dep := e.version(t, i) + deps = append(deps, dep) + e.activate(t, dep) + } + // Roll back to the oldest one, so the deployment being served is also the + // one retention would otherwise drop first. Nothing may collect it. + e.activate(t, deps[0]) + + before := blobCount(t, e) + stats, err := e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + + // 15 deployments, 10 kept by retention plus the active one: 4 collected. + if stats.DeploymentsDeleted != 4 { + t.Errorf("deleted %d deployments, want 4", stats.DeploymentsDeleted) + } + e.mustSurvive(t, deps[0], "the active deployment") + for _, dep := range deps[5:] { + e.mustSurvive(t, dep, "a deployment inside the retention window") + } + for _, dep := range deps[1:5] { + e.mustBeGone(t, dep, "a deployment past the retention window") + } + + if after := blobCount(t, e); after != before-4 { + t.Errorf("blob count went from %d to %d, want %d", before, after, before-4) + } + if stats.BlobsDeleted != 4 { + t.Errorf("collected %d blobs, want the 4 the deleted deployments held", stats.BlobsDeleted) + } + if stats.BytesFreed <= 0 { + t.Errorf("BytesFreed = %d, want the size of what was removed", stats.BytesFreed) + } + for i := 2; i <= 5; i++ { + if hasContent(t, e, "v"+strconv.Itoa(i)) { + t.Errorf("content of the collected deployment v%d is still in the CAS", i) + } + } + // What the survivors reference is untouched, which is what makes a rollback + // to any of them still work. + if !hasContent(t, e, "v1") || !hasContent(t, e, "v15") { + t.Error("content a surviving deployment references was collected") + } + + // Nothing left to do on a second pass. + stats, err = e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if stats.DeploymentsDeleted != 0 || stats.BlobsDeleted != 0 { + t.Errorf("a second pass collected %+v, want nothing", stats) + } +} + +// The grace period is measured from when a deployment stopped being served, so +// a rollback that was a mistake can be undone for a while afterwards. +func TestCollectHonoursTheRetentionGrace(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 0, 3600) + + old := e.version(t, 1) + e.activate(t, old) + current := e.version(t, 2) + e.activate(t, current) + + // retention_count is zero, so only the grace is protecting it. + stats, err := e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if stats.DeploymentsDeleted != 0 { + t.Errorf("deleted %d deployments inside the grace period, want 0", stats.DeploymentsDeleted) + } + e.mustSurvive(t, old, "a deployment retired seconds ago") + + // Backdate the deactivation past the grace and it becomes collectable. + e.exec(t, `UPDATE deployments SET deactivated_at = ? WHERE id = ?`, + time.Now().Add(-2*time.Hour).Unix(), old.ID) + if _, err := e.svc.Collect(t.Context(), false); err != nil { + t.Fatal(err) + } + e.mustBeGone(t, old, "a deployment retired before the grace period") + e.mustSurvive(t, current, "the active deployment") +} + +// A deployment that never finished uploading is not retention's business: it +// has no deactivated_at to measure from and someone may still be pushing to it. +func TestCollectLeavesUnfinishedUploadsToTheExpiry(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 0, 0) + + contents := map[string]string{"index.html": "in progress"} + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { + t.Fatal(err) + } + e.upload(t, contents, "index.html") + + if _, err := e.svc.Collect(t.Context(), false); err != nil { + t.Fatal(err) + } + got, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID) + if err != nil { + t.Fatalf("an in-progress upload was collected: %v", err) + } + if got.State != store.StateUploading { + t.Errorf("state = %q, want it left alone as %q", got.State, store.StateUploading) + } + // Its content is protected too, by the manifest rows that already reference + // it — which is the whole reason the manifest is written before the upload. + if !hasContent(t, e, "in progress") { + t.Error("the content of an in-progress upload was collected") + } + + // Once it is old enough, one pass does the whole job: the expiry fails it + // and drops its manifest, retention finds a failed deployment older than it + // keeps failures for, and the blob pass then reaches what only it + // referenced. That the three run in that order is why it takes one pass and + // not three. + e.exec(t, `UPDATE deployments SET created_at = ? WHERE id = ?`, + time.Now().Add(-48*time.Hour).Unix(), dep.ID) + stats, err := e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if stats.DeploymentsDeleted != 1 || stats.BlobsDeleted != 1 { + t.Errorf("collected %+v, want the abandoned upload and its content", stats) + } + if _, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the abandoned upload survived: %v", err) + } + if hasContent(t, e, "in progress") { + t.Error("content nothing references any more survived the sweep") + } +} + +// A deployment that failed while being finalized keeps its row for a day, so an +// operator looking into a broken CI job can still see that it failed and why. +func TestCollectKeepsRecentFailuresForInspection(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 0, 0) + + dep := e.create(t) + if err := e.db.MarkDeploymentFailed(t.Context(), dep.ID, "assembly failed"); err != nil { + t.Fatal(err) + } + + if _, err := e.svc.Collect(t.Context(), false); err != nil { + t.Fatal(err) + } + got, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID) + if err != nil { + t.Fatalf("a deployment that failed moments ago was collected: %v", err) + } + if got.Error != "assembly failed" { + t.Errorf("error = %q, want the reason still readable", got.Error) + } + + e.exec(t, `UPDATE deployments SET created_at = ? WHERE id = ?`, + time.Now().Add(-48*time.Hour).Unix(), dep.ID) + if _, err := e.svc.Collect(t.Context(), false); err != nil { + t.Fatal(err) + } + if _, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID); !errors.Is(err, store.ErrNotFound) { + t.Errorf("a failed row older than the retention survived: %v", err) + } +} + +// Content two deployments share outlives the first of them. This is what makes +// cross-deployment deduplication safe to rely on. +func TestCollectKeepsSharedContent(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 0, 0) + + shared := "console.log(1)" + old := e.deployReady(t, map[string]string{"index.html": "v1", "app.js": shared}) + e.activate(t, old) + current := e.deployReady(t, map[string]string{"index.html": "v2", "app.js": shared}) + e.activate(t, current) + + if _, err := e.svc.Collect(t.Context(), false); err != nil { + t.Fatal(err) + } + e.mustBeGone(t, old, "the superseded deployment") + if hasContent(t, e, "v1") { + t.Error("content only the collected deployment referenced survived") + } + if !hasContent(t, e, shared) { + t.Fatal("content the active deployment still references was collected") + } + // And it is still readable through the deployment that survived, which is + // the property the assembled tree shares an inode for. + body, err := os.ReadFile(filepath.Join(DeploymentDir(e.dir, e.p.ID, current.PublicID), "app.js")) + if err != nil || string(body) != shared { + t.Errorf("reading shared content from the surviving tree = %q, %v", body, err) + } +} + +// The grace is what makes the read path safe without per-request reference +// counting, so it has to actually hold content back. +func TestCollectHoldsRecentlyDereferencedBlobs(t *testing.T) { + e := newEnv(t) + e.setRetention(t, 0, 0) // BlobGrace left at its default hour. + + old := e.version(t, 1) + e.activate(t, old) + e.activate(t, e.version(t, 2)) + + stats, err := e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + e.mustBeGone(t, old, "the superseded deployment") + if stats.BlobsDeleted != 0 { + t.Errorf("collected %d blobs, want them held by the grace period", stats.BlobsDeleted) + } + if !hasContent(t, e, "v1") { + t.Error("content dereferenced moments ago was removed inside the grace period") + } +} + +func TestCollectDryRunChangesNothing(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 0, 0) + + old := e.version(t, 1) + e.activate(t, old) + current := e.version(t, 2) + e.activate(t, current) + + stats, err := e.svc.Collect(t.Context(), true) + if err != nil { + t.Fatal(err) + } + if !stats.DryRun { + t.Error("the report does not say it was a dry run") + } + if stats.DeploymentsDeleted != 1 { + t.Errorf("reported %d deployments, want the 1 that would be deleted", stats.DeploymentsDeleted) + } + e.mustSurvive(t, old, "a deployment a dry run only reported on") + e.mustSurvive(t, current, "the active deployment") + if !hasContent(t, e, "v1") { + t.Error("a dry run removed content") + } + + // Blobs the reported deletions would free are not counted: nothing was + // deleted, so they are all still referenced. The number is a floor. + if stats.BlobsDeleted != 0 { + t.Errorf("a dry run reported %d collectable blobs, want 0 while everything is referenced", + stats.BlobsDeleted) + } + + // The real pass then does what the dry run said it would. + stats, err = e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if stats.DryRun { + t.Error("a real pass reported itself as a dry run") + } + if stats.DeploymentsDeleted != 1 { + t.Errorf("deleted %d deployments, want 1", stats.DeploymentsDeleted) + } + e.mustBeGone(t, old, "the superseded deployment") +} + +func TestDeleteRemovesRowAndTree(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + + dep := e.version(t, 1) + current := e.version(t, 2) + e.activate(t, current) + + if err := e.svc.Delete(t.Context(), e.p, dep); err != nil { + t.Fatalf("Delete: %v", err) + } + e.mustBeGone(t, dep, "the deleted deployment") + + // Deleting it dropped the manifest, so the next collection reaches what only + // it referenced. + if _, err := e.svc.Collect(t.Context(), false); err != nil { + t.Fatal(err) + } + if hasContent(t, e, "v1") { + t.Error("content the deleted deployment held survived collection") + } + + if err := e.svc.Delete(t.Context(), e.p, dep); apiCode(err) != api.CodeNotFound { + t.Errorf("deleting it again = %v, want %q", err, api.CodeNotFound) + } +} + +// Deleting what a project is serving is a conflict, not a permission problem: +// the client is told to activate something else first. +func TestDeleteRefusesTheActiveDeployment(t *testing.T) { + e := newEnv(t) + dep := e.version(t, 1) + e.activate(t, dep) + + err := e.svc.Delete(t.Context(), e.p, dep) + wantCode(t, err, api.CodeDeploymentActive) + e.mustSurvive(t, dep, "the active deployment") + + // It becomes deletable the moment something else is being served, which is + // the sequence the error message describes. + e.activate(t, e.version(t, 2)) + if err := e.svc.Delete(t.Context(), e.p, dep); err != nil { + t.Fatalf("Delete after activating another deployment: %v", err) + } + e.mustBeGone(t, dep, "the deployment that was superseded and then deleted") +} + +func TestRemoveProjectTrees(t *testing.T) { + e := newEnv(t) + dep := e.version(t, 1) + e.activate(t, dep) + + other := store.DefaultProject("other") + if err := e.db.CreateProject(t.Context(), other); err != nil { + t.Fatal(err) + } + otherDir := DeploymentDir(e.dir, other.ID, "dpl_0000000000000000") + if err := os.MkdirAll(otherDir, 0o755); err != nil { + t.Fatal(err) + } + + if err := e.svc.RemoveProjectTrees(e.p.ID); err != nil { + t.Fatalf("RemoveProjectTrees: %v", err) + } + if _, err := os.Stat(filepath.Join(e.dir, strconv.FormatInt(e.p.ID, 10))); !os.IsNotExist(err) { + t.Errorf("the project's directory survived: %v", err) + } + if _, err := os.Stat(otherDir); err != nil { + t.Errorf("another project's directory was removed: %v", err) + } + + // Idempotent: recovery may run it again after a crash partway through. + if err := e.svc.RemoveProjectTrees(e.p.ID); err != nil { + t.Errorf("a second removal: %v", err) + } +} + +// Deleting the project takes its deployments with it, and the collector then +// reclaims everything they referenced. +func TestCollectAfterProjectDeletion(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + + dep := e.version(t, 1) + e.activate(t, dep) + + if err := e.db.DeleteProject(t.Context(), e.p.ID); err != nil { + t.Fatalf("DeleteProject: %v", err) + } + if err := e.svc.RemoveProjectTrees(e.p.ID); err != nil { + t.Fatalf("RemoveProjectTrees: %v", err) + } + + stats, err := e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if stats.BlobsDeleted != 1 { + t.Errorf("collected %d blobs, want the 1 the deleted project held", stats.BlobsDeleted) + } + if hasContent(t, e, "v1") { + t.Error("content of a deleted project survived collection") + } + if n := blobCount(t, e); n != 0 { + t.Errorf("%d blob rows left after the project was deleted", n) + } +} + +// A deployment claimed by a sweep that was interrupted is finished by the next +// one, whatever its retention would otherwise have said. +func TestCollectResumesAnInterruptedDeletion(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 10, 3600) // Generous: retention alone would keep it. + + dep := e.version(t, 1) + e.activate(t, e.version(t, 2)) + if err := e.db.MarkDeploymentDeleting(t.Context(), dep.ID); err != nil { + t.Fatal(err) + } + + stats, err := e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if stats.DeploymentsDeleted != 1 { + t.Errorf("deleted %d deployments, want the claimed one", stats.DeploymentsDeleted) + } + e.mustBeGone(t, dep, "a deployment a previous sweep had claimed") +} + +// The activation path and the collector both take the project lock, and the +// claim rechecks active = 0 under it. A deployment that becomes active between +// being listed and being claimed is therefore refused rather than deleted. +func TestCollectSkipsADeploymentActivatedUnderIt(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 0, 0) + + old := e.version(t, 1) + e.activate(t, e.version(t, 2)) + + // Stand in for the interleaving: retention has decided to drop `old`, and a + // rollback activates it before the claim runs. + e.activate(t, old) + if err := e.svc.claim(t.Context(), e.p, old); !errors.Is(err, store.ErrConflict) { + t.Fatalf("claiming a deployment that became active = %v, want ErrConflict", err) + } + e.mustSurvive(t, old, "a deployment activated during collection") + + stats, err := e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if stats.DeploymentsDeleted != 1 { + t.Errorf("deleted %d deployments, want only the one that is no longer served", stats.DeploymentsDeleted) + } + e.mustSurvive(t, old, "the deployment the rollback made active") +} + +// Collection walks every project, not just the one a request happened to name. +func TestCollectSpansProjects(t *testing.T) { + e := newEnv(t) + e.svc.BlobGrace = collectNow + e.setRetention(t, 0, 0) + + first := e.p + firstOld := e.version(t, 1) + e.activate(t, e.version(t, 2)) + + second := store.DefaultProject("second") + second.RetentionCount = 0 + second.RetentionGraceS = 0 + if err := e.db.CreateProject(t.Context(), second); err != nil { + t.Fatal(err) + } + e.p = second + secondOld := e.version(t, 3) + e.activate(t, e.version(t, 4)) + e.p = first + + stats, err := e.svc.Collect(t.Context(), false) + if err != nil { + t.Fatal(err) + } + if stats.DeploymentsDeleted != 2 { + t.Errorf("deleted %d deployments, want one from each project", stats.DeploymentsDeleted) + } + e.mustBeGone(t, firstOld, "the first project's superseded deployment") + e.p = second + e.mustBeGone(t, secondOld, "the second project's superseded deployment") +} diff --git a/internal/deploy/recover.go b/internal/deploy/recover.go new file mode 100644 index 0000000..0f419a7 --- /dev/null +++ b/internal/deploy/recover.go @@ -0,0 +1,213 @@ +package deploy + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/store" +) + +// staleUploadAge is how long a deployment may sit unfinished before recovery +// gives up on it. Generous on purpose: the only thing separating a CI job that +// died from one that is uploading a large site over a slow link is how long it +// has been, and expiring the second kind turns a slow deploy into a failed one. +const staleUploadAge = 24 * time.Hour + +// Recover makes the filesystem and the database agree again after a crash or an +// unclean shutdown. +// +// It runs once at startup, before any listener exists and before the registry is +// built, so it can assume it is the only thing touching either. Everything it +// does is idempotent: being killed halfway through only means the next start +// finds a little more to do. +// +// Nothing here is allowed to be fatal. A server that refuses to start because +// one directory could not be swept is worse than one that starts and logs it — +// the deployments themselves are already durable, and every inconsistency this +// looks for is one the running system tolerates. +func (s *Service) Recover(ctx context.Context) error { + // An upload that was in progress is unreferenced by construction: a blob only + // becomes reachable by being renamed out of the temp directory. + if n, err := s.CAS.PurgeTemp(); err != nil { + s.Log.Warn("could not clear interrupted uploads", "err", err) + } else if n > 0 { + s.Log.Info("cleared interrupted uploads", "count", n) + } + + if err := s.recoverBlobs(ctx); err != nil { + return err + } + + // Then the deployments that were still being written when the process went + // away, so their manifest rows are gone before the blob collector next runs + // and can reclaim whatever only they referenced. + n, err := s.DB.ExpireStaleDeployments(ctx, time.Now().Add(-staleUploadAge), + "abandoned: no activity for "+staleUploadAge.String()) + if err != nil { + return err + } + if n > 0 { + s.Log.Info("expired unfinished deployments", "count", n) + } + + if err := s.resumeDeletions(ctx); err != nil { + return err + } + return s.sweepTrees(ctx) +} + +// recoverBlobs finds content the database believes is on disk and is not. +// +// This is the "restored the database, lost the disk" case, and also what a +// half-finished collector sweep leaves behind. Marking the rows absent is +// enough to fix it: the next deploy naming one of these digests is asked to +// upload it, and every deployment that referenced it becomes deployable again as +// soon as one does. +func (s *Service) recoverBlobs(ctx context.Context) error { + var absent []cas.Digest + err := s.DB.EachPresentBlob(ctx, func(d cas.Digest) error { + has, err := s.CAS.Has(d) + if err != nil { + return err + } + if !has { + absent = append(absent, d) + } + return nil + }) + if err != nil { + return err + } + if len(absent) == 0 { + return nil + } + if err := s.DB.MarkBlobsAbsent(ctx, absent); err != nil { + return err + } + // Loud, because on a healthy server this number is zero. Anything else means + // the content store lost data, and an operator wants to hear about it before + // a deploy fails for a reason that looks like the client's fault. + s.Log.Warn("content is missing from the store and was marked for re-upload", + "blobs", len(absent)) + return nil +} + +// resumeDeletions finishes what a collector sweep was doing when it stopped. +// +// A deployment enters the deleting state before anything of it is removed, so a +// row still in that state is a tree that may be half gone. Half a tree is +// exactly what nothing may serve, which is why the state is committed first: it +// makes an interrupted deletion recognisable rather than indistinguishable from +// a healthy deployment. +func (s *Service) resumeDeletions(ctx context.Context) error { + deps, err := s.DB.DeploymentsInState(ctx, store.StateDeleting, 0) + if err != nil { + return err + } + for _, dep := range deps { + if err := s.removeDeployment(ctx, dep); err != nil { + s.Log.Warn("could not finish deleting a deployment", + "deployment", dep.PublicID, "err", err) + continue + } + s.Log.Info("finished deleting a deployment", "deployment", dep.PublicID) + } + return nil +} + +// removeDeployment takes a claimed deployment the rest of the way: its tree +// first, then its rows. Tree before rows, because a row without a tree is a +// deployment that simply cannot be activated, whereas a tree without a row is +// disk nobody will ever account for again. +func (s *Service) removeDeployment(ctx context.Context, dep *store.Deployment) error { + if s.Dir != "" { + dir := DeploymentDir(s.Dir, dep.ProjectID, dep.PublicID) + if err := os.RemoveAll(dir); err != nil { + return err + } + if err := os.RemoveAll(dir + stagingSuffix); err != nil { + return err + } + } + return s.DB.DeleteDeployment(ctx, dep.ID) +} + +// sweepTrees removes assembled trees that nothing refers to. +// +// Two kinds: staging directories, which are by definition a build that never +// finished, and directories whose deployment row is gone — the reverse of the +// blob audit above, and the residue of a deletion that removed rows before it +// removed files, or of a database restored from an older backup than the disk. +func (s *Service) sweepTrees(ctx context.Context) error { + if s.Dir == "" { + return nil + } + refs, err := s.DB.AllDeploymentRefs(ctx) + if err != nil { + return err + } + known := make(map[store.DeploymentRef]struct{}, len(refs)) + for _, r := range refs { + known[r] = struct{}{} + } + + projects, err := os.ReadDir(s.Dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + + var staging, orphans int + for _, pe := range projects { + // Only the layout this package writes is ever considered for removal: + // one directory per project id, named by the id. Anything else under + // $DATA_DIR/deployments was put there by someone else and is left alone. + projectID, err := strconv.ParseInt(pe.Name(), 10, 64) + if err != nil || !pe.IsDir() { + continue + } + projectDir := filepath.Join(s.Dir, pe.Name()) + entries, err := os.ReadDir(projectDir) + if err != nil { + s.Log.Warn("could not read a project's deployment directory", "dir", projectDir, "err", err) + continue + } + for _, e := range entries { + if !e.IsDir() { + continue + } + name := e.Name() + target := filepath.Join(projectDir, name) + switch { + case strings.HasSuffix(name, stagingSuffix): + if err := os.RemoveAll(target); err != nil { + s.Log.Warn("could not remove a staging directory", "dir", target, "err", err) + continue + } + staging++ + default: + if _, ok := known[store.DeploymentRef{ProjectID: projectID, PublicID: name}]; ok { + continue + } + if err := os.RemoveAll(target); err != nil { + s.Log.Warn("could not remove an orphaned deployment tree", "dir", target, "err", err) + continue + } + orphans++ + } + } + } + if staging > 0 || orphans > 0 { + s.Log.Info("swept unreferenced deployment trees", "staging", staging, "orphaned", orphans) + } + return nil +} diff --git a/internal/deploy/service.go b/internal/deploy/service.go new file mode 100644 index 0000000..2efe27b --- /dev/null +++ b/internal/deploy/service.go @@ -0,0 +1,311 @@ +package deploy + +import ( + "context" + "errors" + "io" + "log/slog" + "sync" + "time" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/site" + "github.com/iceBear67/simplepages/internal/store" + "github.com/iceBear67/simplepages/internal/webroot" +) + +// Service runs the deployment lifecycle. +// +// Client-visible failures are returned as *api.Error so the HTTP layer stays a +// translation of shapes rather than a second copy of the rules; anything else +// is an internal error and is rendered as an opaque 500 by httpx.WriteError. +type Service struct { + DB *store.DB + CAS *cas.Store + Log *slog.Logger + + // Dir is the root of the assembled deployment trees. Empty means the + // operator chose assemble_mode=none: content is served straight from the + // CAS and nothing is built on disk. + Dir string + + // Sites is the in-memory state the HTTP site handler reads. Activation + // publishes into it; nil leaves the service usable without a serving layer, + // which is what the store-level tests want. + Sites *site.Registry + + // Webroot maintains the $WEBROOT/~project symlinks. Nil when the operator + // configured no webroot. Nothing here depends on it succeeding. + Webroot *webroot.Webroot + + // BlobGrace is how long content must have been unreferenced before the + // collector removes it. Zero means defaultBlobGrace. Negative collects + // immediately, which is what the tests want and what an operator reclaiming + // space on a server they know is idle might ask for. + BlobGrace time.Duration + + locks projectLocks +} + +// Create starts a deployment. Nothing touches the filesystem until a manifest +// arrives, so an abandoned create costs one row. +func (s *Service) Create(ctx context.Context, p *store.Project, keyID string, meta map[string]string) (*store.Deployment, error) { + dep := &store.Deployment{ProjectID: p.ID, CreatedByKey: keyID, Meta: meta} + if err := s.DB.CreateDeployment(ctx, dep); err != nil { + return nil, err + } + return dep, nil +} + +// SetManifest records the file list and reports which blobs still have to be +// uploaded. files must already be validated: paths through pathutil and sizes +// against the project's limits. +func (s *Service) SetManifest(ctx context.Context, dep *store.Deployment, files []store.FileRow) (missing []cas.Digest, missingBytes int64, err error) { + missing, missingBytes, err = s.DB.SetManifest(ctx, dep.ID, files) + if err != nil { + switch { + case errors.Is(err, store.ErrConflict): + return nil, 0, api.Errorf(api.CodeConflict, + "this deployment can no longer accept a manifest; create a new one") + case errors.Is(err, cas.ErrSizeMismatch): + return nil, 0, api.Errorf(api.CodeSizeMismatch, "%s", err) + } + return nil, 0, err + } + return missing, missingBytes, nil +} + +// Upload stores one blob's content. +// +// The digest must already be named by some manifest. That check is what keeps +// the endpoint from being general-purpose storage: content nobody declared can +// never be written, and the length it must have is the one the manifest agreed +// on rather than whatever Content-Length claims. +// +// Reports whether the content was newly stored; a blob that is already present +// is a success without reading the body, which is what makes a retried deploy +// cheap. +func (s *Service) Upload(ctx context.Context, digest cas.Digest, body io.Reader) (size int64, stored bool, err error) { + b, err := s.DB.Blob(ctx, digest) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return 0, false, api.Errorf(api.CodeNotFound, + "no manifest references this digest; send the manifest first") + } + return 0, false, err + } + if b.Present { + return b.Size, false, nil + } + + // The blob's declared length is both the expectation and the ceiling: Put + // reads one byte past it and rejects anything longer, so a client cannot + // spend more of the disk than its manifest was accepted for. Put requires a + // positive ceiling, hence the floor of one byte for an empty blob. + limit := b.Size + if limit < 1 { + limit = 1 + } + n, err := s.CAS.Put(ctx, digest, b.Size, limit, body) + if err != nil { + switch { + case errors.Is(err, cas.ErrDigestMismatch): + return 0, false, api.Errorf(api.CodeDigestMismatch, "%s", err) + case errors.Is(err, cas.ErrSizeMismatch): + return 0, false, api.Errorf(api.CodeSizeMismatch, "%s", err) + case errors.Is(err, cas.ErrTooLarge): + return 0, false, api.Errorf(api.CodeLimitExceeded, "%s", err) + } + return 0, false, err + } + if err := s.DB.MarkBlobPresent(ctx, digest, n); err != nil { + // The content is on disk and verified; only the row disagrees. A retry + // finds the blob already stored and updates the row then. + return 0, false, err + } + return n, true, nil +} + +// Finalize checks that every blob arrived, assembles the tree, and marks the +// deployment ready. It does not activate it: a ready deployment is one that +// could be served, and choosing when to serve it is a separate decision. +// +// Retrying is safe. An assembled tree is left as it is, and a deployment that +// is already ready simply stays ready. +func (s *Service) Finalize(ctx context.Context, p *store.Project, dep *store.Deployment) (*store.Deployment, error) { + // One finalize per project at a time. Two concurrent CI jobs for one + // project are ordered rather than racing over the same directory. + unlock, err := s.locks.lock(ctx, p.ID) + if err != nil { + return nil, err + } + defer unlock() + + // Re-read under the lock: the state may have moved since the handler + // resolved it. + dep, err = s.DB.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + return nil, mapNotFound(err) + } + switch dep.State { + case store.StateUploading, store.StateReady: + default: + return nil, api.Errorf(api.CodeConflict, "cannot finalize a deployment that is %s", dep.State) + } + + missing, err := s.DB.MissingBlobs(ctx, dep.ID) + if err != nil { + return nil, err + } + if len(missing) > 0 { + hex := make([]string, len(missing)) + for i, d := range missing { + hex[i] = d.String() + } + return nil, api.Errorf(api.CodeBlobsMissing, + "%d blobs have not been uploaded", len(missing)).WithDetail("missing", hex) + } + + if s.Dir != "" { + files, err := s.DB.DeploymentFiles(ctx, dep.ID) + if err != nil { + return nil, err + } + if err := Assemble(ctx, s.CAS, files, DeploymentDir(s.Dir, p.ID, dep.PublicID)); err != nil { + // A cancelled request is not a broken deployment: leave it uploading + // so the client can simply try again. + if ctx.Err() != nil { + return nil, err + } + s.Log.ErrorContext(ctx, "assembling deployment tree failed", + "project", p.Name, "deployment", dep.PublicID, "err", err) + if ferr := s.DB.MarkDeploymentFailed(context.WithoutCancel(ctx), dep.ID, err.Error()); ferr != nil { + s.Log.ErrorContext(ctx, "recording the failure failed too", + "deployment", dep.PublicID, "err", ferr) + } + return nil, api.Errorf(api.CodeInternal, "could not assemble the deployment tree") + } + } + + if err := s.DB.MarkDeploymentReady(ctx, dep.ID); err != nil { + if errors.Is(err, store.ErrConflict) { + return nil, api.Errorf(api.CodeConflict, "%s", err) + } + return nil, err + } + return s.DB.DeploymentByPublicID(ctx, p.ID, dep.PublicID) +} + +// Activate makes a ready deployment the one the project serves. Activating an +// older deployment is how a rollback works, and costs exactly the same. +// +// The order of the steps is the entire correctness argument: +// +// 1. Take the project lock, so two activations of one project are ordered. +// 2. Re-read the deployment under it and require that it is ready. +// 3. Build the snapshot's index — the one step that reads the manifest and can +// fail — *before* anything has changed. A failure here leaves the currently +// served deployment exactly as it was. +// 4. Commit the database transaction. From here on SQLite is the truth. +// 5. Store the pointer. This single store is the switch: requests that started +// earlier finish on the old snapshot, later ones see the new one, and no +// request can ever observe a mixture of the two. +// 6. Repoint the symlink, best effort. +// +// Database before memory matters: a crash between 4 and 5 restarts into a +// process that serves what the database says. The reverse order would leave a +// process serving something the database disagrees with. +func (s *Service) Activate(ctx context.Context, p *store.Project, dep *store.Deployment) (*store.Deployment, error) { + unlock, err := s.locks.lock(ctx, p.ID) + if err != nil { + return nil, err + } + defer unlock() + + dep, err = s.DB.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + return nil, mapNotFound(err) + } + if dep.State != store.StateReady { + return nil, api.Errorf(api.CodeDeploymentNotReady, + "cannot activate a deployment that is %s", dep.State) + } + + idx, err := s.index(ctx, dep) + if err != nil { + return nil, err + } + + if err := s.DB.ActivateDeployment(ctx, p.ID, dep.ID); err != nil { + switch { + case errors.Is(err, store.ErrNotFound): + return nil, api.Errorf(api.CodeNotFound, "no such deployment") + case errors.Is(err, store.ErrConflict): + return nil, api.Errorf(api.CodeConflict, "%s", err) + } + return nil, err + } + + // Re-read so the snapshot and the response carry the timestamps the + // transaction actually wrote. + dep, err = s.DB.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + return nil, mapNotFound(err) + } + + dir := s.deploymentDir(p, dep) + if s.Sites != nil { + s.Sites.Put(p).Activate(site.NewDeployment(dep, idx, dir)) + } + if s.Webroot != nil && dir != "" { + if err := s.Webroot.Point(p.Name, dir); err != nil { + // The site is already being served from memory; the symlink is for + // everything else and the reconciler will fix it. + s.Log.ErrorContext(ctx, "could not repoint the webroot symlink", + "project", p.Name, "deployment", dep.PublicID, "err", err) + } + } + s.Log.InfoContext(ctx, "deployment activated", + "project", p.Name, "deployment", dep.PublicID, + "files", dep.FileCount, "bytes", dep.TotalBytes) + return dep, nil +} + +func mapNotFound(err error) error { + if errors.Is(err, store.ErrNotFound) { + return api.Errorf(api.CodeNotFound, "no such deployment") + } + return err +} + +// projectLocks serialises the mutating operations of one project against each +// other. Entries are keyed by row id and are never evicted: there is one per +// project that has ever been written to, which is bounded by the number of +// projects. +type projectLocks struct { + mu sync.Mutex + m map[int64]chan struct{} +} + +// lock acquires the project's lock, or gives up if ctx is done first — a +// client that has already hung up should not keep a slow assembly waiting. +func (l *projectLocks) lock(ctx context.Context, id int64) (func(), error) { + l.mu.Lock() + if l.m == nil { + l.m = make(map[int64]chan struct{}) + } + ch, ok := l.m[id] + if !ok { + ch = make(chan struct{}, 1) + l.m[id] = ch + } + l.mu.Unlock() + + select { + case ch <- struct{}{}: + return func() { <-ch }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} diff --git a/internal/deploy/service_test.go b/internal/deploy/service_test.go new file mode 100644 index 0000000..40607bf --- /dev/null +++ b/internal/deploy/service_test.go @@ -0,0 +1,468 @@ +package deploy + +import ( + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/store" +) + +// env is a service on a real database and a real CAS, which is what these tests +// are about: every interesting rule here is enforced by a trigger, a unique +// index or the filesystem, and a fake would only assert that the fake agrees +// with itself. +type env struct { + svc *Service + db *store.DB + cas *cas.Store + p *store.Project + dir string // deployments root +} + +func newEnv(t *testing.T) *env { + t.Helper() + base := t.TempDir() + log := slog.New(slog.DiscardHandler) + + db, err := store.Open(t.Context(), filepath.Join(base, "pages.db"), log) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + t.Cleanup(func() { db.Close() }) + + deployDir := filepath.Join(base, "deployments") + cs, err := cas.Open(filepath.Join(base, "cas"), cas.Options{ProbeDir: deployDir, Log: log}) + if err != nil { + t.Fatalf("cas.Open: %v", err) + } + t.Cleanup(func() { cs.Close() }) + + p := store.DefaultProject("demo") + if err := db.CreateProject(t.Context(), p); err != nil { + t.Fatalf("CreateProject: %v", err) + } + return &env{ + svc: &Service{DB: db, CAS: cs, Log: log, Dir: deployDir}, + db: db, cas: cs, p: p, dir: deployDir, + } +} + +func (e *env) create(t *testing.T) *store.Deployment { + t.Helper() + dep, err := e.svc.Create(t.Context(), e.p, "", nil) + if err != nil { + t.Fatalf("Create: %v", err) + } + return dep +} + +// manifest turns path->content into the rows a client would have sent. +func manifest(contents map[string]string) []store.FileRow { + files := make([]store.FileRow, 0, len(contents)) + for p, c := range contents { + files = append(files, store.FileRow{Path: p, Digest: cas.Sum([]byte(c)), Size: int64(len(c))}) + } + return files +} + +// upload pushes every named blob through the service, as a client would. +func (e *env) upload(t *testing.T, contents map[string]string, want ...string) { + t.Helper() + for _, p := range want { + c := contents[p] + if _, _, err := e.svc.Upload(t.Context(), cas.Sum([]byte(c)), strings.NewReader(c)); err != nil { + t.Fatalf("Upload %s: %v", p, err) + } + } +} + +// apiCode returns the wire code of err, or "" if it is not a client error. +func apiCode(err error) api.Code { + var e *api.Error + if errors.As(err, &e) { + return e.Code + } + return "" +} + +func wantCode(t *testing.T, err error, want api.Code) { + t.Helper() + if got := apiCode(err); got != want { + t.Fatalf("error = %v (code %q), want code %q", err, got, want) + } +} + +func TestFullDeploymentRoundTrip(t *testing.T) { + e := newEnv(t) + contents := map[string]string{ + "index.html": "

hello

", + "assets/app.js": "console.log(1)", + "copy.html": "

hello

", // same blob as index.html + } + dep := e.create(t) + if dep.State != store.StatePending { + t.Fatalf("state = %s, want pending", dep.State) + } + + files := manifest(contents) + missing, missingBytes, err := e.svc.SetManifest(t.Context(), dep, files) + if err != nil { + t.Fatalf("SetManifest: %v", err) + } + // Two unique blobs for three files: the shared one is only asked for once. + if len(missing) != 2 { + t.Fatalf("missing = %d digests, want 2", len(missing)) + } + if want := int64(len("

hello

") + len("console.log(1)")); missingBytes != want { + t.Errorf("missingBytes = %d, want %d", missingBytes, want) + } + + // Finalize before the content arrives names what is still outstanding rather + // than failing opaquely, because that list is what the client retries. + _, err = e.svc.Finalize(t.Context(), e.p, dep) + wantCode(t, err, api.CodeBlobsMissing) + var apiErr *api.Error + if errors.As(err, &apiErr) { + list, _ := apiErr.Details["missing"].([]string) + if len(list) != 2 { + t.Errorf("details.missing = %v, want 2 digests", apiErr.Details["missing"]) + } + } + + e.upload(t, contents, "index.html", "assets/app.js") + + dep, err = e.svc.Finalize(t.Context(), e.p, dep) + if err != nil { + t.Fatalf("Finalize: %v", err) + } + if dep.State != store.StateReady { + t.Fatalf("state = %s, want ready", dep.State) + } + if dep.FileCount != 3 || dep.TotalBytes != int64(len(contents["index.html"])*2+len(contents["assets/app.js"])) { + t.Errorf("file_count = %d, total_bytes = %d", dep.FileCount, dep.TotalBytes) + } + if dep.FinalizedAt == nil { + t.Error("finalized_at was not recorded") + } + if dep.Active { + t.Error("finalize activated the deployment; that is a separate decision") + } + + // The tree on disk is the deployment, byte for byte. + got := walk(t, DeploymentDir(e.dir, e.p.ID, dep.PublicID)) + if len(got) != len(contents) { + t.Fatalf("assembled %v", got) + } + for p, c := range contents { + if got[p] != c { + t.Errorf("%s = %q, want %q", p, got[p], c) + } + } +} + +// A second deployment of a mostly-unchanged site is the case the whole +// content-addressed protocol exists for: only what actually changed is asked +// for, across deployments and across projects. +func TestASecondDeploymentOnlyAsksForWhatChanged(t *testing.T) { + e := newEnv(t) + first := map[string]string{"index.html": "v1", "assets/app.js": "shared"} + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(first)); err != nil { + t.Fatal(err) + } + e.upload(t, first, "index.html", "assets/app.js") + if _, err := e.svc.Finalize(t.Context(), e.p, dep); err != nil { + t.Fatal(err) + } + + second := map[string]string{"index.html": "v2", "assets/app.js": "shared"} + dep2 := e.create(t) + missing, missingBytes, err := e.svc.SetManifest(t.Context(), dep2, manifest(second)) + if err != nil { + t.Fatal(err) + } + if len(missing) != 1 || missing[0] != cas.Sum([]byte("v2")) { + t.Fatalf("missing = %v, want just the changed index.html", digests(missing)) + } + if missingBytes != 2 { + t.Errorf("missingBytes = %d, want 2", missingBytes) + } + + e.upload(t, second, "index.html") + if _, err := e.svc.Finalize(t.Context(), e.p, dep2); err != nil { + t.Fatal(err) + } + // Both trees exist and disagree, which is what makes a rollback a rollback. + if got := walk(t, DeploymentDir(e.dir, e.p.ID, dep.PublicID)); got["index.html"] != "v1" { + t.Errorf("first deployment = %v, want v1 intact", got) + } + if got := walk(t, DeploymentDir(e.dir, e.p.ID, dep2.PublicID)); got["index.html"] != "v2" { + t.Errorf("second deployment = %v", got) + } +} + +func digests(ds []cas.Digest) []string { + out := make([]string, len(ds)) + for i, d := range ds { + out[i] = d.String() + } + return out +} + +// The upload endpoint is not general-purpose storage. Content nobody declared +// has no size the server agreed to and no deployment that would ever reference +// it, so it is refused before a byte is read. +func TestUploadRejectsContentNoManifestAskedFor(t *testing.T) { + e := newEnv(t) + body := &countingReader{r: strings.NewReader("unsolicited")} + + _, _, err := e.svc.Upload(t.Context(), cas.Sum([]byte("unsolicited")), body) + wantCode(t, err, api.CodeNotFound) + if body.n != 0 { + t.Errorf("read %d bytes of a body it had already decided to refuse", body.n) + } + if has, _ := e.cas.Has(cas.Sum([]byte("unsolicited"))); has { + t.Error("the content was stored anyway") + } +} + +// The claimed digest is only ever a claim. Without this check a client could +// declare another project's digest and poison every project sharing that blob. +func TestUploadRejectsContentThatDoesNotHashToItsDigest(t *testing.T) { + e := newEnv(t) + contents := map[string]string{"index.html": "honest"} + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { + t.Fatal(err) + } + + claimed := cas.Sum([]byte("honest")) + _, _, err := e.svc.Upload(t.Context(), claimed, strings.NewReader("forged")) + wantCode(t, err, api.CodeDigestMismatch) + if has, _ := e.cas.Has(claimed); has { + t.Fatal("the forged content was stored under the honest digest") + } + + // And the deployment is still deployable once the real bytes arrive. + if _, _, err := e.svc.Upload(t.Context(), claimed, strings.NewReader("honest")); err != nil { + t.Fatalf("honest upload after a forged one: %v", err) + } + if _, err := e.svc.Finalize(t.Context(), e.p, dep); err != nil { + t.Fatalf("Finalize: %v", err) + } +} + +// The manifest's size is the ceiling, so a client cannot spend more disk than +// the manifest it got accepted for. Content-Length is never consulted. +func TestUploadRejectsMoreBytesThanTheManifestDeclared(t *testing.T) { + e := newEnv(t) + dep := e.create(t) + body := strings.Repeat("x", 4096) + // Declare a small file, then send a large one under the same digest. + files := []store.FileRow{{Path: "a.txt", Digest: cas.Sum([]byte(body)), Size: 4}} + if _, _, err := e.svc.SetManifest(t.Context(), dep, files); err != nil { + t.Fatal(err) + } + + _, _, err := e.svc.Upload(t.Context(), files[0].Digest, strings.NewReader(body)) + if code := apiCode(err); code != api.CodeSizeMismatch && code != api.CodeLimitExceeded { + t.Fatalf("error = %v (code %q), want a size rejection", err, code) + } + if has, _ := e.cas.Has(files[0].Digest); has { + t.Error("the oversized content was stored") + } +} + +// Re-uploading a blob the server already has is the fast path a retried deploy +// depends on: no body is read and nothing is rewritten. +func TestUploadOfAPresentBlobDoesNotReadTheBody(t *testing.T) { + e := newEnv(t) + contents := map[string]string{"index.html": "hello"} + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { + t.Fatal(err) + } + d := cas.Sum([]byte("hello")) + + size, stored, err := e.svc.Upload(t.Context(), d, strings.NewReader("hello")) + if err != nil || !stored || size != 5 { + t.Fatalf("first upload: size=%d stored=%v err=%v", size, stored, err) + } + body := &countingReader{r: strings.NewReader("hello")} + size, stored, err = e.svc.Upload(t.Context(), d, body) + if err != nil { + t.Fatal(err) + } + if stored { + t.Error("the second upload claimed to have stored content the server already had") + } + if size != 5 { + t.Errorf("size = %d, want 5", size) + } + if body.n != 0 { + t.Errorf("read %d bytes of a blob it already had", body.n) + } +} + +func TestUploadStoresAnEmptyBlob(t *testing.T) { + e := newEnv(t) + dep := e.create(t) + files := []store.FileRow{{Path: "empty", Digest: cas.Sum(nil), Size: 0}} + if _, _, err := e.svc.SetManifest(t.Context(), dep, files); err != nil { + t.Fatal(err) + } + // Put insists on a positive ceiling; the service floors it at one byte so a + // legitimately empty file is still uploadable. + size, stored, err := e.svc.Upload(t.Context(), files[0].Digest, strings.NewReader("")) + if err != nil || !stored || size != 0 { + t.Fatalf("size=%d stored=%v err=%v", size, stored, err) + } + if _, err := e.svc.Finalize(t.Context(), e.p, dep); err != nil { + t.Fatalf("Finalize: %v", err) + } + got := walk(t, DeploymentDir(e.dir, e.p.ID, dep.PublicID)) + if c, ok := got["empty"]; !ok || c != "" { + t.Errorf("tree = %v, want one empty file", got) + } +} + +// Finalizing twice must be a no-op rather than a second assembly: the client +// that lost its response to a timeout retries, and the tree may already be live. +func TestFinalizeIsIdempotent(t *testing.T) { + e := newEnv(t) + contents := map[string]string{"index.html": "hello"} + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { + t.Fatal(err) + } + e.upload(t, contents, "index.html") + + first, err := e.svc.Finalize(t.Context(), e.p, dep) + if err != nil { + t.Fatal(err) + } + second, err := e.svc.Finalize(t.Context(), e.p, dep) + if err != nil { + t.Fatalf("second Finalize: %v", err) + } + if !first.FinalizedAt.Equal(*second.FinalizedAt) { + t.Errorf("finalized_at moved from %v to %v on a retry", first.FinalizedAt, second.FinalizedAt) + } +} + +func TestFinalizeRejectsADeploymentWithNoManifest(t *testing.T) { + e := newEnv(t) + dep := e.create(t) // still pending + + _, err := e.svc.Finalize(t.Context(), e.p, dep) + wantCode(t, err, api.CodeConflict) + if _, err := os.Stat(DeploymentDir(e.dir, e.p.ID, dep.PublicID)); !errors.Is(err, os.ErrNotExist) { + t.Error("a tree was built for a deployment that never had a manifest") + } +} + +// A deployment belongs to exactly one project. Finalizing another project's +// deployment must not work even when the caller knows its id. +func TestFinalizeIsProjectScoped(t *testing.T) { + e := newEnv(t) + other := store.DefaultProject("other") + if err := e.db.CreateProject(t.Context(), other); err != nil { + t.Fatal(err) + } + contents := map[string]string{"index.html": "hello"} + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { + t.Fatal(err) + } + e.upload(t, contents, "index.html") + + _, err := e.svc.Finalize(t.Context(), other, dep) + wantCode(t, err, api.CodeNotFound) +} + +// assemble_mode=none: content is served straight from the CAS, so finalize must +// still succeed and must not build anything on disk. +func TestFinalizeWithoutAssemblyBuildsNothing(t *testing.T) { + e := newEnv(t) + e.svc.Dir = "" + contents := map[string]string{"index.html": "hello"} + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { + t.Fatal(err) + } + e.upload(t, contents, "index.html") + + dep, err := e.svc.Finalize(t.Context(), e.p, dep) + if err != nil { + t.Fatalf("Finalize: %v", err) + } + if dep.State != store.StateReady { + t.Fatalf("state = %s, want ready", dep.State) + } + if _, err := os.Stat(DeploymentDir(e.dir, e.p.ID, dep.PublicID)); !errors.Is(err, os.ErrNotExist) { + t.Error("a tree was assembled despite assemble_mode=none") + } +} + +// If assembly fails the deployment must end up failed rather than ready — a +// ready deployment is one that could be served, and this one could not be. +func TestFinalizeMarksTheDeploymentFailedWhenAssemblyCannotProceed(t *testing.T) { + e := newEnv(t) + contents := map[string]string{"index.html": "hello"} + dep := e.create(t) + if _, _, err := e.svc.SetManifest(t.Context(), dep, manifest(contents)); err != nil { + t.Fatal(err) + } + e.upload(t, contents, "index.html") + + // A plain file where the project's directory has to go: MkdirAll cannot get + // past it, so assembly fails for a reason that is not the client's fault. + if err := os.MkdirAll(e.dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Dir(DeploymentDir(e.dir, e.p.ID, dep.PublicID)), []byte("in the way"), 0o644); err != nil { + t.Fatal(err) + } + + _, err := e.svc.Finalize(t.Context(), e.p, dep) + wantCode(t, err, api.CodeInternal) + // The reason stays server-side: the client is told nothing about the layout + // of the server's disk. + if strings.Contains(err.Error(), e.dir) { + t.Errorf("the error exposes a server path: %v", err) + } + + after, err := e.db.DeploymentByPublicID(t.Context(), e.p.ID, dep.PublicID) + if err != nil { + t.Fatal(err) + } + if after.State != store.StateFailed { + t.Fatalf("state = %s, want failed", after.State) + } + if after.Error == "" { + t.Error("no reason was recorded for the failure") + } + // And it stays failed: a failed deployment is never resurrected. + _, err = e.svc.Finalize(t.Context(), e.p, dep) + wantCode(t, err, api.CodeConflict) +} + +// countingReader reports whether a body was read at all, which is how the tests +// tell "refused up front" apart from "read and then discarded". +type countingReader struct { + r io.Reader + n int +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += n + return n, err +} diff --git a/internal/deploy/sites.go b/internal/deploy/sites.go new file mode 100644 index 0000000..dcff2a6 --- /dev/null +++ b/internal/deploy/sites.go @@ -0,0 +1,127 @@ +package deploy + +import ( + "context" + "errors" + "time" + + "github.com/iceBear67/simplepages/internal/site" + "github.com/iceBear67/simplepages/internal/store" +) + +// index reads a deployment's manifest and turns it into the lookup structures +// the serving path needs. This is the expensive, failure-prone half of building +// a snapshot, which is why it is separable: an activation runs it before it +// changes anything. +func (s *Service) index(ctx context.Context, dep *store.Deployment) (*site.Index, error) { + files, err := s.DB.DeploymentFiles(ctx, dep.ID) + if err != nil { + return nil, err + } + return site.NewIndex(files), nil +} + +// deploymentDir is where a deployment's tree was assembled, or "" under +// assemble_mode=none, where nothing was. +func (s *Service) deploymentDir(p *store.Project, dep *store.Deployment) string { + if s.Dir == "" { + return "" + } + return DeploymentDir(s.Dir, p.ID, dep.PublicID) +} + +// LoadSites builds the registry from the database and makes the webroot agree +// with it. It runs once, before the listeners start. +// +// Without this a restart would serve 404 for every project until each one was +// deployed again. It also has to complete before the registry can stand in as +// the API's project resolver: a half-built registry would report a caller's own +// project as one that does not exist. +func (s *Service) LoadSites(ctx context.Context) error { + projects, err := s.DB.AllProjects(ctx) + if err != nil { + return err + } + + entries := make([]*site.Project, 0, len(projects)) + var active int + for _, p := range projects { + sp := site.NewProject(p) + entries = append(entries, sp) + + dep, err := s.DB.ActiveDeployment(ctx, p.ID) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + // A project that has never been deployed to. It exists, it + // resolves, and it answers 503 until something is activated. + continue + } + return err + } + idx, err := s.index(ctx, dep) + if err != nil { + return err + } + sp.Activate(site.NewDeployment(dep, idx, s.deploymentDir(p, dep))) + active++ + } + s.Sites.Replace(entries) + s.Log.Info("site registry loaded", "projects", len(entries), "active", active) + + // Best effort, like every other webroot operation: the server serves its own + // content and a stale symlink is not a reason to refuse to start. + if err := s.Reconcile(); err != nil { + s.Log.Warn("could not fully reconcile the webroot", "err", err) + } + return nil +} + +// Reconcile makes $WEBROOT agree with what this process is serving: one symlink +// per project with an active deployment, and nothing of ours left over for +// projects that no longer have one. +// +// The wanted set comes from the registry rather than from the database, because +// the registry is what requests are actually being answered from. The symlinks +// exist so that an external reader — a reverse proxy, a backup job — sees the +// same version this process does, and taking them from anywhere else would let +// the two disagree. +func (s *Service) Reconcile() error { + if s.Webroot == nil { + return nil + } + want := make(map[string]string) + for _, p := range s.Sites.Projects() { + // One load per project, and the value is used for both the name and the + // directory: reading Active() twice could straddle an activation and + // produce a link to a directory the other half of the pair disagrees with. + if d := p.Active(); d != nil && d.Dir != "" { + want[p.Name] = d.Dir + } + } + return s.Webroot.Reconcile(want) +} + +// RunReconciler repairs the webroot on a timer until ctx is done. +// +// Activation repoints a symlink itself and logs when it cannot, so this is for +// the cases where nothing was there to notice: a link an operator deleted or +// edited, one whose repoint failed on a full disk, or one left pointing at a +// deployment that has since been collected. Serving does not depend on any of +// it, which is why a failure here is a warning and never stops the loop. +func (s *Service) RunReconciler(ctx context.Context, every time.Duration) { + if s.Webroot == nil || every <= 0 { + return + } + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := s.Reconcile(); err != nil { + s.Log.Warn("could not fully reconcile the webroot", "err", err) + } + } + } +} diff --git a/internal/httpx/httpx_test.go b/internal/httpx/httpx_test.go new file mode 100644 index 0000000..aef3892 --- /dev/null +++ b/internal/httpx/httpx_test.go @@ -0,0 +1,287 @@ +package httpx + +import ( + "bytes" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + + "github.com/iceBear67/simplepages/api" +) + +func prefixes(t *testing.T, ss ...string) []netip.Prefix { + t.Helper() + out := make([]netip.Prefix, 0, len(ss)) + for _, s := range ss { + p, err := netip.ParsePrefix(s) + if err != nil { + t.Fatalf("ParsePrefix(%q): %v", s, err) + } + out = append(out, p) + } + return out +} + +// TestLogNeverContainsCredentials is the load-bearing test of this package: +// an access log that leaks a bearer token turns every log shipper, backup and +// support ticket into a credential store. +func TestLogNeverContainsCredentials(t *testing.T) { + const token = "pgs_k7m2q4x9v0zt3b8w_S3cr3tVa1ueThatMustNeverBeLogged00000000" + + var buf bytes.Buffer + log := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + h := Chain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A handler that legitimately annotates the log must still not be able to + // smuggle the secret in: it logs the key id, which is public. + LogAttr(r.Context(), "key_id", "k7m2q4x9v0zt3b8w") + LogAttr(r.Context(), "project", "demo") + w.WriteHeader(http.StatusNoContent) + }), WithRequestID(nil), AccessLog(log, nil), Recover(log)) + + r := httptest.NewRequest(http.MethodGet, "/api/v1/whoami?access_token="+token, nil) + r.Header.Set("Authorization", "Bearer "+token) + r.Header.Set("Cookie", "session="+token) + h.ServeHTTP(httptest.NewRecorder(), r) + + out := buf.String() + if out == "" { + t.Fatal("no log output produced") + } + for _, needle := range []string{token, "S3cr3tVa1ue", "Bearer", "Authorization", "session="} { + if strings.Contains(out, needle) { + t.Errorf("log contains %q\nlog: %s", needle, out) + } + } + for _, want := range []string{`"key_id":"k7m2q4x9v0zt3b8w"`, `"project":"demo"`, `"status":204`} { + if !strings.Contains(out, want) { + t.Errorf("log missing %s\nlog: %s", want, out) + } + } + // The raw query string is not logged either: tokens end up there when someone + // ignores the docs, and we would rather drop the field than log the secret. + if strings.Contains(out, "access_token") { + t.Errorf("log contains the query string\nlog: %s", out) + } +} + +func TestAccessLogRecordsStatusAndBytes(t *testing.T) { + var buf bytes.Buffer + log := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + h := Chain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTeapot) + fmt.Fprint(w, "hello") + }), AccessLog(log, nil)) + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/x", nil)) + + var rec map[string]any + if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &rec); err != nil { + t.Fatalf("log line is not JSON: %v (%s)", err, buf.String()) + } + if got := rec["status"]; got != float64(http.StatusTeapot) { + t.Errorf("status = %v, want 418", got) + } + if got := rec["bytes"]; got != float64(5) { + t.Errorf("bytes = %v, want 5", got) + } + if rec["level"] != "WARN" { + t.Errorf("level = %v, want WARN for a 4xx", rec["level"]) + } +} + +func TestRecoverReturnsOpaque500(t *testing.T) { + var buf bytes.Buffer + log := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + h := Chain(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + panic("database password is hunter2") + }), WithRequestID(nil), AccessLog(log, nil), Recover(log)) + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/boom", nil)) + + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", w.Code) + } + // The panicking request must still produce an access-log line, which is only + // true while Recover runs inside AccessLog. + if !strings.Contains(buf.String(), `"msg":"request"`) { + t.Errorf("no access-log line for the panicking request\nlog: %s", buf.String()) + } + if strings.Contains(w.Body.String(), "hunter2") { + t.Errorf("panic value leaked to the client: %s", w.Body.String()) + } + var env api.ErrorEnvelope + if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil { + t.Fatalf("body is not an error envelope: %v (%s)", err, w.Body.String()) + } + if env.Error.Code != api.CodeInternal { + t.Errorf("code = %q, want %q", env.Error.Code, api.CodeInternal) + } + if env.Error.Details["request_id"] == nil { + t.Error("500 body carries no request_id, so the log line cannot be found") + } + if !strings.Contains(buf.String(), "hunter2") { + t.Error("panic value was not logged; it must reach the operator even though it must not reach the client") + } +} + +func TestClientIP(t *testing.T) { + trusted := prefixes(t, "127.0.0.1/32", "10.0.0.0/8") + + cases := []struct { + name string + remote string + xff []string + want string + }{ + {"untrusted peer, header ignored", "203.0.113.9:1234", []string{"9.9.9.9"}, "203.0.113.9"}, + {"trusted peer, single hop", "127.0.0.1:1234", []string{"198.51.100.7"}, "198.51.100.7"}, + {"trusted peer, chain", "10.1.2.3:1234", []string{"198.51.100.7, 10.4.5.6"}, "198.51.100.7"}, + {"forged prefix ignored", "127.0.0.1:1234", []string{"1.2.3.4, 198.51.100.7"}, "198.51.100.7"}, + {"multiple headers", "127.0.0.1:1234", []string{"1.1.1.1", "198.51.100.7"}, "198.51.100.7"}, + {"garbage entries skipped", "127.0.0.1:1234", []string{"not-an-ip, 198.51.100.7, 10.0.0.1"}, "198.51.100.7"}, + {"all trusted, falls back to peer", "127.0.0.1:1234", []string{"10.0.0.1"}, "127.0.0.1"}, + {"no header", "127.0.0.1:1234", nil, "127.0.0.1"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.RemoteAddr = tc.remote + for _, v := range tc.xff { + r.Header.Add("X-Forwarded-For", v) + } + got, ok := ClientIP(r, trusted) + if !ok { + t.Fatal("ClientIP reported no address") + } + if got.String() != tc.want { + t.Errorf("ClientIP = %s, want %s", got, tc.want) + } + }) + } +} + +func TestRequestIDAdoption(t *testing.T) { + trusted := prefixes(t, "127.0.0.1/32") + + cases := []struct { + name string + remote string + header string + wantSet bool // true = the inbound value is echoed back verbatim + }{ + {"trusted peer, sane id", "127.0.0.1:1", "deadbeef-42", true}, + {"untrusted peer", "203.0.113.9:1", "deadbeef-42", false}, + {"trusted peer, space injected", "127.0.0.1:1", "id with space", false}, + {"trusted peer, newline injected", "127.0.0.1:1", "id\nlevel=INFO", false}, + {"trusted peer, over-long", "127.0.0.1:1", strings.Repeat("a", 65), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var seen string + h := WithRequestID(trusted)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = RequestIDFrom(r.Context()) + })) + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.RemoteAddr = tc.remote + r.Header.Set("X-Request-Id", tc.header) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + + if tc.wantSet { + if seen != tc.header { + t.Errorf("request id = %q, want the inbound %q", seen, tc.header) + } + } else if seen == tc.header { + t.Errorf("adopted an untrustworthy inbound id %q", tc.header) + } + if seen == "" { + t.Error("no request id assigned") + } + if got := w.Header().Get("X-Request-Id"); got != seen { + t.Errorf("echoed %q but handler saw %q", got, seen) + } + }) + } +} + +func TestWriteErrorMapsCodes(t *testing.T) { + cases := []struct { + err error + want int + }{ + {api.Errorf(api.CodeNotFound, "nope"), http.StatusNotFound}, + {api.Errorf(api.CodeDeploymentActive, "still active"), http.StatusConflict}, + {api.Errorf(api.CodeInvalidPath, "bad"), http.StatusBadRequest}, + {api.Errorf(api.CodeLimitExceeded, "too big"), http.StatusRequestEntityTooLarge}, + {fmt.Errorf("wrapped: %w", api.Errorf(api.CodeForbidden, "no")), http.StatusForbidden}, + {fmt.Errorf("open /var/lib/pages-server/secret: permission denied"), http.StatusInternalServerError}, + } + log := slog.New(slog.NewJSONHandler(&bytes.Buffer{}, nil)) + for _, tc := range cases { + w := httptest.NewRecorder() + WriteError(w, httptest.NewRequest(http.MethodGet, "/", nil), log, tc.err) + if w.Code != tc.want { + t.Errorf("WriteError(%v) = %d, want %d", tc.err, w.Code, tc.want) + } + if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") { + t.Errorf("Content-Type = %q", ct) + } + } +} + +// A non-api error must not put internal detail in the response body. +func TestWriteErrorHidesInternalDetail(t *testing.T) { + log := slog.New(slog.NewJSONHandler(&bytes.Buffer{}, nil)) + w := httptest.NewRecorder() + WriteError(w, httptest.NewRequest(http.MethodGet, "/", nil), log, + fmt.Errorf("sql: no rows in /var/lib/pages-server/pages.db")) + if strings.Contains(w.Body.String(), "pages.db") { + t.Errorf("internal detail leaked: %s", w.Body.String()) + } +} + +func TestDecodeJSON(t *testing.T) { + type payload struct { + Name string `json:"name"` + } + cases := []struct { + name string + body string + max int64 + want api.Code + }{ + {"ok", `{"name":"demo"}`, 1024, ""}, + {"unknown field", `{"name":"demo","nmae":"typo"}`, 1024, api.CodeBadRequest}, + {"wrong type", `{"name":42}`, 1024, api.CodeBadRequest}, + {"malformed", `{"name":`, 1024, api.CodeBadRequest}, + {"trailing content", `{"name":"a"} {"name":"b"}`, 1024, api.CodeBadRequest}, + {"too large", `{"name":"` + strings.Repeat("x", 200) + `"}`, 32, api.CodePayloadTooLarge}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tc.body)) + var v payload + err := DecodeJSON(httptest.NewRecorder(), r, tc.max, &v) + if tc.want == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatal("expected an error") + } + if got := api.CodeOf(err); got != tc.want { + t.Errorf("code = %q, want %q (%v)", got, tc.want, err) + } + }) + } +} diff --git a/internal/httpx/json.go b/internal/httpx/json.go new file mode 100644 index 0000000..22350c0 --- /dev/null +++ b/internal/httpx/json.go @@ -0,0 +1,127 @@ +// Package httpx holds the HTTP plumbing shared by both listeners: the JSON +// error envelope, the middleware chain, and server lifecycle management. +package httpx + +import ( + "encoding/json" + "errors" + "log/slog" + "net/http" + + "github.com/iceBear67/simplepages/api" +) + +// StatusFor maps a machine-readable code to its HTTP status. +func StatusFor(code api.Code) int { + switch code { + case api.CodeBadRequest, api.CodeInvalidProjectName, api.CodeInvalidPath, + api.CodeDigestMismatch, api.CodeSizeMismatch: + return http.StatusBadRequest + case api.CodeUnauthorized: + return http.StatusUnauthorized + case api.CodeForbidden: + return http.StatusForbidden + case api.CodeNotFound: + return http.StatusNotFound + case api.CodeMethodNotAllowed: + return http.StatusMethodNotAllowed + case api.CodeConflict, api.CodeProjectExists, api.CodeDeploymentNotReady, + api.CodeDeploymentActive, api.CodeBlobsMissing: + return http.StatusConflict + case api.CodePayloadTooLarge, api.CodeLimitExceeded: + return http.StatusRequestEntityTooLarge + case api.CodeRateLimited: + return http.StatusTooManyRequests + case api.CodeUnavailable: + return http.StatusServiceUnavailable + default: + return http.StatusInternalServerError + } +} + +// WriteJSON writes v as JSON with the given status. +func WriteJSON(w http.ResponseWriter, status int, v any) { + buf, err := json.Marshal(v) + if err != nil { + // Marshalling our own response types should never fail; if it does the + // handler already wrote nothing, so a bare 500 is the honest answer. + http.Error(w, `{"error":{"code":"internal","message":"response encoding failed"}}`, + http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _, _ = w.Write(buf) + _, _ = w.Write([]byte("\n")) +} + +// WriteError renders err as the standard error envelope. Non-api errors become +// an opaque 500: internal failure text may name paths or SQL and must not reach +// the client. The full error is logged instead. +func WriteError(w http.ResponseWriter, r *http.Request, log *slog.Logger, err error) { + var apiErr *api.Error + if !errors.As(err, &apiErr) { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + apiErr = api.Errorf(api.CodePayloadTooLarge, "request body exceeds %d bytes", maxErr.Limit) + } else { + if log != nil { + log.ErrorContext(r.Context(), "unhandled error", "err", err, + "method", r.Method, "path", r.URL.Path) + } + apiErr = api.Errorf(api.CodeInternal, "internal error") + } + } + status := StatusFor(apiErr.Code) + if status >= 500 && log != nil { + log.ErrorContext(r.Context(), "request failed", "err", err, + "code", string(apiErr.Code), "method", r.Method, "path", r.URL.Path) + } + WriteJSON(w, status, api.ErrorEnvelope{Error: *apiErr}) +} + +// DecodeJSON reads a JSON body into v, capped at maxBytes. It rejects unknown +// fields (a misspelled key in a deploy script should fail, not be ignored) and +// trailing content after the top-level value. +func DecodeJSON(w http.ResponseWriter, r *http.Request, maxBytes int64, v any) error { + r.Body = http.MaxBytesReader(w, r.Body, maxBytes) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + return api.Errorf(api.CodePayloadTooLarge, "request body exceeds %d bytes", maxErr.Limit) + } + var syn *json.SyntaxError + if errors.As(err, &syn) { + return api.Errorf(api.CodeBadRequest, "malformed JSON at byte %d", syn.Offset) + } + var typeErr *json.UnmarshalTypeError + if errors.As(err, &typeErr) { + return api.Errorf(api.CodeBadRequest, "field %q: want %s", typeErr.Field, typeErr.Type) + } + return api.Errorf(api.CodeBadRequest, "%s", err) + } + if dec.More() { + return api.Errorf(api.CodeBadRequest, "unexpected content after JSON value") + } + return nil +} + +// newInternalEnvelope builds the opaque 500 body. The request id is included so +// an operator can find the corresponding log line, which holds the real cause. +func newInternalEnvelope(reqID string) api.ErrorEnvelope { + e := api.Errorf(api.CodeInternal, "internal error") + if reqID != "" { + e = e.WithDetail("request_id", reqID) + } + return api.ErrorEnvelope{Error: *e} +} + +// NoBody rejects requests that carry a body where none is expected. +func NoBody(r *http.Request) error { + if r.ContentLength > 0 { + return api.Errorf(api.CodeBadRequest, "unexpected request body") + } + return nil +} diff --git a/internal/httpx/middleware.go b/internal/httpx/middleware.go new file mode 100644 index 0000000..aaffa9d --- /dev/null +++ b/internal/httpx/middleware.go @@ -0,0 +1,332 @@ +package httpx + +import ( + "bufio" + "context" + "crypto/rand" + "encoding/hex" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/netip" + "runtime/debug" + "strings" + "sync" + "time" +) + +// Middleware wraps a handler. Chain applies them so that the first listed runs +// outermost. +type Middleware func(http.Handler) http.Handler + +// Chain wraps h with mw, outermost first. +func Chain(h http.Handler, mw ...Middleware) http.Handler { + for i := len(mw) - 1; i >= 0; i-- { + h = mw[i](h) + } + return h +} + +type ctxKey int + +const ( + ctxKeyRequestID ctxKey = iota + ctxKeyLogState +) + +// ---------------------------------------------------------------- request id + +// RequestIDFrom returns the request id assigned by WithRequestID. +func RequestIDFrom(ctx context.Context) string { + id, _ := ctx.Value(ctxKeyRequestID).(string) + return id +} + +// WithRequestID assigns each request an id, echoes it in X-Request-Id, and makes +// it available to handlers and the access log. +// +// An inbound X-Request-Id is adopted only when the peer is a trusted proxy and +// the value is short and printable: it ends up in log records, and an arbitrary +// client-controlled string there is a log-forging primitive. +func WithRequestID(trusted []netip.Prefix) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := "" + if in := r.Header.Get("X-Request-Id"); in != "" && sanitaryID(in) { + if addr, ok := peerAddr(r); ok && inAny(addr, trusted) { + id = in + } + } + if id == "" { + id = newID() + } + w.Header().Set("X-Request-Id", id) + ctx := context.WithValue(r.Context(), ctxKeyRequestID, id) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} + +func newID() string { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + // crypto/rand cannot fail on any supported platform; if it somehow does, + // an empty id is better than taking down the request. + return "" + } + return hex.EncodeToString(b[:]) +} + +func sanitaryID(s string) bool { + if len(s) == 0 || len(s) > 64 { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + if c < 0x21 || c > 0x7e { + return false + } + } + return true +} + +// ------------------------------------------------------------- log enrichment + +// logState collects attributes that handlers discover mid-request (project, +// deployment, key id) so the single access-log line can carry them. +type logState struct { + mu sync.Mutex + attrs []slog.Attr +} + +// LogAttr attaches a key/value pair to this request's access-log line. It is a +// no-op outside the middleware chain, so handlers may call it unconditionally. +// +// Never pass a token, an Authorization header, or any part of either. +func LogAttr(ctx context.Context, key string, value any) { + st, ok := ctx.Value(ctxKeyLogState).(*logState) + if !ok { + return + } + st.mu.Lock() + st.attrs = append(st.attrs, slog.Any(key, value)) + st.mu.Unlock() +} + +// -------------------------------------------------------------- access logger + +// AccessLog emits exactly one record per request. +// +// It logs the method, the URL path, and nothing else from the request: headers +// are never logged (Authorization carries a bearer token) and neither is the raw +// query string. Handlers add their own context with LogAttr. +func AccessLog(log *slog.Logger, trusted []netip.Prefix) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + st := &logState{} + ctx := context.WithValue(r.Context(), ctxKeyLogState, st) + r = r.WithContext(ctx) + + rec := &recorder{ResponseWriter: w, status: http.StatusOK} + start := time.Now() + next.ServeHTTP(rec, r) + dur := time.Since(start) + + level := slog.LevelInfo + switch { + case rec.status >= 500: + level = slog.LevelError + case rec.status == http.StatusNotFound || rec.status == http.StatusMethodNotAllowed: + // Ordinary outcomes on a public listener: a crawler, a stale link, + // a missing favicon. Logging them as warnings would make warnings + // the bulk of the file and hide the ones that mean something. + level = slog.LevelInfo + case rec.status >= 400: + level = slog.LevelWarn + case r.URL.Path == "/healthz" || r.URL.Path == "/readyz": + level = slog.LevelDebug + } + if !log.Enabled(ctx, level) { + return + } + + st.mu.Lock() + extra := st.attrs + st.mu.Unlock() + + attrs := make([]slog.Attr, 0, 8+len(extra)) + attrs = append(attrs, + slog.String("method", r.Method), + slog.String("path", r.URL.Path), + slog.Int("status", rec.status), + slog.Int64("bytes", rec.written), + slog.Float64("dur_ms", float64(dur.Microseconds())/1000), + ) + if ip, ok := ClientIP(r, trusted); ok { + attrs = append(attrs, slog.String("ip", ip.String())) + } + if id := RequestIDFrom(ctx); id != "" { + attrs = append(attrs, slog.String("req_id", id)) + } + attrs = append(attrs, extra...) + log.LogAttrs(ctx, level, "request", attrs...) + }) + } +} + +// ------------------------------------------------------------------ recovery + +// Recover turns a handler panic into a 500 instead of tearing down the process, +// logging the stack. The panic value itself never reaches the client. +func Recover(log *slog.Logger) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + v := recover() + if v == nil { + return + } + // ErrAbortHandler is the documented way to abort a response; + // net/http expects to handle it and logs nothing. + if err, ok := v.(error); ok && errors.Is(err, http.ErrAbortHandler) { + panic(v) + } + log.ErrorContext(r.Context(), "handler panic", + "panic", v, + "method", r.Method, + "path", r.URL.Path, + "req_id", RequestIDFrom(r.Context()), + "stack", string(debug.Stack()), + ) + if rec, ok := w.(*recorder); ok && rec.wroteHeader { + return // response already begun; nothing safe left to send + } + WriteJSON(w, http.StatusInternalServerError, + newInternalEnvelope(RequestIDFrom(r.Context()))) + }() + next.ServeHTTP(w, r) + }) + } +} + +// ---------------------------------------------------------------- client IP + +// ClientIP returns the address to attribute the request to. X-Forwarded-For is +// honoured only when the direct peer is itself a trusted proxy; otherwise any +// client could forge its own source address and defeat per-IP rate limiting. +func ClientIP(r *http.Request, trusted []netip.Prefix) (netip.Addr, bool) { + peer, ok := peerAddr(r) + if !ok { + return netip.Addr{}, false + } + if !inAny(peer, trusted) { + return peer, true + } + // Walk right to left and take the first address that is not itself trusted: + // everything to its right was appended by infrastructure we control, and + // everything to its left may have been forged by the client. + xff := r.Header.Values("X-Forwarded-For") + for i := len(xff) - 1; i >= 0; i-- { + parts := strings.Split(xff[i], ",") + for j := len(parts) - 1; j >= 0; j-- { + addr, err := netip.ParseAddr(strings.TrimSpace(parts[j])) + if err != nil { + continue + } + addr = addr.Unmap() + if !inAny(addr, trusted) { + return addr, true + } + } + } + return peer, true +} + +func peerAddr(r *http.Request) (netip.Addr, bool) { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + addr, err := netip.ParseAddr(host) + if err != nil { + return netip.Addr{}, false + } + return addr.Unmap(), true +} + +func inAny(addr netip.Addr, prefixes []netip.Prefix) bool { + for _, p := range prefixes { + if p.Contains(addr) { + return true + } + } + return false +} + +// ------------------------------------------------------------ response record + +// recorder observes the status and byte count without altering behaviour. +// +// It implements Unwrap, ReadFrom, Flush and Hijack so that wrapping costs +// nothing: without ReadFrom the static handler would lose the sendfile fast +// path that io.Copy takes when the destination is the raw *http.response. +type recorder struct { + http.ResponseWriter + status int + written int64 + wroteHeader bool +} + +func (r *recorder) WriteHeader(status int) { + if r.wroteHeader { + return + } + r.status = status + r.wroteHeader = true + r.ResponseWriter.WriteHeader(status) +} + +func (r *recorder) Write(b []byte) (int, error) { + if !r.wroteHeader { + r.WriteHeader(http.StatusOK) + } + n, err := r.ResponseWriter.Write(b) + r.written += int64(n) + return n, err +} + +func (r *recorder) ReadFrom(src io.Reader) (int64, error) { + if !r.wroteHeader { + r.WriteHeader(http.StatusOK) + } + rf, ok := r.ResponseWriter.(io.ReaderFrom) + if !ok { + n, err := io.Copy(r.ResponseWriter, src) + r.written += n + return n, err + } + n, err := rf.ReadFrom(src) + r.written += n + return n, err +} + +func (r *recorder) Unwrap() http.ResponseWriter { return r.ResponseWriter } + +func (r *recorder) Flush() { + if f, ok := r.ResponseWriter.(http.Flusher); ok { + if !r.wroteHeader { + r.WriteHeader(http.StatusOK) + } + f.Flush() + } +} + +func (r *recorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { + h, ok := r.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, errors.New("httpx: ResponseWriter does not support hijacking") + } + return h.Hijack() +} diff --git a/internal/httpx/server.go b/internal/httpx/server.go new file mode 100644 index 0000000..0c89800 --- /dev/null +++ b/internal/httpx/server.go @@ -0,0 +1,143 @@ +package httpx + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "time" +) + +// Timeouts configures a listener's deadlines. +// +// WriteTimeout is deliberately optional and left at zero for the static +// listener: a global write deadline covers the whole response, so a large file +// over a slow link gets its connection torn down mid-download even though +// nothing is wrong. Per-response deadlines belong to the handler, via +// http.ResponseController. +type Timeouts struct { + ReadHeader time.Duration + Read time.Duration + Idle time.Duration + Write time.Duration // 0 = none +} + +// Server is an http.Server whose listener is already bound. +// +// Binding at construction time means a port conflict is reported before any +// background work starts, and it lets a caller pass ":0" and read back the +// chosen address — which is what the integration tests do. +type Server struct { + Name string + + srv *http.Server + ln net.Listener + log *slog.Logger +} + +// Listen binds addr and prepares a server for h. +func Listen(name, addr string, h http.Handler, t Timeouts, log *slog.Logger) (*Server, error) { + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + s := &Server{ + Name: name, + ln: ln, + log: log, + srv: &http.Server{ + Handler: h, + ReadHeaderTimeout: t.ReadHeader, + ReadTimeout: t.Read, + IdleTimeout: t.Idle, + WriteTimeout: t.Write, + // Route net/http's own errors (malformed requests, TLS handshake + // failures) into the structured log rather than bare stderr. + ErrorLog: slog.NewLogLogger(log.With("listener", name).Handler(), slog.LevelWarn), + }, + } + return s, nil +} + +// Addr is the address actually bound, which differs from the requested one when +// port 0 was asked for. +func (s *Server) Addr() string { return s.ln.Addr().String() } + +// Serve blocks until the server stops. It returns nil on a graceful shutdown. +func (s *Server) Serve() error { + err := s.srv.Serve(s.ln) + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} + +// Shutdown stops accepting connections and waits for in-flight requests, up to +// ctx's deadline. Past the deadline the remaining connections are closed. +func (s *Server) Shutdown(ctx context.Context) error { + err := s.srv.Shutdown(ctx) + if err != nil { + // Shutdown only fails by running out of time; Close is then the only way + // to release the port. + s.log.Warn("graceful shutdown timed out, closing connections", + "listener", s.Name, "err", err) + return s.srv.Close() + } + return nil +} + +// Group runs several servers with a shared lifetime: if one fails, all stop. +type Group struct { + Servers []*Server + Grace time.Duration + Log *slog.Logger +} + +// Run serves until ctx is cancelled or a server fails, then shuts every server +// down within Grace. It returns the first non-nil error. +func (g *Group) Run(ctx context.Context) error { + errs := make(chan error, len(g.Servers)) + for _, s := range g.Servers { + go func() { + g.Log.Info("listening", "listener", s.Name, "addr", s.Addr()) + errs <- s.Serve() + }() + } + + var first error + done := 0 + select { + case <-ctx.Done(): + g.Log.Info("shutting down", "grace", g.Grace) + case err := <-errs: + done++ + first = err + if err != nil { + g.Log.Error("listener failed, stopping", "err", err) + } + } + + // The shutdown deadline must survive the cancellation that triggered it, + // otherwise ctx.Done() would make Shutdown return immediately. + shutCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), g.Grace) + defer cancel() + for _, s := range g.Servers { + if err := s.Shutdown(shutCtx); err != nil && first == nil { + first = err + } + } + + // Drain the remaining Serve results so no goroutine is left blocked on send. + for ; done < len(g.Servers); done++ { + select { + case err := <-errs: + if err != nil && first == nil { + first = err + } + case <-time.After(5 * time.Second): + return first + } + } + return first +} diff --git a/internal/pathutil/pathutil.go b/internal/pathutil/pathutil.go new file mode 100644 index 0000000..1acbd2f --- /dev/null +++ b/internal/pathutil/pathutil.go @@ -0,0 +1,171 @@ +// Package pathutil validates the site-relative paths a deployment manifest may +// declare. +// +// It depends on the standard library only. Both sides run these checks: the CLI +// so a bad path is caught before anything is uploaded, and the server because a +// client is never a trust boundary. cmd/pages/deps_test.go fails the build if +// anything server-side reaches the CLI through this package. +package pathutil + +import ( + "errors" + "fmt" + "io/fs" + "path" + "path/filepath" + "strings" + "unicode/utf8" +) + +const ( + // MaxPathBytes caps a whole path. Linux's PATH_MAX is 4096 including the + // deployment directory this gets joined onto, so anything approaching it is + // already not a real build artifact. + MaxPathBytes = 4096 + // MaxSegmentBytes caps one component, matching the NAME_MAX of every + // filesystem a deployment tree is plausibly assembled on. + MaxSegmentBytes = 255 +) + +// Why a path can be refused. Callers match on these with errors.Is; the message +// carries the specifics. +var ( + ErrEmpty = errors.New("path is empty") + ErrTooLong = errors.New("path is too long") + ErrSegmentTooLong = errors.New("path component is too long") + ErrNotUTF8 = errors.New("path is not valid UTF-8") + ErrControlChar = errors.New("path contains a control character") + ErrBackslash = errors.New("path contains a backslash") + ErrNotRelative = errors.New("path must be relative and slash-separated, with no empty, \".\" or \"..\" component") + ErrNotLocal = errors.New("path is not usable as a local filename") + + ErrDuplicate = errors.New("path appears twice in the manifest") + ErrCaseCollision = errors.New("path differs from another only by letter case") + ErrPathConflict = errors.New("path is used as both a file and a directory") +) + +// Validate reports whether p may appear in a manifest. +// +// The checks overlap on purpose. fs.ValidPath states the semantic rule — +// relative, slash-separated, no empty or "." or ".." component, no trailing +// slash — and filepath.Localize answers the question that actually matters when +// the tree is assembled: can this become a filename that filepath.Join is +// unable to walk out of the destination directory. On Windows Localize also +// rejects the reserved device names. Running both means that a future change to +// either one cannot quietly widen what is accepted. +func Validate(p string) error { + if p == "" { + return ErrEmpty + } + if len(p) > MaxPathBytes { + return fmt.Errorf("%w: %d bytes, limit is %d", ErrTooLong, len(p), MaxPathBytes) + } + if !utf8.ValidString(p) { + // These bytes would be a perfectly legal filename on Linux, but the path + // is also a value in a TEXT column, and SQLite's comparison and + // collation behaviour on non-UTF-8 is undefined. The database is the + // stricter of the two, so it sets the rule. + return ErrNotUTF8 + } + for i := 0; i < len(p); i++ { + // A newline would forge a second line in the access log; an ESC would + // let a manifest repaint the operator's terminal during `pages + // deployment show --files`. NUL would truncate the path at the syscall + // boundary while the database kept the whole thing. + if c := p[i]; c < 0x20 || c == 0x7f { + return ErrControlChar + } + } + if strings.ContainsRune(p, '\\') { + // An ordinary filename character on Linux and a separator on Windows. + // Refusing it outright is what makes a manifest mean the same thing + // wherever the tree is assembled. + return ErrBackslash + } + if !fs.ValidPath(p) || p == "." { + return ErrNotRelative + } + for _, seg := range strings.Split(p, "/") { + if len(seg) > MaxSegmentBytes { + return fmt.Errorf("%w: %d bytes, limit is %d", ErrSegmentTooLong, len(seg), MaxSegmentBytes) + } + } + if _, err := filepath.Localize(p); err != nil { + return fmt.Errorf("%w: %v", ErrNotLocal, err) + } + return nil +} + +// Set accumulates one manifest's paths and rejects the pairs that cannot +// coexist in a single directory tree. +// +// Three kinds, each of which would otherwise surface far from its cause: +// +// - An exact duplicate declares two digests for one path. Assembly would +// write whichever arrived last and the site would be subtly wrong. +// - A case-only collision ("App.js" and "app.js") assembles fine on ext4 and +// silently loses a file on APFS or NTFS — a failure that reproduces on the +// server and on nobody's laptop. +// - A file that another entry needs as a directory ("a" and "a/b") fails +// mid-assembly with a bare ENOTDIR, after part of the tree already exists. +// +// Catching all three at manifest time turns each into one clear error naming +// both paths, before a single byte is uploaded. +type Set struct { + files map[string]string // folded path -> the path that claimed it + dirs map[string]string // folded directory prefix -> a path that needs it +} + +// NewSet returns a Set sized for an expected number of files. +func NewSet(size int) *Set { + return &Set{ + files: make(map[string]string, size), + dirs: make(map[string]string, size/4+1), + } +} + +// Add validates p and records it. The error names the path it conflicts with, +// which is the only detail that makes a collision fixable. +func (s *Set) Add(p string) error { + if err := Validate(p); err != nil { + return err + } + folded := fold(p) + if prev, ok := s.files[folded]; ok { + if prev == p { + return ErrDuplicate + } + return fmt.Errorf("%w: %q", ErrCaseCollision, prev) + } + if prev, ok := s.dirs[folded]; ok { + return fmt.Errorf("%w: %q needs it as a directory", ErrPathConflict, prev) + } + + // Check every ancestor before recording anything, so a rejected path leaves + // the set exactly as it was. + for dir := path.Dir(p); dir != "."; dir = path.Dir(dir) { + if prev, ok := s.files[fold(dir)]; ok { + return fmt.Errorf("%w: %q is a file", ErrPathConflict, prev) + } + } + s.files[folded] = p + for dir := path.Dir(p); dir != "."; dir = path.Dir(dir) { + fd := fold(dir) + if _, ok := s.dirs[fd]; ok { + // Every shallower ancestor is already recorded too. + break + } + s.dirs[fd] = p + } + return nil +} + +// Len is how many paths have been accepted. +func (s *Set) Len() int { return len(s.files) } + +// fold is the key under which two names collide on a case-insensitive +// filesystem. strings.ToLower is not byte-for-byte what APFS or NTFS do — they +// each pin a particular Unicode version's table — but it agrees with both on +// everything a build tool emits, and disagreeing by being too strict is the +// safe direction. +func fold(p string) string { return strings.ToLower(p) } diff --git a/internal/pathutil/pathutil_test.go b/internal/pathutil/pathutil_test.go new file mode 100644 index 0000000..71d0684 --- /dev/null +++ b/internal/pathutil/pathutil_test.go @@ -0,0 +1,180 @@ +package pathutil + +import ( + "errors" + "strings" + "testing" +) + +func TestValidateAccepts(t *testing.T) { + ok := []string{ + "index.html", + "assets/app.js", + "a/b/c/d/e/f/g.txt", + ".well-known/acme-challenge/token", + "_next/static/chunks/main-abc123.js", + "文档/说明.html", + "a file with spaces.html", + "weird!@#$%^&()+=[]{};'\",<>?.html", + "..hidden", // only an exact ".." component is a traversal + "a/..b/c", // + "...", // not "." and not ".." + "~tilde.html", // no special meaning below a project prefix + strings.Repeat("a", MaxSegmentBytes), + } + for _, p := range ok { + if err := Validate(p); err != nil { + t.Errorf("Validate(%q) = %v, want nil", p, err) + } + } +} + +func TestValidateRejects(t *testing.T) { + cases := []struct { + path string + want error + }{ + {"", ErrEmpty}, + {".", ErrNotRelative}, + {"..", ErrNotRelative}, + {"../etc/passwd", ErrNotRelative}, + {"a/../../etc/passwd", ErrNotRelative}, + {"a/./b", ErrNotRelative}, + {"/etc/passwd", ErrNotRelative}, + {"a//b", ErrNotRelative}, + {"a/", ErrNotRelative}, + {"/", ErrNotRelative}, + {"a/b/..", ErrNotRelative}, + {"a\x00b", ErrControlChar}, + {"a\nb", ErrControlChar}, + {"a\tb", ErrControlChar}, + {"a\x1b[31m", ErrControlChar}, + {"a\x7fb", ErrControlChar}, + {"a\\b", ErrBackslash}, + {"..\\..\\windows", ErrBackslash}, + {"a/\xff\xfe/b", ErrNotUTF8}, + {strings.Repeat("a", MaxSegmentBytes+1), ErrSegmentTooLong}, + {"ok/" + strings.Repeat("b", MaxSegmentBytes+1), ErrSegmentTooLong}, + {strings.Repeat("a/", MaxPathBytes/2) + "b", ErrTooLong}, + } + for _, tc := range cases { + err := Validate(tc.path) + if !errors.Is(err, tc.want) { + t.Errorf("Validate(%q) = %v, want %v", tc.path, err, tc.want) + } + } +} + +// The traversal cases are the ones that matter most, so state them again as an +// executable claim about what a manifest can never make Join produce. +func TestValidateBlocksEscape(t *testing.T) { + for _, p := range []string{ + "../x", "a/../../x", "./../x", "/x", "a/b/../../../x", + "..", "a/..", "\\..\\x", "a\\..\\..\\x", + } { + if err := Validate(p); err == nil { + t.Errorf("Validate(%q) accepted a path that can escape its directory", p) + } + } +} + +func TestSetDetectsCollisions(t *testing.T) { + cases := []struct { + name string + paths []string + want error + }{ + {"duplicate", []string{"a.html", "a.html"}, ErrDuplicate}, + {"case only", []string{"App.js", "app.js"}, ErrCaseCollision}, + {"case in a directory", []string{"Assets/x.js", "assets/y.js"}, nil}, + {"case collision under a folded directory", []string{"Assets/x.js", "assets/X.js"}, ErrCaseCollision}, + {"file then directory", []string{"a", "a/b"}, ErrPathConflict}, + {"directory then file", []string{"a/b", "a"}, ErrPathConflict}, + {"deep file then directory", []string{"a/b/c", "a/b"}, ErrPathConflict}, + {"file then deep directory", []string{"a/b", "a/b/c/d"}, ErrPathConflict}, + {"case-folded file vs directory", []string{"A", "a/b"}, ErrPathConflict}, + {"siblings are fine", []string{"a/b", "a/c", "a/d/e"}, nil}, + {"unrelated", []string{"index.html", "assets/app.js", "assets/app.css"}, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := NewSet(len(tc.paths)) + var err error + for _, p := range tc.paths { + if err = s.Add(p); err != nil { + break + } + } + if tc.want == nil { + if err != nil { + t.Fatalf("Add: %v", err) + } + if s.Len() != len(tc.paths) { + t.Errorf("Len = %d, want %d", s.Len(), len(tc.paths)) + } + return + } + if !errors.Is(err, tc.want) { + t.Fatalf("err = %v, want %v", err, tc.want) + } + }) + } +} + +// A rejected path must not leave a mark: the caller may report the error and +// carry on validating the rest of the manifest. +func TestSetRejectionLeavesNoTrace(t *testing.T) { + s := NewSet(4) + if err := s.Add("a/b"); err != nil { + t.Fatal(err) + } + if err := s.Add("a"); !errors.Is(err, ErrPathConflict) { + t.Fatalf("err = %v", err) + } + if err := s.Add("bad\x00path"); !errors.Is(err, ErrControlChar) { + t.Fatalf("err = %v", err) + } + if s.Len() != 1 { + t.Errorf("Len = %d, want 1", s.Len()) + } + if err := s.Add("a/c"); err != nil { + t.Errorf("a sibling must still be accepted: %v", err) + } +} + +// The message has to name the other path, or a 50,000-file manifest reports a +// collision the operator cannot locate. +func TestCollisionErrorNamesTheOtherPath(t *testing.T) { + s := NewSet(2) + if err := s.Add("Assets/App.js"); err != nil { + t.Fatal(err) + } + err := s.Add("assets/app.js") + if err == nil { + t.Fatal("want a collision") + } + if !strings.Contains(err.Error(), "Assets/App.js") { + t.Errorf("error %q does not name the conflicting path", err) + } +} + +func FuzzValidate(f *testing.F) { + for _, s := range []string{"index.html", "a/b", "../x", "a\\b", "", ".", "a\x00b"} { + f.Add(s) + } + f.Fuzz(func(t *testing.T, p string) { + if Validate(p) != nil { + return + } + // Anything accepted must be safe to join onto a directory. Localize is + // the standard library's own statement of that property. + if strings.HasPrefix(p, "/") || strings.Contains(p, "\\") { + t.Fatalf("Validate accepted %q", p) + } + for _, seg := range strings.Split(p, "/") { + if seg == "" || seg == "." || seg == ".." { + t.Fatalf("Validate accepted %q with segment %q", p, seg) + } + } + }) +} diff --git a/internal/site/deployment.go b/internal/site/deployment.go new file mode 100644 index 0000000..a51080c --- /dev/null +++ b/internal/site/deployment.go @@ -0,0 +1,115 @@ +// Package site turns an activated deployment into HTTP responses. +// +// Everything here is built around one property: a request must never observe a +// mixture of two deployments. That is achieved by making the served state an +// immutable snapshot behind an atomic pointer — a handler loads it once, at the +// top, and every subsequent decision in that request comes from the value it +// loaded. Switching versions is one pointer store, so an in-flight request keeps +// reading the deployment it started on until it finishes. +package site + +import ( + "path" + "time" + + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/store" +) + +// FileEntry is what a request path resolves to: enough to serve the bytes and +// to answer a conditional request, and nothing else. +type FileEntry struct { + Digest cas.Digest + Size int64 +} + +// Index is the part of a snapshot that costs something to build. +// +// It is separate from Deployment so activation can pay for it *before* it +// changes anything: if the manifest cannot be read or turned into these maps, +// the failure happens while the old deployment is still the active one. +type Index struct { + files map[string]FileEntry + dirs map[string]struct{} + totalBytes int64 +} + +// NewIndex builds the lookup structures for one manifest. +// +// Dirs holds every directory prefix, which is what lets Resolve tell "no such +// path" apart from "that is a directory, redirect to it with a trailing slash" +// without scanning the file map for prefixes on every miss. +func NewIndex(files []store.FileRow) *Index { + idx := &Index{ + files: make(map[string]FileEntry, len(files)), + dirs: make(map[string]struct{}), + } + for _, f := range files { + idx.files[f.Path] = FileEntry{Digest: f.Digest, Size: f.Size} + idx.totalBytes += f.Size + for dir := path.Dir(f.Path); dir != "." && dir != "/"; dir = path.Dir(dir) { + if _, ok := idx.dirs[dir]; ok { + // Every shorter prefix was added with this one, so there is + // nothing left to walk. + break + } + idx.dirs[dir] = struct{}{} + } + } + return idx +} + +// Deployment is an immutable snapshot of what a project serves. +// +// Nothing mutates one after NewDeployment returns. That is the whole reason a +// version switch can be a bare pointer store: readers need no synchronisation +// beyond the atomic load that handed them the snapshot. +type Deployment struct { + ID string + ProjectID int64 + // Dir is the assembled tree, kept for the webroot symlink and for + // operators. It is empty under assemble_mode=none and is never used to + // serve a request — content comes from the CAS by digest. + Dir string + // CreatedAt is the Last-Modified of every file in the deployment. The CAS + // file's own mtime would be wrong: blobs are shared across projects and + // versions, so their mtime means nothing here and would leak when some + // other project first uploaded the same bytes. + CreatedAt time.Time + ActivatedAt time.Time + FileCount int + TotalBytes int64 + + idx *Index +} + +// NewDeployment pairs a row with an already-built index. It allocates one +// struct and copies no maps, so activation can call it after the database +// commit without any risk of failing there. +func NewDeployment(dep *store.Deployment, idx *Index, dir string) *Deployment { + d := &Deployment{ + ID: dep.PublicID, + ProjectID: dep.ProjectID, + Dir: dir, + CreatedAt: dep.CreatedAt, + FileCount: len(idx.files), + TotalBytes: idx.totalBytes, + idx: idx, + } + if dep.ActivatedAt != nil { + d.ActivatedAt = *dep.ActivatedAt + } + return d +} + +// Lookup finds one file by its slash-separated path within the deployment. +func (d *Deployment) Lookup(rel string) (FileEntry, bool) { + e, ok := d.idx.files[rel] + return e, ok +} + +// IsDir reports whether rel is a directory prefix of some file. +func (d *Deployment) IsDir(rel string) bool { + _, ok := d.idx.dirs[rel] + return ok +} diff --git a/internal/site/registry.go b/internal/site/registry.go new file mode 100644 index 0000000..cb9d283 --- /dev/null +++ b/internal/site/registry.go @@ -0,0 +1,176 @@ +package site + +import ( + "context" + "maps" + "sync" + "sync/atomic" + + "github.com/iceBear67/simplepages/internal/store" +) + +// ProjectConfig is the serving-time part of a project row, copied out so the +// read path never touches the database. It is replaced wholesale, never edited +// in place, so a reader always sees one consistent set of settings. +type ProjectConfig struct { + IndexFile string + NotFoundFile string + SPAFallback bool + CacheControl string +} + +// ConfigOf extracts what serving needs from a project row. +func ConfigOf(p *store.Project) *ProjectConfig { + return &ProjectConfig{ + IndexFile: p.IndexFile, + NotFoundFile: p.NotFoundFile, + SPAFallback: p.SPAFallback, + CacheControl: p.CacheControl, + } +} + +// Project is one project's live serving state. +// +// Both fields are atomic pointers to immutable values, so every read on the +// serving path is one atomic load and nothing else. Writers are already +// serialised per project by the deploy service's lock; the atomics are here for +// the readers, not for mutual exclusion. +type Project struct { + ID int64 + Name string + + cfg atomic.Pointer[ProjectConfig] + active atomic.Pointer[Deployment] +} + +// NewProject builds a registry entry with no deployment activated yet. +func NewProject(p *store.Project) *Project { + sp := &Project{ID: p.ID, Name: p.Name} + sp.cfg.Store(ConfigOf(p)) + return sp +} + +// Active is the deployment this project is serving, or nil if it has never +// activated one. +// +// A request handler calls this exactly once, at the top, and uses the value it +// gets for the rest of the request. Calling it a second time within one request +// is a bug: the two loads could straddle an activation and produce a response +// assembled from two different versions, which is the exact failure this whole +// program exists to prevent. +func (p *Project) Active() *Deployment { return p.active.Load() } + +// Config is the project's serving settings. +func (p *Project) Config() *ProjectConfig { return p.cfg.Load() } + +// Activate publishes a snapshot. This single store is the version switch. +func (p *Project) Activate(d *Deployment) { p.active.Store(d) } + +// SetConfig swaps in new serving settings. +func (p *Project) SetConfig(c *ProjectConfig) { p.cfg.Store(c) } + +// Registry maps project names to their live state. +// +// Reads are overwhelmingly more common than writes — every static request does +// one, while the set of projects changes on the order of minutes to days — so +// the map is copy-on-write behind an atomic pointer: readers get a lock-free, +// allocation-free lookup against a consistent snapshot of the whole set, and +// writers pay O(n) to clone it. A RWMutex would put one shared cache line in +// every request's path for no benefit at this write rate. +// +// Note the layering: activating a deployment does *not* rebuild this map. It +// stores into the Project the map already points at, so the copy is reserved for +// changes to the project set itself. +type Registry struct { + mu sync.Mutex // serialises writers only; readers never take it + byName atomic.Pointer[map[string]*Project] +} + +func NewRegistry() *Registry { + r := &Registry{} + r.byName.Store(&map[string]*Project{}) + return r +} + +// Lookup finds a project by the name in the URL. +func (r *Registry) Lookup(name string) (*Project, bool) { + p, ok := (*r.byName.Load())[name] + return p, ok +} + +// Len is the number of projects the registry knows about. +func (r *Registry) Len() int { return len(*r.byName.Load()) } + +// Projects returns the current entries in no particular order. +func (r *Registry) Projects() []*Project { + m := *r.byName.Load() + out := make([]*Project, 0, len(m)) + for _, p := range m { + out = append(out, p) + } + return out +} + +// ResolveProject satisfies auth.ProjectResolver, which lets the ownership guard +// on the hot deployment endpoints answer from memory instead of querying. +// +// It is only safe in that role because the registry is fully built from the +// database before the listeners start: a partially populated registry would +// report someone's own project as unknown. +func (r *Registry) ResolveProject(_ context.Context, name string) (int64, error) { + p, ok := r.Lookup(name) + if !ok { + return 0, store.ErrNotFound + } + return p.ID, nil +} + +// Replace swaps in a whole new project set, which is how startup publishes the +// registry it built from the database. +func (r *Registry) Replace(ps []*Project) { + m := make(map[string]*Project, len(ps)) + for _, p := range ps { + m[p.Name] = p + } + r.mu.Lock() + defer r.mu.Unlock() + r.byName.Store(&m) +} + +// Put adds or updates a project and returns its live entry. +// +// An entry that is already present is kept rather than rebuilt, so a settings +// change does not disturb the deployment the project is currently serving. +func (r *Registry) Put(p *store.Project) *Project { + r.mu.Lock() + defer r.mu.Unlock() + + old := *r.byName.Load() + if sp, ok := old[p.Name]; ok && sp.ID == p.ID { + sp.SetConfig(ConfigOf(p)) + return sp + } + sp := NewProject(p) + next := maps.Clone(old) + if next == nil { + next = make(map[string]*Project, 1) + } + next[p.Name] = sp + r.byName.Store(&next) + return sp +} + +// Delete drops a project. Requests for it start 404ing as soon as the new map +// is stored; requests already reading a snapshot of it finish normally. +func (r *Registry) Delete(name string) { + r.mu.Lock() + defer r.mu.Unlock() + + old := *r.byName.Load() + if _, ok := old[name]; !ok { + return + } + next := maps.Clone(old) + delete(next, name) + r.byName.Store(&next) +} diff --git a/internal/site/registry_test.go b/internal/site/registry_test.go new file mode 100644 index 0000000..4c570f5 --- /dev/null +++ b/internal/site/registry_test.go @@ -0,0 +1,149 @@ +package site + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/iceBear67/simplepages/internal/store" +) + +func TestRegistryLookupAndDelete(t *testing.T) { + r := NewRegistry() + if _, ok := r.Lookup("demo"); ok { + t.Fatal("an empty registry answered a lookup") + } + if r.Len() != 0 { + t.Fatalf("Len = %d on an empty registry", r.Len()) + } + + sp := r.Put(&store.Project{ID: 7, Name: "demo", IndexFile: "index.html"}) + got, ok := r.Lookup("demo") + if !ok || got != sp { + t.Fatal("Put did not publish the project it returned") + } + if got.ID != 7 || got.Name != "demo" { + t.Errorf("entry = %+v", got) + } + if r.Len() != 1 { + t.Errorf("Len = %d, want 1", r.Len()) + } + + r.Delete("demo") + if _, ok := r.Lookup("demo"); ok { + t.Error("a deleted project still resolves") + } + r.Delete("demo") // deleting twice is not an error + r.Delete("never-existed") +} + +// A settings change must not disturb what the project is currently serving: the +// entry is updated in place rather than rebuilt, so the active pointer survives. +func TestRegistryPutKeepsTheActiveDeployment(t *testing.T) { + r := NewRegistry() + p := &store.Project{ID: 1, Name: "demo", IndexFile: "index.html"} + sp := r.Put(p) + d := fixture("index.html") + sp.Activate(d) + + p2 := &store.Project{ID: 1, Name: "demo", IndexFile: "main.html", SPAFallback: true} + again := r.Put(p2) + if again != sp { + t.Fatal("Put replaced the live entry instead of updating it") + } + if again.Active() != d { + t.Fatal("updating the settings dropped the active deployment") + } + if cfg := again.Config(); cfg.IndexFile != "main.html" || !cfg.SPAFallback { + t.Errorf("config = %+v, want the new settings", cfg) + } +} + +// A project recreated under the same name is a different project, so its entry +// must start empty rather than inherit the old one's deployment. +func TestRegistryPutReplacesAnEntryWithADifferentID(t *testing.T) { + r := NewRegistry() + sp := r.Put(&store.Project{ID: 1, Name: "demo"}) + sp.Activate(fixture("index.html")) + + again := r.Put(&store.Project{ID: 2, Name: "demo"}) + if again == sp { + t.Fatal("a different project id reused the old entry") + } + if again.Active() != nil { + t.Error("the new project inherited the old project's deployment") + } +} + +func TestRegistryReplace(t *testing.T) { + r := NewRegistry() + r.Put(&store.Project{ID: 1, Name: "gone"}) + r.Replace([]*Project{ + NewProject(&store.Project{ID: 2, Name: "a"}), + NewProject(&store.Project{ID: 3, Name: "b"}), + }) + if _, ok := r.Lookup("gone"); ok { + t.Error("Replace kept a project that is not in the new set") + } + if r.Len() != 2 { + t.Fatalf("Len = %d, want 2", r.Len()) + } + names := map[string]bool{} + for _, p := range r.Projects() { + names[p.Name] = true + } + if !names["a"] || !names["b"] { + t.Errorf("Projects = %v", names) + } +} + +// The API's ownership guard compares resolved ids, so this is the lookup that +// decides whether one project's key can touch another's deployment. +func TestRegistryResolveProject(t *testing.T) { + r := NewRegistry() + r.Put(&store.Project{ID: 42, Name: "demo"}) + + id, err := r.ResolveProject(context.Background(), "demo") + if err != nil || id != 42 { + t.Fatalf("ResolveProject = (%d, %v), want (42, nil)", id, err) + } + if _, err := r.ResolveProject(context.Background(), "other"); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("ResolveProject on an unknown name = %v, want ErrNotFound", err) + } +} + +// Readers take no lock at all, so this is worth running under -race: a writer +// mutating the map in place instead of cloning it would show up here. +func TestRegistryConcurrentReadersAndWriters(t *testing.T) { + r := NewRegistry() + r.Put(&store.Project{ID: 1, Name: "stable"}) + stop := make(chan struct{}) + var wg sync.WaitGroup + + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + if p, ok := r.Lookup("stable"); !ok || p.ID != 1 { + t.Error("the stable project vanished while other projects changed") + return + } + r.Lookup("churn") + r.Len() + } + }() + } + for i := range 500 { + r.Put(&store.Project{ID: int64(i + 2), Name: "churn"}) + r.Delete("churn") + } + close(stop) + wg.Wait() +} diff --git a/internal/site/resolve.go b/internal/site/resolve.go new file mode 100644 index 0000000..b7da459 --- /dev/null +++ b/internal/site/resolve.go @@ -0,0 +1,129 @@ +package site + +import ( + "net/http" + "path" + "strings" + + "github.com/iceBear67/simplepages/internal/pathutil" +) + +// Result is what a request path resolved to. It is a value, not a response: +// Resolve performs no I/O and touches nothing, which is what makes the routing +// rules exhaustively testable as a table. +type Result struct { + Entry FileEntry + // Name is the logical path within the deployment, which decides the + // Content-Type. It is never a filesystem path. + Name string + // Status is 200, 301, 400 or 404. A 404 may still carry an Entry, which is + // the project's custom not-found document. + Status int + // Location is the site-absolute path to redirect to, set when Status is 301. + // It is unescaped; the caller is responsible for building the header value. + Location string +} + +// Resolve maps a request onto a file in the deployment. +// +// project and rest are what splitTilde produced, so rest keeps its leading +// slash and is empty for "/~proj". accept is the request's Accept header, which +// only matters for the SPA fallback. +func Resolve(d *Deployment, cfg *ProjectConfig, project, rest, accept string) Result { + // "/~proj" must become "/~proj/" before anything else: without the trailing + // slash every relative link in the page would resolve one level too high. + if rest == "" { + return Result{Status: http.StatusMovedPermanently, Location: "/~" + project + "/"} + } + // splitTilde always leaves the leading slash on, so this cannot happen from + // the serving path. Enforcing it anyway is what keeps every "/~" + project + + // … below a path *inside* the project: without it a rest of "x" would build + // "/~demox" and send the client to a different project entirely. + if rest[0] != '/' { + return Result{Status: http.StatusBadRequest} + } + + // Canonicalise. The trailing slash survives Clean deliberately — it is the + // difference between asking for a directory and asking for a file, and + // dropping it here would redirect "/dir/" to "/dir" only for step 6 to + // redirect it back. + trailing := strings.HasSuffix(rest, "/") + clean := path.Clean(rest) + canon := clean + if trailing && clean != "/" { + canon += "/" + } + if canon != rest { + return Result{Status: http.StatusMovedPermanently, Location: "/~" + project + canon} + } + + rel := strings.TrimPrefix(clean, "/") + if rel == "" { + if cfg.IndexFile == "" { + return Result{Status: http.StatusNotFound} + } + rel = cfg.IndexFile + } + // Clean has already removed any "..", so this is defence in depth rather + // than the primary guard — but it is also what rejects NUL and control + // bytes, and a path that cannot be a manifest entry cannot be a hit. + if err := pathutil.Validate(rel); err != nil { + return Result{Status: http.StatusBadRequest} + } + + if e, ok := d.Lookup(rel); ok { + if trailing && clean != "/" { + // "/page.html/" names a file with a directory's URL. Serving it there + // would make every relative link inside resolve one level too deep, + // and would cache the same bytes under two URLs. + return Result{Status: http.StatusMovedPermanently, Location: "/~" + project + clean} + } + return Result{Entry: e, Name: rel, Status: http.StatusOK} + } + + if d.IsDir(rel) { + if !trailing { + // Redirect rather than serve the index directly, again so relative + // links inside the page resolve against the directory. + return Result{Status: http.StatusMovedPermanently, Location: "/~" + project + clean + "/"} + } + if idx := path.Join(rel, cfg.IndexFile); pathutil.Validate(idx) == nil { + if e, ok := d.Lookup(idx); ok { + return Result{Entry: e, Name: idx, Status: http.StatusOK} + } + } + } + + // SPA fallback, gated on Accept. Without that gate a missing + // /assets/app.js would come back as HTML with status 200, and the failure + // surfaces later as "Unexpected token '<'" somewhere entirely unrelated. + if cfg.SPAFallback && acceptsHTML(accept) { + if e, ok := d.Lookup(cfg.IndexFile); ok { + return Result{Entry: e, Name: cfg.IndexFile, Status: http.StatusOK} + } + } + + if cfg.NotFoundFile != "" { + if e, ok := d.Lookup(cfg.NotFoundFile); ok { + return Result{Entry: e, Name: cfg.NotFoundFile, Status: http.StatusNotFound} + } + } + return Result{Status: http.StatusNotFound} +} + +// acceptsHTML reports whether the client asked for HTML specifically. +// +// "*/*" does not count. A browser navigating to a page sends text/html; a +// fetch() for a script or a JSON document sends */* or something narrower, and +// those are exactly the requests that must keep getting a 404. +func acceptsHTML(accept string) bool { + for len(accept) > 0 { + var field string + field, accept, _ = strings.Cut(accept, ",") + media, _, _ := strings.Cut(field, ";") + if strings.EqualFold(strings.TrimSpace(media), "text/html") { + return true + } + } + return false +} diff --git a/internal/site/resolve_test.go b/internal/site/resolve_test.go new file mode 100644 index 0000000..bc111a4 --- /dev/null +++ b/internal/site/resolve_test.go @@ -0,0 +1,445 @@ +package site + +import ( + "io/fs" + "net/http" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/store" +) + +// siteFiles is one deployment covering every shape Resolve has to tell apart: a +// root index, a directory that has an index, a directory that does not, a custom +// error document, and a non-ASCII name. +var siteFiles = []string{ + "index.html", + "404.html", + "assets/app.js", + "docs/index.html", + "docs/deep/page.html", + "noindex/data.json", + "文档/说明.html", +} + +func fixture(paths ...string) *Deployment { + rows := make([]store.FileRow, len(paths)) + for i, p := range paths { + rows[i] = store.FileRow{Path: p, Digest: cas.Sum([]byte(p)), Size: int64(len(p))} + } + dep := &store.Deployment{PublicID: "dpl_test", ProjectID: 1, CreatedAt: time.Unix(1700000000, 0).UTC()} + return NewDeployment(dep, NewIndex(rows), "") +} + +// plain is the default project: an index, no error document, no fallback. +func plain() *ProjectConfig { return &ProjectConfig{IndexFile: "index.html"} } + +func withNotFound() *ProjectConfig { + c := plain() + c.NotFoundFile = "404.html" + return c +} + +func withSPA() *ProjectConfig { + c := plain() + c.SPAFallback = true + return c +} + +func TestResolve(t *testing.T) { + d := fixture(siteFiles...) + + tests := []struct { + name string + cfg *ProjectConfig + rest string + accept string + status int + file string // Result.Name + location string + }{ + // ------------------------------------------------ canonicalisation + { + name: "bare project name gets a trailing slash", + cfg: plain(), + rest: "", + status: http.StatusMovedPermanently, + location: "/~demo/", + }, + { + name: "root serves the index", + cfg: plain(), + rest: "/", + status: http.StatusOK, + file: "index.html", + }, + { + name: "exact hit", + cfg: plain(), + rest: "/assets/app.js", + status: http.StatusOK, + file: "assets/app.js", + }, + { + name: "unicode path", + cfg: plain(), + rest: "/文档/说明.html", + status: http.StatusOK, + file: "文档/说明.html", + }, + { + name: "double slash is collapsed", + cfg: plain(), + rest: "//assets//app.js", + status: http.StatusMovedPermanently, + location: "/~demo/assets/app.js", + }, + { + name: "dot segment is removed", + cfg: plain(), + rest: "/./assets/app.js", + status: http.StatusMovedPermanently, + location: "/~demo/assets/app.js", + }, + { + // The decoded form of %2e%2e%2f: it arrives here unredirected by the + // mux, and Clean resolves it before anything is looked up. It cannot + // reach outside the deployment because the result is only ever a key + // into a map. + name: "parent segments are resolved, not followed", + cfg: plain(), + rest: "/assets/../index.html", + status: http.StatusMovedPermanently, + location: "/~demo/index.html", + }, + { + name: "parent segments above the root land at the root", + cfg: plain(), + rest: "/../../etc/passwd", + status: http.StatusMovedPermanently, + location: "/~demo/etc/passwd", + }, + { + name: "a file asked for with a directory's URL", + cfg: plain(), + rest: "/index.html/", + status: http.StatusMovedPermanently, + location: "/~demo/index.html", + }, + + // ------------------------------------------------------ directories + { + name: "directory without a trailing slash redirects", + cfg: plain(), + rest: "/docs", + status: http.StatusMovedPermanently, + location: "/~demo/docs/", + }, + { + name: "directory with a trailing slash serves its index", + cfg: plain(), + rest: "/docs/", + status: http.StatusOK, + file: "docs/index.html", + }, + { + name: "intermediate directory redirects too", + cfg: plain(), + rest: "/docs/deep", + status: http.StatusMovedPermanently, + location: "/~demo/docs/deep/", + }, + { + name: "directory with no index is a 404, not a listing", + cfg: plain(), + rest: "/noindex/", + status: http.StatusNotFound, + }, + { + name: "nothing there at all", + cfg: plain(), + rest: "/nope.txt", + status: http.StatusNotFound, + }, + { + name: "an empty index file setting leaves the root a 404", + cfg: &ProjectConfig{}, + rest: "/", + status: http.StatusNotFound, + }, + + // --------------------------------------------------- invalid paths + { + name: "NUL byte", + cfg: plain(), + rest: "/index\x00.html", + status: http.StatusBadRequest, + }, + { + name: "control character", + cfg: plain(), + rest: "/index\n.html", + status: http.StatusBadRequest, + }, + { + name: "backslash", + cfg: plain(), + rest: "/assets\\app.js", + status: http.StatusBadRequest, + }, + { + name: "invalid UTF-8", + cfg: plain(), + rest: "/\xff\xfe.html", + status: http.StatusBadRequest, + }, + { + name: "path longer than the limit", + cfg: plain(), + rest: "/" + strings.Repeat("a", 4097), + status: http.StatusBadRequest, + }, + { + name: "segment longer than the limit", + cfg: plain(), + rest: "/" + strings.Repeat("b", 256) + "/x", + status: http.StatusBadRequest, + }, + + // ------------------------------------------------- custom 404 page + { + name: "the error document is served with a 404 status", + cfg: withNotFound(), + rest: "/nope.txt", + status: http.StatusNotFound, + file: "404.html", + }, + { + name: "an error document that is not in the manifest is skipped", + cfg: &ProjectConfig{IndexFile: "index.html", NotFoundFile: "missing.html"}, + rest: "/nope.txt", + status: http.StatusNotFound, + }, + { + name: "a directory with no index falls to the error document", + cfg: withNotFound(), + rest: "/noindex/", + status: http.StatusNotFound, + file: "404.html", + }, + + // ---------------------------------------------------- SPA fallback + { + name: "a navigation gets the app shell", + cfg: withSPA(), + rest: "/some/client/route", + accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + status: http.StatusOK, + file: "index.html", + }, + { + // The whole point of the Accept gate: without it a missing script + // comes back as HTML with a 200 and fails as a syntax error in some + // unrelated place. + name: "a missing script still 404s", + cfg: withSPA(), + rest: "/assets/missing.js", + accept: "*/*", + status: http.StatusNotFound, + }, + { + name: "no Accept header at all is not a navigation", + cfg: withSPA(), + rest: "/some/client/route", + status: http.StatusNotFound, + }, + { + name: "html anywhere in the list counts", + cfg: withSPA(), + rest: "/some/client/route", + accept: "application/json;q=0.9, text/html;q=0.8", + status: http.StatusOK, + file: "index.html", + }, + { + name: "with the fallback off a navigation 404s", + cfg: plain(), + rest: "/some/client/route", + accept: "text/html", + status: http.StatusNotFound, + }, + { + name: "the fallback does not rescue an invalid path", + cfg: withSPA(), + rest: "/bad\x00path", + accept: "text/html", + status: http.StatusBadRequest, + }, + { + // A directory redirect outranks the fallback: the URL is real, it just + // needs its slash. + name: "the fallback does not swallow a directory redirect", + cfg: withSPA(), + rest: "/docs", + accept: "text/html", + status: http.StatusMovedPermanently, + location: "/~demo/docs/", + }, + { + name: "the error document wins over nothing when the shell is missing", + cfg: &ProjectConfig{IndexFile: "absent.html", NotFoundFile: "404.html", SPAFallback: true}, + rest: "/some/client/route", + accept: "text/html", + status: http.StatusNotFound, + file: "404.html", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := Resolve(d, tc.cfg, "demo", tc.rest, tc.accept) + if got.Status != tc.status { + t.Errorf("status = %d, want %d", got.Status, tc.status) + } + if got.Name != tc.file { + t.Errorf("name = %q, want %q", got.Name, tc.file) + } + if got.Location != tc.location { + t.Errorf("location = %q, want %q", got.Location, tc.location) + } + if tc.file != "" && got.Entry.Digest != cas.Sum([]byte(tc.file)) { + t.Errorf("entry does not belong to %s", tc.file) + } + if tc.file == "" && got.Entry != (FileEntry{}) { + t.Errorf("a response with no file carries an entry: %+v", got.Entry) + } + }) + } +} + +// A redirect must always be a step towards a terminal answer. A pair that +// bounced between two locations would be an infinite loop in a browser. +func TestResolveRedirectsConverge(t *testing.T) { + d := fixture(siteFiles...) + cfgs := map[string]*ProjectConfig{"plain": plain(), "spa": withSPA(), "404": withNotFound()} + paths := []string{ + "", "/", "//", "/.", "/..", "/docs", "/docs/", "/docs//deep", "/docs/deep", + "/index.html", "/index.html/", "/./docs/../docs/", "/noindex", "/nope", + "/文档", "/文档/", + } + for name, cfg := range cfgs { + for _, p := range paths { + rest := p + for hop := 0; ; hop++ { + if hop > 4 { + t.Errorf("%s %q: still redirecting after %d hops", name, p, hop) + break + } + res := Resolve(d, cfg, "demo", rest, "text/html") + if res.Status != http.StatusMovedPermanently { + break + } + next, ok := strings.CutPrefix(res.Location, "/~demo") + if !ok { + t.Fatalf("%s %q: redirect escaped the project: %q", name, p, res.Location) + break + } + if next == rest { + t.Fatalf("%s %q: redirects to itself", name, p) + } + rest = next + } + } + } +} + +// Whatever a request asks for, the answer either names a path that could have +// been a manifest entry or names nothing at all. That is what keeps the serving +// path from ever deriving a filesystem name from user input. +func FuzzResolve(f *testing.F) { + d := fixture(siteFiles...) + cfg := &ProjectConfig{IndexFile: "index.html", NotFoundFile: "404.html", SPAFallback: true} + + for _, s := range []string{ + "", "/", "/index.html", "/../../etc/passwd", "//", "/docs/", "/\x00", + "/文档/说明.html", "/a/b/c/../../..", "/.git/config", + } { + f.Add(s, "text/html") + } + f.Fuzz(func(t *testing.T, rest, accept string) { + res := Resolve(d, cfg, "demo", rest, accept) + switch res.Status { + case http.StatusOK, http.StatusNotFound: + if res.Name == "" { + return + } + if !fs.ValidPath(res.Name) { + t.Fatalf("resolved %q to the invalid name %q", rest, res.Name) + } + if _, ok := d.Lookup(res.Name); !ok { + t.Fatalf("resolved %q to %q, which is not in the manifest", rest, res.Name) + } + case http.StatusMovedPermanently: + if !strings.HasPrefix(res.Location, "/~demo/") && res.Location != "/~demo" { + t.Fatalf("resolved %q to a location outside the project: %q", rest, res.Location) + } + if res.Name != "" { + t.Fatalf("a redirect for %q also named a file: %q", rest, res.Name) + } + case http.StatusBadRequest: + if res.Name != "" { + t.Fatalf("a rejection for %q also named a file: %q", rest, res.Name) + } + default: + t.Fatalf("resolved %q to the unexpected status %d", rest, res.Status) + } + }) +} + +func TestAcceptsHTML(t *testing.T) { + tests := []struct { + accept string + want bool + }{ + {"", false}, + {"*/*", false}, // a fetch() with no opinion is not a navigation + {"text/*", false}, // and neither is a wildcard subtype + {"application/json", false}, + {"text/plain", false}, + {"text/htmlx", false}, + {"text/html", true}, + {"TEXT/HTML", true}, + {"text/html;charset=utf-8", true}, + {"text/html, */*", true}, + {"application/json, text/html;q=0.1", true}, + {" text/html ", true}, + {"application/xhtml+xml,text/html", true}, + } + for _, tc := range tests { + if got := acceptsHTML(tc.accept); got != tc.want { + t.Errorf("acceptsHTML(%q) = %v, want %v", tc.accept, got, tc.want) + } + } +} + +func TestIndexRecordsEveryDirectoryPrefix(t *testing.T) { + d := fixture("a/b/c/d.txt", "top.txt") + for _, dir := range []string{"a", "a/b", "a/b/c"} { + if !d.IsDir(dir) { + t.Errorf("%q is not recorded as a directory", dir) + } + } + for _, notDir := range []string{"", ".", "/", "a/b/c/d.txt", "top.txt", "a/b/c/d"} { + if d.IsDir(notDir) { + t.Errorf("%q is recorded as a directory", notDir) + } + } + if d.FileCount != 2 { + t.Errorf("FileCount = %d, want 2", d.FileCount) + } + if want := int64(len("a/b/c/d.txt") + len("top.txt")); d.TotalBytes != want { + t.Errorf("TotalBytes = %d, want %d", d.TotalBytes, want) + } +} diff --git a/internal/site/route.go b/internal/site/route.go new file mode 100644 index 0000000..2ef8a4c --- /dev/null +++ b/internal/site/route.go @@ -0,0 +1,36 @@ +package site + +import "strings" + +// splitTilde splits a site URL path into its project and the rest. +// +// "/~proj" -> ("proj", "", true) +// "/~proj/" -> ("proj", "/", true) +// "/~proj/a/b" -> ("proj", "/a/b", true) +// "/", "/x", "/~" -> ("", "", false) +// +// The remainder keeps its leading slash, because its absence is what +// distinguishes "/~proj" — which has to be redirected before relative links +// inside the page can resolve — from "/~proj/". +// +// This is hand-parsed rather than expressed as a ServeMux pattern because +// net/http rejects "/~{project}/{path...}": its wildcards must start a path +// segment. Registering "/" instead also keeps the mux's built-in ".." and "//" +// redirects, though those are a convenience and not a defence — a +// percent-encoded traversal arrives here already decoded and unredirected, so +// Resolve does its own normalisation. +func splitTilde(urlPath string) (project, rest string, ok bool) { + if !strings.HasPrefix(urlPath, "/~") { + return "", "", false + } + rest = urlPath[len("/~"):] + if i := strings.IndexByte(rest, '/'); i >= 0 { + project, rest = rest[:i], rest[i:] + } else { + project, rest = rest, "" + } + if project == "" { + return "", "", false + } + return project, rest, true +} diff --git a/internal/site/route_test.go b/internal/site/route_test.go new file mode 100644 index 0000000..4474463 --- /dev/null +++ b/internal/site/route_test.go @@ -0,0 +1,56 @@ +package site + +import "testing" + +func TestSplitTilde(t *testing.T) { + tests := []struct { + path string + project string + rest string + ok bool + }{ + {"/~proj", "proj", "", true}, + {"/~proj/", "proj", "/", true}, + {"/~proj/a/b", "proj", "/a/b", true}, + {"/~proj/a/b/", "proj", "/a/b/", true}, + {"/~proj//a", "proj", "//a", true}, + {"/~p", "p", "", true}, + {"/~proj/~other/x", "proj", "/~other/x", true}, + // A percent-encoded traversal arrives here already decoded; the project + // still ends at the first slash, and Resolve cleans the remainder. + {"/~proj/../../etc/passwd", "proj", "/../../etc/passwd", true}, + + // Not a site request at all. + {"", "", "", false}, + {"/", "", "", false}, + {"/x", "", "", false}, + {"/~", "", "", false}, + {"/~/", "", "", false}, + {"~proj/", "", "", false}, + {"//~proj/", "", "", false}, + {"/favicon.ico", "", "", false}, + {"/api/v1/projects", "", "", false}, + } + for _, tc := range tests { + project, rest, ok := splitTilde(tc.path) + if project != tc.project || rest != tc.rest || ok != tc.ok { + t.Errorf("splitTilde(%q) = (%q, %q, %v), want (%q, %q, %v)", + tc.path, project, rest, ok, tc.project, tc.rest, tc.ok) + } + } +} + +// The project name never contains a slash, which is what makes "~" + name a +// single entry inside the webroot and keeps a redirect built from it inside the +// project's own prefix. +func TestSplitTildeProjectIsOneSegment(t *testing.T) { + for _, p := range []string{"/~a/b", "/~a//b", "/~a/../b", "/~a/b/c/d"} { + project, _, ok := splitTilde(p) + if !ok { + t.Fatalf("splitTilde(%q) refused a site path", p) + } + if project != "a" { + t.Errorf("splitTilde(%q) project = %q, want %q", p, project, "a") + } + } +} diff --git a/internal/site/serve.go b/internal/site/serve.go new file mode 100644 index 0000000..7d41794 --- /dev/null +++ b/internal/site/serve.go @@ -0,0 +1,143 @@ +package site + +import ( + "errors" + "io" + "log/slog" + "mime" + "net/http" + "net/url" + "path" + "strconv" + + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/httpx" +) + +// Handler serves the active deployment of every project. +// +// Content comes from the content store by digest, never from the assembled +// directory. That is a security decision as much as a performance one: the only +// filesystem path this path ever builds is cas/<2>/<2>/<64 hex>, derived from a +// [32]byte that came out of a map lookup. No user-controlled string reaches the +// filesystem at all, so traversal on the read path is not defended against — +// it is structurally impossible. +type Handler struct { + Registry *Registry + CAS *cas.Store + Log *slog.Logger +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + w.Header().Set("Allow", "GET, HEAD") + h.fail(w, http.StatusMethodNotAllowed, "method not allowed\n") + return + } + + project, rest, ok := splitTilde(r.URL.Path) + if !ok { + h.fail(w, http.StatusNotFound, "no site is served at this path; sites live under /~project/\n") + return + } + p, ok := h.Registry.Lookup(project) + if !ok { + h.fail(w, http.StatusNotFound, "no such project\n") + return + } + httpx.LogAttr(r.Context(), "project", project) + + // Loaded exactly once. Every decision below uses this snapshot, so an + // activation that lands mid-request cannot split the response across two + // versions. Calling p.Active() again anywhere in this function would + // reintroduce precisely the failure this program exists to prevent. + d := p.Active() + if d == nil { + h.fail(w, http.StatusServiceUnavailable, "this project has no active deployment\n") + return + } + httpx.LogAttr(r.Context(), "deployment", d.ID) + + cfg := p.Config() + res := Resolve(d, cfg, project, rest, r.Header.Get("Accept")) + switch { + case res.Status == http.StatusMovedPermanently: + // url.URL re-escapes the path, so a project or file name that needed + // escaping in the request does not arrive raw in the Location header. + loc := url.URL{Path: res.Location, RawQuery: r.URL.RawQuery} + w.Header().Set("X-Content-Type-Options", "nosniff") + http.Redirect(w, r, loc.String(), http.StatusMovedPermanently) + case res.Status == http.StatusBadRequest: + h.fail(w, http.StatusBadRequest, "bad request path\n") + case res.Name == "": + h.fail(w, res.Status, "404 page not found\n") + default: + h.serveEntry(w, r, d, cfg, res) + } +} + +func (h *Handler) serveEntry(w http.ResponseWriter, r *http.Request, d *Deployment, cfg *ProjectConfig, res Result) { + f, err := h.CAS.Open(res.Entry.Digest) + if err != nil { + if errors.Is(err, cas.ErrNotFound) { + // The manifest says this blob exists and the store disagrees. GC's + // grace period is supposed to make this unreachable, so it means + // either a bug or an operator who cleared the store by hand. + h.Log.ErrorContext(r.Context(), "blob missing from the content store", + "digest", res.Entry.Digest, "deployment", d.ID, "path", res.Name) + h.fail(w, http.StatusNotFound, "404 page not found\n") + return + } + h.Log.ErrorContext(r.Context(), "opening blob", "err", err, "deployment", d.ID, "path", res.Name) + h.fail(w, http.StatusInternalServerError, "internal server error\n") + return + } + defer f.Close() + + head := w.Header() + head.Set("X-Content-Type-Options", "nosniff") + // The type comes from the logical name, not the CAS path, which has no + // extension at all. + ctype := mime.TypeByExtension(path.Ext(res.Name)) + if ctype == "" { + ctype = "application/octet-stream" + } + head.Set("Content-Type", ctype) + if cfg.CacheControl != "" { + head.Set("Cache-Control", cfg.CacheControl) + } + if cfg.SPAFallback { + // With the fallback on, one URL can answer with the app shell or with a + // 404 depending on Accept, so a shared cache must key on it. + head.Set("Vary", "Accept") + } + + if res.Status != http.StatusOK { + // The custom 404 document. ServeContent always writes 200, so this one + // is written by hand; range requests for an error page are not worth + // the machinery. + head.Set("Content-Length", strconv.FormatInt(res.Entry.Size, 10)) + w.WriteHeader(res.Status) + if r.Method != http.MethodHead { + io.Copy(w, f) + } + return + } + + // A strong validator: the digest *is* the content, so a matching ETag + // cannot be a lie. With it set, ServeContent handles If-None-Match, + // If-Modified-Since, If-Range, Range and multipart ranges by itself. + head.Set("ETag", `"sha256:`+res.Entry.Digest.String()+`"`) + // The empty name keeps ServeContent from sniffing: the type is already set. + // The modtime is the deployment's, never the blob's — blobs are shared + // across projects, so their mtime says when some unrelated project first + // uploaded the same bytes. + http.ServeContent(w, r, "", d.CreatedAt, f) +} + +// fail writes a plain-text response. Site errors are never JSON: whatever is on +// the other end of a static request is a browser or a curl, not an API client. +// http.Error sets the text/plain type and the nosniff header for us. +func (h *Handler) fail(w http.ResponseWriter, status int, msg string) { + http.Error(w, msg, status) +} diff --git a/internal/site/serve_test.go b/internal/site/serve_test.go new file mode 100644 index 0000000..16a3494 --- /dev/null +++ b/internal/site/serve_test.go @@ -0,0 +1,377 @@ +package site + +import ( + "bytes" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/iceBear67/simplepages/internal/cas" + "github.com/iceBear67/simplepages/internal/store" +) + +// depTime is the deployment's creation time, which is the Last-Modified of +// every file it serves. +var depTime = time.Date(2024, 3, 1, 12, 0, 0, 0, time.UTC) + +type serveEnv struct { + h *Handler + cs *cas.Store + reg *Registry + p *Project + log *bytes.Buffer +} + +// newServeEnv puts contents into a real content store and publishes a +// deployment of them. The store is real because the read path's whole shape — +// open by digest, hand the file to ServeContent — only means anything against +// actual files. +func newServeEnv(t *testing.T, cfg *ProjectConfig, contents map[string]string) *serveEnv { + t.Helper() + base := t.TempDir() + buf := &bytes.Buffer{} + log := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + cs, err := cas.Open(base+"/cas", cas.Options{ProbeDir: base + "/deployments", Log: log}) + if err != nil { + t.Fatalf("cas.Open: %v", err) + } + t.Cleanup(func() { cs.Close() }) + + rows := make([]store.FileRow, 0, len(contents)) + for p, c := range contents { + d := cas.Sum([]byte(c)) + limit := int64(len(c)) + if limit < 1 { + limit = 1 + } + if _, err := cs.Put(t.Context(), d, int64(len(c)), limit, strings.NewReader(c)); err != nil { + t.Fatalf("cas.Put %s: %v", p, err) + } + rows = append(rows, store.FileRow{Path: p, Digest: d, Size: int64(len(c))}) + } + + sp := NewProject(&store.Project{ID: 1, Name: "demo"}) + sp.SetConfig(cfg) + sp.Activate(NewDeployment( + &store.Deployment{PublicID: "dpl_0123456789abcdef", ProjectID: 1, CreatedAt: depTime}, + NewIndex(rows), "")) + + reg := NewRegistry() + reg.Replace([]*Project{sp}) + return &serveEnv{ + h: &Handler{Registry: reg, CAS: cs, Log: log}, + cs: cs, reg: reg, p: sp, log: buf, + } +} + +func (e *serveEnv) do(method, target string, header http.Header) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, target, nil) + for k, vs := range header { + req.Header[k] = vs + } + rec := httptest.NewRecorder() + e.h.ServeHTTP(rec, req) + return rec +} + +func (e *serveEnv) get(target string) *httptest.ResponseRecorder { + return e.do(http.MethodGet, target, nil) +} + +func demoSite() map[string]string { + return map[string]string{ + "index.html": "

hello

", + "assets/app.js": "console.log(1)", + "docs/index.html": "

docs

", + "404.html": "

gone

", + "data.bin": strings.Repeat("x", 4096), + "noext": "plain", + } +} + +func TestServeFile(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + + rec := e.get("/~demo/assets/app.js") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if got := rec.Body.String(); got != "console.log(1)" { + t.Errorf("body = %q", got) + } + head := rec.Header() + if ct := head.Get("Content-Type"); !strings.HasPrefix(ct, "text/javascript") && + !strings.HasPrefix(ct, "application/javascript") { + t.Errorf("Content-Type = %q, want a javascript type", ct) + } + if want := `"sha256:` + cas.Sum([]byte("console.log(1)")).String() + `"`; head.Get("Etag") != want { + t.Errorf("ETag = %q, want %q", head.Get("Etag"), want) + } + if head.Get("X-Content-Type-Options") != "nosniff" { + t.Error("nosniff is missing") + } + if head.Get("Cache-Control") != plain().CacheControl { + t.Errorf("Cache-Control = %q", head.Get("Cache-Control")) + } + // The deployment's time, never the blob's: a shared blob's mtime says when + // some unrelated project first uploaded the same bytes. + if got := head.Get("Last-Modified"); got != depTime.Format(http.TimeFormat) { + t.Errorf("Last-Modified = %q, want %q", got, depTime.Format(http.TimeFormat)) + } + if head.Get("Vary") != "" { + t.Errorf("Vary = %q with the SPA fallback off", head.Get("Vary")) + } +} + +func TestServeIndexAndRedirects(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + + if rec := e.get("/~demo/"); rec.Code != http.StatusOK || rec.Body.String() != "

hello

" { + t.Errorf("root: %d %q", rec.Code, rec.Body.String()) + } + if rec := e.get("/~demo"); rec.Code != http.StatusMovedPermanently || + rec.Header().Get("Location") != "/~demo/" { + t.Errorf("bare name: %d %q", rec.Code, rec.Header().Get("Location")) + } + if rec := e.get("/~demo/docs"); rec.Code != http.StatusMovedPermanently || + rec.Header().Get("Location") != "/~demo/docs/" { + t.Errorf("directory: %d %q", rec.Code, rec.Header().Get("Location")) + } + // The query survives the redirect, or a link with parameters loses them. + if rec := e.get("/~demo/docs?a=1&b=2"); rec.Header().Get("Location") != "/~demo/docs/?a=1&b=2" { + t.Errorf("query dropped: %q", rec.Header().Get("Location")) + } + if rec := e.get("/~demo/docs/"); rec.Body.String() != "

docs

" { + t.Errorf("directory index: %q", rec.Body.String()) + } +} + +// A file name that needs escaping must not arrive raw in the Location header. +func TestServeRedirectEscapesTheLocation(t *testing.T) { + e := newServeEnv(t, plain(), map[string]string{"a b/index.html": "spaced"}) + + rec := e.do(http.MethodGet, "/~demo/a%20b", nil) + if rec.Code != http.StatusMovedPermanently { + t.Fatalf("status = %d, want 301", rec.Code) + } + if got := rec.Header().Get("Location"); got != "/~demo/a%20b/" { + t.Errorf("Location = %q, want the space escaped", got) + } +} + +func TestServeMethodGate(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + + rec := e.do(http.MethodHead, "/~demo/index.html", nil) + if rec.Code != http.StatusOK { + t.Errorf("HEAD status = %d", rec.Code) + } + if rec.Body.Len() != 0 { + t.Errorf("HEAD returned %d bytes of body", rec.Body.Len()) + } + + for _, m := range []string{http.MethodPost, http.MethodPut, http.MethodDelete, http.MethodPatch} { + rec := e.do(m, "/~demo/index.html", nil) + if rec.Code != http.StatusMethodNotAllowed { + t.Errorf("%s status = %d, want 405", m, rec.Code) + } + if got := rec.Header().Get("Allow"); got != "GET, HEAD" { + t.Errorf("%s Allow = %q", m, got) + } + } +} + +func TestServeUnknownProjectAndPath(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + + for _, target := range []string{"/~nosuch/", "/~nosuch/index.html"} { + if rec := e.get(target); rec.Code != http.StatusNotFound { + t.Errorf("%s = %d, want 404", target, rec.Code) + } + } + for _, target := range []string{"/", "/index.html", "/api/v1/projects", "/~"} { + rec := e.get(target) + if rec.Code != http.StatusNotFound { + t.Errorf("%s = %d, want 404", target, rec.Code) + } + if !strings.Contains(rec.Body.String(), "/~project/") { + t.Errorf("%s: the 404 does not say where sites live: %q", target, rec.Body.String()) + } + } + // A project with no deployment exists but has nothing to serve, which is a + // different answer from one that does not exist. + e.reg.Put(&store.Project{ID: 2, Name: "empty"}) + if rec := e.get("/~empty/"); rec.Code != http.StatusServiceUnavailable { + t.Errorf("undeployed project = %d, want 503", rec.Code) + } +} + +func TestServeConditionalRequest(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + etag := e.get("/~demo/index.html").Header().Get("Etag") + if etag == "" { + t.Fatal("no ETag to revalidate with") + } + + rec := e.do(http.MethodGet, "/~demo/index.html", http.Header{"If-None-Match": {etag}}) + if rec.Code != http.StatusNotModified { + t.Errorf("status = %d, want 304", rec.Code) + } + if rec.Body.Len() != 0 { + t.Errorf("304 carried %d bytes", rec.Body.Len()) + } + // A stale validator must still transfer the content. + rec = e.do(http.MethodGet, "/~demo/index.html", http.Header{"If-None-Match": {`"sha256:stale"`}}) + if rec.Code != http.StatusOK || rec.Body.String() != "

hello

" { + t.Errorf("stale validator: %d %q", rec.Code, rec.Body.String()) + } + + rec = e.do(http.MethodGet, "/~demo/index.html", + http.Header{"If-Modified-Since": {depTime.Add(time.Hour).Format(http.TimeFormat)}}) + if rec.Code != http.StatusNotModified { + t.Errorf("If-Modified-Since: status = %d, want 304", rec.Code) + } +} + +func TestServeRange(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + + rec := e.do(http.MethodGet, "/~demo/data.bin", http.Header{"Range": {"bytes=10-19"}}) + if rec.Code != http.StatusPartialContent { + t.Fatalf("status = %d, want 206", rec.Code) + } + if got := rec.Body.String(); got != strings.Repeat("x", 10) { + t.Errorf("body = %q (%d bytes)", got, len(got)) + } + if got := rec.Header().Get("Content-Range"); got != "bytes 10-19/4096" { + t.Errorf("Content-Range = %q", got) + } + if rec.Header().Get("Accept-Ranges") != "bytes" { + t.Error("ranges are not advertised") + } +} + +func TestServeCustom404(t *testing.T) { + e := newServeEnv(t, withNotFound(), demoSite()) + + rec := e.get("/~demo/nope.txt") + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", rec.Code) + } + if rec.Body.String() != "

gone

" { + t.Errorf("body = %q, want the project's error document", rec.Body.String()) + } + if got := rec.Header().Get("Content-Type"); !strings.HasPrefix(got, "text/html") { + t.Errorf("Content-Type = %q", got) + } + if got := rec.Header().Get("Content-Length"); got != "11" { + t.Errorf("Content-Length = %q, want 11", got) + } + // An error document is not a cacheable validator target; it must not claim + // to be the requested resource. + if rec.Header().Get("Etag") != "" { + t.Error("the error document was served with an ETag") + } + + // HEAD gets the headers and no body. + rec = e.do(http.MethodHead, "/~demo/nope.txt", nil) + if rec.Code != http.StatusNotFound || rec.Body.Len() != 0 { + t.Errorf("HEAD of a 404: %d, %d bytes", rec.Code, rec.Body.Len()) + } +} + +func TestServeSPAVariesOnAccept(t *testing.T) { + e := newServeEnv(t, withSPA(), demoSite()) + + rec := e.do(http.MethodGet, "/~demo/client/route", http.Header{"Accept": {"text/html"}}) + if rec.Code != http.StatusOK || rec.Body.String() != "

hello

" { + t.Fatalf("navigation: %d %q", rec.Code, rec.Body.String()) + } + if rec.Header().Get("Vary") != "Accept" { + t.Error("a response that depends on Accept did not say so") + } + rec = e.do(http.MethodGet, "/~demo/client/route", http.Header{"Accept": {"*/*"}}) + if rec.Code != http.StatusNotFound { + t.Errorf("fetch: %d, want 404", rec.Code) + } +} + +func TestServeUnknownExtensionIsNotSniffed(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + + rec := e.get("/~demo/noext") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + if got := rec.Header().Get("Content-Type"); got != "application/octet-stream" { + t.Errorf("Content-Type = %q, want application/octet-stream", got) + } +} + +func TestServeBadRequestPath(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + + rec := e.do(http.MethodGet, "/~demo/bad%00path", nil) + if rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want 400", rec.Code) + } +} + +// The grace period is supposed to make this unreachable, so it means a bug or +// an operator who cleared the store by hand. The request must fail cleanly and +// the reason must reach the log. +func TestServeMissingBlobIs404AndLogged(t *testing.T) { + e := newServeEnv(t, plain(), demoSite()) + if err := os.Remove(e.cs.Path(cas.Sum([]byte("

hello

")))); err != nil { + t.Fatal(err) + } + + rec := e.get("/~demo/index.html") + if rec.Code != http.StatusNotFound { + t.Errorf("status = %d, want 404", rec.Code) + } + if !strings.Contains(e.log.String(), "blob missing from the content store") { + t.Errorf("nothing was logged about the missing blob: %s", e.log.String()) + } +} + +// An in-flight response keeps reading the deployment it started on. This is the +// property a symlink swap cannot give, and the reason the switch is a pointer +// store. +func TestServeInFlightResponseSurvivesAnActivation(t *testing.T) { + e := newServeEnv(t, plain(), map[string]string{"index.html": "v1"}) + + // Open the blob the way the handler does, then activate over the top of it. + first := e.p.Active() + entry, ok := first.Lookup("index.html") + if !ok { + t.Fatal("no index.html in the first deployment") + } + f, err := e.cs.Open(entry.Digest) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + d2 := cas.Sum([]byte("v2")) + if _, err := e.cs.Put(t.Context(), d2, 2, 2, strings.NewReader("v2")); err != nil { + t.Fatal(err) + } + e.p.Activate(NewDeployment( + &store.Deployment{PublicID: "dpl_second", ProjectID: 1, CreatedAt: depTime}, + NewIndex([]store.FileRow{{Path: "index.html", Digest: d2, Size: 2}}), "")) + + buf := make([]byte, 8) + n, _ := f.Read(buf) + if string(buf[:n]) != "v1" { + t.Errorf("the open file returned %q, want the deployment it was opened on", buf[:n]) + } + if rec := e.get("/~demo/"); rec.Body.String() != "v2" { + t.Errorf("a new request got %q, want v2", rec.Body.String()) + } +} diff --git a/internal/site/testdata/fuzz/FuzzResolve/dee8cb812bd7d77f b/internal/site/testdata/fuzz/FuzzResolve/dee8cb812bd7d77f new file mode 100644 index 0000000..befbc86 --- /dev/null +++ b/internal/site/testdata/fuzz/FuzzResolve/dee8cb812bd7d77f @@ -0,0 +1,3 @@ +go test fuzz v1 +string("0/.") +string("0") diff --git a/internal/store/blobs.go b/internal/store/blobs.go new file mode 100644 index 0000000..33b367d --- /dev/null +++ b/internal/store/blobs.go @@ -0,0 +1,94 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + + "github.com/iceBear67/simplepages/internal/cas" +) + +// Blob is what the database believes about one piece of content: how long it +// is, and whether its bytes have been verified onto disk yet. +type Blob struct { + Digest cas.Digest + Size int64 + Present bool +} + +// Blob looks up one blob. ErrNotFound means no manifest has ever declared it, +// which is what lets the upload endpoint refuse content nobody asked for. +func (d *DB) Blob(ctx context.Context, digest cas.Digest) (*Blob, error) { + b := Blob{Digest: digest} + err := d.r.QueryRowContext(ctx, + `SELECT size, present FROM blobs WHERE digest = ?`, digest.Bytes()). + Scan(&b.Size, &b.Present) + if err != nil { + return nil, mapErr(err) + } + return &b, nil +} + +// MarkBlobPresent records that a blob's content has been verified onto disk. +// +// Idempotent, because a retried `pages deploy` re-uploads blobs that in the +// meantime became present, and because the row may already say so if a previous +// attempt was interrupted between the rename and this update. +// +// The size is part of the WHERE clause rather than something to overwrite: a +// digest determines its content and therefore its length, so a row that +// disagrees means the manifest and the upload cannot both be describing the same +// bytes, and the safe move is to change nothing. +func (d *DB) MarkBlobPresent(ctx context.Context, digest cas.Digest, size int64) error { + return d.Tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, + `UPDATE blobs SET present = 1 WHERE digest = ? AND size = ?`, digest.Bytes(), size) + if err != nil { + return err + } + if n, err := res.RowsAffected(); err != nil { + return err + } else if n > 0 { + return nil + } + + // Nothing was updated. Distinguish "no such blob" from "the row says a + // different size" so the caller can report something actionable. + var have int64 + err = tx.QueryRowContext(ctx, `SELECT size FROM blobs WHERE digest = ?`, digest.Bytes()).Scan(&have) + if err != nil { + return mapErr(err) + } + return fmt.Errorf("%w: blob %s is %d bytes here, upload declared %d", + cas.ErrSizeMismatch, digest, have, size) + }) +} + +// MissingBlobs lists the distinct digests a deployment needs whose content has +// not been uploaded yet. It is both the manifest response and the check +// finalize runs before assembling anything. +func (d *DB) MissingBlobs(ctx context.Context, deploymentID int64) ([]cas.Digest, error) { + rows, err := d.r.QueryContext(ctx, ` + SELECT DISTINCT f.digest + FROM deployment_files f JOIN blobs b ON b.digest = f.digest + WHERE f.deployment_id = ? AND b.present = 0 + ORDER BY f.digest`, deploymentID) + if err != nil { + return nil, err + } + defer rows.Close() + + var missing []cas.Digest + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return nil, err + } + dg, err := cas.FromBytes(raw) + if err != nil { + return nil, err + } + missing = append(missing, dg) + } + return missing, rows.Err() +} diff --git a/internal/store/db.go b/internal/store/db.go new file mode 100644 index 0000000..a63b5e7 --- /dev/null +++ b/internal/store/db.go @@ -0,0 +1,244 @@ +// Package store owns the SQLite database: connection management, migrations and +// every query the server runs. It is server-side only — the CLI must never reach +// it, and cmd/pages/deps_test.go enforces that. +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "math/rand/v2" + "net/url" + "runtime" + "time" + + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" +) + +const driverName = "sqlite" + +// maxTxAttempts bounds the BEGIN-level retry loop. busy_timeout already handles +// contention inside a transaction; these retries exist for the cases it cannot +// cover (see Tx). +const maxTxAttempts = 5 + +// DB holds the two connection pools SQLite wants under concurrent load. +// +// SQLite permits exactly one writer at a time. Handing database/sql a single +// pool means readers and the writer compete for the same connections and every +// static request can end up queued behind a deployment commit. Two pools make +// the rule explicit instead: W is capped at one connection so writes serialise +// in Go (where waiting is cheap and fair) rather than in SQLite (where it +// surfaces as SQLITE_BUSY), and R holds the concurrent readers that WAL mode +// lets run undisturbed alongside the writer. +type DB struct { + w *sql.DB + r *sql.DB + path string + log *slog.Logger +} + +// Open connects to the database at path, applying the pragmas both pools need, +// and runs any outstanding migrations. +func Open(ctx context.Context, path string, log *slog.Logger) (*DB, error) { + w, err := sql.Open(driverName, dsn(path, false)) + if err != nil { + return nil, fmt.Errorf("open %s: %w", path, err) + } + // One writer, by construction. Idle == open and no lifetime cap: rebuilding a + // SQLite connection means replaying every pragma, and a server-side + // connection has nothing to go stale against. + w.SetMaxOpenConns(1) + w.SetMaxIdleConns(1) + w.SetConnMaxLifetime(0) + w.SetConnMaxIdleTime(0) + + // The writer opens first so the database file, and its WAL, exist before any + // read-only connection tries to attach to them. + if err := w.PingContext(ctx); err != nil { + w.Close() + return nil, fmt.Errorf("open %s: %w", path, err) + } + + readers := max(4, runtime.NumCPU()) + r, err := sql.Open(driverName, dsn(path, true)) + if err != nil { + w.Close() + return nil, fmt.Errorf("open %s (read pool): %w", path, err) + } + r.SetMaxOpenConns(readers) + r.SetMaxIdleConns(readers) + r.SetConnMaxLifetime(0) + r.SetConnMaxIdleTime(0) + if err := r.PingContext(ctx); err != nil { + w.Close() + r.Close() + return nil, fmt.Errorf("open %s (read pool): %w", path, err) + } + + d := &DB{w: w, r: r, path: path, log: log} + if err := d.migrate(ctx); err != nil { + d.Close() + return nil, err + } + return d, nil +} + +// dsn builds the connection string. The pragmas are per-connection state, so +// every connection in both pools must carry them. +func dsn(path string, readOnly bool) string { + q := url.Values{} + // Ordering inside the driver is fixed (busy_timeout first), so the list here + // is grouped by intent rather than by application order. + q.Add("_pragma", "busy_timeout(10000)") // wait for a lock instead of failing + q.Add("_pragma", "journal_mode(WAL)") // readers do not block the writer + q.Add("_pragma", "synchronous(NORMAL)") // the correct pairing with WAL + q.Add("_pragma", "foreign_keys(ON)") // off by default in SQLite + q.Add("_pragma", "recursive_triggers(ON)") + q.Add("_pragma", "temp_store(MEMORY)") + q.Add("_pragma", "wal_autocheckpoint(1000)") + + if readOnly { + // A guard rail, not a security boundary: it turns "this query was meant to + // be a read" from a silent lock contention bug into an immediate error. + q.Set("_query_only", "true") + } else { + // Every transaction takes the write lock at BEGIN. Without this a + // transaction that starts with a SELECT and later writes must upgrade its + // lock, and an upgrade that finds another writer fails with SQLITE_BUSY + // immediately — busy_timeout does not apply to it, because waiting could + // only ever deadlock. + q.Set("_txlock", "immediate") + } + return "file:" + path + "?" + q.Encode() +} + +// Reader returns the read-only pool. Queries that must observe uncommitted +// changes belong inside the writing transaction instead. +func (d *DB) Reader() *sql.DB { return d.r } + +// Path is the database file's location. +func (d *DB) Path() string { return d.path } + +// Tx runs fn inside a single write transaction and commits it, retrying from +// the top when SQLite reports contention. +// +// fn may be called more than once, so it must not have side effects outside the +// transaction — no file writes, no atomic pointer stores, no channel sends. +// Publishing a change to the rest of the process is the caller's job, after Tx +// returns nil. +func (d *DB) Tx(ctx context.Context, fn func(*sql.Tx) error) error { + var err error + for attempt := range maxTxAttempts { + if attempt > 0 { + // Exponential backoff with jitter, so two contending writers do not + // retry in lockstep. + backoff := time.Duration(1< 0 { + b, err := json.Marshal(dep.Meta) + if err != nil { + return err + } + meta = string(b) + } + now := unixNow() + return d.Tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, ` + INSERT INTO deployments (public_id, project_id, state, created_by_key, meta, created_at) + VALUES (?, ?, ?, ?, ?, ?)`, + dep.PublicID, dep.ProjectID, StatePending, nullString(dep.CreatedByKey), meta, now) + if err != nil { + return mapErr(err) + } + id, err := res.LastInsertId() + if err != nil { + return err + } + dep.ID = id + dep.State = StatePending + dep.CreatedAt = time.Unix(now, 0).UTC() + return nil + }) +} + +// DeploymentByPublicID looks up one deployment within one project. +// +// The project is part of the lookup rather than something to check afterwards. +// A handler holding a project-scoped identity therefore cannot name another +// project's deployment at all: the wrong project simply yields ErrNotFound, +// which is also the right answer to give — it reveals nothing about whether the +// id exists elsewhere. +func (d *DB) DeploymentByPublicID(ctx context.Context, projectID int64, publicID string) (*Deployment, error) { + return scanDeployment(d.r.QueryRowContext(ctx, + `SELECT `+deploymentColumns+` FROM deployments WHERE project_id = ? AND public_id = ?`, + projectID, publicID)) +} + +// ActiveDeployment returns the deployment a project is currently serving, or +// ErrNotFound when it has never activated one. The partial unique index makes +// "at most one" a property of the database rather than of this query. +func (d *DB) ActiveDeployment(ctx context.Context, projectID int64) (*Deployment, error) { + return scanDeployment(d.r.QueryRowContext(ctx, + `SELECT `+deploymentColumns+` FROM deployments WHERE project_id = ? AND active = 1`, + projectID)) +} + +// ListDeployments pages one project's deployments, newest first. +// +// The cursor is the public id of the last row of the previous page. An id that +// no longer exists — GC ran between pages — yields an empty page rather than an +// error, which is the behaviour a paging client can actually handle. +func (d *DB) ListDeployments(ctx context.Context, projectID int64, state State, limit int, cursor string) (deployments []*Deployment, next string, err error) { + if limit <= 0 || limit > 500 { + limit = 100 + } + rows, err := d.r.QueryContext(ctx, ` + SELECT `+deploymentColumns+` FROM deployments + WHERE project_id = ? + AND (? = '' OR state = ?) + AND (? = '' OR id < (SELECT id FROM deployments WHERE public_id = ?)) + ORDER BY id DESC LIMIT ?`, + projectID, string(state), string(state), cursor, cursor, limit+1) + if err != nil { + return nil, "", err + } + defer rows.Close() + for rows.Next() { + dep, err := scanDeployment(rows) + if err != nil { + return nil, "", err + } + deployments = append(deployments, dep) + } + if err := rows.Err(); err != nil { + return nil, "", err + } + if len(deployments) > limit { + deployments = deployments[:limit] + next = deployments[len(deployments)-1].PublicID + } + return deployments, next, nil +} + +// SetManifest replaces a deployment's file list and reports which blobs the +// server does not have content for yet. +// +// The ordering here is the point of the whole upload protocol. The manifest +// rows land *before* any blob is uploaded, and the insert triggers bump each +// blob's refcount as they do, so from this transaction's commit onwards every +// blob the deployment needs is protected from GC. That closes the race the +// obvious design has: "the server told me it already had this blob, then +// deleted it while I was uploading the others". +// +// Re-sending a manifest is allowed while the deployment is pending or +// uploading, because a CLI that lost its connection mid-negotiation should be +// able to start over without creating a second deployment. +func (d *DB) SetManifest(ctx context.Context, deploymentID int64, files []FileRow) (missing []cas.Digest, missingBytes int64, err error) { + // One entry per distinct digest, in first-appearance order, so the missing + // list — and therefore the client's upload order — is deterministic. + type need struct { + digest cas.Digest + size int64 + } + var needs []need + seen := make(map[cas.Digest]int64, len(files)) + var totalBytes int64 + for _, f := range files { + totalBytes += f.Size + if prev, ok := seen[f.Digest]; ok { + if prev != f.Size { + return nil, 0, fmt.Errorf("%w: the manifest gives digest %s both %d and %d bytes", + cas.ErrSizeMismatch, f.Digest, prev, f.Size) + } + continue + } + seen[f.Digest] = f.Size + needs = append(needs, need{f.Digest, f.Size}) + } + + now := unixNow() + err = d.Tx(ctx, func(tx *sql.Tx) error { + // Tx re-runs this function on SQLITE_BUSY, so the results are rebuilt + // from scratch each attempt rather than appended to. + missing, missingBytes = nil, 0 + + var state State + if err := tx.QueryRowContext(ctx, + `SELECT state FROM deployments WHERE id = ?`, deploymentID).Scan(&state); err != nil { + return mapErr(err) + } + if state != StatePending && state != StateUploading { + return fmt.Errorf("%w: deployment is %s; a manifest may only be set while pending or uploading", + ErrConflict, state) + } + + // Discard any earlier attempt. A refcount that goes 1 -> 0 -> 1 inside + // this transaction is safe because GC needs the same single write + // connection and so can never observe the zero. + if _, err := tx.ExecContext(ctx, + `DELETE FROM deployment_files WHERE deployment_id = ?`, deploymentID); err != nil { + return err + } + + // Prepared statements rather than batched multi-row VALUES: a 50,000 + // file manifest is a few hundred milliseconds of Exec either way, and + // this version has no chunk arithmetic to get wrong and no bound on how + // many parameters one statement may carry. + insertBlob, err := tx.PrepareContext(ctx, ` + INSERT INTO blobs (digest, size, present, created_at, last_ref_at) + VALUES (?, ?, 0, ?, ?) ON CONFLICT(digest) DO NOTHING`) + if err != nil { + return err + } + defer insertBlob.Close() + readBlob, err := tx.PrepareContext(ctx, `SELECT size, present FROM blobs WHERE digest = ?`) + if err != nil { + return err + } + defer readBlob.Close() + + for _, n := range needs { + if _, err := insertBlob.ExecContext(ctx, n.digest.Bytes(), n.size, now, now); err != nil { + return err + } + var have int64 + var present bool + if err := readBlob.QueryRowContext(ctx, n.digest.Bytes()).Scan(&have, &present); err != nil { + return mapErr(err) + } + if have != n.size { + // A digest fixes its content and therefore its length, so this + // is a client that computed one of the two wrong. Surfacing it + // here beats accepting a manifest whose byte totals are fiction. + return fmt.Errorf("%w: digest %s is %d bytes on this server, the manifest declares %d", + cas.ErrSizeMismatch, n.digest, have, n.size) + } + if !present { + missing = append(missing, n.digest) + missingBytes += n.size + } + } + + insertFile, err := tx.PrepareContext(ctx, ` + INSERT INTO deployment_files (deployment_id, path, digest, size) VALUES (?, ?, ?, ?)`) + if err != nil { + return err + } + defer insertFile.Close() + for _, f := range files { + if _, err := insertFile.ExecContext(ctx, deploymentID, f.Path, f.Digest.Bytes(), f.Size); err != nil { + return mapErr(err) + } + } + + _, err = tx.ExecContext(ctx, + `UPDATE deployments SET state = ?, file_count = ?, total_bytes = ? WHERE id = ?`, + StateUploading, len(files), totalBytes, deploymentID) + return err + }) + if err != nil { + return nil, 0, err + } + return missing, missingBytes, nil +} + +// DeploymentFiles returns the manifest, ordered by path so assembly creates +// each directory once and in a predictable order. +func (d *DB) DeploymentFiles(ctx context.Context, deploymentID int64) ([]FileRow, error) { + rows, err := d.r.QueryContext(ctx, + `SELECT path, digest, size FROM deployment_files WHERE deployment_id = ? ORDER BY path`, + deploymentID) + if err != nil { + return nil, err + } + defer rows.Close() + + var files []FileRow + for rows.Next() { + var f FileRow + var raw []byte + if err := rows.Scan(&f.Path, &raw, &f.Size); err != nil { + return nil, err + } + if f.Digest, err = cas.FromBytes(raw); err != nil { + return nil, err + } + files = append(files, f) + } + return files, rows.Err() +} + +// MarkDeploymentReady records that assembly succeeded. +// +// Idempotent, and deliberately so: the CLI retries finalize after re-uploading +// blobs that went missing, and a retry must not turn a good deployment into an +// error. finalized_at keeps its original value so the timestamp reflects the +// first success rather than the last attempt. +func (d *DB) MarkDeploymentReady(ctx context.Context, id int64) error { + now := unixNow() + return d.Tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, ` + UPDATE deployments SET state = ?, finalized_at = COALESCE(finalized_at, ?), error = NULL + WHERE id = ? AND state IN (?, ?)`, + StateReady, now, id, StateUploading, StateReady) + if err != nil { + return err + } + return requireOneRow(ctx, tx, res, id, "finalize") + }) +} + +// ActivateDeployment makes one ready deployment the project's active one and +// demotes whatever held that role before. +// +// Demotion comes first because deployments_one_active is a partial unique index, +// checked per statement: promoting before demoting would collide with the row +// still holding active = 1. The demotion deliberately excludes the target, so +// re-activating the deployment that is already active is a no-op that refreshes +// its timestamp rather than a transaction that briefly leaves the project with +// nothing active. +// +// The caller is expected to hold the project lock and to have already built +// whatever in-memory snapshot it means to publish: once this commits, SQLite is +// the truth and a crash before the pointer store is recovered from here. +func (d *DB) ActivateDeployment(ctx context.Context, projectID, deploymentID int64) error { + now := unixNow() + return d.Tx(ctx, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE deployments SET active = 0, deactivated_at = ? + WHERE project_id = ? AND active = 1 AND id <> ?`, + now, projectID, deploymentID); err != nil { + return err + } + res, err := tx.ExecContext(ctx, ` + UPDATE deployments SET active = 1, activated_at = ?, deactivated_at = NULL + WHERE id = ? AND project_id = ? AND state = ?`, + now, deploymentID, projectID, StateReady) + if err != nil { + return mapErr(err) + } + return requireOneRow(ctx, tx, res, deploymentID, "activate") + }) +} + +// MarkDeploymentFailed records why a deployment was abandoned. The manifest +// rows stay for now; GC drops them, which is what lets its blobs be reclaimed. +func (d *DB) MarkDeploymentFailed(ctx context.Context, id int64, reason string) error { + return d.Tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, ` + UPDATE deployments SET state = ?, error = ? WHERE id = ? AND state IN (?, ?, ?)`, + StateFailed, truncate(reason, 1024), id, StatePending, StateUploading, StateFailed) + if err != nil { + return err + } + return requireOneRow(ctx, tx, res, id, "fail") + }) +} + +// requireOneRow turns "the UPDATE matched nothing" into the reason it matched +// nothing, which is either a deployment that is gone or one in a state the +// operation does not apply to. +func requireOneRow(ctx context.Context, tx *sql.Tx, res sql.Result, id int64, op string) error { + n, err := res.RowsAffected() + if err != nil { + return err + } + if n > 0 { + return nil + } + var state State + if err := tx.QueryRowContext(ctx, `SELECT state FROM deployments WHERE id = ?`, id).Scan(&state); err != nil { + return mapErr(err) + } + return fmt.Errorf("%w: cannot %s a deployment that is %s", ErrConflict, op, state) +} + +func truncate(s string, max int) string { + if len(s) <= max { + return s + } + return s[:max] + "…" +} diff --git a/internal/store/deployments_test.go b/internal/store/deployments_test.go new file mode 100644 index 0000000..38a1382 --- /dev/null +++ b/internal/store/deployments_test.go @@ -0,0 +1,703 @@ +package store + +import ( + "context" + "errors" + "testing" + + "github.com/iceBear67/simplepages/api" + "github.com/iceBear67/simplepages/internal/cas" +) + +// The lifecycle is spelled out in three places that cannot import each other: +// this package, the CHECK constraint in the schema, and api for the wire. +// Renaming a state in one of them without the others would surface as a +// constraint violation in production rather than at compile time, so it is +// asserted here instead. +func TestStateConstantsMatchTheWire(t *testing.T) { + states := []struct { + store State + wire string + }{ + {StatePending, api.StatePending}, + {StateUploading, api.StateUploading}, + {StateReady, api.StateReady}, + {StateFailed, api.StateFailed}, + {StateDeleting, api.StateDeleting}, + } + + db := testDB(t) + p := testProject(t, db, "demo") + for _, s := range states { + if string(s.store) != s.wire { + t.Errorf("store %q and wire %q disagree", s.store, s.wire) + } + // And the schema accepts it: a state this package can set but the CHECK + // constraint rejects would only fail once something reached that state. + dep := testDeployment(t, db, p.ID) + if _, err := db.w.ExecContext(context.Background(), + `UPDATE deployments SET state = ? WHERE id = ?`, s.store, dep.ID); err != nil { + t.Errorf("the schema rejects state %q: %v", s.store, err) + } + } +} + +func testProject(t *testing.T, db *DB, name string) *Project { + t.Helper() + p := DefaultProject(name) + if err := db.CreateProject(context.Background(), p); err != nil { + t.Fatalf("CreateProject(%q): %v", name, err) + } + return p +} + +func testDeployment(t *testing.T, db *DB, projectID int64) *Deployment { + t.Helper() + dep := &Deployment{ProjectID: projectID} + if err := db.CreateDeployment(context.Background(), dep); err != nil { + t.Fatalf("CreateDeployment: %v", err) + } + return dep +} + +// file builds a manifest entry whose digest really is the digest of content, so +// tests never accidentally assert on an impossible pairing. +func file(path, content string) FileRow { + return FileRow{Path: path, Digest: cas.Sum([]byte(content)), Size: int64(len(content))} +} + +func refcount(t *testing.T, db *DB, d cas.Digest) int { + t.Helper() + var n int + if err := db.Reader().QueryRow(`SELECT refcount FROM blobs WHERE digest = ?`, d.Bytes()).Scan(&n); err != nil { + t.Fatalf("refcount(%s): %v", d, err) + } + return n +} + +func digests(ds []cas.Digest) []string { + out := make([]string, len(ds)) + for i, d := range ds { + out[i] = d.String() + } + return out +} + +func TestCreateDeployment(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + dep := &Deployment{ProjectID: p.ID, CreatedByKey: "k7m2qabcdefghijk", Meta: map[string]string{"git_sha": "abc"}} + // The key must exist: created_by_key is a foreign key. + if _, err := db.w.ExecContext(ctx, + `INSERT INTO api_keys (id, secret_hash, scope, project_id, created_at) VALUES (?, x'00', 'project', ?, 1)`, + dep.CreatedByKey, p.ID); err != nil { + t.Fatal(err) + } + if err := db.CreateDeployment(ctx, dep); err != nil { + t.Fatal(err) + } + if dep.ID == 0 || dep.PublicID == "" || dep.State != StatePending { + t.Fatalf("CreateDeployment left %+v", dep) + } + + got, err := db.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + t.Fatal(err) + } + if got.Meta["git_sha"] != "abc" { + t.Errorf("meta = %v, want git_sha=abc", got.Meta) + } + if got.CreatedByKey != dep.CreatedByKey || got.Active || got.CreatedAt.IsZero() { + t.Errorf("round-tripped as %+v", got) + } +} + +func TestNewDeploymentIDsAreDistinct(t *testing.T) { + seen := make(map[string]bool, 256) + for range 256 { + id, err := NewDeploymentID() + if err != nil { + t.Fatal(err) + } + if len(id) != len("dpl_")+16 { + t.Fatalf("id %q has the wrong shape", id) + } + if seen[id] { + t.Fatalf("NewDeploymentID repeated %q", id) + } + seen[id] = true + } +} + +// Security requirement: a deployment id is only ever resolvable inside its own +// project, so a project-scoped caller cannot name a neighbour's deployment even +// knowing its id exactly. +func TestDeploymentByPublicIDIsProjectScoped(t *testing.T) { + ctx := context.Background() + db := testDB(t) + mine := testProject(t, db, "mine") + theirs := testProject(t, db, "theirs") + dep := testDeployment(t, db, theirs.ID) + + if _, err := db.DeploymentByPublicID(ctx, theirs.ID, dep.PublicID); err != nil { + t.Fatalf("the owning project cannot see its own deployment: %v", err) + } + _, err := db.DeploymentByPublicID(ctx, mine.ID, dep.PublicID) + if !errors.Is(err, ErrNotFound) { + t.Errorf("cross-project lookup = %v, want ErrNotFound", err) + } +} + +// The ordering claim the upload protocol rests on: once SetManifest commits, +// every blob the deployment needs is already refcounted, so nothing GC does +// while the content is still being uploaded can take one away. +func TestSetManifestRefcountsBeforeUpload(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := testDeployment(t, db, p.ID) + + files := []FileRow{ + file("index.html", "

hi

"), + file("assets/app.js", "console.log(1)"), + file("copy.html", "

hi

"), // same content as index.html + } + missing, missingBytes, err := db.SetManifest(ctx, dep.ID, files) + if err != nil { + t.Fatal(err) + } + + // Two distinct digests, each reported once, in first-appearance order. + want := []string{files[0].Digest.String(), files[1].Digest.String()} + if got := digests(missing); len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Errorf("missing = %v, want %v", got, want) + } + if wantBytes := files[0].Size + files[1].Size; missingBytes != wantBytes { + t.Errorf("missingBytes = %d, want %d", missingBytes, wantBytes) + } + + if got := refcount(t, db, files[0].Digest); got != 2 { + t.Errorf("shared blob refcount = %d, want 2 (two paths reference it)", got) + } + if got := refcount(t, db, files[1].Digest); got != 1 { + t.Errorf("refcount = %d, want 1", got) + } + + got, err := db.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + t.Fatal(err) + } + if got.State != StateUploading { + t.Errorf("state = %s, want uploading", got.State) + } + if got.FileCount != 3 { + t.Errorf("file_count = %d, want 3", got.FileCount) + } + if wantTotal := files[0].Size + files[1].Size + files[2].Size; got.TotalBytes != wantTotal { + t.Errorf("total_bytes = %d, want %d", got.TotalBytes, wantTotal) + } +} + +// A CLI that lost its connection mid-negotiation re-sends the manifest. The +// second one must replace the first outright, refcounts included. +func TestSetManifestReplacesTheEarlierAttempt(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := testDeployment(t, db, p.ID) + + dropped := file("old.html", "version one") + kept := file("index.html", "shared") + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{dropped, kept}); err != nil { + t.Fatal(err) + } + added := file("assets/app.js", "version two") + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{kept, added}); err != nil { + t.Fatal(err) + } + + if got := refcount(t, db, dropped.Digest); got != 0 { + t.Errorf("dropped blob refcount = %d, want 0", got) + } + if got := refcount(t, db, kept.Digest); got != 1 { + t.Errorf("kept blob refcount = %d, want 1", got) + } + if got := refcount(t, db, added.Digest); got != 1 { + t.Errorf("added blob refcount = %d, want 1", got) + } + + files, err := db.DeploymentFiles(ctx, dep.ID) + if err != nil { + t.Fatal(err) + } + if len(files) != 2 || files[0].Path != "assets/app.js" || files[1].Path != "index.html" { + t.Errorf("manifest = %+v, want the second attempt ordered by path", files) + } +} + +func TestSetManifestRejectsASizeDisagreement(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + f := file("index.html", "content") + first := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, first.ID, []FileRow{f}); err != nil { + t.Fatal(err) + } + + // Same digest, a different declared length: one of the two numbers is a lie + // and the manifest cannot be accepted either way. + lying := f + lying.Size = f.Size + 1 + second := testDeployment(t, db, p.ID) + _, _, err := db.SetManifest(ctx, second.ID, []FileRow{lying}) + if !errors.Is(err, cas.ErrSizeMismatch) { + t.Errorf("err = %v, want ErrSizeMismatch", err) + } + + // The same disagreement inside one manifest is caught before any write. + third := testDeployment(t, db, p.ID) + _, _, err = db.SetManifest(ctx, third.ID, []FileRow{f, {Path: "other.html", Digest: f.Digest, Size: 99}}) + if !errors.Is(err, cas.ErrSizeMismatch) { + t.Errorf("err = %v, want ErrSizeMismatch", err) + } + if got := refcount(t, db, f.Digest); got != 1 { + t.Errorf("refcount = %d after two rejected manifests, want 1", got) + } +} + +func TestSetManifestRejectsAFinishedDeployment(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := testDeployment(t, db, p.ID) + + f := file("index.html", "content") + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{f}); err != nil { + t.Fatal(err) + } + if err := db.MarkBlobPresent(ctx, f.Digest, f.Size); err != nil { + t.Fatal(err) + } + if err := db.MarkDeploymentReady(ctx, dep.ID); err != nil { + t.Fatal(err) + } + + _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{file("other.html", "x")}) + if !errors.Is(err, ErrConflict) { + t.Errorf("err = %v, want ErrConflict", err) + } +} + +func TestSetManifestOnAMissingDeployment(t *testing.T) { + _, _, err := testDB(t).SetManifest(context.Background(), 424242, []FileRow{file("a", "b")}) + if !errors.Is(err, ErrNotFound) { + t.Errorf("err = %v, want ErrNotFound", err) + } +} + +func TestMissingBlobsShrinksAsContentArrives(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := testDeployment(t, db, p.ID) + + a, b := file("a.html", "aaa"), file("b.html", "bbb") + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{a, b}); err != nil { + t.Fatal(err) + } + if got, err := db.MissingBlobs(ctx, dep.ID); err != nil || len(got) != 2 { + t.Fatalf("MissingBlobs = %v, %v; want 2 digests", digests(got), err) + } + + if err := db.MarkBlobPresent(ctx, a.Digest, a.Size); err != nil { + t.Fatal(err) + } + got, err := db.MissingBlobs(ctx, dep.ID) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0] != b.Digest { + t.Fatalf("MissingBlobs = %v, want just %s", digests(got), b.Digest) + } + + // A second deployment of overlapping content sees only what is genuinely + // new — this is the deduplication the CLI reports as its headline number. + next := testDeployment(t, db, p.ID) + c := file("c.html", "ccc") + missing, _, err := db.SetManifest(ctx, next.ID, []FileRow{a, c}) + if err != nil { + t.Fatal(err) + } + if len(missing) != 1 || missing[0] != c.Digest { + t.Errorf("missing = %v, want just the new blob", digests(missing)) + } + + if err := db.MarkBlobPresent(ctx, b.Digest, b.Size); err != nil { + t.Fatal(err) + } + if got, err := db.MissingBlobs(ctx, dep.ID); err != nil || len(got) != 0 { + t.Errorf("MissingBlobs = %v, %v; want none", digests(got), err) + } +} + +func TestMarkBlobPresent(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := testDeployment(t, db, p.ID) + f := file("index.html", "content") + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{f}); err != nil { + t.Fatal(err) + } + + if err := db.MarkBlobPresent(ctx, f.Digest, f.Size); err != nil { + t.Fatal(err) + } + // Idempotent: a retried upload of a blob that arrived meanwhile is a no-op. + if err := db.MarkBlobPresent(ctx, f.Digest, f.Size); err != nil { + t.Errorf("second MarkBlobPresent: %v", err) + } + b, err := db.Blob(ctx, f.Digest) + if err != nil { + t.Fatal(err) + } + if !b.Present || b.Size != f.Size { + t.Errorf("blob = %+v", b) + } + + if err := db.MarkBlobPresent(ctx, f.Digest, f.Size+1); !errors.Is(err, cas.ErrSizeMismatch) { + t.Errorf("err = %v, want ErrSizeMismatch", err) + } + // Unknown digests are refused, which is what stops the upload endpoint from + // being used as arbitrary storage. + if _, err := db.Blob(ctx, cas.Sum([]byte("never declared"))); !errors.Is(err, ErrNotFound) { + t.Errorf("Blob = %v, want ErrNotFound", err) + } + if err := db.MarkBlobPresent(ctx, cas.Sum([]byte("never declared")), 1); !errors.Is(err, ErrNotFound) { + t.Errorf("MarkBlobPresent = %v, want ErrNotFound", err) + } +} + +func TestMarkDeploymentReadyIsIdempotent(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := testDeployment(t, db, p.ID) + + // A pending deployment has no manifest, so there is nothing to finalize. + if err := db.MarkDeploymentReady(ctx, dep.ID); !errors.Is(err, ErrConflict) { + t.Errorf("finalize while pending = %v, want ErrConflict", err) + } + + f := file("index.html", "content") + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{f}); err != nil { + t.Fatal(err) + } + if err := db.MarkDeploymentReady(ctx, dep.ID); err != nil { + t.Fatal(err) + } + first, err := db.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + t.Fatal(err) + } + if first.State != StateReady || first.FinalizedAt == nil { + t.Fatalf("deployment = %+v", first) + } + + if err := db.MarkDeploymentReady(ctx, dep.ID); err != nil { + t.Errorf("retried finalize: %v", err) + } + again, err := db.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + t.Fatal(err) + } + if !again.FinalizedAt.Equal(*first.FinalizedAt) { + t.Errorf("finalized_at moved from %v to %v", first.FinalizedAt, again.FinalizedAt) + } +} + +func TestMarkDeploymentFailed(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := testDeployment(t, db, p.ID) + + if err := db.MarkDeploymentFailed(ctx, dep.ID, "upload timed out"); err != nil { + t.Fatal(err) + } + got, err := db.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + t.Fatal(err) + } + if got.State != StateFailed || got.Error != "upload timed out" { + t.Errorf("deployment = %+v", got) + } + + // A failed deployment cannot be resurrected by a late manifest or finalize. + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{file("a", "b")}); !errors.Is(err, ErrConflict) { + t.Errorf("SetManifest = %v, want ErrConflict", err) + } + if err := db.MarkDeploymentReady(ctx, dep.ID); !errors.Is(err, ErrConflict) { + t.Errorf("MarkDeploymentReady = %v, want ErrConflict", err) + } +} + +func TestListDeployments(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + other := testProject(t, db, "other") + testDeployment(t, db, other.ID) + + var ids []string + for range 5 { + ids = append(ids, testDeployment(t, db, p.ID).PublicID) + } + // Newest first, so reverse creation order. + var want []string + for i := len(ids) - 1; i >= 0; i-- { + want = append(want, ids[i]) + } + + var seen []string + cursor := "" + for { + page, next, err := db.ListDeployments(ctx, p.ID, "", 2, cursor) + if err != nil { + t.Fatal(err) + } + for _, d := range page { + seen = append(seen, d.PublicID) + } + if next == "" { + break + } + cursor = next + } + if len(seen) != len(want) { + t.Fatalf("paged %v, want %v", seen, want) + } + for i := range want { + if seen[i] != want[i] { + t.Fatalf("paged %v, want %v", seen, want) + } + } + + // Filtering by state. + fifth, err := db.DeploymentByPublicID(ctx, p.ID, ids[4]) + if err != nil { + t.Fatal(err) + } + if err := db.MarkDeploymentFailed(ctx, fifth.ID, "abandoned"); err != nil { + t.Fatal(err) + } + failed, _, err := db.ListDeployments(ctx, p.ID, StateFailed, 10, "") + if err != nil { + t.Fatal(err) + } + if len(failed) != 1 || failed[0].PublicID != ids[4] { + t.Errorf("failed page = %v, want just %s", failed, ids[4]) + } + + // A cursor GC removed between pages yields an empty page, not an error. + gone, _, err := db.ListDeployments(ctx, p.ID, "", 10, "dpl_ffffffffffffffff") + if err != nil { + t.Fatalf("stale cursor: %v", err) + } + if len(gone) != 0 { + t.Errorf("stale cursor returned %d rows", len(gone)) + } +} + +// ready builds a deployment that has a manifest and has been finalized, which +// is the only state activation accepts. +func ready(t *testing.T, db *DB, projectID int64, content string) *Deployment { + t.Helper() + ctx := context.Background() + dep := testDeployment(t, db, projectID) + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{file("index.html", content)}); err != nil { + t.Fatalf("SetManifest: %v", err) + } + if err := db.MarkDeploymentReady(ctx, dep.ID); err != nil { + t.Fatalf("MarkDeploymentReady: %v", err) + } + return dep +} + +func TestActivateDeployment(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + if _, err := db.ActiveDeployment(ctx, p.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("ActiveDeployment on a fresh project = %v, want ErrNotFound", err) + } + + first := ready(t, db, p.ID, "v1") + if err := db.ActivateDeployment(ctx, p.ID, first.ID); err != nil { + t.Fatal(err) + } + got, err := db.ActiveDeployment(ctx, p.ID) + if err != nil { + t.Fatal(err) + } + if got.ID != first.ID || !got.Active || got.ActivatedAt == nil || got.DeactivatedAt != nil { + t.Fatalf("after activation the row is %+v", got) + } + + second := ready(t, db, p.ID, "v2") + if err := db.ActivateDeployment(ctx, p.ID, second.ID); err != nil { + t.Fatal(err) + } + got, err = db.ActiveDeployment(ctx, p.ID) + if err != nil { + t.Fatal(err) + } + if got.ID != second.ID { + t.Fatalf("active = %d, want the second deployment %d", got.ID, second.ID) + } + + // The superseded one stays ready and on disk — that is what makes rollback + // a single activation rather than a redeploy — but it is stamped so GC's + // grace period can start counting. + old, err := db.DeploymentByPublicID(ctx, p.ID, first.PublicID) + if err != nil { + t.Fatal(err) + } + if old.Active || old.State != StateReady || old.DeactivatedAt == nil { + t.Errorf("the superseded deployment is %+v", old) + } + if old.ActivatedAt == nil { + t.Error("deactivation cleared activated_at") + } + + // Rollback. + if err := db.ActivateDeployment(ctx, p.ID, first.ID); err != nil { + t.Fatal(err) + } + got, err = db.ActiveDeployment(ctx, p.ID) + if err != nil { + t.Fatal(err) + } + if got.ID != first.ID || got.DeactivatedAt != nil { + t.Errorf("after rollback the active row is %+v", got) + } +} + +// Re-activating what is already active must not leave the project with nothing +// active in between, which is why the demotion excludes the target row. +func TestActivateDeploymentIsIdempotent(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := ready(t, db, p.ID, "v1") + + for range 3 { + if err := db.ActivateDeployment(ctx, p.ID, dep.ID); err != nil { + t.Fatal(err) + } + got, err := db.ActiveDeployment(ctx, p.ID) + if err != nil { + t.Fatalf("nothing is active after re-activating: %v", err) + } + if got.ID != dep.ID || got.DeactivatedAt != nil { + t.Fatalf("row = %+v", got) + } + } +} + +func TestActivateDeploymentRequiresReady(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + pending := testDeployment(t, db, p.ID) + if err := db.ActivateDeployment(ctx, p.ID, pending.ID); !errors.Is(err, ErrConflict) { + t.Errorf("activating a pending deployment = %v, want ErrConflict", err) + } + + failed := testDeployment(t, db, p.ID) + if err := db.MarkDeploymentFailed(ctx, failed.ID, "assembly failed"); err != nil { + t.Fatal(err) + } + if err := db.ActivateDeployment(ctx, p.ID, failed.ID); !errors.Is(err, ErrConflict) { + t.Errorf("activating a failed deployment = %v, want ErrConflict", err) + } + + if err := db.ActivateDeployment(ctx, p.ID, 9999); !errors.Is(err, ErrNotFound) { + t.Errorf("activating a deployment that does not exist = %v, want ErrNotFound", err) + } + if _, err := db.ActiveDeployment(ctx, p.ID); !errors.Is(err, ErrNotFound) { + t.Error("a refused activation left something active") + } +} + +// The ownership check the API's authorization rests on lives here too: naming +// another project's deployment id must not activate it, and must not disturb +// either project. +func TestActivateDeploymentIsProjectScoped(t *testing.T) { + ctx := context.Background() + db := testDB(t) + victim := testProject(t, db, "victim") + attacker := testProject(t, db, "attacker") + + target := ready(t, db, victim.ID, "secret") + if err := db.ActivateDeployment(ctx, victim.ID, target.ID); err != nil { + t.Fatal(err) + } + mine := ready(t, db, attacker.ID, "mine") + if err := db.ActivateDeployment(ctx, attacker.ID, mine.ID); err != nil { + t.Fatal(err) + } + + if err := db.ActivateDeployment(ctx, attacker.ID, target.ID); err == nil { + t.Fatal("a project activated another project's deployment") + } + got, err := db.ActiveDeployment(ctx, attacker.ID) + if err != nil { + t.Fatal(err) + } + if got.ID != mine.ID { + t.Errorf("the cross-project attempt changed the attacker's active deployment to %d", got.ID) + } + got, err = db.ActiveDeployment(ctx, victim.ID) + if err != nil { + t.Fatal(err) + } + if got.ID != target.ID || !got.Active { + t.Errorf("the cross-project attempt disturbed the victim: %+v", got) + } +} + +// "At most one active deployment per project" is enforced by the database, not +// by the code above it. Writing the second active row by hand is the only way +// to check that: if this ever stops failing, every guarantee that rests on the +// invariant has quietly lost its foundation. +func TestOneActiveDeploymentPerProjectIsEnforcedBySchema(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + other := testProject(t, db, "other") + + first := ready(t, db, p.ID, "v1") + second := ready(t, db, p.ID, "v2") + if err := db.ActivateDeployment(ctx, p.ID, first.ID); err != nil { + t.Fatal(err) + } + + if _, err := db.w.ExecContext(ctx, + `UPDATE deployments SET active = 1 WHERE id = ?`, second.ID); err == nil { + t.Fatal("the schema allowed a project to have two active deployments") + } + + // The index is partial, so it constrains only active rows: any number of + // inactive ones per project, and one active row per *other* project. + elsewhere := ready(t, db, other.ID, "v1") + if err := db.ActivateDeployment(ctx, other.ID, elsewhere.ID); err != nil { + t.Fatalf("the index leaked across projects: %v", err) + } +} diff --git a/internal/store/errors.go b/internal/store/errors.go new file mode 100644 index 0000000..7b2f079 --- /dev/null +++ b/internal/store/errors.go @@ -0,0 +1,73 @@ +package store + +import ( + "database/sql" + "errors" + "time" +) + +// Sentinels the HTTP layer maps to error codes. Callers use errors.Is; the +// store never constructs api.Error values itself, so that the mapping from +// storage failure to wire response lives in exactly one place (internal/adminapi). +var ( + // ErrNotFound is returned instead of sql.ErrNoRows so callers do not have to + // know that the store is backed by database/sql. + ErrNotFound = errors.New("store: not found") + // ErrExists means a uniqueness constraint rejected the write. + ErrExists = errors.New("store: already exists") + // ErrConflict means the row was not in the state the operation required. + ErrConflict = errors.New("store: conflicting state") +) + +// mapErr normalises the errors callers are expected to branch on. +func mapErr(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, sql.ErrNoRows): + return ErrNotFound + case IsConstraint(err): + return errors.Join(ErrExists, err) + default: + return err + } +} + +// ---------------------------------------------------------- null conversions + +func nullTime(t *time.Time) any { + if t == nil { + return nil + } + return t.Unix() +} + +func timePtr(n sql.NullInt64) *time.Time { + if !n.Valid { + return nil + } + t := time.Unix(n.Int64, 0).UTC() + return &t +} + +func nullInt(p *int64) any { + if p == nil { + return nil + } + return *p +} + +func intPtr(n sql.NullInt64) *int64 { + if !n.Valid { + return nil + } + v := n.Int64 + return &v +} + +func nullString(s string) any { + if s == "" { + return nil + } + return s +} diff --git a/internal/store/fsck.go b/internal/store/fsck.go new file mode 100644 index 0000000..9287053 --- /dev/null +++ b/internal/store/fsck.go @@ -0,0 +1,117 @@ +package store + +import ( + "context" + "database/sql" + + "github.com/iceBear67/simplepages/internal/cas" +) + +// maxReportedDrift bounds what a report carries back. A repair fixes every row +// it finds; the list is for a human reading the output, and a human does not +// read ten thousand digests. +const maxReportedDrift = 100 + +// Drift is one blob whose recorded refcount disagrees with the manifests that +// actually name it. +type Drift struct { + Digest cas.Digest + // Stored is what the blobs row claims, Actual what counting the manifest + // rows gives. Stored above Actual wastes disk: the collector will never + // reclaim the blob. Stored below Actual is the dangerous direction — the + // collector may delete content a deployment still needs. + Stored int64 + Actual int64 +} + +// FsckReport is what a consistency check found. +type FsckReport struct { + // Blobs is how many rows were examined. + Blobs int64 + // DriftCount is how many disagreed; Drift lists the first few of them. + DriftCount int + Drift []Drift + // Repaired is how many rows were corrected, and is zero unless the check + // was asked to repair. + Repaired int +} + +// Fsck recomputes every blob's refcount from the manifests and reports the rows +// that disagree, optionally correcting them. +// +// Refcounts are maintained by triggers on deployment_files, so under normal +// operation they cannot drift. This exists for the cases outside normal +// operation: a database restored from a backup taken mid-transaction, a schema +// touched by hand, or a bug in this program. Drift matters because the blob +// collector trusts the counter — a count that reads low is content that will be +// deleted while a deployment still references it, which is the one way this +// design can lose data. +// +// The recount is a single grouped join rather than a query per blob, so a store +// with a million blobs is one table scan and not a million index seeks. +func (d *DB) Fsck(ctx context.Context, repair bool) (FsckReport, error) { + var rep FsckReport + if err := d.r.QueryRowContext(ctx, `SELECT count(*) FROM blobs`).Scan(&rep.Blobs); err != nil { + return rep, err + } + + rows, err := d.r.QueryContext(ctx, ` + SELECT b.digest, b.refcount, coalesce(c.n, 0) + FROM blobs b + LEFT JOIN (SELECT digest, count(*) AS n FROM deployment_files GROUP BY digest) c + ON c.digest = b.digest + WHERE b.refcount <> coalesce(c.n, 0) + ORDER BY b.digest`) + if err != nil { + return rep, err + } + defer rows.Close() + + var drift []Drift + for rows.Next() { + var dr Drift + var raw []byte + if err := rows.Scan(&raw, &dr.Stored, &dr.Actual); err != nil { + return rep, err + } + if dr.Digest, err = cas.FromBytes(raw); err != nil { + return rep, err + } + drift = append(drift, dr) + } + if err := rows.Err(); err != nil { + return rep, err + } + + rep.DriftCount = len(drift) + rep.Drift = drift + if len(rep.Drift) > maxReportedDrift { + rep.Drift = rep.Drift[:maxReportedDrift] + } + if !repair || len(drift) == 0 { + return rep, nil + } + + // Repair writes the recounted value rather than adjusting by the difference: + // the manifests are the definition of the refcount, so the correct value is + // the one just counted, whatever the column happened to say. + err = d.Tx(ctx, func(tx *sql.Tx) error { + rep.Repaired = 0 + stmt, err := tx.PrepareContext(ctx, `UPDATE blobs SET refcount = ? WHERE digest = ?`) + if err != nil { + return err + } + defer stmt.Close() + for _, dr := range drift { + if _, err := stmt.ExecContext(ctx, dr.Actual, dr.Digest.Bytes()); err != nil { + return err + } + rep.Repaired++ + } + return nil + }) + if err != nil { + return rep, err + } + return rep, nil +} diff --git a/internal/store/keys.go b/internal/store/keys.go new file mode 100644 index 0000000..0228af5 --- /dev/null +++ b/internal/store/keys.go @@ -0,0 +1,181 @@ +package store + +import ( + "context" + "database/sql" + "time" +) + +// Scope is what an API key is allowed to touch. +type Scope string + +const ( + // ScopeAdmin may manage every project, key and deployment. + ScopeAdmin Scope = "admin" + // ScopeProject may manage only the deployments of its own project. + ScopeProject Scope = "project" +) + +// APIKey is a row of the api_keys table. The secret itself is never stored — +// only sha256 of it — and is shown to the operator exactly once, at creation. +type APIKey struct { + ID string // public half of the token; safe to display and log + SecretHash []byte // sha256(secret), 32 bytes + Scope Scope + ProjectID *int64 // nil for admin keys + Name string + + CreatedAt time.Time + ExpiresAt *time.Time + LastUsedAt *time.Time + RevokedAt *time.Time +} + +// Usable reports whether the key may authenticate a request at time t. +func (k *APIKey) Usable(t time.Time) bool { + if k.RevokedAt != nil { + return false + } + if k.ExpiresAt != nil && !t.Before(*k.ExpiresAt) { + return false + } + return true +} + +const keyColumns = `id, secret_hash, scope, project_id, name, created_at, expires_at, last_used_at, revoked_at` + +func scanKey(row rowScanner) (*APIKey, error) { + var k APIKey + var projectID, expires, lastUsed, revoked sql.NullInt64 + var created int64 + if err := row.Scan(&k.ID, &k.SecretHash, &k.Scope, &projectID, &k.Name, + &created, &expires, &lastUsed, &revoked); err != nil { + return nil, mapErr(err) + } + k.ProjectID = intPtr(projectID) + k.CreatedAt = time.Unix(created, 0).UTC() + k.ExpiresAt = timePtr(expires) + k.LastUsedAt = timePtr(lastUsed) + k.RevokedAt = timePtr(revoked) + return &k, nil +} + +// CreateKey stores a freshly minted key. +func (d *DB) CreateKey(ctx context.Context, k *APIKey) error { + if k.CreatedAt.IsZero() { + k.CreatedAt = time.Unix(unixNow(), 0).UTC() + } + return d.Tx(ctx, func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` + INSERT INTO api_keys (id, secret_hash, scope, project_id, name, created_at, expires_at) + VALUES (?,?,?,?,?,?,?)`, + k.ID, k.SecretHash, string(k.Scope), nullInt(k.ProjectID), k.Name, + k.CreatedAt.Unix(), nullTime(k.ExpiresAt)) + return mapErr(err) + }) +} + +// KeyByID is the authentication lookup: a primary-key hit on a WITHOUT ROWID +// table, never a scan. +func (d *DB) KeyByID(ctx context.Context, id string) (*APIKey, error) { + return scanKey(d.r.QueryRowContext(ctx, `SELECT `+keyColumns+` FROM api_keys WHERE id = ?`, id)) +} + +// ListKeys returns the keys for one project, or every key when projectID is nil. +// Revoked keys are included so an operator can see what was revoked and when. +func (d *DB) ListKeys(ctx context.Context, projectID *int64) ([]*APIKey, error) { + query := `SELECT ` + keyColumns + ` FROM api_keys` + var args []any + if projectID != nil { + query += ` WHERE project_id = ?` + args = append(args, *projectID) + } + query += ` ORDER BY created_at DESC, id` + + rows, err := d.r.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []*APIKey + for rows.Next() { + k, err := scanKey(rows) + if err != nil { + return nil, err + } + out = append(out, k) + } + return out, rows.Err() +} + +// RevokeKey marks a key unusable. It is idempotent: revoking twice keeps the +// first timestamp, because that is when the key actually stopped working. +// +// The caller must invalidate the auth cache afterwards, or the key stays live +// for up to the cache TTL. +func (d *DB) RevokeKey(ctx context.Context, id string) error { + return d.Tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, + `UPDATE api_keys SET revoked_at = ? WHERE id = ? AND revoked_at IS NULL`, + unixNow(), id) + if err != nil { + return mapErr(err) + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + // Either it does not exist or it was already revoked; distinguish, so + // the API can answer 404 versus 204. + var exists int + if err := tx.QueryRowContext(ctx, `SELECT count(*) FROM api_keys WHERE id = ?`, id).Scan(&exists); err != nil { + return err + } + if exists == 0 { + return ErrNotFound + } + } + return nil + }) +} + +// CountUsableAdminKeys counts admin keys that could authenticate right now. A +// zero result on startup is what triggers minting the bootstrap key. +func (d *DB) CountUsableAdminKeys(ctx context.Context) (int, error) { + var n int + err := d.r.QueryRowContext(ctx, ` + SELECT count(*) FROM api_keys + WHERE scope = 'admin' AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at > ?)`, + unixNow()).Scan(&n) + return n, err +} + +// TouchKeys records last-use times in one transaction. +// +// This is deliberately a batch: updating last_used_at on every request would +// funnel every authenticated read through the single write connection, which is +// exactly the contention the two-pool design exists to avoid. The auth layer +// accumulates the timestamps in memory and flushes them periodically, so the +// column is approximate by design — it answers "is this key still in use?", not +// "when exactly was request N". +func (d *DB) TouchKeys(ctx context.Context, seen map[string]time.Time) error { + if len(seen) == 0 { + return nil + } + return d.Tx(ctx, func(tx *sql.Tx) error { + stmt, err := tx.PrepareContext(ctx, + `UPDATE api_keys SET last_used_at = ? WHERE id = ? AND (last_used_at IS NULL OR last_used_at < ?)`) + if err != nil { + return err + } + defer stmt.Close() + for id, t := range seen { + ts := t.Unix() + if _, err := stmt.ExecContext(ctx, ts, id, ts); err != nil { + return err + } + } + return nil + }) +} diff --git a/internal/store/keys_test.go b/internal/store/keys_test.go new file mode 100644 index 0000000..8e41b15 --- /dev/null +++ b/internal/store/keys_test.go @@ -0,0 +1,329 @@ +package store + +import ( + "bytes" + "context" + "errors" + "fmt" + "testing" + "time" +) + +func mkKey(t *testing.T, db *DB, id string, scope Scope, projectID *int64) *APIKey { + t.Helper() + hash := bytes.Repeat([]byte{byte(len(id))}, 32) + k := &APIKey{ID: id, SecretHash: hash, Scope: scope, ProjectID: projectID, Name: "test " + id} + if err := db.CreateKey(context.Background(), k); err != nil { + t.Fatalf("CreateKey(%s): %v", id, err) + } + return k +} + +func TestCreateAndReadKey(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := DefaultProject("demo") + if err := db.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + + admin := mkKey(t, db, "adminkeyid000000", ScopeAdmin, nil) + proj := mkKey(t, db, "projkeyid0000000", ScopeProject, &p.ID) + + got, err := db.KeyByID(ctx, admin.ID) + if err != nil { + t.Fatalf("KeyByID: %v", err) + } + if got.Scope != ScopeAdmin { + t.Errorf("scope = %q, want admin", got.Scope) + } + if got.ProjectID != nil { + t.Errorf("admin key has project_id %v", *got.ProjectID) + } + if !bytes.Equal(got.SecretHash, admin.SecretHash) { + t.Error("secret hash did not round trip") + } + if got.CreatedAt.IsZero() { + t.Error("created_at not set") + } + if got.RevokedAt != nil || got.ExpiresAt != nil || got.LastUsedAt != nil { + t.Errorf("optional timestamps should be nil: %+v", got) + } + + got, err = db.KeyByID(ctx, proj.ID) + if err != nil { + t.Fatal(err) + } + if got.ProjectID == nil || *got.ProjectID != p.ID { + t.Errorf("project key lost its project: %+v", got) + } +} + +func TestKeyNotFound(t *testing.T) { + db := testDB(t) + if _, err := db.KeyByID(context.Background(), "missing000000000"); !errors.Is(err, ErrNotFound) { + t.Errorf("got %v, want ErrNotFound", err) + } +} + +func TestCreateKeyDuplicateID(t *testing.T) { + ctx := context.Background() + db := testDB(t) + mkKey(t, db, "adminkeyid000000", ScopeAdmin, nil) + err := db.CreateKey(ctx, &APIKey{ID: "adminkeyid000000", SecretHash: make([]byte, 32), Scope: ScopeAdmin}) + if !errors.Is(err, ErrExists) { + t.Fatalf("got %v, want ErrExists", err) + } +} + +func TestRevokeKey(t *testing.T) { + ctx := context.Background() + db := testDB(t) + k := mkKey(t, db, "adminkeyid000000", ScopeAdmin, nil) + + if err := db.RevokeKey(ctx, k.ID); err != nil { + t.Fatalf("RevokeKey: %v", err) + } + got, err := db.KeyByID(ctx, k.ID) + if err != nil { + t.Fatal(err) + } + if got.RevokedAt == nil { + t.Fatal("revoked_at not set") + } + first := *got.RevokedAt + if got.Usable(time.Now()) { + t.Error("a revoked key must not be usable") + } + + // Revoking again must be a no-op, not a moved timestamp: the first time is + // when the key actually stopped working. + if err := db.RevokeKey(ctx, k.ID); err != nil { + t.Fatalf("second RevokeKey: %v", err) + } + got, err = db.KeyByID(ctx, k.ID) + if err != nil { + t.Fatal(err) + } + if !got.RevokedAt.Equal(first) { + t.Errorf("revoked_at moved from %v to %v", first, *got.RevokedAt) + } + + if err := db.RevokeKey(ctx, "missing000000000"); !errors.Is(err, ErrNotFound) { + t.Errorf("revoking an unknown key: got %v, want ErrNotFound", err) + } +} + +func TestKeyUsable(t *testing.T) { + now := time.Unix(1_000_000, 0).UTC() + past := now.Add(-time.Hour) + future := now.Add(time.Hour) + + cases := []struct { + name string + key APIKey + wantUse bool + }{ + {"fresh", APIKey{}, true}, + {"revoked", APIKey{RevokedAt: &past}, false}, + {"expired", APIKey{ExpiresAt: &past}, false}, + {"expires later", APIKey{ExpiresAt: &future}, true}, + {"expires exactly now", APIKey{ExpiresAt: &now}, false}, + {"revoked and unexpired", APIKey{RevokedAt: &past, ExpiresAt: &future}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.key.Usable(now); got != tc.wantUse { + t.Errorf("Usable = %v, want %v", got, tc.wantUse) + } + }) + } +} + +func TestListKeys(t *testing.T) { + ctx := context.Background() + db := testDB(t) + a := DefaultProject("alpha") + b := DefaultProject("beta") + if err := db.CreateProject(ctx, a); err != nil { + t.Fatal(err) + } + if err := db.CreateProject(ctx, b); err != nil { + t.Fatal(err) + } + mkKey(t, db, "adminkeyid000000", ScopeAdmin, nil) + mkKey(t, db, "alphakey00000001", ScopeProject, &a.ID) + mkKey(t, db, "alphakey00000002", ScopeProject, &a.ID) + mkKey(t, db, "betakey000000001", ScopeProject, &b.ID) + + all, err := db.ListKeys(ctx, nil) + if err != nil { + t.Fatal(err) + } + if len(all) != 4 { + t.Errorf("ListKeys(nil) returned %d keys, want 4", len(all)) + } + + forA, err := db.ListKeys(ctx, &a.ID) + if err != nil { + t.Fatal(err) + } + if len(forA) != 2 { + t.Fatalf("ListKeys(alpha) returned %d keys, want 2", len(forA)) + } + for _, k := range forA { + if k.ProjectID == nil || *k.ProjectID != a.ID { + t.Errorf("key %s leaked into alpha's list", k.ID) + } + } + + // Revoked keys stay listed so an operator can see what was revoked and when. + if err := db.RevokeKey(ctx, "alphakey00000001"); err != nil { + t.Fatal(err) + } + forA, err = db.ListKeys(ctx, &a.ID) + if err != nil { + t.Fatal(err) + } + if len(forA) != 2 { + t.Errorf("after revoke, ListKeys(alpha) returned %d keys, want 2", len(forA)) + } +} + +// A zero result here is what makes the server mint a bootstrap token, so the +// "usable" definition has to match what the verifier will accept. +func TestCountUsableAdminKeys(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := DefaultProject("demo") + if err := db.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + + if n, err := db.CountUsableAdminKeys(ctx); err != nil || n != 0 { + t.Fatalf("empty database: n=%d err=%v, want 0", n, err) + } + + // A project key is not an admin key. + mkKey(t, db, "projkeyid0000000", ScopeProject, &p.ID) + if n, _ := db.CountUsableAdminKeys(ctx); n != 0 { + t.Errorf("project key counted as admin: n=%d", n) + } + + mkKey(t, db, "adminkeyid000000", ScopeAdmin, nil) + if n, _ := db.CountUsableAdminKeys(ctx); n != 1 { + t.Errorf("n=%d, want 1", n) + } + + // An expired admin key must not keep the server from bootstrapping. + expired := time.Now().Add(-time.Hour) + if err := db.CreateKey(ctx, &APIKey{ + ID: "expiredadmin0000", SecretHash: make([]byte, 32), Scope: ScopeAdmin, ExpiresAt: &expired, + }); err != nil { + t.Fatal(err) + } + if n, _ := db.CountUsableAdminKeys(ctx); n != 1 { + t.Errorf("expired admin key counted: n=%d", n) + } + + if err := db.RevokeKey(ctx, "adminkeyid000000"); err != nil { + t.Fatal(err) + } + if n, _ := db.CountUsableAdminKeys(ctx); n != 0 { + t.Errorf("revoked admin key counted: n=%d", n) + } +} + +func TestTouchKeys(t *testing.T) { + ctx := context.Background() + db := testDB(t) + mkKey(t, db, "key0000000000001", ScopeAdmin, nil) + mkKey(t, db, "key0000000000002", ScopeAdmin, nil) + + if err := db.TouchKeys(ctx, nil); err != nil { + t.Errorf("empty batch should be a no-op: %v", err) + } + + t1 := time.Unix(1_700_000_000, 0) + if err := db.TouchKeys(ctx, map[string]time.Time{ + "key0000000000001": t1, + "key0000000000002": t1, + // A key that vanished between the request and the flush must not fail + // the whole batch, or one deleted key would stall the flusher forever. + "deletedkey000000": t1, + }); err != nil { + t.Fatalf("TouchKeys: %v", err) + } + lastUsed := func(id string) *time.Time { + t.Helper() + k, err := db.KeyByID(ctx, id) + if err != nil { + t.Fatal(err) + } + return k.LastUsedAt + } + if got := lastUsed("key0000000000001"); got == nil || !got.Equal(t1.UTC()) { + t.Errorf("last_used_at = %v, want %v", got, t1.UTC()) + } + + // Batches can arrive out of order once the flusher runs concurrently with a + // retry; an older timestamp must not walk the column backwards. + older := t1.Add(-time.Hour) + if err := db.TouchKeys(ctx, map[string]time.Time{"key0000000000001": older}); err != nil { + t.Fatal(err) + } + if got := lastUsed("key0000000000001"); !got.Equal(t1.UTC()) { + t.Errorf("last_used_at moved backwards to %v", got) + } + + newer := t1.Add(time.Hour) + if err := db.TouchKeys(ctx, map[string]time.Time{"key0000000000001": newer}); err != nil { + t.Fatal(err) + } + if got := lastUsed("key0000000000001"); !got.Equal(newer.UTC()) { + t.Errorf("last_used_at = %v, want %v", got, newer.UTC()) + } +} + +// The write pool holds a single connection, so a batch flush must not need more +// than one; this would deadlock if TouchKeys opened a nested transaction. +func TestTouchKeysLargeBatch(t *testing.T) { + ctx := context.Background() + db := testDB(t) + seen := map[string]time.Time{} + now := time.Unix(1_700_000_000, 0) + for i := 0; i < 200; i++ { + id := fmt.Sprintf("key%013d", i) + mkKey(t, db, id, ScopeAdmin, nil) + seen[id] = now + } + if err := db.TouchKeys(ctx, seen); err != nil { + t.Fatalf("TouchKeys: %v", err) + } + var n int + if err := db.Reader().QueryRow( + `SELECT count(*) FROM api_keys WHERE last_used_at = ?`, now.Unix()).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 200 { + t.Errorf("%d keys touched, want 200", n) + } +} + +// Keys must die with their project, or a project name could be recreated and +// inherit the old owner's credentials. +func TestKeysCascadeWithProject(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := DefaultProject("demo") + if err := db.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + mkKey(t, db, "projkeyid0000000", ScopeProject, &p.ID) + if err := db.DeleteProject(ctx, p.ID); err != nil { + t.Fatal(err) + } + if _, err := db.KeyByID(ctx, "projkeyid0000000"); !errors.Is(err, ErrNotFound) { + t.Errorf("key survived its project: %v", err) + } +} diff --git a/internal/store/maintenance.go b/internal/store/maintenance.go new file mode 100644 index 0000000..3be0c2e --- /dev/null +++ b/internal/store/maintenance.go @@ -0,0 +1,344 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/iceBear67/simplepages/internal/cas" +) + +// This file holds the queries the background jobs use: startup recovery, the +// retention sweep and the blob collector. +// +// They are kept apart from the request-path queries because they answer a +// different question. A handler works on state a transaction has just +// established; these run against whatever a crash, a restored backup or a +// half-finished sweep left behind, so each one has to be safe to run against a +// world that already disagrees with itself, and safe to run again after being +// interrupted partway. + +// DeploymentRef names a deployment by the two values its directory is built +// from, which is all an orphan sweep needs to know. +type DeploymentRef struct { + ProjectID int64 + PublicID string +} + +// AllDeploymentRefs lists every deployment the database knows about. +// +// Only the identifying pair, because the caller compares it against directory +// names: reading full rows for a sweep that will normally delete nothing would +// be a lot of scanning for no answer that changes. +func (d *DB) AllDeploymentRefs(ctx context.Context) ([]DeploymentRef, error) { + rows, err := d.r.QueryContext(ctx, `SELECT project_id, public_id FROM deployments`) + if err != nil { + return nil, err + } + defer rows.Close() + + var refs []DeploymentRef + for rows.Next() { + var ref DeploymentRef + if err := rows.Scan(&ref.ProjectID, &ref.PublicID); err != nil { + return nil, err + } + refs = append(refs, ref) + } + return refs, rows.Err() +} + +// DeploymentsInState lists deployments in one state across every project, +// oldest first. Recovery uses it to find the rows a previous sweep was in the +// middle of deleting. +func (d *DB) DeploymentsInState(ctx context.Context, state State, limit int) ([]*Deployment, error) { + if limit <= 0 { + limit = 1000 + } + rows, err := d.r.QueryContext(ctx, + `SELECT `+deploymentColumns+` FROM deployments WHERE state = ? ORDER BY id LIMIT ?`, + string(state), limit) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []*Deployment + for rows.Next() { + dep, err := scanDeployment(rows) + if err != nil { + return nil, err + } + out = append(out, dep) + } + return out, rows.Err() +} + +// InactiveDeployments lists one project's deployments that are not the one it +// serves, newest first. This is the input to the retention decision, and the +// active deployment is excluded here rather than filtered later so that no +// arithmetic on the caller's side can ever select it. +func (d *DB) InactiveDeployments(ctx context.Context, projectID int64) ([]*Deployment, error) { + rows, err := d.r.QueryContext(ctx, + `SELECT `+deploymentColumns+` FROM deployments + WHERE project_id = ? AND active = 0 ORDER BY id DESC`, projectID) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []*Deployment + for rows.Next() { + dep, err := scanDeployment(rows) + if err != nil { + return nil, err + } + out = append(out, dep) + } + return out, rows.Err() +} + +// ExpireStaleDeployments fails uploads that were started before cutoff and +// never finished, dropping their manifests so the blobs only they referenced +// become collectable. +// +// Nothing distinguishes a CI job that died from one that is merely slow except +// how long it has been, which is why the cutoff wants to be generous: expiring +// an upload that was still going to succeed turns a slow deploy into a failed +// one. Their blobs survive regardless — they are content-addressed, so the +// retry finds them already present and skips them. +func (d *DB) ExpireStaleDeployments(ctx context.Context, cutoff time.Time, reason string) (int, error) { + var n int + err := d.Tx(ctx, func(tx *sql.Tx) error { + // Tx re-runs this on a busy database, so the count is rebuilt from + // scratch on each attempt rather than added to. + n = 0 + + rows, err := tx.QueryContext(ctx, + `SELECT id FROM deployments WHERE state IN (?, ?) AND created_at < ?`, + StatePending, StateUploading, cutoff.Unix()) + if err != nil { + return err + } + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + rows.Close() + return err + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + + for _, id := range ids { + // The manifest rows go first: their delete trigger is what takes the + // refcounts back down, and it is the only reason expiry frees + // anything. + if _, err := tx.ExecContext(ctx, + `DELETE FROM deployment_files WHERE deployment_id = ?`, id); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, + `UPDATE deployments SET state = ?, error = ? WHERE id = ?`, + StateFailed, truncate(reason, 1024), id); err != nil { + return err + } + n++ + } + return nil + }) + if err != nil { + return 0, err + } + return n, nil +} + +// MarkDeploymentDeleting claims a deployment for deletion. +// +// The state change is committed before any file is removed, so a crash midway +// through leaves a row that says what was happening and recovery can finish the +// job. The active deployment can never be claimed: that check is here, in the +// same statement, rather than in the caller. +func (d *DB) MarkDeploymentDeleting(ctx context.Context, id int64) error { + return d.Tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, + `UPDATE deployments SET state = ? WHERE id = ? AND active = 0`, StateDeleting, id) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n > 0 { + return nil + } + // Nothing matched, so either the row is gone — mapErr turns that into + // ErrNotFound — or it is the active one, which is the only condition the + // statement excludes. + var active bool + if err := tx.QueryRowContext(ctx, + `SELECT active FROM deployments WHERE id = ?`, id).Scan(&active); err != nil { + return mapErr(err) + } + return fmt.Errorf("%w: this deployment is the one the project is serving", ErrConflict) + }) +} + +// DeleteDeployment removes a deployment's rows once its tree is gone. +// +// The manifest rows are deleted explicitly rather than left to ON DELETE +// CASCADE. SQLite does not fire a child table's triggers for cascaded deletes +// unless recursive_triggers is on, and relying on that pragma would make every +// blob's refcount depend on a connection setting; deleting the rows here makes +// the decrement unconditional. +func (d *DB) DeleteDeployment(ctx context.Context, id int64) error { + return d.Tx(ctx, func(tx *sql.Tx) error { + var active bool + if err := tx.QueryRowContext(ctx, + `SELECT active FROM deployments WHERE id = ?`, id).Scan(&active); err != nil { + return mapErr(err) + } + if active { + return fmt.Errorf("%w: this deployment is the one the project is serving", ErrConflict) + } + if _, err := tx.ExecContext(ctx, + `DELETE FROM deployment_files WHERE deployment_id = ?`, id); err != nil { + return err + } + _, err := tx.ExecContext(ctx, `DELETE FROM deployments WHERE id = ?`, id) + return err + }) +} + +// EachPresentBlob calls fn for every digest the database believes is on disk. +// +// Streamed rather than returned as a slice because the caller wants to stat each +// one and a large store has a lot of them; a read here never blocks a write, so +// holding the cursor open across the filesystem calls costs nothing. +func (d *DB) EachPresentBlob(ctx context.Context, fn func(cas.Digest) error) error { + rows, err := d.r.QueryContext(ctx, `SELECT digest FROM blobs WHERE present = 1`) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return err + } + dg, err := cas.FromBytes(raw) + if err != nil { + return err + } + if err := fn(dg); err != nil { + return err + } + } + return rows.Err() +} + +// MarkBlobsAbsent records that content the database claimed is on disk is not. +// +// The rows stay. A manifest may still reference them, and the fix is not to +// forget the blob but to ask for it again: the next deploy that names one of +// these digests is told to upload it, and every deployment that referenced it +// becomes deployable again as soon as one does. +func (d *DB) MarkBlobsAbsent(ctx context.Context, digests []cas.Digest) error { + if len(digests) == 0 { + return nil + } + return d.Tx(ctx, func(tx *sql.Tx) error { + stmt, err := tx.PrepareContext(ctx, `UPDATE blobs SET present = 0 WHERE digest = ?`) + if err != nil { + return err + } + defer stmt.Close() + for _, dg := range digests { + if _, err := stmt.ExecContext(ctx, dg.Bytes()); err != nil { + return err + } + } + return nil + }) +} + +// UnreferencedBlobs lists blobs no manifest has referenced since before cutoff. +// +// Both halves of the condition matter. A refcount of zero says nothing points at +// the content now; the cutoff adds that nothing has pointed at it for a while, +// which is what gives a request that has already resolved a digest and is about +// to open it time to finish. Oldest first, so a backlog drains in a stable order +// rather than the collector revisiting the same rows every sweep. +func (d *DB) UnreferencedBlobs(ctx context.Context, cutoff time.Time, limit int) ([]Blob, error) { + if limit <= 0 { + limit = 5000 + } + rows, err := d.r.QueryContext(ctx, ` + SELECT digest, size, present FROM blobs + WHERE refcount = 0 AND last_ref_at < ? + ORDER BY last_ref_at LIMIT ?`, cutoff.Unix(), limit) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Blob + for rows.Next() { + var b Blob + var raw []byte + if err := rows.Scan(&raw, &b.Size, &b.Present); err != nil { + return nil, err + } + if b.Digest, err = cas.FromBytes(raw); err != nil { + return nil, err + } + out = append(out, b) + } + return out, rows.Err() +} + +// DeleteBlob removes one blob's row and its content together, reporting whether +// there was anything to remove. +// +// remove runs inside the write transaction, and that is the whole point of this +// signature. There is exactly one write connection, so no manifest can be +// accepted between the row disappearing and the file doing so — which closes the +// window where a deployment could come to reference content that was already on +// its way out. remove has to be idempotent, because Tx re-runs its function on a +// busy database; cas.Store.Remove is, deliberately. +// +// A blob that gained a reference since it was listed is left alone and reported +// as false. If remove fails the row survives with it, and the next sweep finds +// the pair again. +func (d *DB) DeleteBlob(ctx context.Context, digest cas.Digest, remove func() error) (bool, error) { + var deleted bool + err := d.Tx(ctx, func(tx *sql.Tx) error { + deleted = false + res, err := tx.ExecContext(ctx, + `DELETE FROM blobs WHERE digest = ? AND refcount = 0`, digest.Bytes()) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + return nil + } + if err := remove(); err != nil { + return err + } + deleted = true + return nil + }) + return deleted, err +} diff --git a/internal/store/maintenance_test.go b/internal/store/maintenance_test.go new file mode 100644 index 0000000..0a0383f --- /dev/null +++ b/internal/store/maintenance_test.go @@ -0,0 +1,623 @@ +package store + +import ( + "context" + "errors" + "strconv" + "testing" + "time" + + "github.com/iceBear67/simplepages/internal/cas" +) + +// blobRow reads a blob's bookkeeping directly, so a test can assert on the +// columns the collector reads rather than on what a helper reports. +func blobRow(t *testing.T, db *DB, d cas.Digest) (refcount int64, present bool, lastRef int64) { + t.Helper() + err := db.Reader().QueryRow( + `SELECT refcount, present, last_ref_at FROM blobs WHERE digest = ?`, d.Bytes()). + Scan(&refcount, &present, &lastRef) + if err != nil { + t.Fatalf("blob %s: %v", d, err) + } + return +} + +func blobExists(t *testing.T, db *DB, d cas.Digest) bool { + t.Helper() + var n int + if err := db.Reader().QueryRow( + `SELECT count(*) FROM blobs WHERE digest = ?`, d.Bytes()).Scan(&n); err != nil { + t.Fatal(err) + } + return n == 1 +} + +func TestAllDeploymentRefs(t *testing.T) { + ctx := context.Background() + db := testDB(t) + a := testProject(t, db, "a") + b := testProject(t, db, "b") + + want := map[string]int64{} + for range 3 { + dep := testDeployment(t, db, a.ID) + want[dep.PublicID] = a.ID + } + dep := testDeployment(t, db, b.ID) + want[dep.PublicID] = b.ID + + refs, err := db.AllDeploymentRefs(ctx) + if err != nil { + t.Fatal(err) + } + if len(refs) != len(want) { + t.Fatalf("got %d refs, want %d", len(refs), len(want)) + } + for _, ref := range refs { + pid, ok := want[ref.PublicID] + if !ok { + t.Errorf("unexpected public id %q", ref.PublicID) + continue + } + if ref.ProjectID != pid { + t.Errorf("%s belongs to project %d, want %d", ref.PublicID, ref.ProjectID, pid) + } + delete(want, ref.PublicID) + } +} + +func TestDeploymentsInStateSpansProjects(t *testing.T) { + ctx := context.Background() + db := testDB(t) + a := testProject(t, db, "a") + b := testProject(t, db, "b") + + // Recovery has to find interrupted deletions wherever they are, so this + // query is deliberately not project-scoped. + first := testDeployment(t, db, a.ID) + second := testDeployment(t, db, b.ID) + for _, dep := range []*Deployment{first, second} { + if _, err := db.w.ExecContext(ctx, + `UPDATE deployments SET state = ? WHERE id = ?`, StateDeleting, dep.ID); err != nil { + t.Fatal(err) + } + } + testDeployment(t, db, a.ID) // still pending; must not be listed + + got, err := db.DeploymentsInState(ctx, StateDeleting, 0) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("got %d deleting deployments, want 2", len(got)) + } + if got[0].ID != first.ID || got[1].ID != second.ID { + t.Errorf("order is %d,%d; want oldest first (%d,%d)", + got[0].ID, got[1].ID, first.ID, second.ID) + } +} + +// The active deployment is excluded in SQL rather than filtered by the caller, +// so no retention arithmetic can select the one being served. +func TestInactiveDeploymentsExcludesTheActiveOne(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + other := testProject(t, db, "other") + testDeployment(t, db, other.ID) + + var deps []*Deployment + for i := range 3 { + deps = append(deps, ready(t, db, p.ID, string(rune('a'+i)))) + } + if err := db.ActivateDeployment(ctx, p.ID, deps[1].ID); err != nil { + t.Fatal(err) + } + + got, err := db.InactiveDeployments(ctx, p.ID) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("got %d inactive deployments, want 2", len(got)) + } + if got[0].ID != deps[2].ID || got[1].ID != deps[0].ID { + t.Errorf("order is %d,%d; want newest first (%d,%d)", + got[0].ID, got[1].ID, deps[2].ID, deps[0].ID) + } +} + +func TestExpireStaleDeployments(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + shared := file("shared.js", "shared") + only := file("only.html", "abandoned") + + // One upload that stalled, one that is merely young, one that finished. + stale := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, stale.ID, []FileRow{shared, only}); err != nil { + t.Fatal(err) + } + young := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, young.ID, []FileRow{shared}); err != nil { + t.Fatal(err) + } + done := ready(t, db, p.ID, "finished") + + // Age the stalled one past the cutoff. Backdating the row is the only way + // to test this without the test sleeping. + if _, err := db.w.ExecContext(ctx, + `UPDATE deployments SET created_at = ? WHERE id = ?`, + time.Now().Add(-48*time.Hour).Unix(), stale.ID); err != nil { + t.Fatal(err) + } + + n, err := db.ExpireStaleDeployments(ctx, time.Now().Add(-24*time.Hour), "abandoned") + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("expired %d deployments, want 1", n) + } + + got, err := db.DeploymentByPublicID(ctx, p.ID, stale.PublicID) + if err != nil { + t.Fatal(err) + } + if got.State != StateFailed { + t.Errorf("state = %q, want %q", got.State, StateFailed) + } + if got.Error != "abandoned" { + t.Errorf("error = %q, want the reason to be recorded", got.Error) + } + + // Dropping the manifest is the point: it is what takes the refcounts back + // down so the collector can reach the content. + if rc, _, _ := blobRow(t, db, only.Digest); rc != 0 { + t.Errorf("refcount of the abandoned file = %d, want 0", rc) + } + // Content the surviving upload also names keeps its reference. + if rc, _, _ := blobRow(t, db, shared.Digest); rc != 1 { + t.Errorf("refcount of the shared file = %d, want 1 (the young upload still names it)", rc) + } + + for _, dep := range []*Deployment{young, done} { + got, err := db.DeploymentByPublicID(ctx, p.ID, dep.PublicID) + if err != nil { + t.Fatal(err) + } + if got.State == StateFailed { + t.Errorf("deployment %s was expired but should not have been", dep.PublicID) + } + } + + // Running it again finds nothing left to do. + n, err = db.ExpireStaleDeployments(ctx, time.Now().Add(-24*time.Hour), "abandoned") + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("a second pass expired %d more, want 0", n) + } +} + +func TestMarkDeploymentDeleting(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + active := ready(t, db, p.ID, "v1") + spare := ready(t, db, p.ID, "v2") + if err := db.ActivateDeployment(ctx, p.ID, active.ID); err != nil { + t.Fatal(err) + } + + // The one being served is refused, and by the same statement that would + // have claimed it — there is no window between the check and the claim. + if err := db.MarkDeploymentDeleting(ctx, active.ID); !errors.Is(err, ErrConflict) { + t.Fatalf("claiming the active deployment = %v, want ErrConflict", err) + } + got, err := db.DeploymentByPublicID(ctx, p.ID, active.PublicID) + if err != nil { + t.Fatal(err) + } + if got.State != StateReady { + t.Errorf("the refused claim changed the state to %q", got.State) + } + + if err := db.MarkDeploymentDeleting(ctx, spare.ID); err != nil { + t.Fatal(err) + } + got, err = db.DeploymentByPublicID(ctx, p.ID, spare.PublicID) + if err != nil { + t.Fatal(err) + } + if got.State != StateDeleting { + t.Errorf("state = %q, want %q", got.State, StateDeleting) + } + + // Re-claiming is fine: a collector that was interrupted after the claim and + // before the removal has to be able to pick the row up again. + if err := db.MarkDeploymentDeleting(ctx, spare.ID); err != nil { + t.Errorf("re-claiming a deployment already being deleted: %v", err) + } + + if err := db.MarkDeploymentDeleting(ctx, 99999); !errors.Is(err, ErrNotFound) { + t.Errorf("claiming a missing deployment = %v, want ErrNotFound", err) + } +} + +func TestDeleteDeploymentDropsManifestRefsExplicitly(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + shared := file("shared.js", "shared") + only := file("index.html", "gone") + dep := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{shared, only}); err != nil { + t.Fatal(err) + } + keeper := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, keeper.ID, []FileRow{shared}); err != nil { + t.Fatal(err) + } + + if err := db.DeleteDeployment(ctx, dep.ID); err != nil { + t.Fatal(err) + } + if _, err := db.DeploymentByPublicID(ctx, p.ID, dep.PublicID); !errors.Is(err, ErrNotFound) { + t.Errorf("the row survived deletion: %v", err) + } + // The AFTER DELETE trigger has to have fired. A cascaded delete would not + // have fired it, which is why the manifest rows are deleted by hand first. + if rc, _, _ := blobRow(t, db, only.Digest); rc != 0 { + t.Errorf("refcount = %d after the only reference was deleted, want 0", rc) + } + if rc, _, _ := blobRow(t, db, shared.Digest); rc != 1 { + t.Errorf("refcount of shared content = %d, want 1", rc) + } + + if err := db.DeleteDeployment(ctx, dep.ID); !errors.Is(err, ErrNotFound) { + t.Errorf("deleting again = %v, want ErrNotFound", err) + } +} + +func TestDeleteDeploymentRefusesTheActiveOne(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + dep := ready(t, db, p.ID, "v1") + if err := db.ActivateDeployment(ctx, p.ID, dep.ID); err != nil { + t.Fatal(err) + } + + if err := db.DeleteDeployment(ctx, dep.ID); !errors.Is(err, ErrConflict) { + t.Fatalf("deleting the active deployment = %v, want ErrConflict", err) + } + if _, err := db.ActiveDeployment(ctx, p.ID); err != nil { + t.Errorf("the project stopped serving anything: %v", err) + } +} + +func TestEachPresentBlobAndMarkBlobsAbsent(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + here := file("here.txt", "here") + gone := file("gone.txt", "gone") + pending := file("pending.txt", "pending") + dep := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{here, gone, pending}); err != nil { + t.Fatal(err) + } + for _, f := range []FileRow{here, gone} { + if err := db.MarkBlobPresent(ctx, f.Digest, f.Size); err != nil { + t.Fatal(err) + } + } + + seen := map[string]bool{} + if err := db.EachPresentBlob(ctx, func(d cas.Digest) error { + seen[d.String()] = true + return nil + }); err != nil { + t.Fatal(err) + } + if len(seen) != 2 || !seen[here.Digest.String()] || !seen[gone.Digest.String()] { + t.Fatalf("present blobs = %v, want exactly the two uploaded ones", seen) + } + if seen[pending.Digest.String()] { + t.Error("a blob that was never uploaded was reported as present") + } + + if err := db.MarkBlobsAbsent(ctx, []cas.Digest{gone.Digest}); err != nil { + t.Fatal(err) + } + // The row stays and keeps its reference: the fix for missing content is to + // ask for it again, not to forget the deployment needs it. + rc, present, _ := blobRow(t, db, gone.Digest) + if present { + t.Error("the blob is still marked present") + } + if rc != 1 { + t.Errorf("refcount = %d, want the manifest reference to survive", rc) + } + if _, present, _ := blobRow(t, db, here.Digest); !present { + t.Error("an unrelated blob was marked absent") + } + + // An empty list is a no-op rather than a statement with no arguments. + if err := db.MarkBlobsAbsent(ctx, nil); err != nil { + t.Errorf("MarkBlobsAbsent(nil): %v", err) + } + + // The callback's error stops the walk and reaches the caller. + stop := errors.New("stop") + if err := db.EachPresentBlob(ctx, func(cas.Digest) error { return stop }); !errors.Is(err, stop) { + t.Errorf("EachPresentBlob swallowed the callback error: %v", err) + } +} + +func TestUnreferencedBlobs(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + kept := file("kept.txt", "kept") + dropped := file("dropped.txt", "dropped") + dep := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{kept, dropped}); err != nil { + t.Fatal(err) + } + if err := db.MarkBlobPresent(ctx, dropped.Digest, dropped.Size); err != nil { + t.Fatal(err) + } + + // Referenced content is never listed, however old. + got, err := db.UnreferencedBlobs(ctx, time.Now().Add(time.Hour), 0) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("listed %d referenced blobs, want 0", len(got)) + } + + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{kept}); err != nil { + t.Fatal(err) + } + + // Freshly unreferenced content is protected by the cutoff, which is what + // gives a request that has already resolved the digest time to open it. + got, err = db.UnreferencedBlobs(ctx, time.Now().Add(-time.Hour), 0) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("listed %d blobs inside the grace period, want 0", len(got)) + } + + got, err = db.UnreferencedBlobs(ctx, time.Now().Add(time.Hour), 0) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("listed %d blobs past the cutoff, want 1", len(got)) + } + if got[0].Digest != dropped.Digest { + t.Errorf("listed %s, want the dereferenced %s", got[0].Digest, dropped.Digest) + } + if got[0].Size != dropped.Size || !got[0].Present { + t.Errorf("listed blob = %+v, want the size and presence the collector needs", got[0]) + } +} + +func TestDeleteBlobRemovesRowAndContentTogether(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + orphan := file("orphan.txt", "orphan") + held := file("held.txt", "held") + dep := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{orphan, held}); err != nil { + t.Fatal(err) + } + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{held}); err != nil { + t.Fatal(err) + } + + removed := 0 + deleted, err := db.DeleteBlob(ctx, orphan.Digest, func() error { removed++; return nil }) + if err != nil { + t.Fatal(err) + } + if !deleted || removed != 1 { + t.Fatalf("deleted = %v, remove called %d times; want true and once", deleted, removed) + } + if blobExists(t, db, orphan.Digest) { + t.Error("the row survived") + } + + // A blob that gained a reference since it was listed is left alone, and the + // content is not touched — this is the check that keeps a deploy racing the + // collector from losing its files. + removed = 0 + deleted, err = db.DeleteBlob(ctx, held.Digest, func() error { removed++; return nil }) + if err != nil { + t.Fatal(err) + } + if deleted || removed != 0 { + t.Errorf("a referenced blob was collected: deleted = %v, remove called %d times", deleted, removed) + } + if !blobExists(t, db, held.Digest) { + t.Error("a referenced blob's row was deleted") + } + + // Nothing to delete is not an error; the previous sweep already did it. + deleted, err = db.DeleteBlob(ctx, orphan.Digest, func() error { + t.Error("remove was called for a row that no longer exists") + return nil + }) + if err != nil || deleted { + t.Errorf("re-deleting = (%v, %v), want (false, nil)", deleted, err) + } +} + +// If the content cannot be removed the row has to survive with it, so the next +// sweep finds the pair again rather than leaving a file nothing points at. +func TestDeleteBlobKeepsTheRowWhenRemovalFails(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + f := file("orphan.txt", "orphan") + dep := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, dep.ID, []FileRow{f}); err != nil { + t.Fatal(err) + } + if _, _, err := db.SetManifest(ctx, dep.ID, nil); err != nil { + t.Fatal(err) + } + + boom := errors.New("disk is having a day") + deleted, err := db.DeleteBlob(ctx, f.Digest, func() error { return boom }) + if !errors.Is(err, boom) { + t.Fatalf("DeleteBlob = %v, want the removal error", err) + } + if deleted { + t.Error("reported a deletion that was rolled back") + } + if !blobExists(t, db, f.Digest) { + t.Fatal("the row was deleted even though the content was not") + } +} + +func TestFsckFindsAndRepairsDrift(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + shared := file("shared.js", "shared") + one := file("one.html", "one") + two := file("two.html", "two") + first := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, first.ID, []FileRow{shared, one}); err != nil { + t.Fatal(err) + } + second := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, second.ID, []FileRow{shared, two}); err != nil { + t.Fatal(err) + } + + // Triggers maintain these counters, so a healthy store never drifts. + rep, err := db.Fsck(ctx, false) + if err != nil { + t.Fatal(err) + } + if rep.Blobs != 3 { + t.Errorf("examined %d blobs, want 3", rep.Blobs) + } + if rep.DriftCount != 0 { + t.Fatalf("a healthy store reported %d drifting blobs: %+v", rep.DriftCount, rep.Drift) + } + + // Injecting drift in both directions. Low is the dangerous one: the + // collector trusts the counter, so a count that reads low is content it + // will delete while a deployment still names it. + if _, err := db.w.ExecContext(ctx, + `UPDATE blobs SET refcount = 0 WHERE digest = ?`, shared.Digest.Bytes()); err != nil { + t.Fatal(err) + } + if _, err := db.w.ExecContext(ctx, + `UPDATE blobs SET refcount = 7 WHERE digest = ?`, one.Digest.Bytes()); err != nil { + t.Fatal(err) + } + + rep, err = db.Fsck(ctx, false) + if err != nil { + t.Fatal(err) + } + if rep.DriftCount != 2 { + t.Fatalf("found %d drifting blobs, want 2: %+v", rep.DriftCount, rep.Drift) + } + if rep.Repaired != 0 { + t.Errorf("a report-only check repaired %d rows", rep.Repaired) + } + found := map[string]Drift{} + for _, dr := range rep.Drift { + found[dr.Digest.String()] = dr + } + if dr := found[shared.Digest.String()]; dr.Stored != 0 || dr.Actual != 2 { + t.Errorf("shared drift = %+v, want stored 0 and actual 2", dr) + } + if dr := found[one.Digest.String()]; dr.Stored != 7 || dr.Actual != 1 { + t.Errorf("one.html drift = %+v, want stored 7 and actual 1", dr) + } + // Report-only means exactly that. + if rc, _, _ := blobRow(t, db, shared.Digest); rc != 0 { + t.Errorf("refcount = %d, want the injected value left alone", rc) + } + + rep, err = db.Fsck(ctx, true) + if err != nil { + t.Fatal(err) + } + if rep.Repaired != 2 { + t.Errorf("repaired %d rows, want 2", rep.Repaired) + } + if rc, _, _ := blobRow(t, db, shared.Digest); rc != 2 { + t.Errorf("refcount after repair = %d, want the recounted 2", rc) + } + if rc, _, _ := blobRow(t, db, one.Digest); rc != 1 { + t.Errorf("refcount after repair = %d, want the recounted 1", rc) + } + + rep, err = db.Fsck(ctx, true) + if err != nil { + t.Fatal(err) + } + if rep.DriftCount != 0 || rep.Repaired != 0 { + t.Errorf("a repaired store still reports %d drift, %d repaired", rep.DriftCount, rep.Repaired) + } +} + +// The list is for a human to read; the repair is not bounded by it. +func TestFsckCapsTheReportedDriftButRepairsEverything(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := testProject(t, db, "demo") + + const n = maxReportedDrift + 10 + rows := make([]FileRow, 0, n) + for i := range n { + rows = append(rows, file("f"+strconv.Itoa(i)+".txt", "content-"+strconv.Itoa(i))) + } + dep := testDeployment(t, db, p.ID) + if _, _, err := db.SetManifest(ctx, dep.ID, rows); err != nil { + t.Fatal(err) + } + if _, err := db.w.ExecContext(ctx, `UPDATE blobs SET refcount = 42`); err != nil { + t.Fatal(err) + } + + rep, err := db.Fsck(ctx, true) + if err != nil { + t.Fatal(err) + } + if rep.DriftCount != n { + t.Errorf("DriftCount = %d, want the full %d", rep.DriftCount, n) + } + if len(rep.Drift) != maxReportedDrift { + t.Errorf("listed %d drifting blobs, want the report capped at %d", len(rep.Drift), maxReportedDrift) + } + if rep.Repaired != n { + t.Errorf("repaired %d rows, want all %d", rep.Repaired, n) + } + if rep, err = db.Fsck(ctx, false); err != nil || rep.DriftCount != 0 { + t.Errorf("after repair: %d drift, err %v", rep.DriftCount, err) + } +} diff --git a/internal/store/migrate.go b/internal/store/migrate.go new file mode 100644 index 0000000..dcbdcb0 --- /dev/null +++ b/internal/store/migrate.go @@ -0,0 +1,182 @@ +package store + +import ( + "context" + "crypto/sha256" + "database/sql" + "embed" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "path" + "sort" + "strconv" + "strings" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +// migration is one numbered file from migrations/. +type migration struct { + version int + name string + body string + checksum string +} + +// migrate applies every migration the database has not seen yet. +// +// SQLite runs DDL inside transactions, so each migration either lands whole or +// not at all — there is no dirty state for an operator to repair by hand, which +// is the main thing an external migration tool buys elsewhere. What is worth +// keeping is the checksum: it catches an already-applied migration file being +// edited afterwards, which otherwise produces two divergent schemas that both +// claim to be at the same version. +func (d *DB) migrate(ctx context.Context) error { + migrations, err := loadMigrations() + if err != nil { + return err + } + + if err := d.Tx(ctx, func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` + CREATE TABLE IF NOT EXISTS schema_version ( + version INTEGER NOT NULL PRIMARY KEY, + checksum TEXT NOT NULL, + applied_at INTEGER NOT NULL + )`) + return err + }); err != nil { + return fmt.Errorf("create schema_version: %w", err) + } + + applied := map[int]string{} + rows, err := d.r.QueryContext(ctx, `SELECT version, checksum FROM schema_version`) + if err != nil { + return fmt.Errorf("read schema_version: %w", err) + } + defer rows.Close() + for rows.Next() { + var v int + var sum string + if err := rows.Scan(&v, &sum); err != nil { + return err + } + applied[v] = sum + } + if err := rows.Err(); err != nil { + return err + } + rows.Close() + + for _, m := range migrations { + if sum, ok := applied[m.version]; ok { + if sum != m.checksum { + return fmt.Errorf( + "migration %s was modified after it was applied (recorded %s, now %s); "+ + "add a new migration instead of editing an applied one", + m.name, sum, m.checksum) + } + continue + } + if d.log != nil { + d.log.Info("applying migration", "version", m.version, "name", m.name) + } + if err := d.Tx(ctx, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, m.body); err != nil { + return err + } + _, err := tx.ExecContext(ctx, + `INSERT INTO schema_version (version, checksum, applied_at) VALUES (?, ?, ?)`, + m.version, m.checksum, unixNow()) + return err + }); err != nil { + return fmt.Errorf("migration %s: %w", m.name, err) + } + } + + // A database from a newer build is not something this binary can serve + // safely: it may be missing columns the newer code added. + for v := range applied { + if !hasVersion(migrations, v) { + return fmt.Errorf( + "database is at schema version %d, which this build does not know about; "+ + "it was probably written by a newer pages-server", v) + } + } + return nil +} + +func hasVersion(ms []migration, v int) bool { + for _, m := range ms { + if m.version == v { + return true + } + } + return false +} + +// loadMigrations reads the embedded files, ordered by version. +func loadMigrations() ([]migration, error) { + entries, err := fs.ReadDir(migrationsFS, "migrations") + if err != nil { + return nil, err + } + var out []migration + seen := map[int]string{} + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") { + continue + } + version, err := parseVersion(e.Name()) + if err != nil { + return nil, err + } + if prev, dup := seen[version]; dup { + return nil, fmt.Errorf("migrations %s and %s share version %d", prev, e.Name(), version) + } + seen[version] = e.Name() + + body, err := migrationsFS.ReadFile(path.Join("migrations", e.Name())) + if err != nil { + return nil, err + } + sum := sha256.Sum256(body) + out = append(out, migration{ + version: version, + name: e.Name(), + body: string(body), + checksum: hex.EncodeToString(sum[:]), + }) + } + if len(out) == 0 { + return nil, errors.New("no migrations embedded") + } + sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version }) + return out, nil +} + +// parseVersion reads the leading number of "0001_init.sql". +func parseVersion(name string) (int, error) { + base, _, ok := strings.Cut(name, "_") + if !ok { + return 0, fmt.Errorf("migration %q: want NNNN_name.sql", name) + } + v, err := strconv.Atoi(base) + if err != nil || v <= 0 { + return 0, fmt.Errorf("migration %q: want a positive leading version number", name) + } + return v, nil +} + +// SchemaVersion reports the highest applied migration version. +func (d *DB) SchemaVersion(ctx context.Context) (int, error) { + var v sql.NullInt64 + err := d.r.QueryRowContext(ctx, `SELECT max(version) FROM schema_version`).Scan(&v) + if err != nil { + return 0, err + } + return int(v.Int64), nil +} diff --git a/internal/store/migrations/0001_init.sql b/internal/store/migrations/0001_init.sql new file mode 100644 index 0000000..c6b1c02 --- /dev/null +++ b/internal/store/migrations/0001_init.sql @@ -0,0 +1,109 @@ +-- Initial schema. +-- +-- Timestamps are unix seconds (INTEGER), matching what the triggers' unixepoch() +-- produces. Digests are stored as raw 32-byte BLOBs, not hex text: half the +-- index size and no conversion on the read path. Hex exists only at the API +-- boundary. + +CREATE TABLE projects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, -- ^[a-z0-9][a-z0-9._-]{0,62}$ + display_name TEXT NOT NULL DEFAULT '', + index_file TEXT NOT NULL DEFAULT 'index.html', + not_found_file TEXT, -- NULL => bare 404 + spa_fallback INTEGER NOT NULL DEFAULT 0 CHECK (spa_fallback IN (0,1)), + cache_control TEXT NOT NULL DEFAULT 'public, max-age=0, must-revalidate', + retention_count INTEGER NOT NULL DEFAULT 10, + retention_grace_s INTEGER NOT NULL DEFAULT 3600, + max_files INTEGER NOT NULL DEFAULT 50000, + max_file_bytes INTEGER NOT NULL DEFAULT 268435456, -- 256 MiB + max_total_bytes INTEGER NOT NULL DEFAULT 2147483648, -- 2 GiB + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +-- API keys. The id is the public lookup handle (safe to display); secret_hash is +-- sha256 of the secret half of the token. See internal/auth/token.go for why a +-- plain hash is the right choice for a 256-bit random secret. +CREATE TABLE api_keys ( + id TEXT NOT NULL PRIMARY KEY, -- 16 chars of base32, no padding + secret_hash BLOB NOT NULL, -- sha256(secret), 32 raw bytes + scope TEXT NOT NULL CHECK (scope IN ('admin','project')), + project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE, + name TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + expires_at INTEGER, + last_used_at INTEGER, + revoked_at INTEGER, + -- An admin key is not scoped to a project and a project key must be. + CHECK ((scope = 'admin' AND project_id IS NULL) + OR (scope = 'project' AND project_id IS NOT NULL)) +) WITHOUT ROWID; +CREATE INDEX api_keys_project ON api_keys(project_id) WHERE project_id IS NOT NULL; + +CREATE TABLE deployments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + public_id TEXT NOT NULL UNIQUE, -- "dpl_" + 16 lowercase hex + project_id INTEGER NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + state TEXT NOT NULL CHECK (state IN + ('pending','uploading','ready','failed','deleting')), + active INTEGER NOT NULL DEFAULT 0 CHECK (active IN (0,1)), + file_count INTEGER NOT NULL DEFAULT 0, + total_bytes INTEGER NOT NULL DEFAULT 0, + created_by_key TEXT REFERENCES api_keys(id) ON DELETE SET NULL, + meta TEXT NOT NULL DEFAULT '{}', -- JSON: git_sha/branch/ci_url/actor + error TEXT, + created_at INTEGER NOT NULL, + finalized_at INTEGER, + activated_at INTEGER, + deactivated_at INTEGER +); + +-- "At most one active deployment per project" is an invariant, so the database +-- enforces it rather than the application. A projects.active_deployment_id +-- column would have needed a circular foreign key and deferred constraints to +-- say the same thing. +CREATE UNIQUE INDEX deployments_one_active ON deployments(project_id) WHERE active = 1; +CREATE INDEX deployments_proj_created ON deployments(project_id, created_at DESC); +CREATE INDEX deployments_state_created ON deployments(state, created_at); +CREATE INDEX deployments_gc ON deployments(deactivated_at) + WHERE active = 0 AND deactivated_at IS NOT NULL; + +CREATE TABLE blobs ( + digest BLOB NOT NULL PRIMARY KEY, -- sha256, 32 raw bytes + size INTEGER NOT NULL, + present INTEGER NOT NULL DEFAULT 0 CHECK (present IN (0,1)), + refcount INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + last_ref_at INTEGER NOT NULL +) WITHOUT ROWID; +CREATE INDEX blobs_gc ON blobs(last_ref_at) WHERE refcount = 0; +CREATE INDEX blobs_pending ON blobs(created_at) WHERE present = 0; + +-- The file manifest of each deployment. +-- +-- encoding is always '' in v1. It is part of the primary key so that a +-- pre-compressed sibling (.br/.gz) can be added later without a table rewrite. +CREATE TABLE deployment_files ( + deployment_id INTEGER NOT NULL REFERENCES deployments(id) ON DELETE CASCADE, + path TEXT NOT NULL, -- slash-separated, fs.ValidPath + encoding TEXT NOT NULL DEFAULT '', -- '' | 'gzip' | 'br' + digest BLOB NOT NULL REFERENCES blobs(digest) ON DELETE RESTRICT, + size INTEGER NOT NULL, + PRIMARY KEY (deployment_id, path, encoding) +) WITHOUT ROWID; +CREATE INDEX deployment_files_digest ON deployment_files(digest); + +-- Refcounts are maintained by the database so that no code path can forget. +-- +-- Caution: ON DELETE CASCADE does not fire these triggers unless +-- recursive_triggers is ON (it is, see internal/store/db.go), and relying on +-- that alone is fragile — delete the manifest rows explicitly before deleting a +-- deployment and let the cascade be the backstop. fsck recomputes every refcount +-- from deployment_files and reports drift. +CREATE TRIGGER deployment_files_ai AFTER INSERT ON deployment_files BEGIN + UPDATE blobs SET refcount = refcount + 1, last_ref_at = unixepoch() WHERE digest = NEW.digest; +END; +CREATE TRIGGER deployment_files_ad AFTER DELETE ON deployment_files BEGIN + UPDATE blobs SET refcount = refcount - 1, last_ref_at = unixepoch() WHERE digest = OLD.digest; +END; diff --git a/internal/store/projects.go b/internal/store/projects.go new file mode 100644 index 0000000..b356fce --- /dev/null +++ b/internal/store/projects.go @@ -0,0 +1,230 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "time" +) + +// Project is a row of the projects table. +// +// The serving-related fields (IndexFile, NotFoundFile, SPAFallback, +// CacheControl) are copied into the in-memory site registry; the limits are +// enforced when a manifest is accepted. +type Project struct { + ID int64 + Name string + DisplayName string + IndexFile string + NotFoundFile string // "" means no custom 404 document + SPAFallback bool + CacheControl string + + RetentionCount int + RetentionGraceS int + MaxFiles int + MaxFileBytes int64 + MaxTotalBytes int64 + + CreatedAt time.Time + UpdatedAt time.Time +} + +// DefaultProject returns a project carrying the same defaults the schema does, +// as the starting point for a create request. +func DefaultProject(name string) *Project { + return &Project{ + Name: name, + IndexFile: "index.html", + CacheControl: "public, max-age=0, must-revalidate", + RetentionCount: 10, + RetentionGraceS: 3600, + MaxFiles: 50000, + MaxFileBytes: 256 << 20, + MaxTotalBytes: 2 << 30, + } +} + +const projectColumns = `id, name, display_name, index_file, not_found_file, spa_fallback, + cache_control, retention_count, retention_grace_s, max_files, max_file_bytes, + max_total_bytes, created_at, updated_at` + +type rowScanner interface { + Scan(dest ...any) error +} + +func scanProject(row rowScanner) (*Project, error) { + var p Project + var notFound sql.NullString + var created, updated int64 + err := row.Scan(&p.ID, &p.Name, &p.DisplayName, &p.IndexFile, ¬Found, &p.SPAFallback, + &p.CacheControl, &p.RetentionCount, &p.RetentionGraceS, &p.MaxFiles, &p.MaxFileBytes, + &p.MaxTotalBytes, &created, &updated) + if err != nil { + return nil, mapErr(err) + } + p.NotFoundFile = notFound.String + p.CreatedAt = time.Unix(created, 0).UTC() + p.UpdatedAt = time.Unix(updated, 0).UTC() + return &p, nil +} + +// CreateProject inserts p and fills in its ID and timestamps. Name uniqueness is +// enforced by the schema, so a duplicate returns ErrExists rather than racing. +func (d *DB) CreateProject(ctx context.Context, p *Project) error { + now := unixNow() + return d.Tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, ` + INSERT INTO projects (name, display_name, index_file, not_found_file, spa_fallback, + cache_control, retention_count, retention_grace_s, max_files, + max_file_bytes, max_total_bytes, created_at, updated_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)`, + p.Name, p.DisplayName, p.IndexFile, nullString(p.NotFoundFile), p.SPAFallback, + p.CacheControl, p.RetentionCount, p.RetentionGraceS, p.MaxFiles, + p.MaxFileBytes, p.MaxTotalBytes, now, now) + if err != nil { + return mapErr(err) + } + id, err := res.LastInsertId() + if err != nil { + return err + } + p.ID = id + p.CreatedAt = time.Unix(now, 0).UTC() + p.UpdatedAt = p.CreatedAt + return nil + }) +} + +// ProjectByName looks a project up by its URL name. +func (d *DB) ProjectByName(ctx context.Context, name string) (*Project, error) { + return scanProject(d.r.QueryRowContext(ctx, + `SELECT `+projectColumns+` FROM projects WHERE name = ?`, name)) +} + +// ProjectByID looks a project up by its primary key. +func (d *DB) ProjectByID(ctx context.Context, id int64) (*Project, error) { + return scanProject(d.r.QueryRowContext(ctx, + `SELECT `+projectColumns+` FROM projects WHERE id = ?`, id)) +} + +// ListProjects returns up to limit projects ordered by name, starting after the +// cursor. The cursor is the last name returned, which is stable under +// concurrent inserts in a way that an offset is not. +func (d *DB) ListProjects(ctx context.Context, limit int, cursor string) (projects []*Project, next string, err error) { + if limit <= 0 || limit > 500 { + limit = 100 + } + // One extra row tells us whether another page exists without a second query. + rows, err := d.r.QueryContext(ctx, + `SELECT `+projectColumns+` FROM projects WHERE name > ? ORDER BY name LIMIT ?`, + cursor, limit+1) + if err != nil { + return nil, "", err + } + defer rows.Close() + for rows.Next() { + p, err := scanProject(rows) + if err != nil { + return nil, "", err + } + projects = append(projects, p) + } + if err := rows.Err(); err != nil { + return nil, "", err + } + if len(projects) > limit { + projects = projects[:limit] + next = projects[len(projects)-1].Name + } + return projects, next, nil +} + +// AllProjects returns every project, for building the in-memory registry at +// startup. The registry holds them all anyway, so paging here would be theatre. +func (d *DB) AllProjects(ctx context.Context) ([]*Project, error) { + rows, err := d.r.QueryContext(ctx, `SELECT `+projectColumns+` FROM projects ORDER BY id`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []*Project + for rows.Next() { + p, err := scanProject(rows) + if err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// UpdateProject writes p's mutable fields back. Name and ID are immutable: a +// rename would invalidate every deployed URL and every cached symlink, and the +// API offers delete-and-recreate instead. +func (d *DB) UpdateProject(ctx context.Context, p *Project) error { + now := unixNow() + return d.Tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, ` + UPDATE projects SET display_name = ?, index_file = ?, not_found_file = ?, + spa_fallback = ?, cache_control = ?, retention_count = ?, + retention_grace_s = ?, max_files = ?, max_file_bytes = ?, + max_total_bytes = ?, updated_at = ? + WHERE id = ?`, + p.DisplayName, p.IndexFile, nullString(p.NotFoundFile), p.SPAFallback, + p.CacheControl, p.RetentionCount, p.RetentionGraceS, p.MaxFiles, + p.MaxFileBytes, p.MaxTotalBytes, now, p.ID) + if err != nil { + return mapErr(err) + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + return ErrNotFound + } + p.UpdatedAt = time.Unix(now, 0).UTC() + return nil + }) +} + +// DeleteProject removes a project and, by cascade, its keys, deployments and +// manifest rows. Blob refcounts fall as the manifest rows go, so the next GC +// pass reclaims the content. +// +// The caller is responsible for the parts the database does not know about: the +// registry entry, the webroot symlink and the assembled directories. +func (d *DB) DeleteProject(ctx context.Context, id int64) error { + return d.Tx(ctx, func(tx *sql.Tx) error { + // Delete the manifest rows explicitly rather than trusting the cascade to + // fire the refcount triggers (see the schema comment). + if _, err := tx.ExecContext(ctx, ` + DELETE FROM deployment_files + WHERE deployment_id IN (SELECT id FROM deployments WHERE project_id = ?)`, id); err != nil { + return err + } + res, err := tx.ExecContext(ctx, `DELETE FROM projects WHERE id = ?`, id) + if err != nil { + return mapErr(err) + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + return ErrNotFound + } + return nil + }) +} + +// CountProjects is used by /api/v1/system/info. +func (d *DB) CountProjects(ctx context.Context) (int64, error) { + var n int64 + if err := d.r.QueryRowContext(ctx, `SELECT count(*) FROM projects`).Scan(&n); err != nil { + return 0, fmt.Errorf("count projects: %w", err) + } + return n, nil +} diff --git a/internal/store/projects_test.go b/internal/store/projects_test.go new file mode 100644 index 0000000..6d01aa1 --- /dev/null +++ b/internal/store/projects_test.go @@ -0,0 +1,285 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "testing" + "time" +) + +func TestCreateAndReadProject(t *testing.T) { + ctx := context.Background() + db := testDB(t) + + p := DefaultProject("demo") + p.DisplayName = "Demo Site" + p.NotFoundFile = "404.html" + p.SPAFallback = true + if err := db.CreateProject(ctx, p); err != nil { + t.Fatalf("CreateProject: %v", err) + } + if p.ID == 0 { + t.Error("CreateProject must fill in the ID") + } + if p.CreatedAt.IsZero() || !p.UpdatedAt.Equal(p.CreatedAt) { + t.Errorf("timestamps not set: created=%v updated=%v", p.CreatedAt, p.UpdatedAt) + } + + got, err := db.ProjectByName(ctx, "demo") + if err != nil { + t.Fatalf("ProjectByName: %v", err) + } + if got.ID != p.ID || got.DisplayName != "Demo Site" || got.NotFoundFile != "404.html" || !got.SPAFallback { + t.Errorf("round trip lost data: %+v", got) + } + if got.IndexFile != "index.html" || got.RetentionCount != 10 { + t.Errorf("defaults not persisted: %+v", got) + } + + byID, err := db.ProjectByID(ctx, p.ID) + if err != nil { + t.Fatalf("ProjectByID: %v", err) + } + if byID.Name != "demo" { + t.Errorf("ProjectByID returned %q", byID.Name) + } +} + +// An empty not_found_file must come back as "" rather than as a bogus "NULL" +// string, because the resolver branches on it being empty. +func TestProjectNullNotFoundFile(t *testing.T) { + ctx := context.Background() + db := testDB(t) + if err := db.CreateProject(ctx, DefaultProject("demo")); err != nil { + t.Fatal(err) + } + got, err := db.ProjectByName(ctx, "demo") + if err != nil { + t.Fatal(err) + } + if got.NotFoundFile != "" { + t.Errorf("NotFoundFile = %q, want empty", got.NotFoundFile) + } +} + +func TestCreateProjectDuplicateName(t *testing.T) { + ctx := context.Background() + db := testDB(t) + if err := db.CreateProject(ctx, DefaultProject("demo")); err != nil { + t.Fatal(err) + } + err := db.CreateProject(ctx, DefaultProject("demo")) + if !errors.Is(err, ErrExists) { + t.Fatalf("second create: got %v, want ErrExists", err) + } +} + +func TestProjectNotFound(t *testing.T) { + ctx := context.Background() + db := testDB(t) + if _, err := db.ProjectByName(ctx, "nope"); !errors.Is(err, ErrNotFound) { + t.Errorf("ProjectByName: got %v, want ErrNotFound", err) + } + if _, err := db.ProjectByID(ctx, 404); !errors.Is(err, ErrNotFound) { + t.Errorf("ProjectByID: got %v, want ErrNotFound", err) + } + if err := db.DeleteProject(ctx, 404); !errors.Is(err, ErrNotFound) { + t.Errorf("DeleteProject: got %v, want ErrNotFound", err) + } + if err := db.UpdateProject(ctx, &Project{ID: 404}); !errors.Is(err, ErrNotFound) { + t.Errorf("UpdateProject: got %v, want ErrNotFound", err) + } +} + +func TestUpdateProject(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := DefaultProject("demo") + if err := db.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + + p.SPAFallback = true + p.CacheControl = "public, max-age=31536000, immutable" + p.RetentionCount = 3 + p.NotFoundFile = "404.html" + if err := db.UpdateProject(ctx, p); err != nil { + t.Fatalf("UpdateProject: %v", err) + } + + got, err := db.ProjectByName(ctx, "demo") + if err != nil { + t.Fatal(err) + } + if !got.SPAFallback || got.RetentionCount != 3 || got.NotFoundFile != "404.html" { + t.Errorf("update did not stick: %+v", got) + } + if got.Name != "demo" { + t.Errorf("name must be immutable, got %q", got.Name) + } + if got.CreatedAt.After(got.UpdatedAt) { + t.Errorf("updated_at %v predates created_at %v", got.UpdatedAt, got.CreatedAt) + } +} + +// Clearing not_found_file must write SQL NULL, not the empty string, so the +// column keeps a single representation of "unset". +func TestUpdateProjectClearsNotFoundFile(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := DefaultProject("demo") + p.NotFoundFile = "404.html" + if err := db.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + p.NotFoundFile = "" + if err := db.UpdateProject(ctx, p); err != nil { + t.Fatal(err) + } + var isNull bool + if err := db.Reader().QueryRow( + `SELECT not_found_file IS NULL FROM projects WHERE id = ?`, p.ID).Scan(&isNull); err != nil { + t.Fatal(err) + } + if !isNull { + t.Error("cleared not_found_file should be stored as NULL") + } +} + +func TestListProjectsPaging(t *testing.T) { + ctx := context.Background() + db := testDB(t) + for i := 0; i < 7; i++ { + if err := db.CreateProject(ctx, DefaultProject(fmt.Sprintf("p%d", i))); err != nil { + t.Fatal(err) + } + } + + var names []string + cursor := "" + for pages := 0; ; pages++ { + if pages > 10 { + t.Fatal("paging did not terminate") + } + batch, next, err := db.ListProjects(ctx, 3, cursor) + if err != nil { + t.Fatal(err) + } + for _, p := range batch { + names = append(names, p.Name) + } + if next == "" { + break + } + cursor = next + } + want := []string{"p0", "p1", "p2", "p3", "p4", "p5", "p6"} + if len(names) != len(want) { + t.Fatalf("paged names = %v, want %v", names, want) + } + for i := range want { + if names[i] != want[i] { + t.Fatalf("paged names = %v, want %v", names, want) + } + } + + n, err := db.CountProjects(ctx) + if err != nil { + t.Fatal(err) + } + if n != 7 { + t.Errorf("CountProjects = %d, want 7", n) + } + + all, err := db.AllProjects(ctx) + if err != nil { + t.Fatal(err) + } + if len(all) != 7 { + t.Errorf("AllProjects returned %d rows, want 7", len(all)) + } +} + +// Deleting a project must take its keys, deployments and manifest rows with it, +// and must drop the blob refcounts so the content becomes collectable. +func TestDeleteProjectCascades(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := DefaultProject("demo") + if err := db.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + digest := make([]byte, 32) + digest[0] = 0x7f + + if err := db.Tx(ctx, func(tx *sql.Tx) error { + if _, err := tx.Exec(`INSERT INTO deployments (id, public_id, project_id, state, created_at) + VALUES (1, 'dpl_a', ?, 'ready', 1)`, p.ID); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO blobs (digest, size, present, created_at, last_ref_at) + VALUES (?, 5, 1, 1, 1)`, digest); err != nil { + return err + } + _, err := tx.Exec(`INSERT INTO deployment_files (deployment_id, path, digest, size) + VALUES (1, 'index.html', ?, 5)`, digest) + return err + }); err != nil { + t.Fatal(err) + } + + pid := p.ID + if err := db.CreateKey(ctx, &APIKey{ + ID: "keyaaaaaaaaaaaaa", SecretHash: make([]byte, 32), Scope: ScopeProject, ProjectID: &pid, + }); err != nil { + t.Fatal(err) + } + + if err := db.DeleteProject(ctx, p.ID); err != nil { + t.Fatalf("DeleteProject: %v", err) + } + + count := func(query string, args ...any) int { + t.Helper() + var n int + if err := db.Reader().QueryRow(query, args...).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + if n := count(`SELECT count(*) FROM deployments`); n != 0 { + t.Errorf("%d deployments survived", n) + } + if n := count(`SELECT count(*) FROM deployment_files`); n != 0 { + t.Errorf("%d manifest rows survived", n) + } + if n := count(`SELECT count(*) FROM api_keys`); n != 0 { + t.Errorf("%d keys survived", n) + } + if n := count(`SELECT refcount FROM blobs WHERE digest = ?`, digest); n != 0 { + t.Errorf("blob refcount = %d, want 0 (content would never be collected)", n) + } + // The blob row itself stays: it is now unreferenced, and reclaiming it is + // GC's job, not the delete path's. + if n := count(`SELECT count(*) FROM blobs`); n != 1 { + t.Errorf("blob row count = %d, want 1", n) + } +} + +func TestProjectTimestampsAreUTC(t *testing.T) { + ctx := context.Background() + db := testDB(t) + p := DefaultProject("demo") + if err := db.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + got, err := db.ProjectByName(ctx, "demo") + if err != nil { + t.Fatal(err) + } + if got.CreatedAt.Location() != time.UTC { + t.Errorf("CreatedAt location = %v, want UTC", got.CreatedAt.Location()) + } +} diff --git a/internal/store/stats.go b/internal/store/stats.go new file mode 100644 index 0000000..5dad0f7 --- /dev/null +++ b/internal/store/stats.go @@ -0,0 +1,30 @@ +package store + +import "context" + +// Counts is the summary behind GET /api/v1/system/info. +type Counts struct { + Projects int64 + Deployments int64 + Blobs int64 + // CASBytes is the size of the blobs the store believes are on disk. It is + // the deduplicated total, so it is smaller — usually much smaller — than the + // sum of the deployments' sizes. + CASBytes int64 +} + +// Counts gathers the summary in one round trip. +// +// The subqueries are counted separately rather than joined: a join would have +// to fan out over deployment_files and then collapse again, which on a large +// manifest is thousands of times the work for the same four numbers. +func (d *DB) Counts(ctx context.Context) (Counts, error) { + var c Counts + err := d.r.QueryRowContext(ctx, ` + SELECT (SELECT count(*) FROM projects), + (SELECT count(*) FROM deployments), + (SELECT count(*) FROM blobs WHERE present = 1), + (SELECT coalesce(sum(size), 0) FROM blobs WHERE present = 1)`). + Scan(&c.Projects, &c.Deployments, &c.Blobs, &c.CASBytes) + return c, err +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..afd899b --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,301 @@ +package store + +import ( + "context" + "database/sql" + "io" + "log/slog" + "path/filepath" + "strings" + "testing" +) + +func testDB(t *testing.T) *DB { + t.Helper() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + db, err := Open(context.Background(), filepath.Join(t.TempDir(), "pages.db"), log) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestOpenAppliesMigrations(t *testing.T) { + db := testDB(t) + v, err := db.SchemaVersion(context.Background()) + if err != nil { + t.Fatal(err) + } + if v != 1 { + t.Errorf("schema version = %d, want 1", v) + } + for _, table := range []string{"projects", "api_keys", "deployments", "blobs", "deployment_files"} { + var n int + if err := db.Reader().QueryRow( + `SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("table %s missing", table) + } + } +} + +func TestMigrateIsIdempotent(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "pages.db") + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + + db, err := Open(ctx, path, log) + if err != nil { + t.Fatal(err) + } + if _, err := db.w.ExecContext(ctx, + `INSERT INTO projects (name, created_at, updated_at) VALUES ('demo', 1, 1)`); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + // Reopening must not re-run migrations, and must not lose data. + db2, err := Open(ctx, path, log) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer db2.Close() + var name string + if err := db2.Reader().QueryRow(`SELECT name FROM projects`).Scan(&name); err != nil { + t.Fatal(err) + } + if name != "demo" { + t.Errorf("project name = %q", name) + } +} + +func TestMigrateDetectsChecksumDrift(t *testing.T) { + ctx := context.Background() + db := testDB(t) + if _, err := db.w.ExecContext(ctx, + `UPDATE schema_version SET checksum = 'tampered' WHERE version = 1`); err != nil { + t.Fatal(err) + } + err := db.migrate(ctx) + if err == nil { + t.Fatal("editing an applied migration must be detected") + } + if !strings.Contains(err.Error(), "modified after it was applied") { + t.Errorf("error = %v", err) + } +} + +func TestMigrateRejectsUnknownFutureVersion(t *testing.T) { + ctx := context.Background() + db := testDB(t) + if _, err := db.w.ExecContext(ctx, + `INSERT INTO schema_version (version, checksum, applied_at) VALUES (99, 'x', 1)`); err != nil { + t.Fatal(err) + } + if err := db.migrate(ctx); err == nil { + t.Fatal("a database from a newer build must be refused") + } +} + +// The partial unique index is what actually guarantees "one active deployment +// per project"; the application layer only has to avoid fighting it. +func TestOneActiveDeploymentPerProject(t *testing.T) { + ctx := context.Background() + db := testDB(t) + + if err := db.Tx(ctx, func(tx *sql.Tx) error { + if _, err := tx.Exec(`INSERT INTO projects (id, name, created_at, updated_at) VALUES (1, 'demo', 1, 1)`); err != nil { + return err + } + _, err := tx.Exec(`INSERT INTO deployments (public_id, project_id, state, active, created_at) + VALUES ('dpl_a', 1, 'ready', 1, 1)`) + return err + }); err != nil { + t.Fatal(err) + } + + err := db.Tx(ctx, func(tx *sql.Tx) error { + _, err := tx.Exec(`INSERT INTO deployments (public_id, project_id, state, active, created_at) + VALUES ('dpl_b', 1, 'ready', 1, 2)`) + return err + }) + if err == nil { + t.Fatal("a second active deployment must be rejected by the database") + } + if !IsConstraint(err) { + t.Errorf("want a constraint violation, got %v", err) + } + + // Demoting the old one first is the supported path. + if err := db.Tx(ctx, func(tx *sql.Tx) error { + if _, err := tx.Exec(`UPDATE deployments SET active = 0 WHERE project_id = 1 AND active = 1`); err != nil { + return err + } + _, err := tx.Exec(`INSERT INTO deployments (public_id, project_id, state, active, created_at) + VALUES ('dpl_b', 1, 'ready', 1, 2)`) + return err + }); err != nil { + t.Fatalf("demote-then-promote must be allowed: %v", err) + } +} + +func TestRefcountTriggers(t *testing.T) { + ctx := context.Background() + db := testDB(t) + digest := make([]byte, 32) + digest[0] = 0xab + + setup := func(tx *sql.Tx) error { + if _, err := tx.Exec(`INSERT INTO projects (id, name, created_at, updated_at) VALUES (1, 'demo', 1, 1)`); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO deployments (id, public_id, project_id, state, created_at) + VALUES (1, 'dpl_a', 1, 'uploading', 1), (2, 'dpl_b', 1, 'uploading', 2)`); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO blobs (digest, size, present, created_at, last_ref_at) + VALUES (?, 10, 0, 1, 1)`, digest); err != nil { + return err + } + _, err := tx.Exec(`INSERT INTO deployment_files (deployment_id, path, digest, size) + VALUES (1, 'index.html', ?, 10), (2, 'index.html', ?, 10)`, digest, digest) + return err + } + if err := db.Tx(ctx, setup); err != nil { + t.Fatal(err) + } + + refcount := func() int { + t.Helper() + var n int + if err := db.Reader().QueryRow(`SELECT refcount FROM blobs WHERE digest = ?`, digest).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + if got := refcount(); got != 2 { + t.Fatalf("refcount after 2 inserts = %d, want 2", got) + } + + if err := db.Tx(ctx, func(tx *sql.Tx) error { + _, err := tx.Exec(`DELETE FROM deployment_files WHERE deployment_id = 1`) + return err + }); err != nil { + t.Fatal(err) + } + if got := refcount(); got != 1 { + t.Errorf("refcount after explicit delete = %d, want 1", got) + } + + // The cascade path: deleting the deployment row must also decrement, which + // only holds because recursive_triggers is ON. + if err := db.Tx(ctx, func(tx *sql.Tx) error { + _, err := tx.Exec(`DELETE FROM deployments WHERE id = 2`) + return err + }); err != nil { + t.Fatal(err) + } + if got := refcount(); got != 0 { + t.Errorf("refcount after cascade = %d, want 0 (recursive_triggers not in effect?)", got) + } +} + +// A blob may not be dropped while a manifest still points at it. +func TestBlobDeleteRestricted(t *testing.T) { + ctx := context.Background() + db := testDB(t) + digest := make([]byte, 32) + + if err := db.Tx(ctx, func(tx *sql.Tx) error { + if _, err := tx.Exec(`INSERT INTO projects (id, name, created_at, updated_at) VALUES (1, 'demo', 1, 1)`); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO deployments (id, public_id, project_id, state, created_at) + VALUES (1, 'dpl_a', 1, 'ready', 1)`); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO blobs (digest, size, present, created_at, last_ref_at) VALUES (?, 1, 1, 1, 1)`, digest); err != nil { + return err + } + _, err := tx.Exec(`INSERT INTO deployment_files (deployment_id, path, digest, size) VALUES (1, 'a', ?, 1)`, digest) + return err + }); err != nil { + t.Fatal(err) + } + + err := db.Tx(ctx, func(tx *sql.Tx) error { + _, err := tx.Exec(`DELETE FROM blobs WHERE digest = ?`, digest) + return err + }) + if err == nil { + t.Fatal("deleting a referenced blob must fail") + } +} + +func TestScopeCheckConstraint(t *testing.T) { + ctx := context.Background() + db := testDB(t) + cases := []struct { + name string + scope string + proj any + ok bool + }{ + {"admin without project", "admin", nil, true}, + {"admin with project", "admin", int64(1), false}, + {"project without project", "project", nil, false}, + {"project with project", "project", int64(1), true}, + {"unknown scope", "root", nil, false}, + } + if err := db.Tx(ctx, func(tx *sql.Tx) error { + _, err := tx.Exec(`INSERT INTO projects (id, name, created_at, updated_at) VALUES (1, 'demo', 1, 1)`) + return err + }); err != nil { + t.Fatal(err) + } + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := db.Tx(ctx, func(tx *sql.Tx) error { + _, err := tx.Exec( + `INSERT INTO api_keys (id, secret_hash, scope, project_id, created_at) VALUES (?, ?, ?, ?, 1)`, + "key"+string(rune('a'+i)), make([]byte, 32), tc.scope, tc.proj) + return err + }) + if tc.ok && err != nil { + t.Errorf("insert should have been accepted: %v", err) + } + if !tc.ok && err == nil { + t.Error("insert should have been rejected") + } + }) + } +} + +// The write pool is capped at one connection, so a transaction that never +// returns would deadlock the server. This asserts that a plain read does not +// need the write pool. +func TestReadsDoNotBlockOnWriter(t *testing.T) { + ctx := context.Background() + db := testDB(t) + done := make(chan struct{}) + go func() { + defer close(done) + _ = db.Tx(ctx, func(tx *sql.Tx) error { + if _, err := tx.Exec(`INSERT INTO projects (name, created_at, updated_at) VALUES ('slow', 1, 1)`); err != nil { + return err + } + var n int + // While this transaction is open, a reader must still make progress. + if err := db.Reader().QueryRow(`SELECT count(*) FROM projects`).Scan(&n); err != nil { + return err + } + return nil + }) + }() + <-done +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..2a08a2b --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,69 @@ +// Package version exposes build metadata injected at link time. +package version + +import ( + "fmt" + "runtime" + "runtime/debug" + "sync" +) + +// Injected via -ldflags "-X github.com/iceBear67/simplepages/internal/version.Version=...". +var ( + Version = "dev" + Commit = "" + Date = "" +) + +var once sync.Once + +// vcsFromBuildInfo fills Commit/Date from the embedded build info when the +// linker flags were not supplied (the usual case for `go run` and `go install`). +func vcsFromBuildInfo() { + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + for _, s := range info.Settings { + switch s.Key { + case "vcs.revision": + if Commit == "" { + Commit = s.Value + } + case "vcs.time": + if Date == "" { + Date = s.Value + } + } + } +} + +// String renders a one-line human-readable version banner. +func String() string { + once.Do(vcsFromBuildInfo) + s := Version + if Commit != "" { + short := Commit + if len(short) > 12 { + short = short[:12] + } + s += "+" + short + } + if Date != "" { + s += " (" + Date + ")" + } + return fmt.Sprintf("%s %s/%s %s", s, runtime.GOOS, runtime.GOARCH, runtime.Version()) +} + +// Short returns just the version string, for API responses. +func Short() string { + once.Do(vcsFromBuildInfo) + if Commit == "" { + return Version + } + short := Commit + if len(short) > 12 { + short = short[:12] + } + return Version + "+" + short +} diff --git a/internal/webroot/webroot.go b/internal/webroot/webroot.go new file mode 100644 index 0000000..c49e16e --- /dev/null +++ b/internal/webroot/webroot.go @@ -0,0 +1,207 @@ +// Package webroot maintains the $WEBROOT/~project symlinks. +// +// The server serves its own content, so these links are not on any request +// path. They exist for everything else: an nginx that would rather serve the +// files itself, a backup job, an operator running ls. That makes every +// operation here best-effort — a failure is logged and reconciled later, never +// a reason to fail a deployment that the database already committed. +package webroot + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/iceBear67/simplepages/internal/config" +) + +// prefix is what marks an entry as ours. Project names cannot contain a slash +// or start with a dot — config.ProjectNamePattern sees to that — so "~" + name +// is always a single entry directly inside the webroot and can never escape it. +const prefix = "~" + +// tmpMarker separates a half-built link from a live one. Point never renames a +// name containing it into place, and Reconcile sweeps any that a crash left. +const tmpMarker = ".tmp." + +// Webroot is a directory of symlinks pointing into the deployments directory. +type Webroot struct { + dir string + deployDir string +} + +// Open prepares the webroot directory. deployDir bounds what Reconcile is +// willing to delete: an entry that does not point inside it belongs to the +// operator, not to us. +func Open(dir, deployDir string) (*Webroot, error) { + if dir == "" { + return nil, errors.New("webroot: no directory configured") + } + abs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + if err := os.MkdirAll(abs, 0o755); err != nil { + return nil, err + } + deployAbs, err := filepath.Abs(deployDir) + if err != nil { + return nil, err + } + return &Webroot{dir: abs, deployDir: deployAbs}, nil +} + +// Dir is the directory being maintained. +func (w *Webroot) Dir() string { return w.dir } + +// Point makes ~project refer to target. +// +// The link is created under a temporary name and renamed into place. rename(2) +// within one directory replaces an existing symlink atomically, so an external +// reader never observes the link missing, dangling or half-written — which is +// the same guarantee, at a coarser grain, that the in-process pointer swap +// gives HTTP clients. +func (w *Webroot) Point(project, target string) error { + link, err := w.linkPath(project) + if err != nil { + return err + } + if cur, err := os.Readlink(link); err == nil && cur == target { + return nil + } + + tmp, err := w.tempLink(project, target) + if err != nil { + return err + } + if err := os.Rename(tmp, link); err != nil { + os.Remove(tmp) + return fmt.Errorf("webroot: point %s: %w", project, err) + } + return nil +} + +// Unpoint removes ~project, if it is one of ours. +func (w *Webroot) Unpoint(project string) error { + link, err := w.linkPath(project) + if err != nil { + return err + } + if !w.ours(link) { + return nil + } + if err := os.Remove(link); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("webroot: unpoint %s: %w", project, err) + } + return nil +} + +// Reconcile makes the directory match want, a project name to target directory +// map, and reports the first thing that went wrong after attempting all of it. +// +// It only ever deletes an entry that is itself a symlink pointing inside the +// deployments directory. An operator's unrelated file, directory or link that +// happens to share the webroot is left strictly alone: this program owns the +// names it created, not the directory. +func (w *Webroot) Reconcile(want map[string]string) error { + entries, err := os.ReadDir(w.dir) + if err != nil { + return err + } + var errs []error + for _, e := range entries { + name := e.Name() + if !strings.HasPrefix(name, prefix) || e.Type()&fs.ModeSymlink == 0 { + continue + } + full := filepath.Join(w.dir, name) + if !w.pointsIntoDeployments(full) { + continue + } + project := name[len(prefix):] + if strings.Contains(project, tmpMarker) { + // A Point that died between symlink and rename. + if err := os.Remove(full); err != nil { + errs = append(errs, err) + } + continue + } + if _, keep := want[project]; keep { + continue + } + if err := os.Remove(full); err != nil && !errors.Is(err, fs.ErrNotExist) { + errs = append(errs, err) + } + } + for project, target := range want { + if err := w.Point(project, target); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// linkPath validates the project name and returns the path of its link. +func (w *Webroot) linkPath(project string) (string, error) { + if !config.ProjectNamePattern.MatchString(project) { + return "", fmt.Errorf("webroot: refusing to touch %q: not a project name", project) + } + return filepath.Join(w.dir, prefix+project), nil +} + +// tempLink creates a symlink under a name that Reconcile will recognise as +// abandoned if this process dies before the rename. +func (w *Webroot) tempLink(project, target string) (string, error) { + base, err := w.linkPath(project) + if err != nil { + return "", err + } + var buf [6]byte + for range 8 { + if _, err := rand.Read(buf[:]); err != nil { + return "", err + } + tmp := base + tmpMarker + hex.EncodeToString(buf[:]) + err := os.Symlink(target, tmp) + if err == nil { + return tmp, nil + } + if !errors.Is(err, fs.ErrExist) { + return "", fmt.Errorf("webroot: link %s: %w", project, err) + } + } + return "", fmt.Errorf("webroot: could not create a temporary link for %s", project) +} + +// ours reports whether path is a symlink this package would have created. +func (w *Webroot) ours(path string) bool { + fi, err := os.Lstat(path) + if err != nil || fi.Mode()&fs.ModeSymlink == 0 { + return false + } + return w.pointsIntoDeployments(path) +} + +// pointsIntoDeployments reports whether a symlink's target is inside the +// deployments directory. The target is compared lexically after cleaning: it is +// the string this program wrote, and resolving it would answer a different +// question about a link that may well dangle. +func (w *Webroot) pointsIntoDeployments(path string) bool { + target, err := os.Readlink(path) + if err != nil { + return false + } + if !filepath.IsAbs(target) { + target = filepath.Join(w.dir, target) + } + target = filepath.Clean(target) + if target == w.deployDir { + return false + } + return strings.HasPrefix(target, w.deployDir+string(filepath.Separator)) +} diff --git a/internal/webroot/webroot_test.go b/internal/webroot/webroot_test.go new file mode 100644 index 0000000..59767fb --- /dev/null +++ b/internal/webroot/webroot_test.go @@ -0,0 +1,395 @@ +package webroot + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +// env is a webroot plus the deployments directory it is allowed to point at. +type env struct { + *Webroot + dir string + deployDir string +} + +func newEnv(t *testing.T) *env { + t.Helper() + base := t.TempDir() + dir := filepath.Join(base, "www") + deployDir := filepath.Join(base, "deployments") + if err := os.MkdirAll(deployDir, 0o755); err != nil { + t.Fatal(err) + } + w, err := Open(dir, deployDir) + if err != nil { + t.Fatal(err) + } + return &env{Webroot: w, dir: dir, deployDir: deployDir} +} + +// deployment creates a deployment directory and returns its path. +func (e *env) deployment(t *testing.T, rel string) string { + t.Helper() + p := filepath.Join(e.deployDir, rel) + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + return p +} + +func (e *env) readlink(t *testing.T, name string) string { + t.Helper() + target, err := os.Readlink(filepath.Join(e.dir, name)) + if err != nil { + t.Fatalf("readlink %s: %v", name, err) + } + return target +} + +func (e *env) names(t *testing.T) []string { + t.Helper() + entries, err := os.ReadDir(e.dir) + if err != nil { + t.Fatal(err) + } + out := make([]string, 0, len(entries)) + for _, entry := range entries { + out = append(out, entry.Name()) + } + return out +} + +func TestOpenCreatesTheDirectory(t *testing.T) { + base := t.TempDir() + dir := filepath.Join(base, "a", "b", "www") + w, err := Open(dir, filepath.Join(base, "deployments")) + if err != nil { + t.Fatal(err) + } + if fi, err := os.Stat(dir); err != nil || !fi.IsDir() { + t.Fatalf("Open did not create %s: %v", dir, err) + } + if !filepath.IsAbs(w.Dir()) { + t.Errorf("Dir() = %q, want an absolute path", w.Dir()) + } + if _, err := Open("", base); err == nil { + t.Error("Open accepted an empty directory") + } +} + +func TestPoint(t *testing.T) { + e := newEnv(t) + first := e.deployment(t, "1/dpl_aaaa") + + if err := e.Point("demo", first); err != nil { + t.Fatal(err) + } + if got := e.readlink(t, "~demo"); got != first { + t.Errorf("~demo -> %q, want %q", got, first) + } + + // Pointing at the same target again is a no-op, not an error. + if err := e.Point("demo", first); err != nil { + t.Fatal(err) + } + + second := e.deployment(t, "1/dpl_bbbb") + if err := e.Point("demo", second); err != nil { + t.Fatal(err) + } + if got := e.readlink(t, "~demo"); got != second { + t.Errorf("after repoint ~demo -> %q, want %q", got, second) + } + // The rename must not leave the temporary link behind. + if names := e.names(t); len(names) != 1 || names[0] != "~demo" { + t.Errorf("webroot contains %v, want just [~demo]", names) + } +} + +// Point replaces the link with rename(2), which is atomic within a directory: +// an external reader either sees the old target or the new one, never a missing +// or half-written link. Observing that from a test means checking that the name +// resolves to a valid deployment at every moment, which the atomicity suite +// does concurrently; here we pin the mechanism it relies on — the link is never +// unlinked first. +func TestPointNeverUnlinksBeforeRenaming(t *testing.T) { + e := newEnv(t) + first := e.deployment(t, "1/dpl_aaaa") + second := e.deployment(t, "1/dpl_bbbb") + if err := e.Point("demo", first); err != nil { + t.Fatal(err) + } + link := filepath.Join(e.dir, "~demo") + before, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + if err := e.Point("demo", second); err != nil { + t.Fatal(err) + } + after, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + if os.SameFile(before, after) { + t.Error("Point reused the same inode; it must create a new link and rename over") + } +} + +func TestPointRefusesANameThatIsNotAProject(t *testing.T) { + e := newEnv(t) + target := e.deployment(t, "1/dpl_aaaa") + // Anything that could escape the webroot, plus the merely invalid. + for _, name := range []string{"", ".", "..", "../evil", "a/b", "/abs", "UPPER", ".hidden", "~demo", strings.Repeat("a", 64)} { + if err := e.Point(name, target); err == nil { + t.Errorf("Point(%q) was accepted", name) + } + if err := e.Unpoint(name); err == nil { + t.Errorf("Unpoint(%q) was accepted", name) + } + } + if names := e.names(t); len(names) != 0 { + t.Errorf("a refused name still created %v", names) + } +} + +func TestUnpoint(t *testing.T) { + e := newEnv(t) + if err := e.Unpoint("demo"); err != nil { + t.Fatalf("Unpoint on a missing link = %v, want nil", err) + } + + if err := e.Point("demo", e.deployment(t, "1/dpl_aaaa")); err != nil { + t.Fatal(err) + } + if err := e.Unpoint("demo"); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(filepath.Join(e.dir, "~demo")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("~demo survived Unpoint: %v", err) + } +} + +// The webroot is very likely a directory the operator also keeps other things +// in. Unpoint must not remove a name that this package did not create, even +// though the name matches the pattern it uses. +func TestUnpointLeavesForeignEntriesAlone(t *testing.T) { + e := newEnv(t) + regular := filepath.Join(e.dir, "~demo") + if err := os.WriteFile(regular, []byte("operator's file"), 0o644); err != nil { + t.Fatal(err) + } + if err := e.Unpoint("demo"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(regular); err != nil { + t.Fatalf("Unpoint deleted a regular file: %v", err) + } + + outside := filepath.Join(t.TempDir(), "elsewhere") + if err := os.MkdirAll(outside, 0o755); err != nil { + t.Fatal(err) + } + foreign := filepath.Join(e.dir, "~other") + if err := os.Symlink(outside, foreign); err != nil { + t.Fatal(err) + } + if err := e.Unpoint("other"); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(foreign); err != nil { + t.Fatalf("Unpoint deleted a symlink pointing outside the deployments dir: %v", err) + } +} + +func TestReconcile(t *testing.T) { + e := newEnv(t) + a := e.deployment(t, "1/dpl_a") + b := e.deployment(t, "2/dpl_b") + c := e.deployment(t, "3/dpl_c") + + if err := e.Point("stale", a); err != nil { + t.Fatal(err) + } + if err := e.Point("moved", a); err != nil { + t.Fatal(err) + } + + want := map[string]string{"moved": b, "fresh": c} + if err := e.Reconcile(want); err != nil { + t.Fatal(err) + } + + if _, err := os.Lstat(filepath.Join(e.dir, "~stale")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("~stale survived Reconcile: %v", err) + } + if got := e.readlink(t, "~moved"); got != b { + t.Errorf("~moved -> %q, want %q", got, b) + } + if got := e.readlink(t, "~fresh"); got != c { + t.Errorf("~fresh -> %q, want %q", got, c) + } +} + +// The guarantee that makes Reconcile safe to run on a shared webroot: it +// deletes only symlinks of its own that point inside the deployments directory. +func TestReconcileLeavesForeignEntriesAlone(t *testing.T) { + e := newEnv(t) + elsewhere := t.TempDir() + + keep := []string{ + "index.html", // a file the operator serves directly + "~notes.txt", // a file whose name happens to start with ~ + "~static", // a directory under a name we would use + "~elsewhere", // a symlink out of our control + "~dangling", // a symlink to nothing + "other-thing", // anything else + } + if err := os.WriteFile(filepath.Join(e.dir, "index.html"), []byte("hi"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(e.dir, "~notes.txt"), []byte("hi"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(e.dir, "~static"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(elsewhere, filepath.Join(e.dir, "~elsewhere")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(elsewhere, "nope"), filepath.Join(e.dir, "~dangling")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(e.dir, "other-thing"), []byte("hi"), 0o644); err != nil { + t.Fatal(err) + } + + // An empty want set is the harshest case: every project is gone, so + // anything Reconcile is willing to delete, it deletes now. + if err := e.Reconcile(nil); err != nil { + t.Fatal(err) + } + for _, name := range keep { + if _, err := os.Lstat(filepath.Join(e.dir, name)); err != nil { + t.Errorf("Reconcile removed %s: %v", name, err) + } + } +} + +// A symlink pointing at the deployments directory itself is not one of ours: +// nothing here ever creates one, so removing it would be removing an +// operator's entry. +func TestReconcileIgnoresALinkToTheDeploymentsRoot(t *testing.T) { + e := newEnv(t) + link := filepath.Join(e.dir, "~all") + if err := os.Symlink(e.deployDir, link); err != nil { + t.Fatal(err) + } + if err := e.Reconcile(nil); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(link); err != nil { + t.Errorf("Reconcile removed a link to the deployments root: %v", err) + } +} + +// A prefix match on the string would treat /deployments-old as inside +// /deployments. It is a sibling directory and must be left alone. +func TestReconcileIgnoresASiblingDirectoryWithASharedPrefix(t *testing.T) { + e := newEnv(t) + sibling := e.deployDir + "-old" + if err := os.MkdirAll(filepath.Join(sibling, "dpl_x"), 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(e.dir, "~archived") + if err := os.Symlink(filepath.Join(sibling, "dpl_x"), link); err != nil { + t.Fatal(err) + } + if err := e.Reconcile(nil); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(link); err != nil { + t.Errorf("Reconcile removed a link into a sibling directory: %v", err) + } +} + +// A crash between symlink() and rename() leaves ~name.tmp. behind. It is +// ours by construction, so Reconcile sweeps it. +func TestReconcileSweepsAbandonedTemporaryLinks(t *testing.T) { + e := newEnv(t) + target := e.deployment(t, "1/dpl_a") + leftover := filepath.Join(e.dir, "~demo"+tmpMarker+"0123456789ab") + if err := os.Symlink(target, leftover); err != nil { + t.Fatal(err) + } + + if err := e.Reconcile(map[string]string{"demo": target}); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(leftover); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("an abandoned temporary link survived Reconcile: %v", err) + } + if got := e.readlink(t, "~demo"); got != target { + t.Errorf("~demo -> %q, want %q", got, target) + } +} + +// A relative target is still ours if it lands inside the deployments directory. +// Reconcile resolves it against the webroot before deciding. +func TestReconcileUnderstandsRelativeTargets(t *testing.T) { + e := newEnv(t) + e.deployment(t, "1/dpl_a") + rel, err := filepath.Rel(e.dir, filepath.Join(e.deployDir, "1", "dpl_a")) + if err != nil { + t.Fatal(err) + } + link := filepath.Join(e.dir, "~gone") + if err := os.Symlink(rel, link); err != nil { + t.Fatal(err) + } + if err := e.Reconcile(nil); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(link); !errors.Is(err, fs.ErrNotExist) { + t.Error("a relative link into the deployments dir survived Reconcile") + } +} + +// Reconcile reports what failed but keeps going: one bad project must not stop +// the rest of the webroot from converging. +func TestReconcileContinuesAfterAFailure(t *testing.T) { + e := newEnv(t) + good := e.deployment(t, "1/dpl_a") + if err := e.Reconcile(map[string]string{"good": good, "Bad Name": good}); err == nil { + t.Fatal("Reconcile hid a failure") + } + if got := e.readlink(t, "~good"); got != good { + t.Errorf("~good -> %q, want %q", got, good) + } +} + +func TestReconcileIsIdempotent(t *testing.T) { + e := newEnv(t) + want := map[string]string{ + "a": e.deployment(t, "1/dpl_a"), + "b": e.deployment(t, "2/dpl_b"), + } + for range 3 { + if err := e.Reconcile(want); err != nil { + t.Fatal(err) + } + } + names := e.names(t) + if len(names) != 2 { + t.Fatalf("webroot contains %v, want two links", names) + } + for project, target := range want { + if got := e.readlink(t, "~"+project); got != target { + t.Errorf("~%s -> %q, want %q", project, got, target) + } + } +}