package store import ( "context" "crypto/rand" "database/sql" "encoding/hex" "encoding/json" "fmt" "time" "github.com/iceBear67/simplepages/internal/cas" ) // State is where a deployment sits in its lifecycle. The values mirror the // CHECK constraint in the schema; api carries the same set for the wire and // TestStateConstantsMatchTheWire asserts the two never drift. type State string const ( // StatePending: the row exists and nothing else does. StatePending State = "pending" // StateUploading: a manifest has been accepted, so every blob it names is // already refcounted and safe from GC while its content is still arriving. StateUploading State = "uploading" // StateReady: every blob is present and the directory tree is assembled. // Only a ready deployment may be activated. StateReady State = "ready" // StateFailed: abandoned. Its manifest rows get dropped so refcounts fall // and GC can reclaim whatever only it referenced. StateFailed State = "failed" // StateDeleting: GC has claimed it and is removing its tree. StateDeleting State = "deleting" ) // Deployment is a row of the deployments table. // // ID is internal and never leaves the server; PublicID is what the API and the // CLI use. Keeping them separate means a URL cannot be walked to a neighbouring // project's deployment by incrementing an integer. type Deployment struct { ID int64 PublicID string ProjectID int64 State State Active bool FileCount int TotalBytes int64 // CreatedByKey is empty once that key has been deleted; the schema sets it // to NULL rather than losing the deployment. CreatedByKey string Meta map[string]string Error string CreatedAt time.Time FinalizedAt *time.Time ActivatedAt *time.Time DeactivatedAt *time.Time } // FileRow is one manifest entry as stored. // // There is deliberately no mode: a client-supplied permission bit must never // reach the filesystem, which is why the schema has no column for one either. // The encoding column is likewise not represented — v1 only ever writes ” — // but it stays in the primary key so pre-compressed variants can be added later // without a painful migration. type FileRow struct { Path string Digest cas.Digest Size int64 } const deploymentColumns = `id, public_id, project_id, state, active, file_count, total_bytes, created_by_key, meta, error, created_at, finalized_at, activated_at, deactivated_at` func scanDeployment(row rowScanner) (*Deployment, error) { var d Deployment var createdBy, meta, errText sql.NullString var created int64 var finalized, activated, deactivated sql.NullInt64 err := row.Scan(&d.ID, &d.PublicID, &d.ProjectID, &d.State, &d.Active, &d.FileCount, &d.TotalBytes, &createdBy, &meta, &errText, &created, &finalized, &activated, &deactivated) if err != nil { return nil, mapErr(err) } d.CreatedByKey = createdBy.String d.Error = errText.String d.CreatedAt = time.Unix(created, 0).UTC() d.FinalizedAt = timePtr(finalized) d.ActivatedAt = timePtr(activated) d.DeactivatedAt = timePtr(deactivated) if meta.String != "" && meta.String != "{}" { // Metadata is whatever the CI job attached. A row that will not parse is // not worth failing a rollback over, so it is dropped rather than // returned as an error. _ = json.Unmarshal([]byte(meta.String), &d.Meta) } return &d, nil } // NewDeploymentID mints a deployment's public identifier. // // 64 bits from crypto/rand, which is not a capability and is not treated as // one: every endpoint taking a deployment id sits behind RequireProject, and // DeploymentByPublicID is always scoped by project, so knowing an id grants // nothing. The randomness is here so two concurrent CI jobs never collide and // so ids carry no information about how many deployments the server has seen. func NewDeploymentID() (string, error) { var buf [8]byte if _, err := rand.Read(buf[:]); err != nil { return "", err } return "dpl_" + hex.EncodeToString(buf[:]), nil } // CreateDeployment inserts a pending deployment and fills in its ID, PublicID // and CreatedAt. Nothing touches the filesystem until a manifest arrives. func (d *DB) CreateDeployment(ctx context.Context, dep *Deployment) error { if dep.PublicID == "" { id, err := NewDeploymentID() if err != nil { return err } dep.PublicID = id } meta := "{}" if len(dep.Meta) > 0 { b, err := json.Marshal(dep.Meta) if err != nil { return err } meta = string(b) } now := unixNow() return d.Tx(ctx, func(tx *sql.Tx) error { res, err := tx.ExecContext(ctx, ` INSERT INTO deployments (public_id, project_id, state, created_by_key, meta, created_at) VALUES (?, ?, ?, ?, ?, ?)`, dep.PublicID, dep.ProjectID, StatePending, nullString(dep.CreatedByKey), meta, now) if err != nil { return mapErr(err) } id, err := res.LastInsertId() if err != nil { return err } dep.ID = id dep.State = StatePending dep.CreatedAt = time.Unix(now, 0).UTC() return nil }) } // DeploymentByPublicID looks up one deployment within one project. // // The project is part of the lookup rather than something to check afterwards. // A handler holding a project-scoped identity therefore cannot name another // project's deployment at all: the wrong project simply yields ErrNotFound, // which is also the right answer to give — it reveals nothing about whether the // id exists elsewhere. func (d *DB) DeploymentByPublicID(ctx context.Context, projectID int64, publicID string) (*Deployment, error) { return scanDeployment(d.r.QueryRowContext(ctx, `SELECT `+deploymentColumns+` FROM deployments WHERE project_id = ? AND public_id = ?`, projectID, publicID)) } // ActiveDeployment returns the deployment a project is currently serving, or // ErrNotFound when it has never activated one. The partial unique index makes // "at most one" a property of the database rather than of this query. func (d *DB) ActiveDeployment(ctx context.Context, projectID int64) (*Deployment, error) { return scanDeployment(d.r.QueryRowContext(ctx, `SELECT `+deploymentColumns+` FROM deployments WHERE project_id = ? AND active = 1`, projectID)) } // ListDeployments pages one project's deployments, newest first. // // The cursor is the public id of the last row of the previous page. An id that // no longer exists — GC ran between pages — yields an empty page rather than an // error, which is the behaviour a paging client can actually handle. func (d *DB) ListDeployments(ctx context.Context, projectID int64, state State, limit int, cursor string) (deployments []*Deployment, next string, err error) { if limit <= 0 || limit > 500 { limit = 100 } rows, err := d.r.QueryContext(ctx, ` SELECT `+deploymentColumns+` FROM deployments WHERE project_id = ? AND (? = '' OR state = ?) AND (? = '' OR id < (SELECT id FROM deployments WHERE public_id = ?)) ORDER BY id DESC LIMIT ?`, projectID, string(state), string(state), cursor, cursor, limit+1) if err != nil { return nil, "", err } defer rows.Close() for rows.Next() { dep, err := scanDeployment(rows) if err != nil { return nil, "", err } deployments = append(deployments, dep) } if err := rows.Err(); err != nil { return nil, "", err } if len(deployments) > limit { deployments = deployments[:limit] next = deployments[len(deployments)-1].PublicID } return deployments, next, nil } // SetManifest replaces a deployment's file list and reports which blobs the // server does not have content for yet. // // The ordering here is the point of the whole upload protocol. The manifest // rows land *before* any blob is uploaded, and the insert triggers bump each // blob's refcount as they do, so from this transaction's commit onwards every // blob the deployment needs is protected from GC. That closes the race the // obvious design has: "the server told me it already had this blob, then // deleted it while I was uploading the others". // // Re-sending a manifest is allowed while the deployment is pending or // uploading, because a CLI that lost its connection mid-negotiation should be // able to start over without creating a second deployment. func (d *DB) SetManifest(ctx context.Context, deploymentID int64, files []FileRow) (missing []cas.Digest, missingBytes int64, err error) { // One entry per distinct digest, in first-appearance order, so the missing // list — and therefore the client's upload order — is deterministic. type need struct { digest cas.Digest size int64 } var needs []need seen := make(map[cas.Digest]int64, len(files)) var totalBytes int64 for _, f := range files { totalBytes += f.Size if prev, ok := seen[f.Digest]; ok { if prev != f.Size { return nil, 0, fmt.Errorf("%w: the manifest gives digest %s both %d and %d bytes", cas.ErrSizeMismatch, f.Digest, prev, f.Size) } continue } seen[f.Digest] = f.Size needs = append(needs, need{f.Digest, f.Size}) } now := unixNow() err = d.Tx(ctx, func(tx *sql.Tx) error { // Tx re-runs this function on SQLITE_BUSY, so the results are rebuilt // from scratch each attempt rather than appended to. missing, missingBytes = nil, 0 var state State if err := tx.QueryRowContext(ctx, `SELECT state FROM deployments WHERE id = ?`, deploymentID).Scan(&state); err != nil { return mapErr(err) } if state != StatePending && state != StateUploading { return fmt.Errorf("%w: deployment is %s; a manifest may only be set while pending or uploading", ErrConflict, state) } // Discard any earlier attempt. A refcount that goes 1 -> 0 -> 1 inside // this transaction is safe because GC needs the same single write // connection and so can never observe the zero. if _, err := tx.ExecContext(ctx, `DELETE FROM deployment_files WHERE deployment_id = ?`, deploymentID); err != nil { return err } // Prepared statements rather than batched multi-row VALUES: a 50,000 // file manifest is a few hundred milliseconds of Exec either way, and // this version has no chunk arithmetic to get wrong and no bound on how // many parameters one statement may carry. insertBlob, err := tx.PrepareContext(ctx, ` INSERT INTO blobs (digest, size, present, created_at, last_ref_at) VALUES (?, ?, 0, ?, ?) ON CONFLICT(digest) DO NOTHING`) if err != nil { return err } defer insertBlob.Close() readBlob, err := tx.PrepareContext(ctx, `SELECT size, present FROM blobs WHERE digest = ?`) if err != nil { return err } defer readBlob.Close() for _, n := range needs { if _, err := insertBlob.ExecContext(ctx, n.digest.Bytes(), n.size, now, now); err != nil { return err } var have int64 var present bool if err := readBlob.QueryRowContext(ctx, n.digest.Bytes()).Scan(&have, &present); err != nil { return mapErr(err) } if have != n.size { // A digest fixes its content and therefore its length, so this // is a client that computed one of the two wrong. Surfacing it // here beats accepting a manifest whose byte totals are fiction. return fmt.Errorf("%w: digest %s is %d bytes on this server, the manifest declares %d", cas.ErrSizeMismatch, n.digest, have, n.size) } if !present { missing = append(missing, n.digest) missingBytes += n.size } } insertFile, err := tx.PrepareContext(ctx, ` INSERT INTO deployment_files (deployment_id, path, digest, size) VALUES (?, ?, ?, ?)`) if err != nil { return err } defer insertFile.Close() for _, f := range files { if _, err := insertFile.ExecContext(ctx, deploymentID, f.Path, f.Digest.Bytes(), f.Size); err != nil { return mapErr(err) } } _, err = tx.ExecContext(ctx, `UPDATE deployments SET state = ?, file_count = ?, total_bytes = ? WHERE id = ?`, StateUploading, len(files), totalBytes, deploymentID) return err }) if err != nil { return nil, 0, err } return missing, missingBytes, nil } // DeploymentFiles returns the manifest, ordered by path so assembly creates // each directory once and in a predictable order. func (d *DB) DeploymentFiles(ctx context.Context, deploymentID int64) ([]FileRow, error) { rows, err := d.r.QueryContext(ctx, `SELECT path, digest, size FROM deployment_files WHERE deployment_id = ? ORDER BY path`, deploymentID) if err != nil { return nil, err } defer rows.Close() var files []FileRow for rows.Next() { var f FileRow var raw []byte if err := rows.Scan(&f.Path, &raw, &f.Size); err != nil { return nil, err } if f.Digest, err = cas.FromBytes(raw); err != nil { return nil, err } files = append(files, f) } return files, rows.Err() } // MarkDeploymentReady records that assembly succeeded. // // Idempotent, and deliberately so: the CLI retries finalize after re-uploading // blobs that went missing, and a retry must not turn a good deployment into an // error. finalized_at keeps its original value so the timestamp reflects the // first success rather than the last attempt. func (d *DB) MarkDeploymentReady(ctx context.Context, id int64) error { now := unixNow() return d.Tx(ctx, func(tx *sql.Tx) error { res, err := tx.ExecContext(ctx, ` UPDATE deployments SET state = ?, finalized_at = COALESCE(finalized_at, ?), error = NULL WHERE id = ? AND state IN (?, ?)`, StateReady, now, id, StateUploading, StateReady) if err != nil { return err } return requireOneRow(ctx, tx, res, id, "finalize") }) } // ActivateDeployment makes one ready deployment the project's active one and // demotes whatever held that role before. // // Demotion comes first because deployments_one_active is a partial unique index, // checked per statement: promoting before demoting would collide with the row // still holding active = 1. The demotion deliberately excludes the target, so // re-activating the deployment that is already active is a no-op that refreshes // its timestamp rather than a transaction that briefly leaves the project with // nothing active. // // The caller is expected to hold the project lock and to have already built // whatever in-memory snapshot it means to publish: once this commits, SQLite is // the truth and a crash before the pointer store is recovered from here. func (d *DB) ActivateDeployment(ctx context.Context, projectID, deploymentID int64) error { now := unixNow() return d.Tx(ctx, func(tx *sql.Tx) error { if _, err := tx.ExecContext(ctx, ` UPDATE deployments SET active = 0, deactivated_at = ? WHERE project_id = ? AND active = 1 AND id <> ?`, now, projectID, deploymentID); err != nil { return err } res, err := tx.ExecContext(ctx, ` UPDATE deployments SET active = 1, activated_at = ?, deactivated_at = NULL WHERE id = ? AND project_id = ? AND state = ?`, now, deploymentID, projectID, StateReady) if err != nil { return mapErr(err) } return requireOneRow(ctx, tx, res, deploymentID, "activate") }) } // MarkDeploymentFailed records why a deployment was abandoned. The manifest // rows stay for now; GC drops them, which is what lets its blobs be reclaimed. func (d *DB) MarkDeploymentFailed(ctx context.Context, id int64, reason string) error { return d.Tx(ctx, func(tx *sql.Tx) error { res, err := tx.ExecContext(ctx, ` UPDATE deployments SET state = ?, error = ? WHERE id = ? AND state IN (?, ?, ?)`, StateFailed, truncate(reason, 1024), id, StatePending, StateUploading, StateFailed) if err != nil { return err } return requireOneRow(ctx, tx, res, id, "fail") }) } // requireOneRow turns "the UPDATE matched nothing" into the reason it matched // nothing, which is either a deployment that is gone or one in a state the // operation does not apply to. func requireOneRow(ctx context.Context, tx *sql.Tx, res sql.Result, id int64, op string) error { n, err := res.RowsAffected() if err != nil { return err } if n > 0 { return nil } var state State if err := tx.QueryRowContext(ctx, `SELECT state FROM deployments WHERE id = ?`, id).Scan(&state); err != nil { return mapErr(err) } return fmt.Errorf("%w: cannot %s a deployment that is %s", ErrConflict, op, state) } func truncate(s string, max int) string { if len(s) <= max { return s } return s[:max] + "…" }