init

internal/store/fsck.go

@@ -0,0 +1,117 @@
+package store
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/iceBear67/simplepages/internal/cas"
+)
+
+// maxReportedDrift bounds what a report carries back. A repair fixes every row
+// it finds; the list is for a human reading the output, and a human does not
+// read ten thousand digests.
+const maxReportedDrift = 100
+
+// Drift is one blob whose recorded refcount disagrees with the manifests that
+// actually name it.
+type Drift struct {
+	Digest cas.Digest
+	// Stored is what the blobs row claims, Actual what counting the manifest
+	// rows gives. Stored above Actual wastes disk: the collector will never
+	// reclaim the blob. Stored below Actual is the dangerous direction — the
+	// collector may delete content a deployment still needs.
+	Stored int64
+	Actual int64
+}
+
+// FsckReport is what a consistency check found.
+type FsckReport struct {
+	// Blobs is how many rows were examined.
+	Blobs int64
+	// DriftCount is how many disagreed; Drift lists the first few of them.
+	DriftCount int
+	Drift      []Drift
+	// Repaired is how many rows were corrected, and is zero unless the check
+	// was asked to repair.
+	Repaired int
+}
+
+// Fsck recomputes every blob's refcount from the manifests and reports the rows
+// that disagree, optionally correcting them.
+//
+// Refcounts are maintained by triggers on deployment_files, so under normal
+// operation they cannot drift. This exists for the cases outside normal
+// operation: a database restored from a backup taken mid-transaction, a schema
+// touched by hand, or a bug in this program. Drift matters because the blob
+// collector trusts the counter — a count that reads low is content that will be
+// deleted while a deployment still references it, which is the one way this
+// design can lose data.
+//
+// The recount is a single grouped join rather than a query per blob, so a store
+// with a million blobs is one table scan and not a million index seeks.
+func (d *DB) Fsck(ctx context.Context, repair bool) (FsckReport, error) {
+	var rep FsckReport
+	if err := d.r.QueryRowContext(ctx, `SELECT count(*) FROM blobs`).Scan(&rep.Blobs); err != nil {
+		return rep, err
+	}
+
+	rows, err := d.r.QueryContext(ctx, `
+		SELECT b.digest, b.refcount, coalesce(c.n, 0)
+		FROM blobs b
+		LEFT JOIN (SELECT digest, count(*) AS n FROM deployment_files GROUP BY digest) c
+		       ON c.digest = b.digest
+		WHERE b.refcount <> coalesce(c.n, 0)
+		ORDER BY b.digest`)
+	if err != nil {
+		return rep, err
+	}
+	defer rows.Close()
+
+	var drift []Drift
+	for rows.Next() {
+		var dr Drift
+		var raw []byte
+		if err := rows.Scan(&raw, &dr.Stored, &dr.Actual); err != nil {
+			return rep, err
+		}
+		if dr.Digest, err = cas.FromBytes(raw); err != nil {
+			return rep, err
+		}
+		drift = append(drift, dr)
+	}
+	if err := rows.Err(); err != nil {
+		return rep, err
+	}
+
+	rep.DriftCount = len(drift)
+	rep.Drift = drift
+	if len(rep.Drift) > maxReportedDrift {
+		rep.Drift = rep.Drift[:maxReportedDrift]
+	}
+	if !repair || len(drift) == 0 {
+		return rep, nil
+	}
+
+	// Repair writes the recounted value rather than adjusting by the difference:
+	// the manifests are the definition of the refcount, so the correct value is
+	// the one just counted, whatever the column happened to say.
+	err = d.Tx(ctx, func(tx *sql.Tx) error {
+		rep.Repaired = 0
+		stmt, err := tx.PrepareContext(ctx, `UPDATE blobs SET refcount = ? WHERE digest = ?`)
+		if err != nil {
+			return err
+		}
+		defer stmt.Close()
+		for _, dr := range drift {
+			if _, err := stmt.ExecContext(ctx, dr.Actual, dr.Digest.Bytes()); err != nil {
+				return err
+			}
+			rep.Repaired++
+		}
+		return nil
+	})
+	if err != nil {
+		return rep, err
+	}
+	return rep, nil
+}