package store import ( "context" "database/sql" "github.com/iceBear67/simplepages/internal/cas" ) // maxReportedDrift bounds what a report carries back. A repair fixes every row // it finds; the list is for a human reading the output, and a human does not // read ten thousand digests. const maxReportedDrift = 100 // Drift is one blob whose recorded refcount disagrees with the manifests that // actually name it. type Drift struct { Digest cas.Digest // Stored is what the blobs row claims, Actual what counting the manifest // rows gives. Stored above Actual wastes disk: the collector will never // reclaim the blob. Stored below Actual is the dangerous direction — the // collector may delete content a deployment still needs. Stored int64 Actual int64 } // FsckReport is what a consistency check found. type FsckReport struct { // Blobs is how many rows were examined. Blobs int64 // DriftCount is how many disagreed; Drift lists the first few of them. DriftCount int Drift []Drift // Repaired is how many rows were corrected, and is zero unless the check // was asked to repair. Repaired int } // Fsck recomputes every blob's refcount from the manifests and reports the rows // that disagree, optionally correcting them. // // Refcounts are maintained by triggers on deployment_files, so under normal // operation they cannot drift. This exists for the cases outside normal // operation: a database restored from a backup taken mid-transaction, a schema // touched by hand, or a bug in this program. Drift matters because the blob // collector trusts the counter — a count that reads low is content that will be // deleted while a deployment still references it, which is the one way this // design can lose data. // // The recount is a single grouped join rather than a query per blob, so a store // with a million blobs is one table scan and not a million index seeks. func (d *DB) Fsck(ctx context.Context, repair bool) (FsckReport, error) { var rep FsckReport if err := d.r.QueryRowContext(ctx, `SELECT count(*) FROM blobs`).Scan(&rep.Blobs); err != nil { return rep, err } rows, err := d.r.QueryContext(ctx, ` SELECT b.digest, b.refcount, coalesce(c.n, 0) FROM blobs b LEFT JOIN (SELECT digest, count(*) AS n FROM deployment_files GROUP BY digest) c ON c.digest = b.digest WHERE b.refcount <> coalesce(c.n, 0) ORDER BY b.digest`) if err != nil { return rep, err } defer rows.Close() var drift []Drift for rows.Next() { var dr Drift var raw []byte if err := rows.Scan(&raw, &dr.Stored, &dr.Actual); err != nil { return rep, err } if dr.Digest, err = cas.FromBytes(raw); err != nil { return rep, err } drift = append(drift, dr) } if err := rows.Err(); err != nil { return rep, err } rep.DriftCount = len(drift) rep.Drift = drift if len(rep.Drift) > maxReportedDrift { rep.Drift = rep.Drift[:maxReportedDrift] } if !repair || len(drift) == 0 { return rep, nil } // Repair writes the recounted value rather than adjusting by the difference: // the manifests are the definition of the refcount, so the correct value is // the one just counted, whatever the column happened to say. err = d.Tx(ctx, func(tx *sql.Tx) error { rep.Repaired = 0 stmt, err := tx.PrepareContext(ctx, `UPDATE blobs SET refcount = ? WHERE digest = ?`) if err != nil { return err } defer stmt.Close() for _, dr := range drift { if _, err := stmt.ExecContext(ctx, dr.Actual, dr.Digest.Bytes()); err != nil { return err } rep.Repaired++ } return nil }) if err != nil { return rep, err } return rep, nil }