pages/internal/cas/cas.go

package cas

import (
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"log/slog"
	"os"
	"path"
	"path/filepath"
)

// LinkMode is how an assembled deployment tree gets at a blob's content.
type LinkMode string

const (
	// LinkHard hardlinks. A deployment tree then costs directory entries and
	// nothing else, however many deployments share the same files.
	LinkHard LinkMode = "hardlink"
	// LinkCopy copies. Correct everywhere, at the cost of disk proportional to
	// deployed content rather than to unique content.
	LinkCopy LinkMode = "copy"
)

// Failures a caller distinguishes. Everything else is an I/O error and is
// returned as it came back from the operating system.
var (
	ErrDigestMismatch = errors.New("cas: content does not hash to the declared digest")
	ErrSizeMismatch   = errors.New("cas: content length does not match the declared size")
	ErrTooLarge       = errors.New("cas: content exceeds the maximum file size")
	ErrNotFound       = errors.New("cas: no such blob")
)

const (
	// tmpDir holds uploads in progress. Anything in it is unreferenced by
	// definition: a blob only becomes reachable by being renamed out of here.
	tmpDir = "tmp"

	// blobMode is read-only for everyone, and that is load-bearing rather than
	// tidy. An assembled deployment file is usually a hardlink to the blob —
	// the same inode — so anything that writes through the copy in $WEBROOT
	// corrupts the blob itself, and with it every project that shares that
	// content. Read-only is the cheap barrier; docs/operations.md carries the
	// warning that $WEBROOT is read-only to outside consumers.
	blobMode = 0o444
	dirMode  = 0o755
	tmpMode  = 0o600
)

// Options configure Open.
type Options struct {
	// Mode forces a link mode. The zero value probes and falls back to copying.
	Mode LinkMode
	// ProbeDir is where the probe tries to place a link. It must be the
	// directory deployment trees are assembled in: hardlinks cannot cross
	// filesystems, so probing anywhere else answers a different question.
	ProbeDir string
	Log      *slog.Logger
}

// Store is the on-disk blob store.
type Store struct {
	dir      string
	root     *os.Root
	linkMode LinkMode
	log      *slog.Logger
}

// Open prepares the store at dir, creating it if necessary.
func Open(dir string, opt Options) (*Store, error) {
	if dir == "" {
		return nil, errors.New("cas: directory is required")
	}
	abs, err := filepath.Abs(dir)
	if err != nil {
		return nil, err
	}
	if err := os.MkdirAll(filepath.Join(abs, tmpDir), dirMode); err != nil {
		return nil, err
	}
	log := opt.Log
	if log == nil {
		log = slog.New(slog.DiscardHandler)
	}

	// Every name handed to root below is derived from a digest, so traversal is
	// not the threat being defended against here. What os.Root buys is that a
	// symlink planted inside the store — by a restore from a bad backup, by a
	// misdirected rsync — cannot make a write land outside it.
	root, err := os.OpenRoot(abs)
	if err != nil {
		return nil, err
	}
	s := &Store{dir: abs, root: root, log: log}

	switch opt.Mode {
	case LinkCopy:
		s.linkMode = LinkCopy
	case LinkHard:
		if err := probeLink(abs, opt.ProbeDir); err != nil {
			root.Close()
			return nil, fmt.Errorf("cas: hardlinks were requested but %s cannot be hardlinked into %s: %w", abs, opt.ProbeDir, err)
		}
		s.linkMode = LinkHard
	case "":
		if err := probeLink(abs, opt.ProbeDir); err != nil {
			s.linkMode = LinkCopy
			log.Warn("hardlinks unavailable, deployment trees will be copies; disk use will be proportional to deployed content rather than to unique content",
				"cas_dir", abs, "deployments_dir", opt.ProbeDir, "err", err)
		} else {
			s.linkMode = LinkHard
		}
	default:
		root.Close()
		return nil, fmt.Errorf("cas: unknown link mode %q", opt.Mode)
	}
	return s, nil
}

// Dir is the store's absolute root directory.
func (s *Store) Dir() string { return s.dir }

// LinkMode reports how LinkInto will place content.
func (s *Store) LinkMode() LinkMode { return s.linkMode }

// Close releases the store's directory handle.
func (s *Store) Close() error { return s.root.Close() }

// Path is where a blob lives on disk. Only for hardlinking, which needs a name
// the kernel resolves from the process's own root; every other operation goes
// through the store's directory handle.
func (s *Store) Path(d Digest) string {
	return filepath.Join(s.dir, filepath.FromSlash(d.Rel()))
}

// Put stores r's bytes under want, if and only if they really hash to want.
//
// The verification is the reason cross-project deduplication is safe at all.
// Blobs are shared: if the server took the caller's word for the digest, a
// client could declare the digest of another project's index.html, upload
// whatever it liked, and every project referencing that content would start
// serving the attacker's bytes. So the hash is recomputed over the stream as it
// arrives and the upload is discarded unless it matches — the claimed digest is
// only ever a claim.
//
// declaredSize may be -1 when the caller genuinely does not know the length; any
// other value must match what arrives. maxBytes is a hard ceiling and is
// enforced by reading one byte past it rather than by trusting Content-Length.
//
// Returns the number of bytes stored. A blob that is already present is left
// alone and the upload discarded, which is safe precisely because both are known
// to hash to want.
func (s *Store) Put(ctx context.Context, want Digest, declaredSize, maxBytes int64, r io.Reader) (int64, error) {
	if maxBytes <= 0 {
		return 0, fmt.Errorf("cas: maxBytes must be positive, got %d", maxBytes)
	}
	if declaredSize > maxBytes {
		return 0, fmt.Errorf("%w: declared %d bytes, limit is %d", ErrTooLarge, declaredSize, maxBytes)
	}

	tmpRel, f, err := s.newTemp()
	if err != nil {
		return 0, err
	}
	// Every path out of this function that is not a successful rename has to
	// remove the temporary file, including the ones that are not our error — a
	// client that hung up mid-upload, a cancelled context. One defer covers all
	// of them.
	committed := false
	defer func() {
		f.Close() // no-op if already closed below
		if !committed {
			if err := s.root.Remove(tmpRel); err != nil && !errors.Is(err, fs.ErrNotExist) {
				s.log.Warn("removing abandoned upload", "path", tmpRel, "err", err)
			}
		}
	}()

	h := sha256.New()
	n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(&ctxReader{ctx: ctx, r: r}, maxBytes+1))
	if err != nil {
		return 0, err
	}
	if n > maxBytes {
		return 0, fmt.Errorf("%w: limit is %d bytes", ErrTooLarge, maxBytes)
	}
	if declaredSize >= 0 && n != declaredSize {
		return 0, fmt.Errorf("%w: declared %d bytes, received %d", ErrSizeMismatch, declaredSize, n)
	}
	if got := Digest(h.Sum(nil)); got != want {
		return 0, fmt.Errorf("%w: declared %s, computed %s", ErrDigestMismatch, want, got)
	}

	// Durability before visibility. Losing the content while keeping the
	// directory entry would be corruption — the database would say present=1
	// and the file would be a hole — whereas losing the rename is merely a blob
	// that has to be uploaded again, which deploy.Recover already handles by
	// setting present=0 for digests whose file is missing. So the file is
	// fsynced and the containing directory deliberately is not.
	if err := f.Sync(); err != nil {
		return 0, err
	}
	if err := f.Chmod(blobMode); err != nil {
		return 0, err
	}
	if err := f.Close(); err != nil {
		return 0, err
	}

	rel := want.Rel()
	if err := s.root.MkdirAll(path.Dir(rel), dirMode); err != nil {
		return 0, err
	}
	if _, err := s.root.Stat(rel); err == nil {
		// Two clients uploading identical content at once is a benign race, not
		// a conflict: keep what is there and drop ours. Losing this race to a
		// writer that renames between the Stat and here is equally benign,
		// since rename is atomic and both files hold the same bytes.
		return n, nil
	} else if !errors.Is(err, fs.ErrNotExist) {
		return 0, err
	}
	if err := s.root.Rename(tmpRel, rel); err != nil {
		return 0, err
	}
	committed = true
	return n, nil
}

// Has reports whether the blob's bytes are on disk.
func (s *Store) Has(d Digest) (bool, error) {
	if _, err := s.root.Stat(d.Rel()); err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

// Open returns the blob for reading; the caller closes it.
//
// This is the read path of every request the server serves, and it is
// deliberately the only filesystem call on it. The name comes from Rel(), which
// is derived from 32 bytes that came out of an in-memory map, so no byte of
// user input reaches the filesystem here. Traversal on the read path is not so
// much prevented as inexpressible.
func (s *Store) Open(d Digest) (*os.File, error) {
	f, err := s.root.Open(d.Rel())
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return nil, fmt.Errorf("%w: %s", ErrNotFound, d)
		}
		return nil, err
	}
	return f, nil
}

// LinkInto places the blob's content at destDir/relPath.
//
// destDir is a staging directory the caller has just created and relPath has
// passed pathutil.Validate, so the join cannot leave destDir and there is no
// pre-existing symlink for it to follow.
func (s *Store) LinkInto(d Digest, destDir, relPath string) error {
	dest := filepath.Join(destDir, filepath.FromSlash(relPath))
	if s.linkMode == LinkHard {
		err := os.Link(s.Path(d), dest)
		switch {
		case err == nil:
			return nil
		case errors.Is(err, fs.ErrNotExist):
			// The blob is gone, or a parent directory was never created. Either
			// is a bug here, not a filesystem limitation, and copying would only
			// fail again with a less informative error.
			return fmt.Errorf("%w: %s: %w", ErrNotFound, d, err)
		case errors.Is(err, fs.ErrExist):
			// pathutil.Set rejects duplicate paths within a manifest, so two
			// files claiming one name means the manifest was not checked.
			return fmt.Errorf("cas: %s already exists in the deployment tree: %w", relPath, err)
		}
		// Anything else — most often EMLINK, since ext4 caps a file at 65,000
		// links and a blob shared by enough deployments does reach that — is a
		// per-file limitation rather than a reason to fail the deployment.
		s.log.Debug("hardlink failed, copying this file instead", "digest", d, "path", relPath, "err", err)
	}
	return s.copyInto(d, dest)
}

func (s *Store) copyInto(d Digest, dest string) error {
	src, err := s.Open(d)
	if err != nil {
		return err
	}
	defer src.Close()

	// O_EXCL because a collision means two manifest entries claimed one path,
	// which pathutil.Set is supposed to have made impossible.
	dst, err := os.OpenFile(dest, os.O_CREATE|os.O_EXCL|os.O_WRONLY, blobMode)
	if err != nil {
		return err
	}
	defer dst.Close()
	if _, err := io.Copy(dst, src); err != nil {
		return err
	}
	// A copy is the only content in the deployment tree that is not already
	// durable — a hardlink shares the inode that Put fsynced.
	if err := dst.Sync(); err != nil {
		return err
	}
	return dst.Close()
}

// Remove deletes a blob. Missing is success, so GC can be re-run after a crash.
func (s *Store) Remove(d Digest) error {
	if err := s.root.Remove(d.Rel()); err != nil && !errors.Is(err, fs.ErrNotExist) {
		return err
	}
	return nil
}

// PurgeTemp deletes every upload in progress and reports how many. Startup
// calls it: an interrupted upload is unreferenced garbage by construction.
func (s *Store) PurgeTemp() (int, error) {
	entries, err := os.ReadDir(filepath.Join(s.dir, tmpDir))
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return 0, nil
		}
		return 0, err
	}
	n := 0
	for _, e := range entries {
		if err := s.root.RemoveAll(tmpDir + "/" + e.Name()); err != nil {
			return n, err
		}
		n++
	}
	return n, nil
}

// newTemp creates a uniquely named file under tmp/ and returns its store-relative
// name. 128 bits of randomness make a collision impossible; the retry loop costs
// nothing and means a hypothetical one is not a failed upload.
func (s *Store) newTemp() (string, *os.File, error) {
	var buf [16]byte
	for attempt := 0; attempt < 3; attempt++ {
		if _, err := rand.Read(buf[:]); err != nil {
			return "", nil, err
		}
		name := tmpDir + "/" + hex.EncodeToString(buf[:])
		f, err := s.root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, tmpMode)
		if err == nil {
			return name, f, nil
		}
		if !errors.Is(err, fs.ErrExist) {
			return "", nil, err
		}
	}
	return "", nil, errors.New("cas: could not create a temporary file")
}

// probeLink answers whether a hardlink from the blob store into the deployment
// tree actually works here.
//
// It has to be a real attempt rather than a check of the filesystem type.
// overlayfs — Docker's default, and this machine's /home — can fail or silently
// copy up across layers; a bind mount or a separate volume for the deployment
// tree puts the two directories on different devices and link(2) returns EXDEV;
// some hardened mounts refuse link(2) outright. The only reliable answer is to
// try it against the directory that will actually be used.
func probeLink(casDir, probeDir string) error {
	if probeDir == "" {
		return errors.New("no deployments directory to probe against")
	}
	if err := os.MkdirAll(probeDir, dirMode); err != nil {
		return err
	}
	var buf [8]byte
	if _, err := rand.Read(buf[:]); err != nil {
		return err
	}
	suffix := hex.EncodeToString(buf[:])
	src := filepath.Join(casDir, tmpDir, ".link-probe-"+suffix)
	dst := filepath.Join(probeDir, ".link-probe-"+suffix)
	if err := os.WriteFile(src, []byte("probe"), tmpMode); err != nil {
		return err
	}
	defer os.Remove(src)
	if err := os.Link(src, dst); err != nil {
		return err
	}
	return os.Remove(dst)
}

// ctxReader makes a long upload cancellable. The HTTP server closes the body
// when a client disappears, but Put also runs against local readers on recovery
// paths where nothing else would notice a shutdown.
type ctxReader struct {
	ctx context.Context
	r   io.Reader
}

func (c *ctxReader) Read(p []byte) (int, error) {
	if err := c.ctx.Err(); err != nil {
		return 0, err
	}
	return c.r.Read(p)
}