package cas import ( "context" "crypto/rand" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "io/fs" "log/slog" "os" "path" "path/filepath" ) // LinkMode is how an assembled deployment tree gets at a blob's content. type LinkMode string const ( // LinkHard hardlinks. A deployment tree then costs directory entries and // nothing else, however many deployments share the same files. LinkHard LinkMode = "hardlink" // LinkCopy copies. Correct everywhere, at the cost of disk proportional to // deployed content rather than to unique content. LinkCopy LinkMode = "copy" ) // Failures a caller distinguishes. Everything else is an I/O error and is // returned as it came back from the operating system. var ( ErrDigestMismatch = errors.New("cas: content does not hash to the declared digest") ErrSizeMismatch = errors.New("cas: content length does not match the declared size") ErrTooLarge = errors.New("cas: content exceeds the maximum file size") ErrNotFound = errors.New("cas: no such blob") ) const ( // tmpDir holds uploads in progress. Anything in it is unreferenced by // definition: a blob only becomes reachable by being renamed out of here. tmpDir = "tmp" // blobMode is read-only for everyone, and that is load-bearing rather than // tidy. An assembled deployment file is usually a hardlink to the blob — // the same inode — so anything that writes through the copy in $WEBROOT // corrupts the blob itself, and with it every project that shares that // content. Read-only is the cheap barrier; docs/operations.md carries the // warning that $WEBROOT is read-only to outside consumers. blobMode = 0o444 dirMode = 0o755 tmpMode = 0o600 ) // Options configure Open. type Options struct { // Mode forces a link mode. The zero value probes and falls back to copying. Mode LinkMode // ProbeDir is where the probe tries to place a link. It must be the // directory deployment trees are assembled in: hardlinks cannot cross // filesystems, so probing anywhere else answers a different question. ProbeDir string Log *slog.Logger } // Store is the on-disk blob store. type Store struct { dir string root *os.Root linkMode LinkMode log *slog.Logger } // Open prepares the store at dir, creating it if necessary. func Open(dir string, opt Options) (*Store, error) { if dir == "" { return nil, errors.New("cas: directory is required") } abs, err := filepath.Abs(dir) if err != nil { return nil, err } if err := os.MkdirAll(filepath.Join(abs, tmpDir), dirMode); err != nil { return nil, err } log := opt.Log if log == nil { log = slog.New(slog.DiscardHandler) } // Every name handed to root below is derived from a digest, so traversal is // not the threat being defended against here. What os.Root buys is that a // symlink planted inside the store — by a restore from a bad backup, by a // misdirected rsync — cannot make a write land outside it. root, err := os.OpenRoot(abs) if err != nil { return nil, err } s := &Store{dir: abs, root: root, log: log} switch opt.Mode { case LinkCopy: s.linkMode = LinkCopy case LinkHard: if err := probeLink(abs, opt.ProbeDir); err != nil { root.Close() return nil, fmt.Errorf("cas: hardlinks were requested but %s cannot be hardlinked into %s: %w", abs, opt.ProbeDir, err) } s.linkMode = LinkHard case "": if err := probeLink(abs, opt.ProbeDir); err != nil { s.linkMode = LinkCopy log.Warn("hardlinks unavailable, deployment trees will be copies; disk use will be proportional to deployed content rather than to unique content", "cas_dir", abs, "deployments_dir", opt.ProbeDir, "err", err) } else { s.linkMode = LinkHard } default: root.Close() return nil, fmt.Errorf("cas: unknown link mode %q", opt.Mode) } return s, nil } // Dir is the store's absolute root directory. func (s *Store) Dir() string { return s.dir } // LinkMode reports how LinkInto will place content. func (s *Store) LinkMode() LinkMode { return s.linkMode } // Close releases the store's directory handle. func (s *Store) Close() error { return s.root.Close() } // Path is where a blob lives on disk. Only for hardlinking, which needs a name // the kernel resolves from the process's own root; every other operation goes // through the store's directory handle. func (s *Store) Path(d Digest) string { return filepath.Join(s.dir, filepath.FromSlash(d.Rel())) } // Put stores r's bytes under want, if and only if they really hash to want. // // The verification is the reason cross-project deduplication is safe at all. // Blobs are shared: if the server took the caller's word for the digest, a // client could declare the digest of another project's index.html, upload // whatever it liked, and every project referencing that content would start // serving the attacker's bytes. So the hash is recomputed over the stream as it // arrives and the upload is discarded unless it matches — the claimed digest is // only ever a claim. // // declaredSize may be -1 when the caller genuinely does not know the length; any // other value must match what arrives. maxBytes is a hard ceiling and is // enforced by reading one byte past it rather than by trusting Content-Length. // // Returns the number of bytes stored. A blob that is already present is left // alone and the upload discarded, which is safe precisely because both are known // to hash to want. func (s *Store) Put(ctx context.Context, want Digest, declaredSize, maxBytes int64, r io.Reader) (int64, error) { if maxBytes <= 0 { return 0, fmt.Errorf("cas: maxBytes must be positive, got %d", maxBytes) } if declaredSize > maxBytes { return 0, fmt.Errorf("%w: declared %d bytes, limit is %d", ErrTooLarge, declaredSize, maxBytes) } tmpRel, f, err := s.newTemp() if err != nil { return 0, err } // Every path out of this function that is not a successful rename has to // remove the temporary file, including the ones that are not our error — a // client that hung up mid-upload, a cancelled context. One defer covers all // of them. committed := false defer func() { f.Close() // no-op if already closed below if !committed { if err := s.root.Remove(tmpRel); err != nil && !errors.Is(err, fs.ErrNotExist) { s.log.Warn("removing abandoned upload", "path", tmpRel, "err", err) } } }() h := sha256.New() n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(&ctxReader{ctx: ctx, r: r}, maxBytes+1)) if err != nil { return 0, err } if n > maxBytes { return 0, fmt.Errorf("%w: limit is %d bytes", ErrTooLarge, maxBytes) } if declaredSize >= 0 && n != declaredSize { return 0, fmt.Errorf("%w: declared %d bytes, received %d", ErrSizeMismatch, declaredSize, n) } if got := Digest(h.Sum(nil)); got != want { return 0, fmt.Errorf("%w: declared %s, computed %s", ErrDigestMismatch, want, got) } // Durability before visibility. Losing the content while keeping the // directory entry would be corruption — the database would say present=1 // and the file would be a hole — whereas losing the rename is merely a blob // that has to be uploaded again, which deploy.Recover already handles by // setting present=0 for digests whose file is missing. So the file is // fsynced and the containing directory deliberately is not. if err := f.Sync(); err != nil { return 0, err } if err := f.Chmod(blobMode); err != nil { return 0, err } if err := f.Close(); err != nil { return 0, err } rel := want.Rel() if err := s.root.MkdirAll(path.Dir(rel), dirMode); err != nil { return 0, err } if _, err := s.root.Stat(rel); err == nil { // Two clients uploading identical content at once is a benign race, not // a conflict: keep what is there and drop ours. Losing this race to a // writer that renames between the Stat and here is equally benign, // since rename is atomic and both files hold the same bytes. return n, nil } else if !errors.Is(err, fs.ErrNotExist) { return 0, err } if err := s.root.Rename(tmpRel, rel); err != nil { return 0, err } committed = true return n, nil } // Has reports whether the blob's bytes are on disk. func (s *Store) Has(d Digest) (bool, error) { if _, err := s.root.Stat(d.Rel()); err != nil { if errors.Is(err, fs.ErrNotExist) { return false, nil } return false, err } return true, nil } // Open returns the blob for reading; the caller closes it. // // This is the read path of every request the server serves, and it is // deliberately the only filesystem call on it. The name comes from Rel(), which // is derived from 32 bytes that came out of an in-memory map, so no byte of // user input reaches the filesystem here. Traversal on the read path is not so // much prevented as inexpressible. func (s *Store) Open(d Digest) (*os.File, error) { f, err := s.root.Open(d.Rel()) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("%w: %s", ErrNotFound, d) } return nil, err } return f, nil } // LinkInto places the blob's content at destDir/relPath. // // destDir is a staging directory the caller has just created and relPath has // passed pathutil.Validate, so the join cannot leave destDir and there is no // pre-existing symlink for it to follow. func (s *Store) LinkInto(d Digest, destDir, relPath string) error { dest := filepath.Join(destDir, filepath.FromSlash(relPath)) if s.linkMode == LinkHard { err := os.Link(s.Path(d), dest) switch { case err == nil: return nil case errors.Is(err, fs.ErrNotExist): // The blob is gone, or a parent directory was never created. Either // is a bug here, not a filesystem limitation, and copying would only // fail again with a less informative error. return fmt.Errorf("%w: %s: %w", ErrNotFound, d, err) case errors.Is(err, fs.ErrExist): // pathutil.Set rejects duplicate paths within a manifest, so two // files claiming one name means the manifest was not checked. return fmt.Errorf("cas: %s already exists in the deployment tree: %w", relPath, err) } // Anything else — most often EMLINK, since ext4 caps a file at 65,000 // links and a blob shared by enough deployments does reach that — is a // per-file limitation rather than a reason to fail the deployment. s.log.Debug("hardlink failed, copying this file instead", "digest", d, "path", relPath, "err", err) } return s.copyInto(d, dest) } func (s *Store) copyInto(d Digest, dest string) error { src, err := s.Open(d) if err != nil { return err } defer src.Close() // O_EXCL because a collision means two manifest entries claimed one path, // which pathutil.Set is supposed to have made impossible. dst, err := os.OpenFile(dest, os.O_CREATE|os.O_EXCL|os.O_WRONLY, blobMode) if err != nil { return err } defer dst.Close() if _, err := io.Copy(dst, src); err != nil { return err } // A copy is the only content in the deployment tree that is not already // durable — a hardlink shares the inode that Put fsynced. if err := dst.Sync(); err != nil { return err } return dst.Close() } // Remove deletes a blob. Missing is success, so GC can be re-run after a crash. func (s *Store) Remove(d Digest) error { if err := s.root.Remove(d.Rel()); err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil } // PurgeTemp deletes every upload in progress and reports how many. Startup // calls it: an interrupted upload is unreferenced garbage by construction. func (s *Store) PurgeTemp() (int, error) { entries, err := os.ReadDir(filepath.Join(s.dir, tmpDir)) if err != nil { if errors.Is(err, fs.ErrNotExist) { return 0, nil } return 0, err } n := 0 for _, e := range entries { if err := s.root.RemoveAll(tmpDir + "/" + e.Name()); err != nil { return n, err } n++ } return n, nil } // newTemp creates a uniquely named file under tmp/ and returns its store-relative // name. 128 bits of randomness make a collision impossible; the retry loop costs // nothing and means a hypothetical one is not a failed upload. func (s *Store) newTemp() (string, *os.File, error) { var buf [16]byte for attempt := 0; attempt < 3; attempt++ { if _, err := rand.Read(buf[:]); err != nil { return "", nil, err } name := tmpDir + "/" + hex.EncodeToString(buf[:]) f, err := s.root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, tmpMode) if err == nil { return name, f, nil } if !errors.Is(err, fs.ErrExist) { return "", nil, err } } return "", nil, errors.New("cas: could not create a temporary file") } // probeLink answers whether a hardlink from the blob store into the deployment // tree actually works here. // // It has to be a real attempt rather than a check of the filesystem type. // overlayfs — Docker's default, and this machine's /home — can fail or silently // copy up across layers; a bind mount or a separate volume for the deployment // tree puts the two directories on different devices and link(2) returns EXDEV; // some hardened mounts refuse link(2) outright. The only reliable answer is to // try it against the directory that will actually be used. func probeLink(casDir, probeDir string) error { if probeDir == "" { return errors.New("no deployments directory to probe against") } if err := os.MkdirAll(probeDir, dirMode); err != nil { return err } var buf [8]byte if _, err := rand.Read(buf[:]); err != nil { return err } suffix := hex.EncodeToString(buf[:]) src := filepath.Join(casDir, tmpDir, ".link-probe-"+suffix) dst := filepath.Join(probeDir, ".link-probe-"+suffix) if err := os.WriteFile(src, []byte("probe"), tmpMode); err != nil { return err } defer os.Remove(src) if err := os.Link(src, dst); err != nil { return err } return os.Remove(dst) } // ctxReader makes a long upload cancellable. The HTTP server closes the body // when a client disappears, but Put also runs against local readers on recovery // paths where nothing else would notice a shutdown. type ctxReader struct { ctx context.Context r io.Reader } func (c *ctxReader) Read(p []byte) (int, error) { if err := c.ctx.Err(); err != nil { return 0, err } return c.r.Read(p) }