pages/internal/client/scan.go

package client

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"slices"

	"golang.org/x/sync/errgroup"

	"github.com/iceBear67/simplepages/api"
	"github.com/iceBear67/simplepages/internal/pathutil"
)

// skipDirs are never walked into. A build output directory should not contain
// them at all, but "pages deploy ." on a repository root is a mistake people
// make once, and uploading a .git directory publishes the whole history.
var skipDirs = []string{".git", ".hg", ".svn"}

// LocalFile is one file of a scanned directory, already hashed.
type LocalFile struct {
	// Path is site-relative and slash-separated: it is what the URL will be.
	Path string `json:"path"`
	// Digest is the lowercase hex SHA-256 of the contents.
	Digest string `json:"digest"`
	Size   int64  `json:"size"`
}

// ScanOptions filters and paces a scan.
type ScanOptions struct {
	// Include, when non-empty, keeps only files matching at least one pattern.
	// Exclude drops files matching any pattern, and prunes whole directories.
	// A pattern is path.Match syntax, tried against the site-relative path and
	// against the base name, so both "assets/*.map" and "*.map" work.
	Include []string
	Exclude []string

	// FollowSymlinks reads through symbolic links instead of refusing them.
	// Links are still resolved inside the scanned directory, so one pointing at
	// /etc/passwd fails rather than publishing it.
	FollowSymlinks bool

	// Concurrency bounds the hashing goroutines. Zero means GOMAXPROCS.
	Concurrency int
}

// Source is a scanned directory: the manifest it produced, plus the handle the
// upload path reads the contents back through.
//
// Files are read through an os.Root rather than by path, so a symlink swapped
// in between the scan and the upload still cannot reach outside the directory
// the user named.
type Source struct {
	Dir        string
	Files      []LocalFile
	TotalBytes int64

	root *os.Root
}

// Scan walks dir, hashes what it finds, and returns the result. The caller must
// Close the Source.
func Scan(ctx context.Context, dir string, opts ScanOptions) (*Source, error) {
	if err := checkPatterns("include", opts.Include); err != nil {
		return nil, err
	}
	if err := checkPatterns("exclude", opts.Exclude); err != nil {
		return nil, err
	}

	abs, err := filepath.Abs(dir)
	if err != nil {
		return nil, fmt.Errorf("%s: %w", dir, err)
	}
	fi, err := os.Stat(abs)
	if err != nil {
		return nil, err
	}
	if !fi.IsDir() {
		return nil, fmt.Errorf("%s is not a directory", dir)
	}
	root, err := os.OpenRoot(abs)
	if err != nil {
		return nil, err
	}

	s := &Source{Dir: abs, root: root}
	if err := s.walk(ctx, opts); err != nil {
		root.Close()
		return nil, err
	}
	if len(s.Files) == 0 {
		root.Close()
		return nil, fmt.Errorf("%s contains no files to deploy", dir)
	}
	if err := s.hash(ctx, opts.Concurrency); err != nil {
		root.Close()
		return nil, err
	}
	return s, nil
}

// Close releases the directory handle.
func (s *Source) Close() error {
	if s == nil || s.root == nil {
		return nil
	}
	return s.root.Close()
}

// Open reads one of the scanned files.
func (s *Source) Open(p string) (*os.File, error) {
	return s.root.Open(filepath.FromSlash(p))
}

// Manifest renders the scan as the wire form the server expects.
func (s *Source) Manifest() []api.FileEntry {
	out := make([]api.FileEntry, len(s.Files))
	for i, f := range s.Files {
		out[i] = api.FileEntry{Path: f.Path, Digest: f.Digest, Size: f.Size}
	}
	return out
}

// UniqueBlobs counts distinct digests, which is what the deduplicating upload
// actually has to deal with.
func (s *Source) UniqueBlobs() int {
	seen := make(map[string]struct{}, len(s.Files))
	for _, f := range s.Files {
		seen[f.Digest] = struct{}{}
	}
	return len(seen)
}

// walk collects the paths and sizes. Hashing is a separate pass so it can run
// concurrently over a list that is already known to be valid: finding out on
// file 40,000 that file 3 has an unusable name would waste the whole scan.
func (s *Source) walk(ctx context.Context, opts ScanOptions) error {
	set := pathutil.NewSet(0)
	return filepath.WalkDir(s.Dir, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if err := ctx.Err(); err != nil {
			return err
		}
		rel, err := filepath.Rel(s.Dir, p)
		if err != nil {
			return err
		}
		if rel == "." {
			return nil
		}
		name := filepath.ToSlash(rel)

		if d.IsDir() {
			if slices.Contains(skipDirs, d.Name()) || matchAny(opts.Exclude, name) {
				return fs.SkipDir
			}
			return nil
		}

		// WalkDir reports entry types from Lstat, so a symlink arrives as a
		// symlink and is never silently followed.
		var size int64
		switch {
		case d.Type()&fs.ModeSymlink != 0:
			if !opts.FollowSymlinks {
				return fmt.Errorf("%s is a symbolic link; a deployment holds regular files only "+
					"(pass --follow-symlinks to upload what it points at)", name)
			}
			fi, err := s.root.Stat(filepath.FromSlash(name))
			if err != nil {
				return fmt.Errorf("%s: %w", name, err)
			}
			if !fi.Mode().IsRegular() {
				return fmt.Errorf("%s points at a %s, not a regular file", name, kindOf(fi.Mode()))
			}
			size = fi.Size()
		case d.Type().IsRegular():
			fi, err := d.Info()
			if err != nil {
				return err
			}
			size = fi.Size()
		default:
			return fmt.Errorf("%s is a %s; a deployment holds regular files only",
				name, kindOf(d.Type()))
		}

		if !keep(opts, name) {
			return nil
		}
		// The same checks the server runs, so a name that could never be stored
		// is reported here — with the local path in hand — instead of as a
		// rejected manifest after the walk.
		if err := set.Add(name); err != nil {
			return fmt.Errorf("%s: %w", name, err)
		}
		s.Files = append(s.Files, LocalFile{Path: name, Size: size})
		s.TotalBytes += size
		return nil
	})
}

// hash fills in every digest. It is CPU-bound on small files and IO-bound on
// large ones, so it runs at GOMAXPROCS by default.
func (s *Source) hash(ctx context.Context, concurrency int) error {
	if concurrency <= 0 {
		concurrency = runtime.GOMAXPROCS(0)
	}
	g, ctx := errgroup.WithContext(ctx)
	g.SetLimit(concurrency)
	for i := range s.Files {
		g.Go(func() error {
			if err := ctx.Err(); err != nil {
				return err
			}
			f := &s.Files[i]
			digest, size, err := s.digest(f.Path)
			if err != nil {
				return fmt.Errorf("%s: %w", f.Path, err)
			}
			// The file may have been rewritten between the walk and now. The
			// digest and the size have to describe the same bytes, so take both
			// from the read that produced the digest.
			f.Digest, f.Size = digest, size
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		return err
	}
	// Sorted output makes "pages deploy --dry-run" diffable between runs.
	slices.SortFunc(s.Files, func(a, b LocalFile) int {
		if a.Path < b.Path {
			return -1
		}
		if a.Path > b.Path {
			return 1
		}
		return 0
	})
	s.TotalBytes = 0
	for _, f := range s.Files {
		s.TotalBytes += f.Size
	}
	return nil
}

func (s *Source) digest(p string) (string, int64, error) {
	f, err := s.Open(p)
	if err != nil {
		return "", 0, err
	}
	defer f.Close()
	h := sha256.New()
	n, err := io.Copy(h, f)
	if err != nil {
		return "", 0, err
	}
	return hex.EncodeToString(h.Sum(nil)), n, nil
}

// keep applies the include/exclude filters to a file.
func keep(opts ScanOptions, name string) bool {
	if len(opts.Include) > 0 && !matchAny(opts.Include, name) {
		return false
	}
	return !matchAny(opts.Exclude, name)
}

// matchAny reports whether name matches a pattern, either as a whole path or by
// its base name. Matching the base name too is what makes "--exclude '*.map'"
// behave the way everyone expects, since path.Match's "*" does not cross "/".
func matchAny(patterns []string, name string) bool {
	base := path.Base(name)
	for _, pat := range patterns {
		if ok, _ := path.Match(pat, name); ok {
			return true
		}
		if ok, _ := path.Match(pat, base); ok {
			return true
		}
	}
	return false
}

// checkPatterns rejects malformed globs up front. path.Match reports a bad
// pattern only when it is tried, so an unchecked one would silently match
// nothing and quietly deploy the wrong file set.
func checkPatterns(flag string, patterns []string) error {
	for _, pat := range patterns {
		if _, err := path.Match(pat, "x"); err != nil {
			return fmt.Errorf("--%s %q: %w", flag, pat, err)
		}
	}
	return nil
}

func kindOf(m fs.FileMode) string {
	switch {
	case m&fs.ModeDir != 0:
		return "directory"
	case m&fs.ModeSymlink != 0:
		return "symbolic link"
	case m&fs.ModeDevice != 0:
		return "device file"
	case m&fs.ModeNamedPipe != 0:
		return "named pipe"
	case m&fs.ModeSocket != 0:
		return "socket"
	default:
		return "special file"
	}
}