init

2026-08-15 07:13:00 +00:00
commit dd50674fdc
114 changed files with 26865 additions and 0 deletions
@@ -0,0 +1,323 @@
+package client
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"slices"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/iceBear67/simplepages/api"
+	"github.com/iceBear67/simplepages/internal/pathutil"
+)
+
+// skipDirs are never walked into. A build output directory should not contain
+// them at all, but "pages deploy ." on a repository root is a mistake people
+// make once, and uploading a .git directory publishes the whole history.
+var skipDirs = []string{".git", ".hg", ".svn"}
+
+// LocalFile is one file of a scanned directory, already hashed.
+type LocalFile struct {
+	// Path is site-relative and slash-separated: it is what the URL will be.
+	Path string `json:"path"`
+	// Digest is the lowercase hex SHA-256 of the contents.
+	Digest string `json:"digest"`
+	Size   int64  `json:"size"`
+}
+
+// ScanOptions filters and paces a scan.
+type ScanOptions struct {
+	// Include, when non-empty, keeps only files matching at least one pattern.
+	// Exclude drops files matching any pattern, and prunes whole directories.
+	// A pattern is path.Match syntax, tried against the site-relative path and
+	// against the base name, so both "assets/*.map" and "*.map" work.
+	Include []string
+	Exclude []string
+
+	// FollowSymlinks reads through symbolic links instead of refusing them.
+	// Links are still resolved inside the scanned directory, so one pointing at
+	// /etc/passwd fails rather than publishing it.
+	FollowSymlinks bool
+
+	// Concurrency bounds the hashing goroutines. Zero means GOMAXPROCS.
+	Concurrency int
+}
+
+// Source is a scanned directory: the manifest it produced, plus the handle the
+// upload path reads the contents back through.
+//
+// Files are read through an os.Root rather than by path, so a symlink swapped
+// in between the scan and the upload still cannot reach outside the directory
+// the user named.
+type Source struct {
+	Dir        string
+	Files      []LocalFile
+	TotalBytes int64
+
+	root *os.Root
+}
+
+// Scan walks dir, hashes what it finds, and returns the result. The caller must
+// Close the Source.
+func Scan(ctx context.Context, dir string, opts ScanOptions) (*Source, error) {
+	if err := checkPatterns("include", opts.Include); err != nil {
+		return nil, err
+	}
+	if err := checkPatterns("exclude", opts.Exclude); err != nil {
+		return nil, err
+	}
+
+	abs, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", dir, err)
+	}
+	fi, err := os.Stat(abs)
+	if err != nil {
+		return nil, err
+	}
+	if !fi.IsDir() {
+		return nil, fmt.Errorf("%s is not a directory", dir)
+	}
+	root, err := os.OpenRoot(abs)
+	if err != nil {
+		return nil, err
+	}
+
+	s := &Source{Dir: abs, root: root}
+	if err := s.walk(ctx, opts); err != nil {
+		root.Close()
+		return nil, err
+	}
+	if len(s.Files) == 0 {
+		root.Close()
+		return nil, fmt.Errorf("%s contains no files to deploy", dir)
+	}
+	if err := s.hash(ctx, opts.Concurrency); err != nil {
+		root.Close()
+		return nil, err
+	}
+	return s, nil
+}
+
+// Close releases the directory handle.
+func (s *Source) Close() error {
+	if s == nil || s.root == nil {
+		return nil
+	}
+	return s.root.Close()
+}
+
+// Open reads one of the scanned files.
+func (s *Source) Open(p string) (*os.File, error) {
+	return s.root.Open(filepath.FromSlash(p))
+}
+
+// Manifest renders the scan as the wire form the server expects.
+func (s *Source) Manifest() []api.FileEntry {
+	out := make([]api.FileEntry, len(s.Files))
+	for i, f := range s.Files {
+		out[i] = api.FileEntry{Path: f.Path, Digest: f.Digest, Size: f.Size}
+	}
+	return out
+}
+
+// UniqueBlobs counts distinct digests, which is what the deduplicating upload
+// actually has to deal with.
+func (s *Source) UniqueBlobs() int {
+	seen := make(map[string]struct{}, len(s.Files))
+	for _, f := range s.Files {
+		seen[f.Digest] = struct{}{}
+	}
+	return len(seen)
+}
+
+// walk collects the paths and sizes. Hashing is a separate pass so it can run
+// concurrently over a list that is already known to be valid: finding out on
+// file 40,000 that file 3 has an unusable name would waste the whole scan.
+func (s *Source) walk(ctx context.Context, opts ScanOptions) error {
+	set := pathutil.NewSet(0)
+	return filepath.WalkDir(s.Dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(s.Dir, p)
+		if err != nil {
+			return err
+		}
+		if rel == "." {
+			return nil
+		}
+		name := filepath.ToSlash(rel)
+
+		if d.IsDir() {
+			if slices.Contains(skipDirs, d.Name()) || matchAny(opts.Exclude, name) {
+				return fs.SkipDir
+			}
+			return nil
+		}
+
+		// WalkDir reports entry types from Lstat, so a symlink arrives as a
+		// symlink and is never silently followed.
+		var size int64
+		switch {
+		case d.Type()&fs.ModeSymlink != 0:
+			if !opts.FollowSymlinks {
+				return fmt.Errorf("%s is a symbolic link; a deployment holds regular files only "+
+					"(pass --follow-symlinks to upload what it points at)", name)
+			}
+			fi, err := s.root.Stat(filepath.FromSlash(name))
+			if err != nil {
+				return fmt.Errorf("%s: %w", name, err)
+			}
+			if !fi.Mode().IsRegular() {
+				return fmt.Errorf("%s points at a %s, not a regular file", name, kindOf(fi.Mode()))
+			}
+			size = fi.Size()
+		case d.Type().IsRegular():
+			fi, err := d.Info()
+			if err != nil {
+				return err
+			}
+			size = fi.Size()
+		default:
+			return fmt.Errorf("%s is a %s; a deployment holds regular files only",
+				name, kindOf(d.Type()))
+		}
+
+		if !keep(opts, name) {
+			return nil
+		}
+		// The same checks the server runs, so a name that could never be stored
+		// is reported here — with the local path in hand — instead of as a
+		// rejected manifest after the walk.
+		if err := set.Add(name); err != nil {
+			return fmt.Errorf("%s: %w", name, err)
+		}
+		s.Files = append(s.Files, LocalFile{Path: name, Size: size})
+		s.TotalBytes += size
+		return nil
+	})
+}
+
+// hash fills in every digest. It is CPU-bound on small files and IO-bound on
+// large ones, so it runs at GOMAXPROCS by default.
+func (s *Source) hash(ctx context.Context, concurrency int) error {
+	if concurrency <= 0 {
+		concurrency = runtime.GOMAXPROCS(0)
+	}
+	g, ctx := errgroup.WithContext(ctx)
+	g.SetLimit(concurrency)
+	for i := range s.Files {
+		g.Go(func() error {
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+			f := &s.Files[i]
+			digest, size, err := s.digest(f.Path)
+			if err != nil {
+				return fmt.Errorf("%s: %w", f.Path, err)
+			}
+			// The file may have been rewritten between the walk and now. The
+			// digest and the size have to describe the same bytes, so take both
+			// from the read that produced the digest.
+			f.Digest, f.Size = digest, size
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return err
+	}
+	// Sorted output makes "pages deploy --dry-run" diffable between runs.
+	slices.SortFunc(s.Files, func(a, b LocalFile) int {
+		if a.Path < b.Path {
+			return -1
+		}
+		if a.Path > b.Path {
+			return 1
+		}
+		return 0
+	})
+	s.TotalBytes = 0
+	for _, f := range s.Files {
+		s.TotalBytes += f.Size
+	}
+	return nil
+}
+
+func (s *Source) digest(p string) (string, int64, error) {
+	f, err := s.Open(p)
+	if err != nil {
+		return "", 0, err
+	}
+	defer f.Close()
+	h := sha256.New()
+	n, err := io.Copy(h, f)
+	if err != nil {
+		return "", 0, err
+	}
+	return hex.EncodeToString(h.Sum(nil)), n, nil
+}
+
+// keep applies the include/exclude filters to a file.
+func keep(opts ScanOptions, name string) bool {
+	if len(opts.Include) > 0 && !matchAny(opts.Include, name) {
+		return false
+	}
+	return !matchAny(opts.Exclude, name)
+}
+
+// matchAny reports whether name matches a pattern, either as a whole path or by
+// its base name. Matching the base name too is what makes "--exclude '*.map'"
+// behave the way everyone expects, since path.Match's "*" does not cross "/".
+func matchAny(patterns []string, name string) bool {
+	base := path.Base(name)
+	for _, pat := range patterns {
+		if ok, _ := path.Match(pat, name); ok {
+			return true
+		}
+		if ok, _ := path.Match(pat, base); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// checkPatterns rejects malformed globs up front. path.Match reports a bad
+// pattern only when it is tried, so an unchecked one would silently match
+// nothing and quietly deploy the wrong file set.
+func checkPatterns(flag string, patterns []string) error {
+	for _, pat := range patterns {
+		if _, err := path.Match(pat, "x"); err != nil {
+			return fmt.Errorf("--%s %q: %w", flag, pat, err)
+		}
+	}
+	return nil
+}
+
+func kindOf(m fs.FileMode) string {
+	switch {
+	case m&fs.ModeDir != 0:
+		return "directory"
+	case m&fs.ModeSymlink != 0:
+		return "symbolic link"
+	case m&fs.ModeDevice != 0:
+		return "device file"
+	case m&fs.ModeNamedPipe != 0:
+		return "named pipe"
+	case m&fs.ModeSocket != 0:
+		return "socket"
+	default:
+		return "special file"
+	}
+}