package client import ( "context" "crypto/sha256" "encoding/hex" "fmt" "io" "io/fs" "os" "path" "path/filepath" "runtime" "slices" "golang.org/x/sync/errgroup" "github.com/iceBear67/simplepages/api" "github.com/iceBear67/simplepages/internal/pathutil" ) // skipDirs are never walked into. A build output directory should not contain // them at all, but "pages deploy ." on a repository root is a mistake people // make once, and uploading a .git directory publishes the whole history. var skipDirs = []string{".git", ".hg", ".svn"} // LocalFile is one file of a scanned directory, already hashed. type LocalFile struct { // Path is site-relative and slash-separated: it is what the URL will be. Path string `json:"path"` // Digest is the lowercase hex SHA-256 of the contents. Digest string `json:"digest"` Size int64 `json:"size"` } // ScanOptions filters and paces a scan. type ScanOptions struct { // Include, when non-empty, keeps only files matching at least one pattern. // Exclude drops files matching any pattern, and prunes whole directories. // A pattern is path.Match syntax, tried against the site-relative path and // against the base name, so both "assets/*.map" and "*.map" work. Include []string Exclude []string // FollowSymlinks reads through symbolic links instead of refusing them. // Links are still resolved inside the scanned directory, so one pointing at // /etc/passwd fails rather than publishing it. FollowSymlinks bool // Concurrency bounds the hashing goroutines. Zero means GOMAXPROCS. Concurrency int } // Source is a scanned directory: the manifest it produced, plus the handle the // upload path reads the contents back through. // // Files are read through an os.Root rather than by path, so a symlink swapped // in between the scan and the upload still cannot reach outside the directory // the user named. type Source struct { Dir string Files []LocalFile TotalBytes int64 root *os.Root } // Scan walks dir, hashes what it finds, and returns the result. The caller must // Close the Source. func Scan(ctx context.Context, dir string, opts ScanOptions) (*Source, error) { if err := checkPatterns("include", opts.Include); err != nil { return nil, err } if err := checkPatterns("exclude", opts.Exclude); err != nil { return nil, err } abs, err := filepath.Abs(dir) if err != nil { return nil, fmt.Errorf("%s: %w", dir, err) } fi, err := os.Stat(abs) if err != nil { return nil, err } if !fi.IsDir() { return nil, fmt.Errorf("%s is not a directory", dir) } root, err := os.OpenRoot(abs) if err != nil { return nil, err } s := &Source{Dir: abs, root: root} if err := s.walk(ctx, opts); err != nil { root.Close() return nil, err } if len(s.Files) == 0 { root.Close() return nil, fmt.Errorf("%s contains no files to deploy", dir) } if err := s.hash(ctx, opts.Concurrency); err != nil { root.Close() return nil, err } return s, nil } // Close releases the directory handle. func (s *Source) Close() error { if s == nil || s.root == nil { return nil } return s.root.Close() } // Open reads one of the scanned files. func (s *Source) Open(p string) (*os.File, error) { return s.root.Open(filepath.FromSlash(p)) } // Manifest renders the scan as the wire form the server expects. func (s *Source) Manifest() []api.FileEntry { out := make([]api.FileEntry, len(s.Files)) for i, f := range s.Files { out[i] = api.FileEntry{Path: f.Path, Digest: f.Digest, Size: f.Size} } return out } // UniqueBlobs counts distinct digests, which is what the deduplicating upload // actually has to deal with. func (s *Source) UniqueBlobs() int { seen := make(map[string]struct{}, len(s.Files)) for _, f := range s.Files { seen[f.Digest] = struct{}{} } return len(seen) } // walk collects the paths and sizes. Hashing is a separate pass so it can run // concurrently over a list that is already known to be valid: finding out on // file 40,000 that file 3 has an unusable name would waste the whole scan. func (s *Source) walk(ctx context.Context, opts ScanOptions) error { set := pathutil.NewSet(0) return filepath.WalkDir(s.Dir, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } if err := ctx.Err(); err != nil { return err } rel, err := filepath.Rel(s.Dir, p) if err != nil { return err } if rel == "." { return nil } name := filepath.ToSlash(rel) if d.IsDir() { if slices.Contains(skipDirs, d.Name()) || matchAny(opts.Exclude, name) { return fs.SkipDir } return nil } // WalkDir reports entry types from Lstat, so a symlink arrives as a // symlink and is never silently followed. var size int64 switch { case d.Type()&fs.ModeSymlink != 0: if !opts.FollowSymlinks { return fmt.Errorf("%s is a symbolic link; a deployment holds regular files only "+ "(pass --follow-symlinks to upload what it points at)", name) } fi, err := s.root.Stat(filepath.FromSlash(name)) if err != nil { return fmt.Errorf("%s: %w", name, err) } if !fi.Mode().IsRegular() { return fmt.Errorf("%s points at a %s, not a regular file", name, kindOf(fi.Mode())) } size = fi.Size() case d.Type().IsRegular(): fi, err := d.Info() if err != nil { return err } size = fi.Size() default: return fmt.Errorf("%s is a %s; a deployment holds regular files only", name, kindOf(d.Type())) } if !keep(opts, name) { return nil } // The same checks the server runs, so a name that could never be stored // is reported here — with the local path in hand — instead of as a // rejected manifest after the walk. if err := set.Add(name); err != nil { return fmt.Errorf("%s: %w", name, err) } s.Files = append(s.Files, LocalFile{Path: name, Size: size}) s.TotalBytes += size return nil }) } // hash fills in every digest. It is CPU-bound on small files and IO-bound on // large ones, so it runs at GOMAXPROCS by default. func (s *Source) hash(ctx context.Context, concurrency int) error { if concurrency <= 0 { concurrency = runtime.GOMAXPROCS(0) } g, ctx := errgroup.WithContext(ctx) g.SetLimit(concurrency) for i := range s.Files { g.Go(func() error { if err := ctx.Err(); err != nil { return err } f := &s.Files[i] digest, size, err := s.digest(f.Path) if err != nil { return fmt.Errorf("%s: %w", f.Path, err) } // The file may have been rewritten between the walk and now. The // digest and the size have to describe the same bytes, so take both // from the read that produced the digest. f.Digest, f.Size = digest, size return nil }) } if err := g.Wait(); err != nil { return err } // Sorted output makes "pages deploy --dry-run" diffable between runs. slices.SortFunc(s.Files, func(a, b LocalFile) int { if a.Path < b.Path { return -1 } if a.Path > b.Path { return 1 } return 0 }) s.TotalBytes = 0 for _, f := range s.Files { s.TotalBytes += f.Size } return nil } func (s *Source) digest(p string) (string, int64, error) { f, err := s.Open(p) if err != nil { return "", 0, err } defer f.Close() h := sha256.New() n, err := io.Copy(h, f) if err != nil { return "", 0, err } return hex.EncodeToString(h.Sum(nil)), n, nil } // keep applies the include/exclude filters to a file. func keep(opts ScanOptions, name string) bool { if len(opts.Include) > 0 && !matchAny(opts.Include, name) { return false } return !matchAny(opts.Exclude, name) } // matchAny reports whether name matches a pattern, either as a whole path or by // its base name. Matching the base name too is what makes "--exclude '*.map'" // behave the way everyone expects, since path.Match's "*" does not cross "/". func matchAny(patterns []string, name string) bool { base := path.Base(name) for _, pat := range patterns { if ok, _ := path.Match(pat, name); ok { return true } if ok, _ := path.Match(pat, base); ok { return true } } return false } // checkPatterns rejects malformed globs up front. path.Match reports a bad // pattern only when it is tried, so an unchecked one would silently match // nothing and quietly deploy the wrong file set. func checkPatterns(flag string, patterns []string) error { for _, pat := range patterns { if _, err := path.Match(pat, "x"); err != nil { return fmt.Errorf("--%s %q: %w", flag, pat, err) } } return nil } func kindOf(m fs.FileMode) string { switch { case m&fs.ModeDir != 0: return "directory" case m&fs.ModeSymlink != 0: return "symbolic link" case m&fs.ModeDevice != 0: return "device file" case m&fs.ModeNamedPipe != 0: return "named pipe" case m&fs.ModeSocket != 0: return "socket" default: return "special file" } }