package cas import ( "bytes" "context" "errors" "io" "io/fs" "log/slog" "os" "path/filepath" "strings" "sync" "testing" ) func newStore(t *testing.T, mode LinkMode) (*Store, string) { t.Helper() base := t.TempDir() deployDir := filepath.Join(base, "deployments") s, err := Open(filepath.Join(base, "cas"), Options{ Mode: mode, ProbeDir: deployDir, Log: slog.New(slog.NewTextHandler(io.Discard, nil)), }) if err != nil { if mode == LinkHard { // Not every filesystem supports link(2), which is the whole reason // Open probes. Skipping is honest here; TestOpenAutoFallsBackToCopy // covers what happens when it is unavailable. t.Skipf("hardlinks unavailable under %s: %v", base, err) } t.Fatalf("Open: %v", err) } t.Cleanup(func() { s.Close() }) return s, deployDir } // tempFiles is how many uploads are sitting in tmp/. Every test that exercises a // failure path checks this: a leaked temporary file is a slow disk leak that no // other test would notice. func tempFiles(t *testing.T, s *Store) int { t.Helper() entries, err := os.ReadDir(filepath.Join(s.Dir(), tmpDir)) if err != nil { t.Fatal(err) } return len(entries) } func put(t *testing.T, s *Store, content string) Digest { t.Helper() d := Sum([]byte(content)) n, err := s.Put(context.Background(), d, int64(len(content)), 1<<20, strings.NewReader(content)) if err != nil { t.Fatalf("Put(%q): %v", content, err) } if n != int64(len(content)) { t.Fatalf("Put returned %d bytes, want %d", n, len(content)) } return d } func TestParseDigest(t *testing.T) { d := Sum([]byte("hello")) hexForm := d.String() got, err := ParseDigest(hexForm) if err != nil { t.Fatalf("ParseDigest: %v", err) } if got != d { t.Errorf("round trip changed the digest") } bad := []string{ "", hexForm[:HexLen-1], hexForm + "0", strings.ToUpper(hexForm), // uppercase is a second spelling; see ParseDigest strings.Repeat("g", HexLen), hexForm[:HexLen-2] + "!!", "../../../etc/passwd", } for _, s := range bad { if _, err := ParseDigest(s); !errors.Is(err, ErrBadDigest) { t.Errorf("ParseDigest(%q) = %v, want ErrBadDigest", s, err) } } } func TestDigestRelIsSharded(t *testing.T) { d := Sum([]byte("hello")) rel := d.Rel() hexForm := d.String() want := hexForm[0:2] + "/" + hexForm[2:4] + "/" + hexForm if rel != want { t.Errorf("Rel = %q, want %q", rel, want) } if !fs.ValidPath(rel) { t.Errorf("Rel = %q is not a valid path", rel) } } func TestFromBytes(t *testing.T) { d := Sum([]byte("hello")) // database/sql reuses its scan buffers, so a digest must not alias one. buf := append([]byte(nil), d.Bytes()...) got, err := FromBytes(buf) if err != nil { t.Fatal(err) } for i := range buf { buf[i] = 0 } if got != d { t.Error("FromBytes aliased its argument instead of copying") } if _, err := FromBytes(buf[:8]); err == nil { t.Error("a short digest must be rejected") } } func TestPutAndOpen(t *testing.T) { s, _ := newStore(t, LinkCopy) const content = "

hello

" d := put(t, s, content) ok, err := s.Has(d) if err != nil || !ok { t.Fatalf("Has = %v, %v", ok, err) } f, err := s.Open(d) if err != nil { t.Fatal(err) } defer f.Close() got, err := io.ReadAll(f) if err != nil { t.Fatal(err) } if string(got) != content { t.Errorf("read back %q, want %q", got, content) } if tempFiles(t, s) != 0 { t.Error("a successful Put left a temporary file behind") } } func TestPutIsIdempotent(t *testing.T) { s, _ := newStore(t, LinkCopy) const content = "same bytes" d := put(t, s, content) before, err := os.Stat(s.Path(d)) if err != nil { t.Fatal(err) } put(t, s, content) after, err := os.Stat(s.Path(d)) if err != nil { t.Fatal(err) } if !os.SameFile(before, after) { t.Error("re-uploading identical content replaced the blob instead of keeping it") } if tempFiles(t, s) != 0 { t.Error("a redundant Put left a temporary file behind") } } func TestOpenMissingBlob(t *testing.T) { s, _ := newStore(t, LinkCopy) d := Sum([]byte("never uploaded")) if ok, err := s.Has(d); err != nil || ok { t.Fatalf("Has = %v, %v", ok, err) } if _, err := s.Open(d); !errors.Is(err, ErrNotFound) { t.Errorf("Open = %v, want ErrNotFound", err) } } // The heart of the store: a caller's digest is a claim, never a fact. func TestPutRejectsAClaimedDigest(t *testing.T) { s, _ := newStore(t, LinkCopy) ctx := context.Background() // Stand in for another project's file, already deduplicated into the store. victim := "the real index.html" victimDigest := put(t, s, victim) attack := "" _, err := s.Put(ctx, victimDigest, int64(len(attack)), 1<<20, strings.NewReader(attack)) if !errors.Is(err, ErrDigestMismatch) { t.Fatalf("Put = %v, want ErrDigestMismatch", err) } f, err := s.Open(victimDigest) if err != nil { t.Fatal(err) } defer f.Close() got, _ := io.ReadAll(f) if string(got) != victim { t.Fatalf("the existing blob was overwritten: %q", got) } if tempFiles(t, s) != 0 { t.Error("the rejected upload left a temporary file behind") } } type errReader struct{ err error } func (e errReader) Read([]byte) (int, error) { return 0, e.err } // Every way Put can fail has to clean up after itself, or a store slowly fills // with the debris of clients that hung up. func TestPutCleansUpOnEveryFailure(t *testing.T) { s, _ := newStore(t, LinkCopy) const content = "some content" d := Sum([]byte(content)) cancelled, cancel := context.WithCancel(context.Background()) cancel() cases := []struct { name string ctx context.Context want Digest size int64 max int64 body io.Reader wantErr error }{ {"digest mismatch", context.Background(), Sum([]byte("other")), int64(len(content)), 1 << 20, strings.NewReader(content), ErrDigestMismatch}, {"size mismatch", context.Background(), d, int64(len(content)) + 1, 1 << 20, strings.NewReader(content), ErrSizeMismatch}, {"over the limit", context.Background(), d, -1, 4, strings.NewReader(content), ErrTooLarge}, {"declared over the limit", context.Background(), d, 1 << 30, 4, strings.NewReader(content), ErrTooLarge}, {"reader failed", context.Background(), d, -1, 1 << 20, errReader{errors.New("connection reset")}, nil}, {"context cancelled", cancelled, d, -1, 1 << 20, strings.NewReader(content), context.Canceled}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { _, err := s.Put(tc.ctx, tc.want, tc.size, tc.max, tc.body) if err == nil { t.Fatal("Put should have failed") } if tc.wantErr != nil && !errors.Is(err, tc.wantErr) { t.Errorf("err = %v, want %v", err, tc.wantErr) } if n := tempFiles(t, s); n != 0 { t.Errorf("%d temporary file(s) left behind", n) } if ok, _ := s.Has(tc.want); ok { t.Error("a failed Put made a blob visible") } }) } } // A client that lies in Content-Length must not get past the ceiling; the limit // is enforced against the bytes that arrive, not against the declaration. func TestPutEnforcesTheLimitAgainstActualBytes(t *testing.T) { s, _ := newStore(t, LinkCopy) body := strings.Repeat("x", 100) d := Sum([]byte(body)) // Declares that it fits, then sends more. _, err := s.Put(context.Background(), d, 4, 8, strings.NewReader(body)) if err == nil { t.Fatal("Put should have failed") } if tempFiles(t, s) != 0 { t.Error("temporary file left behind") } } func TestPutExactlyAtTheLimit(t *testing.T) { s, _ := newStore(t, LinkCopy) body := strings.Repeat("x", 64) d := Sum([]byte(body)) if _, err := s.Put(context.Background(), d, 64, 64, strings.NewReader(body)); err != nil { t.Errorf("a file exactly at the limit must be accepted: %v", err) } } func TestPutEmptyBlob(t *testing.T) { s, _ := newStore(t, LinkCopy) d := put(t, s, "") f, err := s.Open(d) if err != nil { t.Fatal(err) } defer f.Close() b, _ := io.ReadAll(f) if len(b) != 0 { t.Errorf("read %d bytes from the empty blob", len(b)) } } // Concurrent uploads of identical content are expected — two CI jobs deploying // the same vendored asset — and all of them must succeed with one file left. func TestConcurrentIdenticalPut(t *testing.T) { s, _ := newStore(t, LinkCopy) const content = "shared asset" d := Sum([]byte(content)) const n = 16 errs := make([]error, n) var wg sync.WaitGroup start := make(chan struct{}) for i := range n { wg.Add(1) go func() { defer wg.Done() <-start _, errs[i] = s.Put(context.Background(), d, int64(len(content)), 1<<20, strings.NewReader(content)) }() } close(start) wg.Wait() for i, err := range errs { if err != nil { t.Errorf("goroutine %d: %v", i, err) } } if tempFiles(t, s) != 0 { t.Error("temporary files left behind") } f, err := s.Open(d) if err != nil { t.Fatal(err) } defer f.Close() got, _ := io.ReadAll(f) if string(got) != content { t.Errorf("blob = %q, want %q", got, content) } } func TestPurgeTemp(t *testing.T) { s, _ := newStore(t, LinkCopy) d := put(t, s, "keep me") for _, name := range []string{"aaaa", "bbbb"} { if err := os.WriteFile(filepath.Join(s.Dir(), tmpDir, name), []byte("interrupted"), 0o600); err != nil { t.Fatal(err) } } n, err := s.PurgeTemp() if err != nil { t.Fatal(err) } if n != 2 { t.Errorf("purged %d, want 2", n) } if tempFiles(t, s) != 0 { t.Error("tmp is not empty") } if ok, _ := s.Has(d); !ok { t.Error("PurgeTemp removed a committed blob") } if n, err := s.PurgeTemp(); err != nil || n != 0 { t.Errorf("second PurgeTemp = %d, %v", n, err) } } func TestRemoveIsIdempotent(t *testing.T) { s, _ := newStore(t, LinkCopy) d := put(t, s, "temporary") if err := s.Remove(d); err != nil { t.Fatal(err) } if ok, _ := s.Has(d); ok { t.Error("blob still present after Remove") } // GC re-running after a crash must not fail on what it already deleted. if err := s.Remove(d); err != nil { t.Errorf("second Remove: %v", err) } } func TestLinkIntoCopyMode(t *testing.T) { s, deployDir := newStore(t, LinkCopy) const content = "console.log(1)\n" d := put(t, s, content) dest := filepath.Join(deployDir, "assets") if err := os.MkdirAll(dest, 0o755); err != nil { t.Fatal(err) } if err := s.LinkInto(d, dest, "app.js"); err != nil { t.Fatal(err) } got, err := os.ReadFile(filepath.Join(dest, "app.js")) if err != nil { t.Fatal(err) } if string(got) != content { t.Errorf("content = %q, want %q", got, content) } } func TestLinkIntoRejectsAMissingBlob(t *testing.T) { for _, mode := range []LinkMode{LinkCopy, LinkHard} { t.Run(string(mode), func(t *testing.T) { s, deployDir := newStore(t, mode) if err := os.MkdirAll(deployDir, 0o755); err != nil { t.Fatal(err) } d := Sum([]byte("never uploaded")) if err := s.LinkInto(d, deployDir, "x.html"); !errors.Is(err, ErrNotFound) { t.Errorf("LinkInto = %v, want ErrNotFound", err) } }) } } func TestLinkIntoRejectsADuplicatePath(t *testing.T) { for _, mode := range []LinkMode{LinkCopy, LinkHard} { t.Run(string(mode), func(t *testing.T) { s, deployDir := newStore(t, mode) if err := os.MkdirAll(deployDir, 0o755); err != nil { t.Fatal(err) } d := put(t, s, "x") if err := s.LinkInto(d, deployDir, "x.html"); err != nil { t.Fatal(err) } if err := s.LinkInto(d, deployDir, "x.html"); err == nil { t.Error("writing twice to one path must fail rather than overwrite") } }) } } func TestOpenRejectsAnUnknownMode(t *testing.T) { if _, err := Open(t.TempDir(), Options{Mode: "symlink"}); err == nil { t.Error("an unknown link mode must be refused at startup, not at deploy time") } } // Forcing hardlinks has to fail loudly when they do not work, or an operator who // asked for them silently gets copies and a full disk. func TestOpenHardlinkModeFailsWithoutAProbeTarget(t *testing.T) { if _, err := Open(filepath.Join(t.TempDir(), "cas"), Options{Mode: LinkHard}); err == nil { t.Error("hardlink mode with nothing to probe against must fail") } } func TestOpenAutoFallsBackToCopy(t *testing.T) { // No ProbeDir, so the probe cannot succeed and auto must degrade rather than // refuse to start. s, err := Open(filepath.Join(t.TempDir(), "cas"), Options{}) if err != nil { t.Fatalf("auto mode must start anyway: %v", err) } defer s.Close() if s.LinkMode() != LinkCopy { t.Errorf("LinkMode = %q, want %q", s.LinkMode(), LinkCopy) } } func TestBlobsAreReadOnly(t *testing.T) { s, _ := newStore(t, LinkCopy) d := put(t, s, "immutable") fi, err := os.Stat(s.Path(d)) if err != nil { t.Fatal(err) } // A writable blob is a writable inode shared by every project referencing // that content; see blobMode. if perm := fi.Mode().Perm(); perm != blobMode { t.Errorf("blob mode = %04o, want %04o", perm, blobMode) } } func TestPutLargeStream(t *testing.T) { s, _ := newStore(t, LinkCopy) body := bytes.Repeat([]byte("0123456789abcdef"), 1<<16) // 1 MiB d := Sum(body) n, err := s.Put(context.Background(), d, int64(len(body)), 4<<20, bytes.NewReader(body)) if err != nil { t.Fatal(err) } if n != int64(len(body)) { t.Fatalf("stored %d bytes, want %d", n, len(body)) } f, err := s.Open(d) if err != nil { t.Fatal(err) } defer f.Close() got, _ := io.ReadAll(f) if !bytes.Equal(got, body) { t.Error("stored content differs from what was written") } }