init

internal/manager/http.go

@@ -0,0 +1,152 @@
+package manager
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Handler exposes the manager over HTTP:
+//
+//	/healthz  the process is alive (use this for a container health check)
+//	/readyz   every repo has synced successfully at least once
+//	/status   JSON snapshot of every repo
+//	/metrics  Prometheus text format
+func (m *Manager) Handler(version string) http.Handler {
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
+		writeText(w, http.StatusOK, "ok\n")
+	})
+
+	mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) {
+		var pending []string
+		for _, s := range m.Statuses() {
+			if s.LastSuccess.IsZero() {
+				pending = append(pending, s.Name)
+			}
+		}
+		if len(pending) > 0 {
+			writeText(w, http.StatusServiceUnavailable,
+				"awaiting first successful sync: "+strings.Join(pending, ", ")+"\n")
+			return
+		}
+		writeText(w, http.StatusOK, "ready\n")
+	})
+
+	mux.HandleFunc("GET /status", func(w http.ResponseWriter, r *http.Request) {
+		statuses := m.Statuses()
+		body := struct {
+			Version  string   `json:"version"`
+			Uptime   string   `json:"uptime"`
+			Repos    []Status `json:"repos"`
+			Healthy  bool     `json:"healthy"`
+			Failing  int      `json:"failing"`
+			Reported string   `json:"reported_at"`
+		}{
+			Version:  version,
+			Uptime:   m.Uptime().Round(time.Second).String(),
+			Repos:    statuses,
+			Healthy:  true,
+			Reported: time.Now().UTC().Format(time.RFC3339),
+		}
+		for _, s := range statuses {
+			if s.Failures > 0 {
+				body.Failing++
+				body.Healthy = false
+			}
+		}
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		enc := json.NewEncoder(w)
+		enc.SetIndent("", "  ")
+		_ = enc.Encode(body)
+	})
+
+	mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+		var b strings.Builder
+		fmt.Fprintf(&b, "# HELP syncbot_build_info Version of the running binary.\n")
+		fmt.Fprintf(&b, "# TYPE syncbot_build_info gauge\n")
+		fmt.Fprintf(&b, "syncbot_build_info{version=%q} 1\n", version)
+		fmt.Fprintf(&b, "# HELP syncbot_uptime_seconds Time since start.\n")
+		fmt.Fprintf(&b, "# TYPE syncbot_uptime_seconds gauge\n")
+		fmt.Fprintf(&b, "syncbot_uptime_seconds %s\n", seconds(m.Uptime()))
+
+		metric(&b, "syncbot_sync_total", "counter", "Sync cycles started.")
+		for _, s := range m.Statuses() {
+			fmt.Fprintf(&b, "syncbot_sync_total{repo=%q} %d\n", s.Name, s.Syncs)
+		}
+		metric(&b, "syncbot_push_total", "counter", "Cycles that pushed to dst.")
+		for _, s := range m.Statuses() {
+			fmt.Fprintf(&b, "syncbot_push_total{repo=%q} %d\n", s.Name, s.Pushes)
+		}
+		metric(&b, "syncbot_consecutive_failures", "gauge", "Failed cycles since the last success.")
+		for _, s := range m.Statuses() {
+			fmt.Fprintf(&b, "syncbot_consecutive_failures{repo=%q} %d\n", s.Name, s.Failures)
+		}
+		metric(&b, "syncbot_refs", "gauge", "Mirrored refs.")
+		for _, s := range m.Statuses() {
+			fmt.Fprintf(&b, "syncbot_refs{repo=%q} %d\n", s.Name, s.Refs)
+		}
+		metric(&b, "syncbot_last_success_timestamp_seconds", "gauge", "Unix time of the last successful sync.")
+		for _, s := range m.Statuses() {
+			fmt.Fprintf(&b, "syncbot_last_success_timestamp_seconds{repo=%q} %d\n", s.Name, unix(s.LastSuccess))
+		}
+		metric(&b, "syncbot_last_duration_seconds", "gauge", "Duration of the last sync cycle.")
+		for _, s := range m.Statuses() {
+			fmt.Fprintf(&b, "syncbot_last_duration_seconds{repo=%q} %s\n", s.Name,
+				strconv.FormatFloat(float64(s.LastDurMS)/1000, 'f', 3, 64))
+		}
+		_, _ = w.Write([]byte(b.String()))
+	})
+
+	return mux
+}
+
+// Serve runs the HTTP endpoint until ctx is cancelled.
+func (m *Manager) Serve(ctx context.Context, addr, version string, log *slog.Logger) error {
+	srv := &http.Server{
+		Addr:              addr,
+		Handler:           m.Handler(version),
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	go func() {
+		<-ctx.Done()
+		shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(shutdown)
+	}()
+
+	log.Info("http listening", "addr", addr,
+		"endpoints", "/healthz /readyz /status /metrics")
+	if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+		return err
+	}
+	return nil
+}
+
+func metric(b *strings.Builder, name, kind, help string) {
+	fmt.Fprintf(b, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, kind)
+}
+
+func seconds(d time.Duration) string {
+	return strconv.FormatFloat(d.Seconds(), 'f', 3, 64)
+}
+
+func unix(t time.Time) int64 {
+	if t.IsZero() {
+		return 0
+	}
+	return t.Unix()
+}
+
+func writeText(w http.ResponseWriter, code int, body string) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.WriteHeader(code)
+	_, _ = w.Write([]byte(body))
+}

internal/manager/manager.go

@@ -0,0 +1,412 @@
+// Package manager owns the running set of sync jobs.
+//
+// Each repository gets its own goroutine with its own timer, so a slow or
+// broken repo never delays the others. A shared semaphore caps how many git
+// processes run at once, which is what actually bounds memory use.
+//
+// Reloading is incremental: Apply diffs the new configuration against the
+// running jobs and touches only what changed. Repos whose settings are
+// unchanged keep their timer and their in-flight work.
+package manager
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"sync"
+	"time"
+
+	"syncbot/internal/config"
+	"syncbot/internal/gitx"
+	"syncbot/internal/syncer"
+)
+
+// Status is the observable state of one repository.
+type Status struct {
+	Name        string    `json:"name"`
+	Src         string    `json:"src"`
+	Dst         string    `json:"dst"`
+	Interval    string    `json:"interval"`
+	Syncing     bool      `json:"syncing"`
+	Refs        int       `json:"refs"`
+	Syncs       int64     `json:"syncs"`
+	Pushes      int64     `json:"pushes"`
+	Failures    int       `json:"consecutive_failures"`
+	LastRun     time.Time `json:"last_run,omitzero"`
+	LastSuccess time.Time `json:"last_success,omitzero"`
+	NextRun     time.Time `json:"next_run,omitzero"`
+	LastError   string    `json:"last_error,omitempty"`
+	LastDurMS   int64     `json:"last_duration_ms"`
+}
+
+// Manager supervises one goroutine per repository.
+type Manager struct {
+	ctx context.Context
+	log *slog.Logger
+
+	mu      sync.Mutex
+	jobs    map[string]*job
+	stats   map[string]*Status
+	sem     chan struct{}
+	conc    int
+	workDir string
+	sync    *syncer.Syncer
+	started time.Time
+
+	// wg covers every goroutine ever started, including ones already replaced
+	// by a reload, so shutdown does not leave a git process behind.
+	wg sync.WaitGroup
+}
+
+type job struct {
+	spec   config.Job
+	cancel context.CancelFunc
+	done   chan struct{}
+}
+
+// New returns a Manager whose jobs all derive from ctx.
+func New(ctx context.Context, log *slog.Logger) *Manager {
+	return &Manager{
+		ctx:     ctx,
+		log:     log,
+		jobs:    make(map[string]*job),
+		stats:   make(map[string]*Status),
+		sem:     make(chan struct{}, 1),
+		conc:    1,
+		started: time.Now(),
+	}
+}
+
+// Apply reconciles the running jobs with cfg. It never blocks on in-flight
+// syncs: a job being replaced is cancelled, and its successor waits for the
+// handover before touching the same mirror directory.
+func (m *Manager) Apply(cfg *config.Config) error {
+	home, err := prepareDirs(cfg.WorkDir)
+	if err != nil {
+		return err
+	}
+
+	if cfg.Concurrency <= 0 {
+		cfg.Concurrency = 1
+	}
+
+	m.mu.Lock()
+
+	if m.conc != cfg.Concurrency {
+		// Callers release into the channel they acquired from, so swapping the
+		// semaphore is safe even while syncs are in flight.
+		m.log.Info("concurrency changed", "from", m.conc, "to", cfg.Concurrency)
+		m.sem = make(chan struct{}, cfg.Concurrency)
+		m.conc = cfg.Concurrency
+	}
+
+	workDirChanged := m.workDir != cfg.WorkDir
+	if workDirChanged {
+		m.workDir = cfg.WorkDir
+		m.sync = &syncer.Syncer{Log: m.log, Home: home}
+	}
+
+	want := make(map[string]config.Job, len(cfg.Jobs))
+	for _, j := range cfg.Jobs {
+		want[j.Name] = j
+	}
+
+	// Stop jobs that disappeared or whose resolved spec changed.
+	var stopped []*job
+	for name, j := range m.jobs {
+		w, keep := want[name]
+		if keep && !workDirChanged && reflect.DeepEqual(w, j.spec) {
+			continue
+		}
+		delete(m.jobs, name)
+		stopped = append(stopped, j)
+		if !keep {
+			delete(m.stats, name)
+			m.log.Info("repo removed", "repo", name)
+		}
+	}
+	handover := make(map[string]<-chan struct{}, len(stopped))
+	for _, j := range stopped {
+		j.cancel()
+		handover[j.spec.Name] = j.done
+	}
+
+	// Start whatever is now missing.
+	var added, restarted int
+	for i, spec := range cfg.Jobs {
+		if _, running := m.jobs[spec.Name]; running {
+			continue
+		}
+		prev := handover[spec.Name]
+		if prev != nil {
+			restarted++
+		} else {
+			added++
+		}
+		m.startLocked(spec, prev, stagger(i))
+	}
+	m.mu.Unlock()
+
+	if added+restarted+len(stopped) > 0 {
+		m.log.Info("configuration applied",
+			"repos", len(cfg.Jobs), "started", added, "restarted", restarted,
+			"stopped", len(stopped)-restarted, "concurrency", cfg.Concurrency)
+	}
+	return nil
+}
+
+// startLocked launches a job goroutine. m.mu must be held.
+func (m *Manager) startLocked(spec config.Job, waitFor <-chan struct{}, delay time.Duration) {
+	ctx, cancel := context.WithCancel(m.ctx)
+	j := &job{spec: spec, cancel: cancel, done: make(chan struct{})}
+	m.jobs[spec.Name] = j
+
+	st, ok := m.stats[spec.Name]
+	if !ok {
+		st = &Status{Name: spec.Name}
+		m.stats[spec.Name] = st
+	}
+	// Counters survive a restart; the descriptive fields follow the new spec.
+	st.Src = gitx.RedactURL(spec.Src.URL)
+	st.Dst = gitx.RedactURL(spec.Dst.URL)
+	st.Interval = spec.Interval.String()
+
+	m.wg.Add(1)
+	go m.loop(ctx, j, waitFor, delay)
+}
+
+func (m *Manager) loop(ctx context.Context, j *job, waitFor <-chan struct{}, delay time.Duration) {
+	defer m.wg.Done()
+	defer close(j.done)
+
+	// When replacing a job, let the previous goroutine finish unwinding before
+	// operating on its mirror directory.
+	if waitFor != nil {
+		select {
+		case <-waitFor:
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	log := m.log.With("repo", j.spec.Name)
+	log.Info("watching", "src", gitx.RedactURL(j.spec.Src.URL),
+		"dst", gitx.RedactURL(j.spec.Dst.URL), "interval", j.spec.Interval)
+
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+
+	for {
+		m.update(j.spec.Name, func(s *Status) { s.NextRun = time.Now().Add(delay) })
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+		}
+
+		failures := m.runOnce(ctx, j.spec, log)
+		if ctx.Err() != nil {
+			return
+		}
+
+		delay = j.spec.Interval
+		if failures > 0 {
+			delay = backoff(j.spec.Interval, j.spec.MaxBackoff, failures)
+			if delay > j.spec.Interval {
+				log.Warn("backing off", "failures", failures, "retry_in", delay)
+			}
+		}
+		timer.Reset(delay)
+	}
+}
+
+// runOnce performs a single sync and returns the consecutive failure count.
+func (m *Manager) runOnce(ctx context.Context, spec config.Job, log *slog.Logger) int {
+	release, err := m.acquire(ctx)
+	if err != nil {
+		return 0 // shutting down
+	}
+	defer release()
+
+	m.update(spec.Name, func(s *Status) {
+		s.Syncing = true
+		s.LastRun = time.Now()
+		s.Syncs++
+	})
+
+	res, err := m.syncer().Sync(ctx, spec)
+
+	failures := 0
+	m.update(spec.Name, func(s *Status) {
+		s.Syncing = false
+		s.LastDurMS = res.Duration.Milliseconds()
+		if err != nil {
+			s.Failures++
+			s.LastError = err.Error()
+		} else {
+			s.Failures = 0
+			s.LastError = ""
+			s.LastSuccess = time.Now()
+			s.Refs = res.Refs
+			if res.Pushed {
+				s.Pushes++
+			}
+		}
+		failures = s.Failures
+	})
+
+	switch {
+	case err != nil && ctx.Err() != nil:
+		log.Info("sync cancelled")
+	case err != nil:
+		log.Error("sync failed", "err", err, "consecutive_failures", failures)
+	case res.Pushed:
+		log.Info("pushed", "refs", res.Refs, "changes", len(res.Changes),
+			"dur", res.Duration.Round(time.Millisecond))
+		for _, c := range res.Changes {
+			log.Debug("ref updated", "change", c)
+		}
+	}
+	return failures
+}
+
+// RunOnce syncs every job exactly once and reports whether all succeeded. It
+// is the engine behind the -once flag, for cron-style or one-shot deployments.
+func (m *Manager) RunOnce(cfg *config.Config) error {
+	if err := m.Apply(&config.Config{ // reuse Apply's dir/semaphore setup, no jobs
+		WorkDir: cfg.WorkDir, Concurrency: cfg.Concurrency,
+		ReloadInterval: cfg.ReloadInterval, LogLevel: cfg.LogLevel, LogFormat: cfg.LogFormat,
+	}); err != nil {
+		return err
+	}
+
+	var wg sync.WaitGroup
+	errs := make([]error, len(cfg.Jobs))
+	for i, spec := range cfg.Jobs {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			release, err := m.acquire(m.ctx)
+			if err != nil {
+				errs[i] = err
+				return
+			}
+			defer release()
+
+			log := m.log.With("repo", spec.Name)
+			res, err := m.syncer().Sync(m.ctx, spec)
+			if err != nil {
+				errs[i] = fmt.Errorf("%s: %w", spec.Name, err)
+				log.Error("sync failed", "err", err)
+				return
+			}
+			log.Info("synced", "refs", res.Refs, "pushed", res.Pushed,
+				"dur", res.Duration.Round(time.Millisecond))
+		}()
+	}
+	wg.Wait()
+	return errors.Join(errs...)
+}
+
+// Stop cancels every job and waits up to grace for them to unwind. No further
+// Apply may be called afterwards.
+func (m *Manager) Stop(grace time.Duration) {
+	m.mu.Lock()
+	for _, j := range m.jobs {
+		j.cancel()
+	}
+	m.jobs = make(map[string]*job)
+	m.mu.Unlock()
+
+	// Waiting on the group rather than on the current jobs also covers
+	// goroutines a recent reload replaced but that have not finished unwinding.
+	done := make(chan struct{})
+	go func() {
+		m.wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(grace):
+		m.log.Warn("shutdown grace period expired; abandoning in-flight syncs", "grace", grace)
+	}
+}
+
+// Statuses returns a snapshot of every known repository, sorted by name.
+func (m *Manager) Statuses() []Status {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	out := make([]Status, 0, len(m.stats))
+	for _, s := range m.stats {
+		out = append(out, *s)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// Uptime reports how long the manager has been running.
+func (m *Manager) Uptime() time.Duration { return time.Since(m.started) }
+
+// acquire takes a slot from the concurrency semaphore. The release function
+// returns the slot to the same channel it came from, which keeps the count
+// correct even if Apply swapped the semaphore in the meantime.
+func (m *Manager) acquire(ctx context.Context) (func(), error) {
+	m.mu.Lock()
+	sem := m.sem
+	m.mu.Unlock()
+
+	select {
+	case sem <- struct{}{}:
+		return func() { <-sem }, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+func (m *Manager) syncer() *syncer.Syncer {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.sync
+}
+
+func (m *Manager) update(name string, fn func(*Status)) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if s, ok := m.stats[name]; ok {
+		fn(s)
+	}
+}
+
+// prepareDirs creates the mirror store and the isolated HOME git/ssh will use.
+func prepareDirs(workDir string) (home string, err error) {
+	home = filepath.Join(workDir, "home")
+	for _, d := range []string{filepath.Join(workDir, "mirrors"), home} {
+		if err := os.MkdirAll(d, 0o700); err != nil {
+			return "", fmt.Errorf("prepare %s: %w", d, err)
+		}
+	}
+	return home, nil
+}
+
+// backoff grows the retry delay geometrically, capped at max, so a repo that
+// is down for hours stops hammering it (and our logs) every interval.
+func backoff(interval, max time.Duration, failures int) time.Duration {
+	d := interval
+	for i := 1; i < failures && d < max; i++ {
+		d *= 2
+	}
+	return min(d, max)
+}
+
+// stagger spreads the first run of each repo so a restart with many repos does
+// not fire every git process in the same instant.
+func stagger(index int) time.Duration {
+	return min(time.Duration(index)*250*time.Millisecond, 15*time.Second)
+}

internal/manager/manager_test.go

@@ -0,0 +1,243 @@
+package manager
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"syncbot/internal/config"
+)
+
+func testConfig(workDir string, jobs ...config.Job) *config.Config {
+	return &config.Config{
+		WorkDir:        workDir,
+		Concurrency:    2,
+		ReloadInterval: time.Second,
+		Jobs:           jobs,
+	}
+}
+
+// newJob builds a spec pointing at paths that do not exist. The job goroutine
+// will fail its sync quickly and harmlessly, which is all these tests need —
+// they are about supervision, not about git.
+func newJob(name, workDir string, interval time.Duration) config.Job {
+	return config.Job{
+		Name:       name,
+		Src:        config.Endpoint{URL: filepath.Join(workDir, name+"-src.git")},
+		Dst:        config.Endpoint{URL: filepath.Join(workDir, name+"-dst.git")},
+		Dir:        filepath.Join(workDir, "mirrors", name+".git"),
+		Interval:   interval,
+		Timeout:    5 * time.Second,
+		MaxBackoff: time.Hour,
+		Refs:       config.DefaultRefs,
+		Prune:      true,
+		Force:      true,
+	}
+}
+
+func newTestManager(t *testing.T) (*Manager, string) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	workDir := t.TempDir()
+	m := New(ctx, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	t.Cleanup(func() { m.Stop(5 * time.Second) })
+	return m, workDir
+}
+
+func (m *Manager) jobPointers() map[string]*job {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	out := make(map[string]*job, len(m.jobs))
+	for k, v := range m.jobs {
+		out[k] = v
+	}
+	return out
+}
+
+func TestApplyStartsJobs(t *testing.T) {
+	m, dir := newTestManager(t)
+
+	if err := m.Apply(testConfig(dir, newJob("a", dir, time.Hour), newJob("b", dir, time.Hour))); err != nil {
+		t.Fatal(err)
+	}
+
+	if got := len(m.jobPointers()); got != 2 {
+		t.Fatalf("running jobs = %d, want 2", got)
+	}
+	if got := len(m.Statuses()); got != 2 {
+		t.Errorf("statuses = %d, want 2", got)
+	}
+}
+
+// The whole point of an incremental reload: editing one repo must not disturb
+// the others' timers or in-flight work.
+func TestApplyLeavesUnchangedJobsRunning(t *testing.T) {
+	m, dir := newTestManager(t)
+	cfg := testConfig(dir, newJob("a", dir, time.Hour), newJob("b", dir, time.Hour))
+	if err := m.Apply(cfg); err != nil {
+		t.Fatal(err)
+	}
+	before := m.jobPointers()
+
+	// Identical configuration: nothing should be touched.
+	if err := m.Apply(testConfig(dir, newJob("a", dir, time.Hour), newJob("b", dir, time.Hour))); err != nil {
+		t.Fatal(err)
+	}
+	after := m.jobPointers()
+	for name, j := range before {
+		if after[name] != j {
+			t.Errorf("job %q was restarted despite an identical config", name)
+		}
+	}
+
+	// Change only "a": "b" must survive untouched.
+	if err := m.Apply(testConfig(dir, newJob("a", dir, 30*time.Minute), newJob("b", dir, time.Hour))); err != nil {
+		t.Fatal(err)
+	}
+	changed := m.jobPointers()
+	if changed["a"] == before["a"] {
+		t.Error("job \"a\" should have been restarted after its interval changed")
+	}
+	if changed["b"] != before["b"] {
+		t.Error("job \"b\" should not have been restarted")
+	}
+	if got := changed["a"].spec.Interval; got != 30*time.Minute {
+		t.Errorf("restarted job carries interval %s, want 30m", got)
+	}
+}
+
+func TestApplyStopsRemovedJobs(t *testing.T) {
+	m, dir := newTestManager(t)
+	if err := m.Apply(testConfig(dir, newJob("a", dir, time.Hour), newJob("b", dir, time.Hour))); err != nil {
+		t.Fatal(err)
+	}
+	removed := m.jobPointers()["b"]
+
+	if err := m.Apply(testConfig(dir, newJob("a", dir, time.Hour))); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, still := m.jobPointers()["b"]; still {
+		t.Fatal("job \"b\" is still registered after being removed from the config")
+	}
+	select {
+	case <-removed.done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("goroutine for the removed job did not exit")
+	}
+	for _, s := range m.Statuses() {
+		if s.Name == "b" {
+			t.Error("status for the removed job should be dropped")
+		}
+	}
+}
+
+func TestCountersSurviveJobRestart(t *testing.T) {
+	m, dir := newTestManager(t)
+	if err := m.Apply(testConfig(dir, newJob("a", dir, time.Hour))); err != nil {
+		t.Fatal(err)
+	}
+
+	m.update("a", func(s *Status) { s.Syncs, s.Pushes = 7, 3 })
+
+	if err := m.Apply(testConfig(dir, newJob("a", dir, 5*time.Minute))); err != nil {
+		t.Fatal(err)
+	}
+
+	got := m.Statuses()[0]
+	if got.Syncs != 7 || got.Pushes != 3 {
+		t.Errorf("counters reset on restart: syncs=%d pushes=%d, want 7/3", got.Syncs, got.Pushes)
+	}
+	if got.Interval != "5m0s" {
+		t.Errorf("status interval = %q, want it to follow the new spec", got.Interval)
+	}
+}
+
+func TestConcurrencyChangeKeepsSemaphoreConsistent(t *testing.T) {
+	m, dir := newTestManager(t)
+	cfg := testConfig(dir, newJob("a", dir, time.Hour))
+	cfg.Concurrency = 1
+	if err := m.Apply(cfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Hold a slot from the old semaphore, then resize.
+	release, err := m.acquire(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cfg2 := testConfig(dir, newJob("a", dir, time.Hour))
+	cfg2.Concurrency = 4
+	if err := m.Apply(cfg2); err != nil {
+		t.Fatal(err)
+	}
+	release() // must return the slot to the channel it came from, not the new one
+
+	// The new semaphore should have its full capacity available.
+	var releases []func()
+	for i := 0; i < 4; i++ {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		r, err := m.acquire(ctx)
+		cancel()
+		if err != nil {
+			t.Fatalf("acquire %d/4 blocked after resize: %v", i+1, err)
+		}
+		releases = append(releases, r)
+	}
+	for _, r := range releases {
+		r()
+	}
+}
+
+func TestApplyRejectsUnusableWorkDir(t *testing.T) {
+	m, dir := newTestManager(t)
+	// A regular file cannot host the mirror directories.
+	blocked := filepath.Join(dir, "not-a-dir")
+	if err := os.WriteFile(blocked, []byte("x"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := m.Apply(testConfig(blocked, newJob("a", dir, time.Hour))); err == nil {
+		t.Fatal("want an error for a work_dir that cannot be created")
+	}
+}
+
+func TestBackoff(t *testing.T) {
+	const interval, max = time.Minute, 30 * time.Minute
+	cases := []struct {
+		failures int
+		want     time.Duration
+	}{
+		{0, time.Minute},
+		{1, time.Minute},
+		{2, 2 * time.Minute},
+		{3, 4 * time.Minute},
+		{4, 8 * time.Minute},
+		{6, 30 * time.Minute}, // capped
+		{100, 30 * time.Minute},
+	}
+	for _, tc := range cases {
+		if got := backoff(interval, max, tc.failures); got != tc.want {
+			t.Errorf("backoff(failures=%d) = %s, want %s", tc.failures, got, tc.want)
+		}
+	}
+}
+
+func TestStaggerIsBounded(t *testing.T) {
+	if got := stagger(0); got != 0 {
+		t.Errorf("the first repo should start immediately, got %s", got)
+	}
+	if got := stagger(3); got != 750*time.Millisecond {
+		t.Errorf("stagger(3) = %s, want 750ms", got)
+	}
+	if got := stagger(1000); got != 15*time.Second {
+		t.Errorf("stagger should be capped at 15s, got %s", got)
+	}
+}