// Package config loads, validates and resolves syncbot's TOML configuration. // // The on-disk document is deliberately forgiving: every knob has a default, // [global] supplies fallbacks for all repositories, and each [[repo]] overrides // only what it needs. Load flattens all of that into a []Job in which every // value is already resolved, so the rest of the daemon never has to reason // about defaults or inheritance — and so the hot-reload logic can decide // whether a job changed with a plain reflect.DeepEqual. package config import ( "fmt" "os" "path/filepath" "regexp" "runtime" "sort" "strings" "time" "github.com/BurntSushi/toml" ) // Duration is a time.Duration that decodes from a TOML string such as "5m". type Duration time.Duration // UnmarshalText implements encoding.TextUnmarshaler. func (d *Duration) UnmarshalText(b []byte) error { v, err := time.ParseDuration(string(b)) if err != nil { return fmt.Errorf("invalid duration %q (want e.g. \"30s\", \"5m\", \"2h\")", b) } *d = Duration(v) return nil } // D returns the underlying time.Duration. func (d Duration) D() time.Duration { return time.Duration(d) } // Endpoint is one side of a mirror. In TOML it may be written either as a bare // URL string or as a table when it needs its own credentials: // // dst = "git@github.com:me/repo.git" // // [repo.dst] // url = "git@github.com:me/repo.git" // ssh_key = "/etc/syncbot/keys/repo" type Endpoint struct { URL string `json:"url"` SSHKey string `json:"-"` KnownHosts string `json:"-"` StrictHostKey string `json:"-"` } // UnmarshalTOML accepts both the string and the table spelling of an endpoint. func (e *Endpoint) UnmarshalTOML(v any) error { switch t := v.(type) { case string: e.URL = t return nil case map[string]any: for _, k := range sortedKeys(t) { s, ok := t[k].(string) if !ok { return fmt.Errorf("key %q must be a string", k) } switch k { case "url": e.URL = s case "ssh_key": e.SSHKey = s case "known_hosts": e.KnownHosts = s case "strict_host_key": e.StrictHostKey = s default: return fmt.Errorf("unknown key %q (want url, ssh_key, known_hosts or strict_host_key)", k) } } if e.URL == "" { return fmt.Errorf("missing required key \"url\"") } return nil default: return fmt.Errorf("must be a URL string or a table, got %T", v) } } // Global holds process-wide settings plus the defaults inherited by every repo. type Global struct { WorkDir string `toml:"work_dir"` Listen string `toml:"listen"` LogLevel string `toml:"log_level"` LogFormat string `toml:"log_format"` Concurrency int `toml:"concurrency"` ReloadInterval Duration `toml:"reload_interval"` // Inherited by every [[repo]] unless overridden there. Interval Duration `toml:"interval"` Timeout Duration `toml:"timeout"` MaxBackoff Duration `toml:"max_backoff"` Refs []string `toml:"refs"` Prune *bool `toml:"prune"` Force *bool `toml:"force"` Atomic *bool `toml:"atomic"` AllowEmpty *bool `toml:"allow_empty"` SSHKey string `toml:"ssh_key"` KnownHosts string `toml:"known_hosts"` StrictHostKey string `toml:"strict_host_key"` GitConfig []string `toml:"git_config"` } // Repo is one [[repo]] block as written by the user. type Repo struct { Name string `toml:"name"` Src Endpoint `toml:"src"` Dst Endpoint `toml:"dst"` Enabled *bool `toml:"enabled"` Interval Duration `toml:"interval"` Timeout Duration `toml:"timeout"` MaxBackoff Duration `toml:"max_backoff"` Refs []string `toml:"refs"` Prune *bool `toml:"prune"` Force *bool `toml:"force"` Atomic *bool `toml:"atomic"` AllowEmpty *bool `toml:"allow_empty"` SSHKey string `toml:"ssh_key"` KnownHosts string `toml:"known_hosts"` StrictHostKey string `toml:"strict_host_key"` GitConfig []string `toml:"git_config"` } // file mirrors the TOML document itself. type file struct { Global Global `toml:"global"` Repos []Repo `toml:"repo"` } // Job is a fully resolved sync unit: all defaults folded in, ready to run. // Every field is comparable with reflect.DeepEqual, which is how the manager // detects that a reloaded config actually changed something for this repo. type Job struct { Name string Src Endpoint Dst Endpoint Dir string // local bare mirror Interval time.Duration Timeout time.Duration MaxBackoff time.Duration Refs []string Prune bool Force bool Atomic bool AllowEmpty bool GitConfig []string } // Config is the resolved configuration the daemon runs on. type Config struct { WorkDir string Listen string LogLevel string LogFormat string Concurrency int ReloadInterval time.Duration Jobs []Job } // Defaults applied when the document leaves a value out. const ( DefaultWorkDir = "/var/lib/syncbot" DefaultInterval = 5 * time.Minute DefaultTimeout = 30 * time.Minute DefaultMaxBackoff = time.Hour DefaultReloadInterval = 5 * time.Second DefaultLogLevel = "info" DefaultLogFormat = "text" ) // DefaultRefs mirrors branches and tags, which is what almost everyone wants. var DefaultRefs = []string{"refs/heads/*", "refs/tags/*"} // nameRe keeps repo names usable as directory names and Prometheus labels. var nameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`) // Load reads, validates and resolves the config file at path. func Load(path string) (*Config, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } return Parse(b) } // Parse resolves an in-memory TOML document. Load is the usual entry point; // Parse exists so tests (and -check) can work without touching disk. func Parse(b []byte) (*Config, error) { var f file md, err := toml.Decode(string(b), &f) if err != nil { return nil, err } if undec := unknownKeys(md); len(undec) > 0 { return nil, fmt.Errorf("unknown config key(s): %s", strings.Join(undec, ", ")) } return resolve(&f) } // unknownKeys reports keys the decoder did not recognise, so a typo like // "intervall" fails loudly at startup instead of silently doing nothing. func unknownKeys(md toml.MetaData) []string { var out []string for _, k := range md.Undecoded() { s := k.String() // Endpoint consumes its own subtree via UnmarshalTOML and validates the // keys itself; the decoder cannot see into it, so ignore those paths. if strings.Contains(s, ".src.") || strings.Contains(s, ".dst.") { continue } out = append(out, s) } sort.Strings(out) return out } func resolve(f *file) (*Config, error) { g := f.Global workDir, err := expandEnv(orString(g.WorkDir, DefaultWorkDir)) if err != nil { return nil, fmt.Errorf("global.work_dir: %w", err) } if !filepath.IsAbs(workDir) { if workDir, err = filepath.Abs(workDir); err != nil { return nil, fmt.Errorf("global.work_dir: %w", err) } } cfg := &Config{ WorkDir: workDir, Listen: g.Listen, LogLevel: strings.ToLower(orString(g.LogLevel, DefaultLogLevel)), LogFormat: strings.ToLower(orString(g.LogFormat, DefaultLogFormat)), Concurrency: g.Concurrency, ReloadInterval: orDuration(g.ReloadInterval, DefaultReloadInterval), } if cfg.Concurrency <= 0 { cfg.Concurrency = min(4, runtime.NumCPU()) } switch cfg.LogLevel { case "debug", "info", "warn", "error": default: return nil, fmt.Errorf("global.log_level: %q is not one of debug, info, warn, error", cfg.LogLevel) } switch cfg.LogFormat { case "text", "json": default: return nil, fmt.Errorf("global.log_format: %q is not one of text, json", cfg.LogFormat) } if cfg.ReloadInterval < time.Second { return nil, fmt.Errorf("global.reload_interval: must be at least 1s") } if len(f.Repos) == 0 { return nil, fmt.Errorf("no [[repo]] blocks defined: nothing to sync") } seen := make(map[string]bool, len(f.Repos)) for i := range f.Repos { r := &f.Repos[i] where := fmt.Sprintf("repo[%d]", i) if r.Name != "" { where = fmt.Sprintf("repo %q", r.Name) } if !nameRe.MatchString(r.Name) { return nil, fmt.Errorf("%s: name must match %s", where, nameRe) } if seen[r.Name] { return nil, fmt.Errorf("%s: duplicate name", where) } seen[r.Name] = true if enabled := r.Enabled; enabled != nil && !*enabled { continue } job, err := resolveJob(r, &g, workDir) if err != nil { return nil, fmt.Errorf("%s: %w", where, err) } cfg.Jobs = append(cfg.Jobs, *job) } if len(cfg.Jobs) == 0 { return nil, fmt.Errorf("every [[repo]] is disabled: nothing to sync") } sort.Slice(cfg.Jobs, func(i, j int) bool { return cfg.Jobs[i].Name < cfg.Jobs[j].Name }) return cfg, nil } func resolveJob(r *Repo, g *Global, workDir string) (*Job, error) { j := &Job{ Name: r.Name, Dir: filepath.Join(workDir, "mirrors", r.Name+".git"), Interval: pick(r.Interval, g.Interval, DefaultInterval), Timeout: pick(r.Timeout, g.Timeout, DefaultTimeout), MaxBackoff: pick(r.MaxBackoff, g.MaxBackoff, DefaultMaxBackoff), Refs: orSlice(r.Refs, g.Refs, DefaultRefs), Prune: orBool(true, r.Prune, g.Prune), Force: orBool(true, r.Force, g.Force), Atomic: orBool(false, r.Atomic, g.Atomic), AllowEmpty: orBool(false, r.AllowEmpty, g.AllowEmpty), GitConfig: orSlice(r.GitConfig, g.GitConfig, nil), } var err error // SSH settings cascade: endpoint table -> [[repo]] -> [global]. if j.Src, err = resolveEndpoint(r.Src, r, g); err != nil { return nil, fmt.Errorf("src: %w", err) } if j.Dst, err = resolveEndpoint(r.Dst, r, g); err != nil { return nil, fmt.Errorf("dst: %w", err) } if j.Src.URL == "" { return nil, fmt.Errorf("src is required") } if j.Dst.URL == "" { return nil, fmt.Errorf("dst is required") } if j.Src.URL == j.Dst.URL { return nil, fmt.Errorf("src and dst are the same repository") } if j.Interval <= 0 { return nil, fmt.Errorf("interval must be positive") } if j.Timeout <= 0 { return nil, fmt.Errorf("timeout must be positive") } if j.MaxBackoff < j.Interval { j.MaxBackoff = j.Interval } if len(j.Refs) == 0 { return nil, fmt.Errorf("refs must not be empty") } for _, p := range j.Refs { if !strings.HasPrefix(p, "refs/") { return nil, fmt.Errorf("refs: %q must start with \"refs/\"", p) } if strings.Count(p, "*") > 1 { return nil, fmt.Errorf("refs: %q may contain at most one \"*\"", p) } } for _, kv := range j.GitConfig { if !strings.Contains(kv, "=") { return nil, fmt.Errorf("git_config: %q must be in key=value form", kv) } } return j, nil } func resolveEndpoint(e Endpoint, r *Repo, g *Global) (Endpoint, error) { out := Endpoint{ URL: e.URL, SSHKey: orString(e.SSHKey, r.SSHKey, g.SSHKey), KnownHosts: orString(e.KnownHosts, r.KnownHosts, g.KnownHosts), StrictHostKey: orString(e.StrictHostKey, r.StrictHostKey, g.StrictHostKey), } var err error for _, p := range []*string{&out.URL, &out.SSHKey, &out.KnownHosts} { if *p, err = expandEnv(*p); err != nil { return out, err } } if out.StrictHostKey == "" { // With a pinned known_hosts file we can afford to be strict; without // one, trust-on-first-use is the only thing that can work unattended. if out.KnownHosts != "" { out.StrictHostKey = "yes" } else { out.StrictHostKey = "accept-new" } } switch out.StrictHostKey { case "yes", "no", "accept-new": default: return out, fmt.Errorf("strict_host_key: %q is not one of yes, no, accept-new", out.StrictHostKey) } if out.SSHKey != "" && !filepath.IsAbs(out.SSHKey) { return out, fmt.Errorf("ssh_key: %q must be an absolute path", out.SSHKey) } return out, nil } // envRe matches ${VAR}. Bare $VAR is deliberately not expanded so that secrets // containing a literal '$' survive unharmed. var envRe = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`) // expandEnv substitutes ${VAR} references, failing loudly on undefined names — // silently expanding to "" would produce a subtly broken URL instead. func expandEnv(s string) (string, error) { if !strings.Contains(s, "${") { return s, nil } var missing []string out := envRe.ReplaceAllStringFunc(s, func(m string) string { name := m[2 : len(m)-1] v, ok := os.LookupEnv(name) if !ok { missing = append(missing, name) return "" } return v }) if len(missing) > 0 { return "", fmt.Errorf("undefined environment variable(s): %s", strings.Join(missing, ", ")) } return out, nil } func orString(vs ...string) string { for _, v := range vs { if v != "" { return v } } return "" } func orBool(def bool, vs ...*bool) bool { for _, v := range vs { if v != nil { return *v } } return def } func orSlice(vs ...[]string) []string { for _, v := range vs { if len(v) > 0 { return append([]string(nil), v...) } } return nil } func orDuration(v Duration, def time.Duration) time.Duration { if v != 0 { return v.D() } return def } func pick(repo, global Duration, def time.Duration) time.Duration { if repo != 0 { return repo.D() } if global != 0 { return global.D() } return def } func sortedKeys(m map[string]any) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out } // orBool with a literal default needs a *bool; these make the call sites read // naturally without sprinkling helper variables around. func boolPtr(b bool) *bool { return &b }