init
ci / test (push) Canceled after 0s
docker / build (push) Canceled after 0s

This commit is contained in:
iceBear67
2026-08-14 07:13:22 +00:00
commit a006483bbc
26 changed files with 3826 additions and 0 deletions
+444
View File
@@ -0,0 +1,444 @@
// Package gitx is a thin, hermetic wrapper around the git command line.
//
// Shelling out to git — rather than linking a pure-Go implementation — keeps
// the binary small and the memory profile flat: the heavy lifting happens in a
// short-lived child process that the kernel reclaims when it exits.
//
// Every invocation runs with an isolated HOME and with the system/global git
// config disabled, so syncbot behaves identically no matter whose account or
// container it runs in.
package gitx
import (
"bytes"
"context"
"fmt"
"io"
"log/slog"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// SSH describes how to authenticate to one endpoint over SSH.
type SSH struct {
KeyPath string // deploy key; empty means "use the agent / default keys"
KnownHosts string // pinned host key file; empty means use Home/known_hosts
StrictHostKey string // yes | no | accept-new
}
// Options carry everything a git invocation needs beyond its arguments.
type Options struct {
Log *slog.Logger
Home string // isolated HOME for git and ssh; must exist
GitConfig []string // extra "key=value" settings passed as -c
SSHCmd string // pre-built GIT_SSH_COMMAND, see PrepareSSH
Secrets []string // substrings scrubbed from logs and error messages
}
// Refs maps a full ref name to the object it points at.
type Refs map[string]string
// Equal reports whether two ref sets are identical.
func (r Refs) Equal(other Refs) bool {
if len(r) != len(other) {
return false
}
for k, v := range r {
if other[k] != v {
return false
}
}
return true
}
// termGrace is how long a git child gets to exit after SIGTERM before we
// escalate to SIGKILL on its whole process group.
const termGrace = 3 * time.Second
// Run executes git with the given arguments and returns its stdout.
//
// The child is put in its own process group so that a timeout or a shutdown
// takes down the helpers git spawns (ssh, git-remote-https) instead of leaking
// them.
func Run(ctx context.Context, o Options, dir string, args ...string) (string, error) {
full := make([]string, 0, len(args)+2*len(o.GitConfig))
for _, kv := range o.GitConfig {
full = append(full, "-c", kv)
}
full = append(full, args...)
cmd := exec.Command("git", full...)
cmd.Dir = dir
cmd.Env = environ(o)
cmd.SysProcAttr = sysProcAttr()
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &limitedWriter{W: &stderr, N: 64 << 10}
started := time.Now()
if err := cmd.Start(); err != nil {
return "", fmt.Errorf("git %s: %w", args[0], err)
}
// Watchdog: translate context cancellation into signals for the group.
watchdogDone := make(chan struct{})
go func() {
select {
case <-ctx.Done():
terminate(cmd.Process.Pid)
case <-watchdogDone:
}
}()
err := cmd.Wait()
close(watchdogDone)
if o.Log != nil && o.Log.Enabled(ctx, slog.LevelDebug) {
o.Log.Debug("git", "args", scrub(strings.Join(args, " "), o.Secrets),
"dur", time.Since(started).Round(time.Millisecond), "err", err)
}
if err != nil {
msg := scrub(strings.TrimSpace(stderr.String()), o.Secrets)
if ctx.Err() != nil {
return "", fmt.Errorf("git %s: %w (%s)", args[0], ctx.Err(), firstLines(msg, 3))
}
if msg == "" {
msg = err.Error()
}
return "", fmt.Errorf("git %s: %s", args[0], firstLines(msg, 8))
}
return stdout.String(), nil
}
// environ builds a deterministic environment: the parent's, minus anything that
// could redirect git's authentication or config, plus our own settings.
func environ(o Options) []string {
drop := map[string]bool{
"HOME": true, "XDG_CONFIG_HOME": true,
"GIT_SSH": true, "GIT_SSH_COMMAND": true, "GIT_ASKPASS": true, "SSH_ASKPASS": true,
"GIT_CONFIG": true, "GIT_CONFIG_GLOBAL": true, "GIT_CONFIG_SYSTEM": true,
"GIT_DIR": true, "GIT_WORK_TREE": true, "GIT_TERMINAL_PROMPT": true,
}
out := make([]string, 0, 16)
for _, kv := range os.Environ() {
if k, _, ok := strings.Cut(kv, "="); ok && !drop[k] {
out = append(out, kv)
}
}
out = append(out,
"HOME="+o.Home,
"GIT_CONFIG_GLOBAL="+os.DevNull,
"GIT_CONFIG_SYSTEM="+os.DevNull,
"GIT_TERMINAL_PROMPT=0", // never block waiting for a password
"SSH_ASKPASS_REQUIRE=never",
"LC_ALL=C",
)
if o.SSHCmd != "" {
out = append(out, "GIT_SSH_COMMAND="+o.SSHCmd)
}
return out
}
// PrepareSSH builds a GIT_SSH_COMMAND for the endpoint.
//
// Deploy keys are usually mounted read-only from a secret store, which often
// means mode 0644 — and ssh flatly refuses group- or world-readable keys. When
// that happens we copy the key into tmpDir at 0600 rather than asking the
// operator to fix permissions they may not control.
func PrepareSSH(s SSH, home, tmpDir string) (string, error) {
knownHosts := s.KnownHosts
if knownHosts == "" {
knownHosts = filepath.Join(home, "known_hosts")
if _, err := os.Stat(knownHosts); os.IsNotExist(err) {
if err := os.WriteFile(knownHosts, nil, 0o600); err != nil {
return "", fmt.Errorf("create known_hosts: %w", err)
}
}
}
strict := s.StrictHostKey
if strict == "" {
strict = "accept-new"
}
args := []string{"ssh",
"-o", "BatchMode=yes",
"-o", "StrictHostKeyChecking=" + strict,
"-o", "UserKnownHostsFile=" + knownHosts,
"-o", "ConnectTimeout=30",
}
if s.KeyPath != "" {
key, err := usableKey(s.KeyPath, tmpDir)
if err != nil {
return "", err
}
// IdentitiesOnly stops ssh from offering an agent's keys first and
// tripping GitHub's "too many authentication failures".
args = append(args, "-i", key, "-o", "IdentitiesOnly=yes")
}
quoted := make([]string, len(args))
for i, a := range args {
quoted[i] = shellQuote(a)
}
return strings.Join(quoted, " "), nil
}
func usableKey(path, tmpDir string) (string, error) {
fi, err := os.Stat(path)
if err != nil {
return "", fmt.Errorf("ssh_key: %w", err)
}
if fi.IsDir() {
return "", fmt.Errorf("ssh_key: %s is a directory", path)
}
if fi.Mode().Perm()&0o077 == 0 {
return path, nil
}
// Too permissive for ssh: stage a private copy.
b, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("ssh_key: %w", err)
}
dst := filepath.Join(tmpDir, "id_"+filepath.Base(path))
if err := os.WriteFile(dst, b, 0o600); err != nil {
return "", fmt.Errorf("ssh_key: stage private copy: %w", err)
}
return dst, nil
}
// EnsureMirror makes sure dir holds a usable bare repository, creating it on
// first run. A directory that exists but is not a bare repo is reported rather
// than deleted — that is almost always a misconfigured mount, and silently
// wiping it would be the wrong kind of helpful.
func EnsureMirror(ctx context.Context, o Options, dir string) error {
if _, err := os.Stat(filepath.Join(dir, "HEAD")); err == nil {
out, err := Run(ctx, o, dir, "rev-parse", "--is-bare-repository")
if err != nil {
return fmt.Errorf("%s exists but is not a git repository: %w", dir, err)
}
if strings.TrimSpace(out) != "true" {
return fmt.Errorf("%s is not a bare repository", dir)
}
return nil
} else if !os.IsNotExist(err) {
return err
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return err
}
_, err := Run(ctx, o, o.Home, "init", "--bare", "--quiet", "--initial-branch=main", dir)
return err
}
// LsRemote asks a remote which refs it currently has, without transferring any
// objects. This is the "check for updates" probe: cheap enough to run on a
// short interval even against large repositories.
func LsRemote(ctx context.Context, o Options, dir, repoURL string, patterns []string) (Refs, error) {
args := append([]string{"ls-remote", "--refs", "--", repoURL}, patterns...)
out, err := Run(ctx, o, dir, args...)
if err != nil {
return nil, err
}
return parseRefs(out, "\t", true, patterns), nil
}
// LocalRefs reads the mirror's own refs, filtered to the managed patterns.
func LocalRefs(ctx context.Context, o Options, dir string, patterns []string) (Refs, error) {
out, err := Run(ctx, o, dir, "for-each-ref", "--format=%(objectname)\t%(refname)")
if err != nil {
return nil, err
}
return parseRefs(out, "\t", true, patterns), nil
}
// Fetch updates the mirror from src. Refs that vanished upstream are pruned so
// the mirror is an exact copy of the managed namespace, not an accumulation.
func Fetch(ctx context.Context, o Options, dir, src string, patterns []string) error {
args := []string{"fetch", "--force", "--no-tags", "--no-write-fetch-head", "--prune", "--quiet", "--", src}
for _, p := range patterns {
args = append(args, "+"+p+":"+p)
}
_, err := Run(ctx, o, dir, args...)
return err
}
// PushOptions controls how the mirror is written to the destination.
type PushOptions struct {
Prune bool // delete destination refs that no longer exist upstream
Force bool // allow non-fast-forward updates (a mirror must)
Atomic bool // all refs update, or none do
}
// Push writes the mirror's managed refs to dst and returns the porcelain lines
// describing what actually changed.
func Push(ctx context.Context, o Options, dir, dst string, patterns []string, po PushOptions) ([]string, error) {
args := []string{"push", "--porcelain"}
if po.Prune {
args = append(args, "--prune")
}
if po.Force {
args = append(args, "--force")
}
if po.Atomic {
args = append(args, "--atomic")
}
args = append(args, "--", dst)
for _, p := range patterns {
args = append(args, p+":"+p)
}
out, err := Run(ctx, o, dir, args...)
if err != nil {
return nil, err
}
var changed []string
for _, line := range strings.Split(out, "\n") {
line = strings.TrimRight(line, "\r")
// Porcelain format: "<flag>\t<from>:<to>\t<summary>". '=' means the ref
// was already up to date, which is the boring majority.
if line == "" || strings.HasPrefix(line, "To ") || line == "Done" || strings.HasPrefix(line, "=\t") {
continue
}
changed = append(changed, scrub(line, o.Secrets))
}
return changed, nil
}
// MatchRef implements git's refspec globbing: at most one "*", which matches
// any run of characters including "/".
func MatchRef(pattern, ref string) bool {
i := strings.IndexByte(pattern, '*')
if i < 0 {
return pattern == ref
}
prefix, suffix := pattern[:i], pattern[i+1:]
return len(ref) >= len(prefix)+len(suffix) &&
strings.HasPrefix(ref, prefix) &&
strings.HasSuffix(ref, suffix)
}
// MatchAny reports whether ref matches any of the patterns.
func MatchAny(patterns []string, ref string) bool {
for _, p := range patterns {
if MatchRef(p, ref) {
return true
}
}
return false
}
// parseRefs reads "<object><sep><ref>" lines, optionally filtering to patterns.
func parseRefs(out, sep string, filter bool, patterns []string) Refs {
refs := make(Refs)
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
obj, ref, ok := strings.Cut(line, sep)
if !ok {
continue
}
// Peeled entries ("refs/tags/v1^{}") describe the commit behind an
// annotated tag; the tag object itself is what we mirror.
if strings.HasSuffix(ref, "^{}") {
continue
}
if filter && !MatchAny(patterns, ref) {
continue
}
refs[ref] = obj
}
return refs
}
// RedactURL strips the password from a URL so it can be logged.
func RedactURL(raw string) string {
if !strings.Contains(raw, "://") {
return raw // scp-style (git@host:path) carries no inline secret
}
u, err := url.Parse(raw)
if err != nil || u.User == nil {
return raw
}
if _, hasPassword := u.User.Password(); hasPassword {
// Plain letters: anything punctuation-ish would come back
// percent-encoded from URL.String() and read as noise in a log line.
u.User = url.UserPassword(u.User.Username(), "redacted")
} else {
u.User = url.User(u.User.Username())
}
return u.String()
}
// URLSecret returns the credential embedded in a URL, if any, so callers can
// register it with Options.Secrets and keep it out of logs.
func URLSecret(raw string) string {
if !strings.Contains(raw, "://") {
return ""
}
u, err := url.Parse(raw)
if err != nil || u.User == nil {
return ""
}
if pw, ok := u.User.Password(); ok && pw != "" {
return pw
}
return ""
}
func scrub(s string, secrets []string) string {
for _, sec := range secrets {
if sec != "" {
s = strings.ReplaceAll(s, sec, "***")
}
}
return s
}
func firstLines(s string, n int) string {
lines := strings.Split(s, "\n")
if len(lines) > n {
lines = append(lines[:n], "...")
}
return strings.Join(lines, "; ")
}
// shellQuote makes a token safe for GIT_SSH_COMMAND, which git hands to a shell.
func shellQuote(s string) string {
if s != "" && !strings.ContainsAny(s, " \t\n\"'\\$`&;|<>()*?[]{}#~!") {
return s
}
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}
// limitedWriter keeps a runaway stderr from growing without bound.
type limitedWriter struct {
W io.Writer
N int
}
// Write always reports the full length: a short write would be treated as an
// error by os/exec and would abort an otherwise healthy git invocation.
func (l *limitedWriter) Write(p []byte) (int, error) {
total := len(p)
if l.N <= 0 {
return total, nil
}
if len(p) > l.N {
p = p[:l.N]
}
n, err := l.W.Write(p)
l.N -= n
return total, err
}
+221
View File
@@ -0,0 +1,221 @@
package gitx
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestMatchRef(t *testing.T) {
cases := []struct {
pattern, ref string
want bool
}{
{"refs/heads/*", "refs/heads/main", true},
{"refs/heads/*", "refs/heads/feature/nested/deep", true}, // '*' spans '/'
{"refs/heads/*", "refs/tags/v1", false},
{"refs/heads/*", "refs/heads/", true},
{"refs/heads/main", "refs/heads/main", true},
{"refs/heads/main", "refs/heads/maint", false},
{"refs/tags/v*", "refs/tags/v1.2.3", true},
{"refs/tags/v*", "refs/tags/rc1", false},
{"refs/heads/*-stable", "refs/heads/2.0-stable", true},
{"refs/heads/*-stable", "refs/heads/2.0-beta", false},
}
for _, tc := range cases {
if got := MatchRef(tc.pattern, tc.ref); got != tc.want {
t.Errorf("MatchRef(%q, %q) = %v, want %v", tc.pattern, tc.ref, got, tc.want)
}
}
}
func TestParseRefsSkipsPeeledAndFilters(t *testing.T) {
out := strings.Join([]string{
"aaa\trefs/heads/main",
"bbb\trefs/tags/v1",
"ccc\trefs/tags/v1^{}", // peeled annotated tag
"ddd\trefs/pull/7/head",
"",
}, "\n")
refs := parseRefs(out, "\t", true, []string{"refs/heads/*", "refs/tags/*"})
if len(refs) != 2 {
t.Fatalf("got %d refs, want 2: %v", len(refs), refs)
}
if refs["refs/heads/main"] != "aaa" || refs["refs/tags/v1"] != "bbb" {
t.Errorf("unexpected refs: %v", refs)
}
if _, ok := refs["refs/tags/v1^{}"]; ok {
t.Error("peeled tag entry should be dropped")
}
if _, ok := refs["refs/pull/7/head"]; ok {
t.Error("unmanaged ref should be filtered out")
}
}
func TestRefsEqual(t *testing.T) {
a := Refs{"refs/heads/main": "1", "refs/tags/v1": "2"}
if !a.Equal(Refs{"refs/tags/v1": "2", "refs/heads/main": "1"}) {
t.Error("same contents should compare equal regardless of order")
}
if a.Equal(Refs{"refs/heads/main": "1"}) {
t.Error("different sizes should not compare equal")
}
if a.Equal(Refs{"refs/heads/main": "1", "refs/tags/v1": "9"}) {
t.Error("different objects should not compare equal")
}
}
func TestRedactURL(t *testing.T) {
cases := []struct{ in, want string }{
{"https://user:token@github.com/me/x.git", "https://user:redacted@github.com/me/x.git"},
{"https://token@github.com/me/x.git", "https://token@github.com/me/x.git"},
{"https://github.com/me/x.git", "https://github.com/me/x.git"},
{"git@github.com:me/x.git", "git@github.com:me/x.git"},
{"/srv/git/local.git", "/srv/git/local.git"},
}
for _, tc := range cases {
if got := RedactURL(tc.in); got != tc.want {
t.Errorf("RedactURL(%q) = %q, want %q", tc.in, got, tc.want)
}
}
}
func TestURLSecret(t *testing.T) {
if got := URLSecret("https://x-access-token:ghp_abc@github.com/me/x.git"); got != "ghp_abc" {
t.Errorf("URLSecret = %q, want ghp_abc", got)
}
if got := URLSecret("git@github.com:me/x.git"); got != "" {
t.Errorf("URLSecret = %q, want empty", got)
}
}
func TestScrubRemovesSecrets(t *testing.T) {
got := scrub("fatal: auth failed for ghp_abc123", []string{"ghp_abc123"})
if strings.Contains(got, "ghp_abc123") {
t.Errorf("secret leaked: %q", got)
}
}
func TestShellQuote(t *testing.T) {
cases := []struct{ in, want string }{
{"ssh", "ssh"},
{"/etc/keys/id_ed25519", "/etc/keys/id_ed25519"},
{"/keys/my key", `'/keys/my key'`},
{"it's", `'it'\''s'`},
{"StrictHostKeyChecking=accept-new", "StrictHostKeyChecking=accept-new"},
}
for _, tc := range cases {
if got := shellQuote(tc.in); got != tc.want {
t.Errorf("shellQuote(%q) = %q, want %q", tc.in, got, tc.want)
}
}
}
// A deploy key mounted from a secret store is often world-readable, which ssh
// rejects outright. PrepareSSH must stage a 0600 copy instead of failing.
func TestPrepareSSHStagesPermissiveKey(t *testing.T) {
home, tmp := t.TempDir(), t.TempDir()
key := filepath.Join(t.TempDir(), "deploy_key")
if err := os.WriteFile(key, []byte("PRIVATE KEY"), 0o644); err != nil {
t.Fatal(err)
}
cmd, err := PrepareSSH(SSH{KeyPath: key, StrictHostKey: "accept-new"}, home, tmp)
if err != nil {
t.Fatal(err)
}
if strings.Contains(cmd, key) {
t.Errorf("should use a staged copy, not the 0644 original: %s", cmd)
}
staged := filepath.Join(tmp, "id_deploy_key")
fi, err := os.Stat(staged)
if err != nil {
t.Fatalf("staged copy missing: %v", err)
}
if perm := fi.Mode().Perm(); perm != 0o600 {
t.Errorf("staged key mode = %o, want 600", perm)
}
for _, want := range []string{"IdentitiesOnly=yes", "BatchMode=yes", "StrictHostKeyChecking=accept-new"} {
if !strings.Contains(cmd, want) {
t.Errorf("ssh command missing %q: %s", want, cmd)
}
}
}
func TestPrepareSSHKeepsPrivateKeyPath(t *testing.T) {
home, tmp := t.TempDir(), t.TempDir()
key := filepath.Join(t.TempDir(), "deploy_key")
if err := os.WriteFile(key, []byte("PRIVATE KEY"), 0o600); err != nil {
t.Fatal(err)
}
cmd, err := PrepareSSH(SSH{KeyPath: key}, home, tmp)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(cmd, key) {
t.Errorf("an already-private key should be used in place: %s", cmd)
}
}
func TestPrepareSSHCreatesKnownHosts(t *testing.T) {
home, tmp := t.TempDir(), t.TempDir()
if _, err := PrepareSSH(SSH{}, home, tmp); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(home, "known_hosts")); err != nil {
t.Errorf("known_hosts not created in HOME: %v", err)
}
}
func TestEnvironIsHermetic(t *testing.T) {
t.Setenv("GIT_SSH_COMMAND", "ssh -i /attacker/key")
t.Setenv("GIT_CONFIG_GLOBAL", "/attacker/gitconfig")
t.Setenv("HTTPS_PROXY", "http://proxy.internal:3128")
env := environ(Options{Home: "/var/lib/syncbot/home", SSHCmd: "ssh -o BatchMode=yes"})
got := map[string]string{}
for _, kv := range env {
if k, v, ok := strings.Cut(kv, "="); ok {
got[k] = v // later entries win, matching exec's behaviour
}
}
if got["GIT_SSH_COMMAND"] != "ssh -o BatchMode=yes" {
t.Errorf("inherited GIT_SSH_COMMAND not overridden: %q", got["GIT_SSH_COMMAND"])
}
if got["GIT_CONFIG_GLOBAL"] != os.DevNull {
t.Errorf("global git config not disabled: %q", got["GIT_CONFIG_GLOBAL"])
}
if got["HOME"] != "/var/lib/syncbot/home" {
t.Errorf("HOME = %q", got["HOME"])
}
if got["GIT_TERMINAL_PROMPT"] != "0" {
t.Error("git must never prompt for credentials")
}
if got["HTTPS_PROXY"] != "http://proxy.internal:3128" {
t.Error("proxy settings should be inherited")
}
}
func TestLimitedWriterCaps(t *testing.T) {
var sb strings.Builder
w := &limitedWriter{W: &sb, N: 10}
n, err := w.Write([]byte(strings.Repeat("x", 100)))
if err != nil {
t.Fatal(err)
}
if n != 100 {
t.Errorf("Write reported %d, want the full 100 so callers do not see a short write", n)
}
if sb.Len() != 10 {
t.Errorf("captured %d bytes, want the 10-byte cap", sb.Len())
}
}
+19
View File
@@ -0,0 +1,19 @@
//go:build !unix
package gitx
import (
"os"
"syscall"
)
// sysProcAttr has no portable equivalent outside unix; the default is fine.
func sysProcAttr() *syscall.SysProcAttr { return nil }
// terminate kills just the child. Helper processes it spawned may outlive it,
// but syncbot is deployed on Linux, where proc_unix.go handles this properly.
func terminate(pid int) {
if p, err := os.FindProcess(pid); err == nil {
_ = p.Kill()
}
}
+24
View File
@@ -0,0 +1,24 @@
//go:build unix
package gitx
import (
"syscall"
"time"
)
// sysProcAttr puts git in its own process group so we can signal the whole
// tree — git itself plus the ssh or git-remote-https helper it spawned.
func sysProcAttr() *syscall.SysProcAttr {
return &syscall.SysProcAttr{Setpgid: true}
}
// terminate asks the process group to exit, then insists.
func terminate(pid int) {
if pid <= 0 {
return
}
_ = syscall.Kill(-pid, syscall.SIGTERM)
time.Sleep(termGrace)
_ = syscall.Kill(-pid, syscall.SIGKILL)
}