From fb0597578f9cff42e17f171b5fd8ddada21591f2 Mon Sep 17 00:00:00 2001 From: iceBear67 Date: Sun, 16 Aug 2026 03:08:29 +0000 Subject: [PATCH] Rework CI: cross Windows on Linux, build only on dispatch/tag Regular push/PR runs the test job only. workflow_dispatch and v* tags then build linux-amd64, windows-amd64, and macos-arm64 as separate jobs. Windows is cargo-xwin from Ubuntu (MSVC ABI, Unix-host protoc) instead of native windows-latest. Registry/git cache is shared on Ubuntu; release target/ is not cached; xwin splat has its own cache. --- .github/workflows/build.yml | 296 ++++++++++++++++++++++++++---------- scripts/ci-apply.sh | 29 ++++ scripts/ci-package.sh | 45 ++++++ 3 files changed, 288 insertions(+), 82 deletions(-) create mode 100755 scripts/ci-apply.sh create mode 100755 scripts/ci-package.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0d2914a..704078f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,9 +1,20 @@ -# Native release builds of the patched grok binary. +# CI for the patched grok binary. # -# Matrix is host=target (no cross): Windows/Linux amd64 and macOS arm64. -# work/ is derived the same way as `make apply`, but the upstream checkout is -# a depth-1 fetch of the pinned SHA so CI does not clone full grok-build history. -name: Build +# Regular push / pull_request: test job only. +# workflow_dispatch or a v* tag: test, then the three release builds. +# Tags also publish the artifacts as a GitHub Release. +# +# Windows is cross-compiled from Ubuntu (cargo-xwin → x86_64-pc-windows-msvc), +# in its own job — not mixed into the native linux-amd64 build. Official proto +# codegen is Unix-host-only (/dev/stdout, Linux protoc); native windows-latest +# is the path that broke. +# +# Cache budget (GitHub's repo cap is 10 GB): +# - cargo registry/git is shared across the Ubuntu jobs (no target/) +# - test keeps a separate debug work/target (small crates only) +# - xwin's MSVC splat is cached on its own +# - release target/ is never cached (multi-GB) +name: CI on: push: @@ -13,8 +24,10 @@ on: workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Distinguish test-only runs from dispatch/tag builds so a release does not + # cancel an in-flight PR test on the same branch name, and vice versa. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} env: CARGO_TERM_COLOR: always @@ -25,30 +38,75 @@ permissions: contents: read jobs: - build: - name: ${{ matrix.artifact }} - runs-on: ${{ matrix.os }} + test: + name: test + runs-on: ubuntu-latest + timeout-minutes: 90 + defaults: + run: + shell: bash + steps: + - name: Disable CRLF conversion + run: git config --global core.autocrlf false + + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@1.94.0 + + - uses: arduino/setup-protoc@v3 + with: + version: "29.3" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Apply patches onto pinned upstream + run: ./scripts/ci-apply.sh + + - name: Export PROTOC + run: echo "PROTOC=$(command -v protoc)" >> "$GITHUB_ENV" + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: work + shared-key: ubuntu-cargo + cache-targets: false + save-if: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }} + + - name: Restore test target cache + id: test-target + uses: actions/cache/restore@v4 + with: + path: work/target + key: test-target-1.94-${{ hashFiles('work/Cargo.lock') }} + restore-keys: | + test-target-1.94- + + - name: Test + working-directory: work + # make test default + the proto crate (exercises protoc / build.rs). + # pager/shell stay off this job: their debug target/ would blow the + # 10 GB cache budget and the runner disk. + run: cargo test --locked -p xai-grok-version -p xai-grok-tools-api + + - name: Save test target cache + if: github.ref == 'refs/heads/main' && github.event_name == 'push' && steps.test-target.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: work/target + key: test-target-1.94-${{ hashFiles('work/Cargo.lock') }} + + build-linux-amd64: + name: grok-linux-amd64 + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/') + needs: test + runs-on: ubuntu-latest timeout-minutes: 180 defaults: run: shell: bash - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - target: x86_64-unknown-linux-gnu - artifact: grok-linux-amd64 - exe: xai-grok-pager - - os: windows-latest - target: x86_64-pc-windows-msvc - artifact: grok-windows-amd64 - exe: xai-grok-pager.exe - - os: macos-14 - target: aarch64-apple-darwin - artifact: grok-macos-arm64 - exe: xai-grok-pager - + env: + TARGET: x86_64-unknown-linux-gnu + ARTIFACT: grok-linux-amd64 + EXE: xai-grok-pager steps: - name: Disable CRLF conversion run: git config --global core.autocrlf false @@ -57,7 +115,7 @@ jobs: - uses: dtolnay/rust-toolchain@1.94.0 with: - targets: ${{ matrix.target }} + targets: x86_64-unknown-linux-gnu - uses: arduino/setup-protoc@v3 with: @@ -65,79 +123,153 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Apply patches onto pinned upstream - run: | - set -euo pipefail - REV="$(grep -vE '^\s*(#|$)' upstream.rev | head -n1 | tr -d '[:space:]')" - [[ -n "$REV" ]] || { echo "upstream.rev has no revision" >&2; exit 1; } + run: ./scripts/ci-apply.sh - git init work - git -C work remote add origin https://github.com/xai-org/grok-build.git - git -C work fetch --depth 1 origin "$REV" - git -C work checkout --force --detach FETCH_HEAD - git -C work checkout -B fork - git -C work tag -f base - git -C work config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git -C work config user.name "github-actions[bot]" - - shopt -s nullglob - patches=("$GITHUB_WORKSPACE"/patches/*.patch) - if ((${#patches[@]})); then - git -C work am --3way --keep-cr --whitespace=nowarn "${patches[@]}" - fi - echo "work/ ready: upstream ${REV:0:12} + ${#patches[@]} patch(es)" + - name: Export PROTOC + run: echo "PROTOC=$(command -v protoc)" >> "$GITHUB_ENV" - uses: Swatinem/rust-cache@v2 with: workspaces: work - key: ${{ matrix.target }} + shared-key: ubuntu-cargo cache-targets: false - name: Release build working-directory: work - run: cargo build --release --locked -p xai-grok-pager-bin --target ${{ matrix.target }} + run: cargo build --release --locked -p xai-grok-pager-bin --target "$TARGET" - name: Package - env: - TARGET: ${{ matrix.target }} - ARTIFACT: ${{ matrix.artifact }} - EXE: ${{ matrix.exe }} - run: | - set -euo pipefail - src="work/target/${TARGET}/release/${EXE}" - [[ -f "$src" ]] || { echo "missing $src" >&2; ls -la "work/target/${TARGET}/release" >&2; exit 1; } - - mkdir -p dist - if [[ "$EXE" == *.exe ]]; then - dest="dist/${ARTIFACT}.exe" - else - dest="dist/${ARTIFACT}" - strip "$src" || true - fi - cp "$src" "$dest" - - { - echo "artifact=$(basename "$dest")" - echo "target=${TARGET}" - echo "git=${GITHUB_SHA}" - echo "upstream=$(grep -vE '^\s*(#|$)' upstream.rev | head -n1 | tr -d '[:space:]')" - echo "rustc=$(rustc --version)" - } > "dist/${ARTIFACT}.txt" - - if command -v sha256sum >/dev/null; then - (cd dist && sha256sum "$(basename "$dest")" "${ARTIFACT}.txt" > "${ARTIFACT}.sha256") - else - (cd dist && shasum -a 256 "$(basename "$dest")" "${ARTIFACT}.txt" > "${ARTIFACT}.sha256") - fi + run: ./scripts/ci-package.sh - uses: actions/upload-artifact@v4 with: - name: ${{ matrix.artifact }} + name: grok-linux-amd64 + path: dist/* + if-no-files-found: error + + build-windows-amd64: + name: grok-windows-amd64 + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/') + needs: test + runs-on: ubuntu-latest + timeout-minutes: 180 + defaults: + run: + shell: bash + env: + TARGET: x86_64-pc-windows-msvc + ARTIFACT: grok-windows-amd64 + EXE: xai-grok-pager.exe + XWIN_CACHE_DIR: ${{ github.workspace }}/.xwin-cache + steps: + - name: Disable CRLF conversion + run: git config --global core.autocrlf false + + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@1.94.0 + with: + targets: x86_64-pc-windows-msvc + + - uses: arduino/setup-protoc@v3 + with: + version: "29.3" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Apply patches onto pinned upstream + run: ./scripts/ci-apply.sh + + - name: Export PROTOC + run: echo "PROTOC=$(command -v protoc)" >> "$GITHUB_ENV" + + - name: Install clang / lld / llvm + run: sudo apt-get update && sudo apt-get install -y clang lld llvm + + - uses: taiki-e/install-action@v2 + with: + tool: cargo-xwin + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: work + shared-key: ubuntu-cargo + cache-targets: false + + - name: Cache xwin MSVC splat + uses: actions/cache@v4 + with: + path: ${{ env.XWIN_CACHE_DIR }} + key: xwin-x86_64-msvc-v1 + + - name: Release build (cross) + working-directory: work + run: cargo xwin build --release --locked -p xai-grok-pager-bin --target "$TARGET" + + - name: Package + run: ./scripts/ci-package.sh + + - uses: actions/upload-artifact@v4 + with: + name: grok-windows-amd64 + path: dist/* + if-no-files-found: error + + build-macos-arm64: + name: grok-macos-arm64 + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/') + needs: test + runs-on: macos-14 + timeout-minutes: 180 + defaults: + run: + shell: bash + env: + TARGET: aarch64-apple-darwin + ARTIFACT: grok-macos-arm64 + EXE: xai-grok-pager + steps: + - name: Disable CRLF conversion + run: git config --global core.autocrlf false + + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@1.94.0 + with: + targets: aarch64-apple-darwin + + - uses: arduino/setup-protoc@v3 + with: + version: "29.3" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Apply patches onto pinned upstream + run: ./scripts/ci-apply.sh + + - name: Export PROTOC + run: echo "PROTOC=$(command -v protoc)" >> "$GITHUB_ENV" + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: work + key: macos-arm64 + cache-targets: false + + - name: Release build + working-directory: work + run: cargo build --release --locked -p xai-grok-pager-bin --target "$TARGET" + + - name: Package + run: ./scripts/ci-package.sh + + - uses: actions/upload-artifact@v4 + with: + name: grok-macos-arm64 path: dist/* if-no-files-found: error release: if: startsWith(github.ref, 'refs/tags/') - needs: build + needs: [build-linux-amd64, build-windows-amd64, build-macos-arm64] runs-on: ubuntu-latest permissions: contents: write diff --git a/scripts/ci-apply.sh b/scripts/ci-apply.sh new file mode 100755 index 0000000..deb62c1 --- /dev/null +++ b/scripts/ci-apply.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Shallow work/ checkout for GitHub Actions. +# +# Unlike apply-patches.sh this does not clone upstream/ (that copy is a full +# history clone for make update). CI only needs the pinned SHA + patches/. + +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +REV="$(read_rev)" + +info "Fetching upstream ${REV:0:12} -> work/" +rm -rf "$WORK_DIR" +git init "$WORK_DIR" +git -C "$WORK_DIR" remote add origin "$UPSTREAM_URL" +git -C "$WORK_DIR" fetch --depth 1 origin "$REV" +git -C "$WORK_DIR" checkout --force --detach FETCH_HEAD +git -C "$WORK_DIR" checkout -B "$WORK_BRANCH" +git -C "$WORK_DIR" tag -f "$BASE_TAG" +git -C "$WORK_DIR" config user.email "41898282+github-actions[bot]@users.noreply.github.com" +git -C "$WORK_DIR" config user.name "github-actions[bot]" + +shopt -s nullglob +patches=("$PATCHES_DIR"/*.patch) +shopt -u nullglob +if ((${#patches[@]})); then + git -C "$WORK_DIR" am --3way --keep-cr --whitespace=nowarn "${patches[@]}" +fi + +info "work/ ready: upstream ${REV:0:12} + ${#patches[@]} patch(es)" diff --git a/scripts/ci-package.sh b/scripts/ci-package.sh new file mode 100755 index 0000000..077559c --- /dev/null +++ b/scripts/ci-package.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Copy a cargo --target release binary into dist/ with a checksum sidecar. +# +# Required env: +# TARGET rustc triple (selects work/target/$TARGET/release/) +# ARTIFACT basename without extension (grok-linux-amd64, …) +# EXE filename cargo wrote (xai-grok-pager or xai-grok-pager.exe) + +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +: "${TARGET:?TARGET is required}" +: "${ARTIFACT:?ARTIFACT is required}" +: "${EXE:?EXE is required}" + +src="$WORK_DIR/target/$TARGET/release/$EXE" +[[ -f "$src" ]] || { + echo "missing $src" >&2 + ls -la "$WORK_DIR/target/$TARGET/release" >&2 || true + exit 1 +} + +mkdir -p "$ROOT/dist" +if [[ "$EXE" == *.exe ]]; then + dest="$ROOT/dist/${ARTIFACT}.exe" +else + dest="$ROOT/dist/${ARTIFACT}" + strip "$src" || true +fi +cp "$src" "$dest" + +{ + echo "artifact=$(basename "$dest")" + echo "target=${TARGET}" + echo "git=${GITHUB_SHA:-}" + echo "upstream=$(read_rev)" + echo "rustc=$(rustc --version)" +} > "$ROOT/dist/${ARTIFACT}.txt" + +if command -v sha256sum >/dev/null; then + (cd "$ROOT/dist" && sha256sum "$(basename "$dest")" "${ARTIFACT}.txt" > "${ARTIFACT}.sha256") +else + (cd "$ROOT/dist" && shasum -a 256 "$(basename "$dest")" "${ARTIFACT}.txt" > "${ARTIFACT}.sha256") +fi + +info "packaged $dest"