From 49a20ed019669fd4deef4fa350a95c57f660f271 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Wed, 15 Apr 2026 18:36:50 -0700 Subject: [PATCH] split arm64 build: Docker on Linux, QEMU snapshot on macOS Docker is difficult to run on macOS CI runners (colima VZ and QEMU backends both crash). Split into two stages: 1. docker-build (Linux): builds arm64 Docker image, exports tarball 2. qemu-snapshot (macOS): provisions QEMU VM under HVF, captures snapshot Add SKIP_DOCKER_BUILD=1 to build-image.sh to reuse a pre-built bundle. --- .../workflows/qemu-emulator-build-arm64.yaml | 119 ++++++++++++------ docker/local-emulator/qemu/build-image.sh | 8 +- 2 files changed, 89 insertions(+), 38 deletions(-) diff --git a/.github/workflows/qemu-emulator-build-arm64.yaml b/.github/workflows/qemu-emulator-build-arm64.yaml index 5018d22c1..d50ed633f 100644 --- a/.github/workflows/qemu-emulator-build-arm64.yaml +++ b/.github/workflows/qemu-emulator-build-arm64.yaml @@ -1,9 +1,11 @@ name: Build QEMU Emulator Image (arm64 / macOS) -# arm64 emulator images are built on a macOS Apple Silicon runner so the -# snapshot is captured under HVF — the same accelerator developer Macs use. -# KVM snapshots (from Linux runners) are NOT resumable under HVF because -# `-cpu max` expands to different feature sets under each accelerator. +# arm64 emulator images are built in two stages: +# 1. docker-build (Linux): builds the Docker container image for arm64 and +# exports a tarball — Docker is painful to run on macOS CI runners. +# 2. qemu-snapshot (macOS): boots the image under HVF on Apple Silicon, +# provisions it, and captures a snapshot. HVF snapshots are portable to +# developer Macs; KVM snapshots are NOT (differing -cpu max features). on: push: @@ -22,14 +24,68 @@ concurrency: env: EMULATOR_IMAGE_NAME: stack-local-emulator - EMULATOR_IMAGE_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/images - EMULATOR_RUN_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/run jobs: - build: - name: Build QEMU Image (arm64) + # ---------- Stage 1: build Docker image on Linux ---------- + docker-build: + name: Build Docker Image (arm64) + runs-on: ubicloud-standard-8 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v6 + + - name: Set up QEMU user-mode emulation + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Generate emulator env + run: node docker/local-emulator/generate-env-development.mjs + + - name: Build arm64 Docker image + run: | + docker buildx build \ + --platform linux/arm64 \ + --tag "$EMULATOR_IMAGE_NAME" \ + --load \ + -f docker/local-emulator/Dockerfile \ + . + + - name: Export Docker image bundle + run: | + mkdir -p /tmp/bundle + docker save "$EMULATOR_IMAGE_NAME" | gzip -c > /tmp/bundle/emulator-arm64-docker-images.tar.gz + docker image inspect --format '{{.ID}}' "$EMULATOR_IMAGE_NAME" > /tmp/bundle/emulator-arm64-docker-images.tar.gz.image-ids + ls -lh /tmp/bundle/ + + - name: Upload Docker bundle + uses: actions/upload-artifact@v4 + with: + name: arm64-docker-bundle + path: /tmp/bundle/ + retention-days: 1 + compression-level: 0 + + # ---------- Stage 2: QEMU provision + snapshot on macOS (HVF) ---------- + qemu-snapshot: + name: QEMU Snapshot (arm64 / HVF) + needs: docker-build runs-on: macos-15 timeout-minutes: 120 + env: + EMULATOR_IMAGE_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/images + EMULATOR_RUN_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/run steps: - uses: actions/checkout@v6 @@ -46,17 +102,6 @@ jobs: - name: Install system dependencies run: brew install qemu socat zstd - - name: Set up Docker via colima - run: | - brew install docker docker-buildx colima - # Wire up buildx as a CLI plugin - mkdir -p ~/.docker - echo '{"cliPluginsExtraDirs":["/opt/homebrew/lib/docker/cli-plugins"]}' > ~/.docker/config.json - # VZ driver doesn't work on GHA macOS runners — use QEMU backend - colima start --vm-type=qemu --cpu 4 --memory 6 --disk 60 --arch aarch64 - docker info - docker buildx version - - name: Verify QEMU + HVF run: | qemu-system-aarch64 --version @@ -67,17 +112,26 @@ jobs: exit 1 fi - - name: Build QEMU image - run: | - chmod +x docker/local-emulator/qemu/build-image.sh - EMULATOR_PROVISION_TIMEOUT=6000 \ - docker/local-emulator/qemu/build-image.sh arm64 + - name: Download Docker bundle + uses: actions/download-artifact@v4 + with: + name: arm64-docker-bundle + path: ${{ env.EMULATOR_IMAGE_DIR }}/ - name: Generate emulator env run: node docker/local-emulator/generate-env-development.mjs - # HVF gives us native-speed arm64 — we can verify the image boots - # and services come up, unlike the old cross-arch TCG path. + - name: Build QEMU image (provision + snapshot) + run: | + chmod +x docker/local-emulator/qemu/build-image.sh + # SKIP_DOCKER_BUILD=1 tells build-image.sh to skip the Docker + # build + export steps — we already have the bundle from stage 1. + EMULATOR_PROVISION_TIMEOUT=6000 \ + SKIP_DOCKER_BUILD=1 \ + docker/local-emulator/qemu/build-image.sh arm64 + + # HVF gives us native-speed arm64 — verify the image boots and + # services come up (previously impossible under cross-arch TCG). - name: Build stack-cli run: | pnpm install --frozen-lockfile --filter '@stackframe/stack-cli...' @@ -87,34 +141,27 @@ jobs: env: EMULATOR_ARCH: arm64 EMULATOR_READY_TIMEOUT: 3200 - EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }} - EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }} run: node packages/stack-cli/dist/index.js emulator start - name: Verify services are healthy env: EMULATOR_ARCH: arm64 - EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }} - EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }} run: node packages/stack-cli/dist/index.js emulator status - name: Stop emulator if: always() env: EMULATOR_ARCH: arm64 - EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }} - EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }} run: node packages/stack-cli/dist/index.js emulator stop - name: Print serial log on failure if: failure() - run: | - tail -100 "$EMULATOR_RUN_DIR/vm/serial.log" 2>/dev/null || true + run: tail -100 "$EMULATOR_RUN_DIR/vm/serial.log" 2>/dev/null || true - name: Package image run: | - BASE_IMG="docker/local-emulator/qemu/images/stack-emulator-arm64.qcow2" - SAVEVM="docker/local-emulator/qemu/images/stack-emulator-arm64.savevm.zst" + BASE_IMG="$EMULATOR_IMAGE_DIR/stack-emulator-arm64.qcow2" + SAVEVM="$EMULATOR_IMAGE_DIR/stack-emulator-arm64.savevm.zst" cp "$BASE_IMG" "stack-emulator-arm64.qcow2" if [ -f "$SAVEVM" ]; then cp "$SAVEVM" "stack-emulator-arm64.savevm.zst" diff --git a/docker/local-emulator/qemu/build-image.sh b/docker/local-emulator/qemu/build-image.sh index 295a7972e..55d0cb293 100755 --- a/docker/local-emulator/qemu/build-image.sh +++ b/docker/local-emulator/qemu/build-image.sh @@ -657,8 +657,12 @@ BUILD_ENV_FILE="$REPO_ROOT/docker/local-emulator/.env.development" for arch in "${TARGET_ARCHS[@]}"; do local_base="$IMAGE_DIR/debian-${DEBIAN_VERSION}-base-${arch}.qcow2" download_cloud_image "$arch" "$local_base" - build_local_emulator_image "$arch" - prepare_bundle_artifacts "$arch" + if [ "${SKIP_DOCKER_BUILD:-0}" = "1" ]; then + log "SKIP_DOCKER_BUILD=1: reusing pre-built Docker bundle" + else + build_local_emulator_image "$arch" + prepare_bundle_artifacts "$arch" + fi build_one "$arch" done