stack/.github/workflows/qemu-emulator-build.yaml

name: Build & Publish QEMU Emulator Images

on:
  push:
    branches:
      - main
      - dev
  pull_request:
    paths:
      - 'docker/local-emulator/**'
      - '.github/workflows/qemu-emulator-build.yaml'
  workflow_dispatch:
    inputs:
      publish:
        description: 'Publish images to GitHub Releases'
        type: boolean
        default: false

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}

env:
  EMULATOR_IMAGE_NAME: stack-local-emulator
  # Shell scripts (build-image.sh, run-emulator.sh) read these directly.
  EMULATOR_IMAGE_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/images
  EMULATOR_RUN_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/run
  # The stack-cli ignores EMULATOR_IMAGE_DIR/RUN_DIR and derives its own paths
  # from HEXCLAVE_EMULATOR_HOME. Point it at the same workspace so `emulator
  # start` finds the freshly-built qcow2 from build-image.sh and cold-boots
  # it, instead of auto-pulling from a prior release. CI doesn't capture a
  # savevm (EMULATOR_CAPTURE_SAVEVM defaults to 0); users capture locally
  # on first `stack emulator pull`.
  HEXCLAVE_EMULATOR_HOME: ${{ github.workspace }}/docker/local-emulator/qemu

jobs:
  build:
    name: Build QEMU Image (${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        include:
          # Both arches build on ubicloud's amd64 runner. amd64 uses KVM;
          # arm64 runs under cross-arch TCG (slow, but only cloud-init
          # provisioning has to complete — the boot/verify smoke test below
          # is gated to amd64 because TCG can't boot Next.js in any
          # reasonable time). Snapshots are NOT published — `stack emulator
          # pull` captures one locally on first run, which is the only way
          # to guarantee KVM/HVF/TCG + `-cpu max` compatibility on the
          # user's machine.
          - arch: amd64
            runner: ubicloud-standard-8
          - arch: arm64
            runner: ubicloud-standard-8

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Set up QEMU user-mode emulation
        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3

      # Node is needed on both arches for generate-env-development.mjs.
      # pnpm is needed for build-image.sh (runs pnpm install inside Docker
      # context via the lockfile).
      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4

      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 22

      - name: Install system dependencies
        run: |
          sudo apt-get update
          # qemu-utils gives us qemu-img; qemu-efi-aarch64 provides the arm64
          # UEFI firmware. The actual qemu-system-* binaries come from the
          # source build below — Ubuntu 24.04 ships QEMU 8.2 which predates
          # the mapped-ram migration capability we rely on.
          sudo apt-get install -y qemu-utils qemu-efi-aarch64 socat genisoimage zstd \
            ninja-build pkg-config python3-venv \
            libglib2.0-dev libpixman-1-dev libslirp-dev libepoxy-dev libgbm-dev          

      # QEMU 10.2.2 is required for the mapped-ram + multifd migration path
      # used by the fast-resume snapshot. Cache the compiled prefix so CI
      # only pays the ~5-8 min build cost once per runner image.
      - name: Restore QEMU 10.2.2 cache
        id: qemu-cache
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          path: /opt/qemu
          key: qemu-10.2.2-${{ runner.os }}-${{ runner.arch }}-v1

      - name: Build QEMU 10.2.2 from source
        if: steps.qemu-cache.outputs.cache-hit != 'true'
        run: |
          set -euxo pipefail
          curl -fsSL https://download.qemu.org/qemu-10.2.2.tar.xz -o /tmp/qemu.tar.xz
          mkdir -p /tmp/qemu-src
          tar -xf /tmp/qemu.tar.xz -C /tmp/qemu-src --strip-components=1
          cd /tmp/qemu-src
          ./configure --prefix=/opt/qemu \
            --target-list=x86_64-softmmu,aarch64-softmmu \
            --enable-kvm --enable-slirp --enable-tcg \
            --disable-docs --disable-gtk --disable-sdl --disable-vnc \
            --disable-guest-agent --disable-tools
          make -j"$(nproc)"
          sudo make install          

      - name: Put QEMU 10.2.2 on PATH
        run: |
          echo "/opt/qemu/bin" >> "$GITHUB_PATH"
          /opt/qemu/bin/qemu-system-x86_64 --version
          /opt/qemu/bin/qemu-system-aarch64 --version          

      - name: Enable KVM access
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm || true
          ls -la /dev/kvm || echo "no /dev/kvm present"
          if [ -w /dev/kvm ]; then
            echo "KVM is writable — hardware acceleration will be used"
          else
            echo "WARNING: /dev/kvm is not writable — will fall back to TCG (very slow)"
          fi          

      - name: Build QEMU image
        run: |
          chmod +x docker/local-emulator/qemu/build-image.sh
          EMULATOR_PROVISION_TIMEOUT=6000 \
            docker/local-emulator/qemu/build-image.sh ${{ matrix.arch }}          

      - name: Generate emulator env
        run: node docker/local-emulator/generate-env-development.mjs

      # amd64 runs under KVM on the runner so we can boot the newly-built
      # image to verify it works end-to-end before publishing. arm64 runs
      # under cross-arch TCG on an amd64 host, which can't reliably boot
      # Next.js within any sane window — skipped.
      - name: Start emulator and verify
        if: matrix.arch == 'amd64'
        env:
          EMULATOR_ARCH: ${{ matrix.arch }}
          EMULATOR_READY_TIMEOUT: 3200
          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
        run: |
          chmod +x docker/local-emulator/qemu/run-emulator.sh
          docker/local-emulator/qemu/run-emulator.sh start          

      - name: Verify services are healthy
        if: matrix.arch == 'amd64'
        env:
          EMULATOR_ARCH: ${{ matrix.arch }}
          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
        run: docker/local-emulator/qemu/run-emulator.sh status

      - name: Stop emulator
        if: always() && matrix.arch == 'amd64'
        env:
          EMULATOR_ARCH: ${{ matrix.arch }}
          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
        run: docker/local-emulator/qemu/run-emulator.sh stop

      - name: Package image
        run: |
          BASE_IMG="docker/local-emulator/qemu/images/stack-emulator-${{ matrix.arch }}.qcow2"
          cp "$BASE_IMG" "stack-emulator-${{ matrix.arch }}.qcow2"
          ls -lh "stack-emulator-${{ matrix.arch }}.qcow2"          

      - name: Upload image artifact
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: qemu-emulator-${{ matrix.arch }}
          path: stack-emulator-${{ matrix.arch }}.qcow2
          if-no-files-found: error
          retention-days: 30
          compression-level: 0

  test:
    name: Smoke Test (${{ matrix.arch }})
    needs: build
    runs-on: ubicloud-standard-8
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: amd64

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y qemu-utils socat zstd \
            ninja-build pkg-config python3-venv \
            libglib2.0-dev libpixman-1-dev libslirp-dev libepoxy-dev libgbm-dev          

      - name: Restore QEMU 10.2.2 cache
        id: qemu-cache
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          path: /opt/qemu
          key: qemu-10.2.2-${{ runner.os }}-${{ runner.arch }}-v1

      - name: Build QEMU 10.2.2 from source
        if: steps.qemu-cache.outputs.cache-hit != 'true'
        run: |
          set -euxo pipefail
          curl -fsSL https://download.qemu.org/qemu-10.2.2.tar.xz -o /tmp/qemu.tar.xz
          mkdir -p /tmp/qemu-src
          tar -xf /tmp/qemu.tar.xz -C /tmp/qemu-src --strip-components=1
          cd /tmp/qemu-src
          ./configure --prefix=/opt/qemu \
            --target-list=x86_64-softmmu,aarch64-softmmu \
            --enable-kvm --enable-slirp --enable-tcg \
            --disable-docs --disable-gtk --disable-sdl --disable-vnc \
            --disable-guest-agent --disable-tools
          make -j"$(nproc)"
          sudo make install          

      - name: Put QEMU 10.2.2 on PATH
        run: |
          echo "/opt/qemu/bin" >> "$GITHUB_PATH"
          /opt/qemu/bin/qemu-system-x86_64 --version          

      - name: Enable KVM access
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm || true
          ls -la /dev/kvm || echo "no /dev/kvm present"
          if [ -w /dev/kvm ]; then
            echo "KVM is writable — hardware acceleration will be used"
          else
            echo "WARNING: /dev/kvm is not writable — will fall back to TCG (very slow)"
          fi          

      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4

      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 22
          cache: pnpm

      - name: Install stack-cli deps + build
        run: |
          # See "Build stack-cli" step comment for why dashboard filter is needed
          pnpm install --frozen-lockfile --filter '@hexclave/cli...' --filter '@hexclave/dashboard...'
          pnpm exec turbo run build --filter='@hexclave/cli...'          

      - name: Download built image
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: qemu-emulator-${{ matrix.arch }}
          path: ${{ github.workspace }}/.stack-emulator-images/

      - name: Place qcow2 into HEXCLAVE_EMULATOR_HOME layout
        run: |
          mkdir -p "$HEXCLAVE_EMULATOR_HOME/images"
          cp "${{ github.workspace }}/.stack-emulator-images/stack-emulator-${{ matrix.arch }}.qcow2" "$HEXCLAVE_EMULATOR_HOME/images/"
          ls -lh "$HEXCLAVE_EMULATOR_HOME/images/"          

      # No savevm.zst artifact (users capture locally via `emulator pull`),
      # so `emulator start` cold-boots the qcow2. Budget accordingly.
      - name: Start emulator via CLI
        run: |
          EMULATOR_ARCH=${{ matrix.arch }} \
          EMULATOR_READY_TIMEOUT=600 \
            node packages/cli/dist/index.js emulator start          

      - name: Verify services are healthy
        run: node packages/cli/dist/index.js emulator status

      - name: Smoke test — backend health
        run: curl -sf http://localhost:26701/health?db=1

      - name: Smoke test — dashboard reachable
        run: curl -sf -o /dev/null -w "HTTP %{http_code}\n" http://localhost:26700/handler/sign-in

      - name: Smoke test — MinIO health
        run: curl -sf http://localhost:26702/minio/health/live

      - name: Smoke test — Inbucket reachable
        run: curl -sf -o /dev/null -w "HTTP %{http_code}\n" http://localhost:26703/

      - name: Stop emulator
        if: always()
        run: node packages/cli/dist/index.js emulator stop

      - name: Print serial log on failure
        if: failure()
        run: tail -100 "$HEXCLAVE_EMULATOR_HOME/run/vm/serial.log" 2>/dev/null || true

  publish:
    name: Publish to GitHub Releases
    needs: [build, test]
    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || (github.event_name == 'workflow_dispatch' && inputs.publish)
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Download all artifacts
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          path: artifacts

      - name: Prepare release assets
        run: |
          mkdir -p release
          SHORT_SHA="${GITHUB_SHA:0:8}"
          BRANCH="${GITHUB_REF_NAME}"
          DATE="$(date -u +%Y%m%d)"
          TAG="emulator-${BRANCH}-${DATE}-${SHORT_SHA}"
          echo "RELEASE_TAG=${TAG}" >> "$GITHUB_ENV"
          echo "SHORT_SHA=${SHORT_SHA}" >> "$GITHUB_ENV"

          for f in artifacts/qemu-emulator-*/*.qcow2; do
            cp "$f" release/
          done

          cat > release-notes.md <<EOF
          ## QEMU Emulator Images

          Built from \`${BRANCH}\` @ \`${GITHUB_SHA}\`

          ### Images
          | File | Description |
          |------|-------------|
          | \`stack-emulator-arm64.qcow2\` | ARM64 disk image |
          | \`stack-emulator-amd64.qcow2\` | AMD64 disk image |

          \`emulator pull\` downloads the qcow2 and captures a local fast-start
          snapshot (~1-3 min). Subsequent \`emulator start\`s resume in ~3-8 s.
          Snapshots are captured locally because QEMU migration state isn't
          portable across accelerators (KVM / HVF / TCG) or \`-cpu max\`
          feature sets.

          ### Usage
          \`\`\`bash
          stack emulator pull
          stack emulator start
          \`\`\`
          EOF

          ls -lh release/          

      - name: Create or update GitHub Release
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          TITLE="QEMU Emulator — ${{ github.ref_name }} ($SHORT_SHA)"

          if gh release view "$RELEASE_TAG" >/dev/null 2>&1; then
            gh release edit "$RELEASE_TAG" \
              --title "$TITLE" \
              --notes-file release-notes.md \
              --prerelease
            gh release upload "$RELEASE_TAG" release/* --clobber
          else
            gh release create "$RELEASE_TAG" \
              --title "$TITLE" \
              --notes-file release-notes.md \
              --prerelease \
              release/*
          fi          

      - name: Update latest tag for branch
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          LATEST_TAG="emulator-${{ github.ref_name }}-latest"
          TITLE="QEMU Emulator — ${{ github.ref_name }} (latest)"
          NOTES="Latest emulator images from \`${{ github.ref_name }}\`. Auto-updated on each build."

          if gh release view "$LATEST_TAG" >/dev/null 2>&1; then
            gh release edit "$LATEST_TAG" \
              --draft \
              --prerelease \
              --target "${{ github.sha }}" \
              --title "$TITLE" \
              --notes "$NOTES"
          else
            gh release create "$LATEST_TAG" \
              --draft \
              --prerelease \
              --target "${{ github.sha }}" \
              --title "$TITLE" \
              --notes "$NOTES" \
            || gh release edit "$LATEST_TAG" \
              --draft \
              --prerelease \
              --target "${{ github.sha }}" \
              --title "$TITLE" \
              --notes "$NOTES"
          fi

          gh release upload "$LATEST_TAG" release/* --clobber
          gh release edit "$LATEST_TAG" --draft=false --prerelease