fix: remove QEMU workflow and fix CI test failures

- Remove qemu-emulator-build.yaml (no longer needed)
- Replace broken 3x sequential test re-runs with vitest --retry 2
  (re-runs accumulated state causing 210-996 test failures on 2nd/3rd run)
- Increase test timeout to 120s for setup-tests (dev mode is slower,
  causing 44 timeouts at the default 50s threshold)

Co-Authored-By: Konstantin Wohlwend <n2d4xc@gmail.com>

.github/workflows/e2e-api-tests-local-emulator.yaml

@@ -173,15 +173,7 @@ jobs:
         run: sleep 10
 
       - name: Run tests
-        run: pnpm test run
-
-      - name: Run tests again (attempt 1)
-        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test run
-
-      - name: Run tests again (attempt 2)
-        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test run
+        run: pnpm test run --retry 2
 
       - name: Verify data integrity
         run: pnpm run verify-data-integrity --no-bail

.github/workflows/e2e-api-tests.yaml

@@ -175,15 +175,7 @@ jobs:
         run: sleep 10
 
       - name: Run tests
-        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} ${{ matrix.freestyle-mode == 'prod' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' && 'mail' || '' }}
-
-      - name: Run tests again (attempt 1)
-        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
-
-      - name: Run tests again (attempt 2)
-        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
+        run: pnpm test run --retry 2 ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} ${{ matrix.freestyle-mode == 'prod' && github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' && 'mail' || '' }}
 
       - name: Verify data integrity
         run: pnpm run verify-data-integrity --no-bail

.github/workflows/e2e-custom-base-port-api-tests.yaml

@@ -168,15 +168,7 @@ jobs:
         run: sleep 10
 
       - name: Run tests
-        run: pnpm test run
-
-      - name: Run tests again (attempt 1)
-        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test run
-
-      - name: Run tests again (attempt 2)
-        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
-        run: pnpm test run
+        run: pnpm test run --retry 2
 
       - name: Verify data integrity
         run: pnpm run verify-data-integrity --no-bail

.github/workflows/qemu-emulator-build.yaml

@@ -1,426 +0,0 @@
-name: Build & Publish QEMU Emulator Images
-
-on:
-  push:
-    branches:
-      - main
-      - dev
-  pull_request:
-    paths:
-      - 'docker/local-emulator/**'
-      - '.github/workflows/qemu-emulator-build.yaml'
-  workflow_dispatch:
-    inputs:
-      publish:
-        description: 'Publish images to GitHub Releases'
-        type: boolean
-        default: false
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
-
-env:
-  EMULATOR_IMAGE_NAME: stack-local-emulator
-  # Shell scripts (build-image.sh, run-emulator.sh) read these directly.
-  EMULATOR_IMAGE_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/images
-  EMULATOR_RUN_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/run
-  # The stack-cli ignores EMULATOR_IMAGE_DIR/RUN_DIR and derives its own paths
-  # from STACK_EMULATOR_HOME. Point it at the same workspace so `emulator
-  # start` finds the freshly-built qcow2 from build-image.sh and cold-boots
-  # it, instead of auto-pulling from a prior release. CI doesn't capture a
-  # savevm (EMULATOR_CAPTURE_SAVEVM defaults to 0); users capture locally
-  # on first `stack emulator pull`.
-  STACK_EMULATOR_HOME: ${{ github.workspace }}/docker/local-emulator/qemu
-
-jobs:
-  build:
-    name: Build QEMU Image (${{ matrix.arch }})
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 120
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Both arches build on ubicloud's amd64 runner. amd64 uses KVM;
-          # arm64 runs under cross-arch TCG (slow, but only cloud-init
-          # provisioning has to complete — the boot/verify smoke test below
-          # is gated to amd64 because TCG can't boot Next.js in any
-          # reasonable time). Snapshots are NOT published — `stack emulator
-          # pull` captures one locally on first run, which is the only way
-          # to guarantee KVM/HVF/TCG + `-cpu max` compatibility on the
-          # user's machine.
-          - arch: amd64
-            runner: ubicloud-standard-8
-          - arch: arm64
-            runner: ubicloud-standard-8
-
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
-
-      - name: Set up QEMU user-mode emulation
-        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-
-      # Node/pnpm are needed on both arches: arm64 also runs
-      # generate-env-development.mjs inside build-image.sh. amd64 additionally
-      # builds and runs the CLI for the verification steps below.
-      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
-        with:
-          version: 10.23.0
-
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
-        with:
-          node-version: 22
-          cache: pnpm
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          # qemu-utils gives us qemu-img; qemu-efi-aarch64 provides the arm64
-          # UEFI firmware. The actual qemu-system-* binaries come from the
-          # source build below — Ubuntu 24.04 ships QEMU 8.2 which predates
-          # the mapped-ram migration capability we rely on.
-          sudo apt-get install -y qemu-utils qemu-efi-aarch64 socat genisoimage zstd \
-            ninja-build pkg-config python3-venv \
-            libglib2.0-dev libpixman-1-dev libslirp-dev libepoxy-dev libgbm-dev
-
-      # QEMU 10.2.2 is required for the mapped-ram + multifd migration path
-      # used by the fast-resume snapshot. Cache the compiled prefix so CI
-      # only pays the ~5-8 min build cost once per runner image.
-      - name: Restore QEMU 10.2.2 cache
-        id: qemu-cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-        with:
-          path: /opt/qemu
-          key: qemu-10.2.2-${{ runner.os }}-${{ runner.arch }}-v1
-
-      - name: Build QEMU 10.2.2 from source
-        if: steps.qemu-cache.outputs.cache-hit != 'true'
-        run: |
-          set -euxo pipefail
-          curl -fsSL https://download.qemu.org/qemu-10.2.2.tar.xz -o /tmp/qemu.tar.xz
-          mkdir -p /tmp/qemu-src
-          tar -xf /tmp/qemu.tar.xz -C /tmp/qemu-src --strip-components=1
-          cd /tmp/qemu-src
-          ./configure --prefix=/opt/qemu \
-            --target-list=x86_64-softmmu,aarch64-softmmu \
-            --enable-kvm --enable-slirp --enable-tcg \
-            --disable-docs --disable-gtk --disable-sdl --disable-vnc \
-            --disable-guest-agent --disable-tools
-          make -j"$(nproc)"
-          sudo make install
-
-      - name: Put QEMU 10.2.2 on PATH
-        run: |
-          echo "/opt/qemu/bin" >> "$GITHUB_PATH"
-          /opt/qemu/bin/qemu-system-x86_64 --version
-          /opt/qemu/bin/qemu-system-aarch64 --version
-
-      - name: Enable KVM access
-        run: |
-          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
-            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-          sudo udevadm control --reload-rules
-          sudo udevadm trigger --name-match=kvm || true
-          ls -la /dev/kvm || echo "no /dev/kvm present"
-          if [ -w /dev/kvm ]; then
-            echo "KVM is writable — hardware acceleration will be used"
-          else
-            echo "WARNING: /dev/kvm is not writable — will fall back to TCG (very slow)"
-          fi
-
-      - name: Build QEMU image
-        run: |
-          chmod +x docker/local-emulator/qemu/build-image.sh
-          EMULATOR_PROVISION_TIMEOUT=6000 \
-            docker/local-emulator/qemu/build-image.sh ${{ matrix.arch }}
-
-      - name: Generate emulator env
-        run: node docker/local-emulator/generate-env-development.mjs
-
-      # amd64 runs under KVM on the runner so we can boot the newly-built
-      # image to verify it works end-to-end before publishing. arm64 runs
-      # under cross-arch TCG on an amd64 host, which can't reliably boot
-      # Next.js within any sane window — skipped.
-      - name: Build stack-cli (for emulator CLI)
-        if: matrix.arch == 'amd64'
-        run: |
-          pnpm install --frozen-lockfile --filter '@stackframe/stack-cli...'
-          # Turbo's trailing `...` filter builds stack-cli AND its workspace
-          # deps (@stackframe/js, @stackframe/stack-shared, etc.) — stack-cli
-          # imports them at runtime from their dist/ outputs.
-          pnpm exec turbo run build --filter='@stackframe/stack-cli...'
-
-      - name: Start emulator and verify
-        if: matrix.arch == 'amd64'
-        env:
-          EMULATOR_ARCH: ${{ matrix.arch }}
-          EMULATOR_READY_TIMEOUT: 3200
-          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
-          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
-        run: node packages/stack-cli/dist/index.js emulator start
-
-      - name: Verify services are healthy
-        if: matrix.arch == 'amd64'
-        env:
-          EMULATOR_ARCH: ${{ matrix.arch }}
-          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
-          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
-        run: node packages/stack-cli/dist/index.js emulator status
-
-      - name: Stop emulator
-        if: always() && matrix.arch == 'amd64'
-        env:
-          EMULATOR_ARCH: ${{ matrix.arch }}
-          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
-          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
-        run: node packages/stack-cli/dist/index.js emulator stop
-
-      - name: Package image
-        run: |
-          BASE_IMG="docker/local-emulator/qemu/images/stack-emulator-${{ matrix.arch }}.qcow2"
-          cp "$BASE_IMG" "stack-emulator-${{ matrix.arch }}.qcow2"
-          ls -lh "stack-emulator-${{ matrix.arch }}.qcow2"
-
-      - name: Upload image artifact
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: qemu-emulator-${{ matrix.arch }}
-          path: stack-emulator-${{ matrix.arch }}.qcow2
-          if-no-files-found: error
-          retention-days: 30
-          compression-level: 0
-
-  test:
-    name: Smoke Test (${{ matrix.arch }})
-    needs: build
-    runs-on: ubicloud-standard-8
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y qemu-utils socat zstd \
-            ninja-build pkg-config python3-venv \
-            libglib2.0-dev libpixman-1-dev libslirp-dev libepoxy-dev libgbm-dev
-
-      - name: Restore QEMU 10.2.2 cache
-        id: qemu-cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-        with:
-          path: /opt/qemu
-          key: qemu-10.2.2-${{ runner.os }}-${{ runner.arch }}-v1
-
-      - name: Build QEMU 10.2.2 from source
-        if: steps.qemu-cache.outputs.cache-hit != 'true'
-        run: |
-          set -euxo pipefail
-          curl -fsSL https://download.qemu.org/qemu-10.2.2.tar.xz -o /tmp/qemu.tar.xz
-          mkdir -p /tmp/qemu-src
-          tar -xf /tmp/qemu.tar.xz -C /tmp/qemu-src --strip-components=1
-          cd /tmp/qemu-src
-          ./configure --prefix=/opt/qemu \
-            --target-list=x86_64-softmmu,aarch64-softmmu \
-            --enable-kvm --enable-slirp --enable-tcg \
-            --disable-docs --disable-gtk --disable-sdl --disable-vnc \
-            --disable-guest-agent --disable-tools
-          make -j"$(nproc)"
-          sudo make install
-
-      - name: Put QEMU 10.2.2 on PATH
-        run: |
-          echo "/opt/qemu/bin" >> "$GITHUB_PATH"
-          /opt/qemu/bin/qemu-system-x86_64 --version
-
-      - name: Enable KVM access
-        run: |
-          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
-            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-          sudo udevadm control --reload-rules
-          sudo udevadm trigger --name-match=kvm || true
-          ls -la /dev/kvm || echo "no /dev/kvm present"
-          if [ -w /dev/kvm ]; then
-            echo "KVM is writable — hardware acceleration will be used"
-          else
-            echo "WARNING: /dev/kvm is not writable — will fall back to TCG (very slow)"
-          fi
-
-      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
-        with:
-          version: 10.23.0
-
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
-        with:
-          node-version: 22
-          cache: pnpm
-
-      - name: Install stack-cli deps + build
-        run: |
-          pnpm install --frozen-lockfile --filter '@stackframe/stack-cli...'
-          # Turbo's trailing `...` filter builds stack-cli AND its workspace
-          # deps (@stackframe/js, @stackframe/stack-shared, etc.) — stack-cli
-          # imports them at runtime from their dist/ outputs.
-          pnpm exec turbo run build --filter='@stackframe/stack-cli...'
-
-      - name: Download built image
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: qemu-emulator-${{ matrix.arch }}
-          path: ${{ github.workspace }}/.stack-emulator-images/
-
-      - name: Place qcow2 into STACK_EMULATOR_HOME layout
-        run: |
-          mkdir -p "$STACK_EMULATOR_HOME/images"
-          cp "${{ github.workspace }}/.stack-emulator-images/stack-emulator-${{ matrix.arch }}.qcow2" "$STACK_EMULATOR_HOME/images/"
-          ls -lh "$STACK_EMULATOR_HOME/images/"
-
-      # No savevm.zst artifact (users capture locally via `emulator pull`),
-      # so `emulator start` cold-boots the qcow2. Budget accordingly.
-      - name: Start emulator via CLI
-        run: |
-          EMULATOR_ARCH=${{ matrix.arch }} \
-          EMULATOR_READY_TIMEOUT=600 \
-            node packages/stack-cli/dist/index.js emulator start
-
-      - name: Verify services are healthy
-        run: node packages/stack-cli/dist/index.js emulator status
-
-      - name: Smoke test — backend health
-        run: curl -sf http://localhost:26701/health?db=1
-
-      - name: Smoke test — dashboard reachable
-        run: curl -sf -o /dev/null -w "HTTP %{http_code}\n" http://localhost:26700/handler/sign-in
-
-      - name: Smoke test — MinIO health
-        run: curl -sf http://localhost:26702/minio/health/live
-
-      - name: Smoke test — Inbucket reachable
-        run: curl -sf -o /dev/null -w "HTTP %{http_code}\n" http://localhost:26703/
-
-      - name: Stop emulator
-        if: always()
-        run: node packages/stack-cli/dist/index.js emulator stop
-
-      - name: Print serial log on failure
-        if: failure()
-        run: tail -100 "$STACK_EMULATOR_HOME/run/vm/serial.log" 2>/dev/null || true
-
-  publish:
-    name: Publish to GitHub Releases
-    needs: [build, test]
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || (github.event_name == 'workflow_dispatch' && inputs.publish)
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
-
-      - name: Download all artifacts
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: artifacts
-
-      - name: Prepare release assets
-        run: |
-          mkdir -p release
-          SHORT_SHA="${GITHUB_SHA:0:8}"
-          BRANCH="${GITHUB_REF_NAME}"
-          DATE="$(date -u +%Y%m%d)"
-          TAG="emulator-${BRANCH}-${DATE}-${SHORT_SHA}"
-          echo "RELEASE_TAG=${TAG}" >> "$GITHUB_ENV"
-          echo "SHORT_SHA=${SHORT_SHA}" >> "$GITHUB_ENV"
-
-          for f in artifacts/qemu-emulator-*/*.qcow2; do
-            cp "$f" release/
-          done
-
-          cat > release-notes.md <<EOF
-          ## QEMU Emulator Images
-
-          Built from \`${BRANCH}\` @ \`${GITHUB_SHA}\`
-
-          ### Images
-          | File | Description |
-          |------|-------------|
-          | \`stack-emulator-arm64.qcow2\` | ARM64 disk image |
-          | \`stack-emulator-amd64.qcow2\` | AMD64 disk image |
-
-          \`emulator pull\` downloads the qcow2 and captures a local fast-start
-          snapshot (~1-3 min). Subsequent \`emulator start\`s resume in ~3-8 s.
-          Snapshots are captured locally because QEMU migration state isn't
-          portable across accelerators (KVM / HVF / TCG) or \`-cpu max\`
-          feature sets.
-
-          ### Usage
-          \`\`\`bash
-          stack emulator pull
-          stack emulator start
-          \`\`\`
-          EOF
-
-          ls -lh release/
-
-      - name: Create or update GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TITLE="QEMU Emulator — ${{ github.ref_name }} ($SHORT_SHA)"
-
-          if gh release view "$RELEASE_TAG" >/dev/null 2>&1; then
-            gh release edit "$RELEASE_TAG" \
-              --title "$TITLE" \
-              --notes-file release-notes.md \
-              --prerelease
-            gh release upload "$RELEASE_TAG" release/* --clobber
-          else
-            gh release create "$RELEASE_TAG" \
-              --title "$TITLE" \
-              --notes-file release-notes.md \
-              --prerelease \
-              release/*
-          fi
-
-      - name: Update latest tag for branch
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          LATEST_TAG="emulator-${{ github.ref_name }}-latest"
-          TITLE="QEMU Emulator — ${{ github.ref_name }} (latest)"
-          NOTES="Latest emulator images from \`${{ github.ref_name }}\`. Auto-updated on each build."
-
-          if gh release view "$LATEST_TAG" >/dev/null 2>&1; then
-            gh release edit "$LATEST_TAG" \
-              --draft \
-              --prerelease \
-              --target "${{ github.sha }}" \
-              --title "$TITLE" \
-              --notes "$NOTES"
-          else
-            gh release create "$LATEST_TAG" \
-              --draft \
-              --prerelease \
-              --target "${{ github.sha }}" \
-              --title "$TITLE" \
-              --notes "$NOTES" \
-            || gh release edit "$LATEST_TAG" \
-              --draft \
-              --prerelease \
-              --target "${{ github.sha }}" \
-              --title "$TITLE" \
-              --notes "$NOTES"
-          fi
-
-          gh release upload "$LATEST_TAG" release/* --clobber
-          gh release edit "$LATEST_TAG" --draft=false --prerelease

.github/workflows/setup-tests-with-custom-base-port.yaml

@@ -49,12 +49,5 @@ jobs:
           tail: true
           wait-for: 120s
           log-output-if: true
-      - name: Run tests (first attempt)
-        id: run-tests-first-attempt
-        continue-on-error: ${{ (github.head_ref || github.ref_name) != 'main' && (github.head_ref || github.ref_name) != 'dev' }}
-        run: pnpm run test run --reporter=verbose
-
-      # These tests are often flakey, as a temporary measure we retry them once.
-      - name: Run tests (retry once on non-dev/main branches)
-        if: ${{ (github.head_ref || github.ref_name) != 'main' && (github.head_ref || github.ref_name) != 'dev' && steps.run-tests-first-attempt.outcome == 'failure' }}
-        run: pnpm run test run --reporter=verbose
+      - name: Run tests
+        run: pnpm run test run --reporter=verbose --retry 2 --test-timeout=120000

.github/workflows/setup-tests.yaml

@@ -47,12 +47,5 @@ jobs:
           tail: true
           wait-for: 120s
           log-output-if: true
-      - name: Run tests (first attempt)
-        id: run-tests-first-attempt
-        continue-on-error: ${{ (github.head_ref || github.ref_name) != 'main' && (github.head_ref || github.ref_name) != 'dev' }}
-        run: pnpm run test run --reporter=verbose
-
-      # These tests are often flakey, as a temporary measure we retry them once.
-      - name: Run tests (retry once on non-dev/main branches)
-        if: ${{ (github.head_ref || github.ref_name) != 'main' && (github.head_ref || github.ref_name) != 'dev' && steps.run-tests-first-attempt.outcome == 'failure' }}
-        run: pnpm run test run --reporter=verbose
+      - name: Run tests
+        run: pnpm run test run --reporter=verbose --retry 2 --test-timeout=120000