From 9e342da0f29a92f4d46fb0ecc6a2506b0aae77b9 Mon Sep 17 00:00:00 2001
From: BilalG1 <bg2002@gmail.com>
Date: Thu, 9 Apr 2026 21:27:18 -0700
Subject: [PATCH] Fix cron jobs using dev env instead of test env in CI
 workflows (#1319)

The custom-base-port and db-migration-backwards-compatibility workflows
were running cron jobs with `with-env:dev` instead of `with-env:test`,
causing ClickHouse sync mismatches in verify-data-integrity.

<!--

Make sure you've read the CONTRIBUTING.md guidelines:
https://github.com/stack-auth/stack-auth/blob/dev/CONTRIBUTING.md

-->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Chores**
* Streamlined CI test workflows to standardize background cron job
startup for more consistent test runs.
* **Tests**
* Improved end-to-end test reliability by aligning background process
behavior across suites.
* **Bug Fixes**
* Enhanced data verification reliability by ensuring external database
sync before integrity checks and tightening comparison ordering for
certain records, reducing false mismatch detections.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
---
 .github/workflows/db-migration-backwards-compatibility.yaml  | 4 ++--
 .github/workflows/e2e-custom-base-port-api-tests.yaml        | 2 +-
 .../verify-data-integrity/clickhouse-sync-verifier.ts        | 2 +-
 apps/backend/scripts/verify-data-integrity/index.ts          | 5 +++++
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/db-migration-backwards-compatibility.yaml b/.github/workflows/db-migration-backwards-compatibility.yaml
index 19b6241d1..122e5d663 100644
--- a/.github/workflows/db-migration-backwards-compatibility.yaml
+++ b/.github/workflows/db-migration-backwards-compatibility.yaml
@@ -200,7 +200,7 @@ jobs:
         uses: JarvusInnovations/background-action@v1.0.7
         if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }}
         with:
-          run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream &
+          run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream &
           wait-on: |
             http://localhost:8102
           tail: true
@@ -394,7 +394,7 @@ jobs:
         uses: JarvusInnovations/background-action@v1.0.7
         if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }}
         with:
-          run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream &
+          run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream &
           wait-on: |
             http://localhost:8102
           tail: true
diff --git a/.github/workflows/e2e-custom-base-port-api-tests.yaml b/.github/workflows/e2e-custom-base-port-api-tests.yaml
index 66517efc4..dc0a7f4c8 100644
--- a/.github/workflows/e2e-custom-base-port-api-tests.yaml
+++ b/.github/workflows/e2e-custom-base-port-api-tests.yaml
@@ -145,7 +145,7 @@ jobs:
       - name: Start run-cron-jobs in background
         uses: JarvusInnovations/background-action@v1.0.7
         with:
-          run: pnpm -C apps/backend run run-cron-jobs --log-order=stream &
+          run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream &
           wait-on: |
             http://localhost:6702
           tail: true
diff --git a/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts b/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts
index 9eeb0104e..dfa48f870 100644
--- a/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts
+++ b/apps/backend/scripts/verify-data-integrity/clickhouse-sync-verifier.ts
@@ -18,7 +18,7 @@ const SORT_KEYS = {
   team_invitations: ["id"],
   email_outboxes: ["id"],
   project_permissions: ["user_id", "permission_id"],
-  notification_preferences: ["id"],
+  notification_preferences: ["user_id", "notification_category_id"],
   refresh_tokens: ["id"],
   connected_accounts: ["user_id", "provider", "provider_account_id"],
 } satisfies Record<keyof typeof DEFAULT_DB_SYNC_MAPPINGS, string[]>;
diff --git a/apps/backend/scripts/verify-data-integrity/index.ts b/apps/backend/scripts/verify-data-integrity/index.ts
index 48cc992cc..f63ae32a7 100644
--- a/apps/backend/scripts/verify-data-integrity/index.ts
+++ b/apps/backend/scripts/verify-data-integrity/index.ts
@@ -1,3 +1,4 @@
+import { syncExternalDatabases } from "@/lib/external-db-sync";
 import { DEFAULT_BRANCH_ID, getSoleTenancyFromProjectBranch } from "@/lib/tenancies";
 import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client";
 import type { OrganizationRenderedConfig } from "@stackframe/stack-shared/dist/config/schema";
@@ -228,6 +229,10 @@ async function main() {
 
       if (!shouldSkipClickhouse && clickhouseAvailable && tenancy) {
         await recurse("[clickhouse sync]", async (recurse) => {
+          // Flush any pending ClickHouse syncs by running a direct sync before verifying.
+          // This avoids race conditions where QStash hasn't delivered all sync callbacks yet.
+          await syncExternalDatabases(tenancy);
+
           await verifyClickhouseSync({
             tenancy,
             projectId,