From cf86ea59528d478ac281bbca0e6cff81879286c3 Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Thu, 5 Feb 2026 11:30:39 -0800 Subject: [PATCH 1/2] Proxy caching --- apps/backend/src/proxy.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/backend/src/proxy.tsx b/apps/backend/src/proxy.tsx index a413d2952..858ec1b9f 100644 --- a/apps/backend/src/proxy.tsx +++ b/apps/backend/src/proxy.tsx @@ -74,6 +74,7 @@ export async function proxy(request: NextRequest) { "Access-Control-Max-Age": "86400", // 1 day (capped to lower values, eg. 10min, by some browsers) "Access-Control-Allow-Headers": corsAllowedRequestHeaders.join(', '), "Access-Control-Expose-Headers": corsAllowedResponseHeaders.join(', '), + "Vary": corsAllowedRequestHeaders.join(', '), } : undefined; // ensure our clients can handle 429 responses From 2055d98dea0ab3ecb3f9e6db8cae8be464d8fa43 Mon Sep 17 00:00:00 2001 From: aadesh18 <110230993+aadesh18@users.noreply.github.com> Date: Thu, 5 Feb 2026 12:04:31 -0800 Subject: [PATCH 2/2] External db sync (#1036) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Screenshot 2026-02-04 at 9 47 16 AM **This PR revolves around the following components** 1. Sequencer - sequences the updates in the internal db 2. Poller - polls for the latest updates to sync with the external db 3. Outgoing Request Handler - essentially a trigger that can make http requests based on a change in the internal db 4. Sync Engine - syncs with the latest changes from the internal db to the external db **What has been done** - Added a global sequence id for ProjectUser, ContactChannel and DeletedRow. - Added the deletedRow table to keep track of the rows that were deleted across ProjectUser and ContactChannel. - Added the OutgoingRequest table to keep track of the outgoing requests - Added function for the sequencer to call to sequence updates - Added a sequencer that sequences all the changes in the internal db every 50 ms - Added a poller that polls for the latest changes in the internal db every 50 ms, and adds to a queue - Added a Vercel cron that calls sequencer and poller every minute - Added a queue that fulfills the outgoing requests by making http calls (for external db sync, it calls the sync engine endpoint) - Added a sync engine that uses the defined sql mapping query in the user's schema to pull in the changes for the user, and sync them with the external db - Added tests to test out each functionality **How to review this PR:** 1. Review the migrations (sequence id, deletedRow, triggers, backlog sync) (all files created under the migrations folder) 2. Review sequencer 3. Review poller 4. Review the changes in schema 5. Review sync-engine (the function, and it's helper file) 6. Review the schema changes, and query mappings 7. Review the tests (basic, advanced and race, along with the helper file) 8. Review the changes made in Dockerfile to support local testing using the postgres docker --- > [!NOTE] > Introduces a cron-driven external DB sync pipeline with global sequencing, internal poller and webhook sync engine, new DB tables/functions, config schema/mappings, and comprehensive e2e tests. > > - **Database (Prisma/Migrations)**: > - Add global sequence (`global_seq_id`) and `sequenceId`/`shouldUpdateSequenceId` to `ProjectUser`, `ContactChannel`, `DeletedRow` with partial indexes. > - Create `DeletedRow` (capture deletes) and `OutgoingRequest` (queue) tables; add unique/indexes. > - Add triggers/functions: `log_deleted_row`, `reset_sequence_id_on_update`, `backfill_null_sequence_ids`, `enqueue_tenant_sync`. > - **Backend/API**: > - New internal routes: `GET /api/latest/internal/external-db-sync/sequencer`, `GET /poller`, `POST /sync-engine` (Upstash-verified) for sync orchestration. > - Add cron wiring: `vercel.json` schedules and local `scripts/run-cron-jobs.ts`; start in dev via `dev` script. > - Tweak route handler (remove noisy logging) without behavior change. > - **Sync Engine**: > - Implement `src/lib/external-db-sync.ts` to read tenant mappings and upsert to external Postgres (schema bootstrap, param checks, sequencing). > - Add default mappings `DEFAULT_DB_SYNC_MAPPINGS` and config schema `dbSync.externalDatabases` in shared config. > - **Testing/Infra**: > - Add extensive e2e tests (basics, advanced, race conditions) for sequencing, idempotency, deletes, pagination, multi-mapping, and permissions. > - Docker compose: add `external-db-test` Postgres for tests; e2e deps for `pg` types. > > Written by [Cursor Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit 3f2a8efcfbd94789c8ffd83cf967e06ca942014f. This will update automatically on new commits. Configure [here](https://cursor.com/dashboard?tab=bugbot). ## Summary by CodeRabbit * **New Features** * External PostgreSQL sync: automatic, batched replication with mappings, resume/idempotency, and on-demand enqueueing. * **Admin UI** * Real-time External DB Sync dashboard and status API showing per-mapping backlog, sequencer/poller/sync-engine telemetry, and fusebox controls. * **Tests** * Large e2e suite: basic, advanced, race, high-volume tests and test utilities for external DB sync. * **Chores** * DB migrations, CI/workflow updates, background cron runner and local/dev test support. --------- Co-authored-by: Konsti Wohlwend Co-authored-by: Bilal Godil --- .../db-migration-backwards-compatibility.yaml | 12 +- .github/workflows/e2e-api-tests.yaml | 25 +- .../e2e-custom-base-port-api-tests.yaml | 25 +- .../e2e-source-of-truth-api-tests.yaml | 26 +- ...rt-dev-and-test-with-custom-base-port.yaml | 5 +- .github/workflows/restart-dev-and-test.yaml | 6 +- .../setup-tests-with-custom-base-port.yaml | 6 +- .github/workflows/setup-tests.yaml | 7 +- apps/backend/.env.development | 6 +- apps/backend/package.json | 5 +- .../migration.sql | 105 ++ .../migration.sql | 24 + apps/backend/prisma/schema.prisma | 57 + apps/backend/scripts/db-migrations.ts | 4 +- .../scripts/db-migrations.tsup.config.ts | 8 +- apps/backend/scripts/run-cron-jobs.ts | 44 + .../app/api/latest/contact-channels/crud.tsx | 31 +- .../verify/verification-code-handler.tsx | 10 +- .../config/override/[level]/route.tsx | 23 +- .../external-db-sync/fusebox/route.ts | 98 ++ .../internal/external-db-sync/poller/route.ts | 289 +++++ .../external-db-sync/sequencer/route.ts | 253 ++++ .../internal/external-db-sync/status/route.ts | 817 ++++++++++++ .../external-db-sync/sync-engine/route.tsx | 68 + .../backend/src/app/api/latest/users/crud.tsx | 24 +- apps/backend/src/lib/contact-channel.tsx | 25 +- .../src/lib/external-db-sync-metadata.ts | 32 + .../backend/src/lib/external-db-sync-queue.ts | 43 + apps/backend/src/lib/external-db-sync.ts | 498 ++++++++ apps/backend/src/lib/tokens.tsx | 5 +- apps/backend/src/proxy.tsx | 2 +- apps/backend/vercel.json | 8 + .../external-db-sync/page-client.tsx | 732 +++++++++++ .../[projectId]/external-db-sync/page.tsx | 9 + apps/e2e/.env.development | 2 + apps/e2e/package.json | 4 +- .../api/v1/auth/sessions/index.test.ts | 4 +- .../api/v1/external-db-sync-advanced.test.ts | 1124 +++++++++++++++++ .../api/v1/external-db-sync-basics.test.ts | 489 +++++++ .../v1/external-db-sync-high-volume.test.ts | 187 +++ .../api/v1/external-db-sync-race.test.ts | 410 ++++++ .../api/v1/external-db-sync-utils.ts | 387 ++++++ .../mock-external-db-sync-projects.sql | 262 ++++ apps/e2e/tests/global-setup.ts | 2 + apps/e2e/tests/helpers.ts | 2 +- docker/dependencies/docker.compose.yaml | 2 + docker/dev-postgres-replica/entrypoint.sh | 4 + .../src/config/db-sync-mappings.ts | 165 +++ .../src/config/schema-fuzzer.test.ts | 11 + packages/stack-shared/src/config/schema.ts | 20 +- pnpm-lock.yaml | 702 +++------- 51 files changed, 6537 insertions(+), 572 deletions(-) create mode 100644 apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql create mode 100644 apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql create mode 100644 apps/backend/scripts/run-cron-jobs.ts create mode 100644 apps/backend/src/app/api/latest/internal/external-db-sync/fusebox/route.ts create mode 100644 apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts create mode 100644 apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts create mode 100644 apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts create mode 100644 apps/backend/src/app/api/latest/internal/external-db-sync/sync-engine/route.tsx create mode 100644 apps/backend/src/lib/external-db-sync-metadata.ts create mode 100644 apps/backend/src/lib/external-db-sync-queue.ts create mode 100644 apps/backend/src/lib/external-db-sync.ts create mode 100644 apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx create mode 100644 apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page.tsx create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-advanced.test.ts create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-high-volume.test.ts create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts create mode 100644 apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql create mode 100644 packages/stack-shared/src/config/db-sync-mappings.ts diff --git a/.github/workflows/db-migration-backwards-compatibility.yaml b/.github/workflows/db-migration-backwards-compatibility.yaml index adcc34c96..c09ce9555 100644 --- a/.github/workflows/db-migration-backwards-compatibility.yaml +++ b/.github/workflows/db-migration-backwards-compatibility.yaml @@ -193,6 +193,17 @@ jobs: wait-for: 30s log-output-if: true + - name: Start run-cron-jobs in background + uses: JarvusInnovations/background-action@v1.0.7 + if: ${{ hashFiles('apps/backend/scripts/run-cron-jobs.ts') != '' }} + with: + run: pnpm -C apps/backend run with-env:dev tsx scripts/run-cron-jobs.ts --log-order=stream & + wait-on: | + http://localhost:8102 + tail: true + wait-for: 30s + log-output-if: true + - name: Wait 10 seconds run: sleep 10 @@ -230,4 +241,3 @@ jobs: steps: - name: No migration changes detected run: echo "No changes to migrations folder detected. Skipping backwards compatibility test." - diff --git a/.github/workflows/e2e-api-tests.yaml b/.github/workflows/e2e-api-tests.yaml index c59c38879..6aa1e55e2 100644 --- a/.github/workflows/e2e-api-tests.yaml +++ b/.github/workflows/e2e-api-tests.yaml @@ -19,6 +19,9 @@ jobs: NODE_ENV: test STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" strategy: matrix: @@ -100,6 +103,9 @@ jobs: - name: Wait on Svix run: pnpx wait-on tcp:localhost:8113 + + - name: Wait on QStash + run: pnpx wait-on tcp:localhost:8125 - name: Initialize database run: pnpm run db:init @@ -140,20 +146,29 @@ jobs: tail: true wait-for: 30s log-output-if: true + - name: Start run-cron-jobs in background + uses: JarvusInnovations/background-action@v1.0.7 + with: + run: pnpm -C apps/backend run run-cron-jobs:test --log-order=stream & + wait-on: | + http://localhost:8102 + tail: true + wait-for: 30s + log-output-if: true - name: Wait 10 seconds run: sleep 10 - name: Run tests - run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} + run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} - - name: Run tests again, to make sure they are stable (attempt 1) + - name: Run tests again (attempt 1) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} + run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} - - name: Run tests again, to make sure they are stable (attempt 2) + - name: Run tests again (attempt 2) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} + run: pnpm test run ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }} - name: Verify data integrity run: pnpm run verify-data-integrity --no-bail diff --git a/.github/workflows/e2e-custom-base-port-api-tests.yaml b/.github/workflows/e2e-custom-base-port-api-tests.yaml index 14802828f..1a1cf52ef 100644 --- a/.github/workflows/e2e-custom-base-port-api-tests.yaml +++ b/.github/workflows/e2e-custom-base-port-api-tests.yaml @@ -19,6 +19,9 @@ jobs: STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:6728/stackframe" NEXT_PUBLIC_STACK_PORT_PREFIX: "67" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" strategy: matrix: @@ -94,6 +97,9 @@ jobs: - name: Wait on Svix run: pnpx wait-on tcp:localhost:6713 + + - name: Wait on QStash + run: pnpx wait-on tcp:localhost:6725 - name: Initialize database run: pnpm run db:init @@ -134,20 +140,29 @@ jobs: tail: true wait-for: 30s log-output-if: true + - name: Start run-cron-jobs in background + uses: JarvusInnovations/background-action@v1.0.7 + with: + run: pnpm -C apps/backend run run-cron-jobs --log-order=stream & + wait-on: | + http://localhost:6702 + tail: true + wait-for: 30s + log-output-if: true - name: Wait 10 seconds run: sleep 10 - name: Run tests - run: pnpm test + run: pnpm test run - - name: Run tests again, to make sure they are stable (attempt 1) + - name: Run tests again (attempt 1) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test + run: pnpm test run - - name: Run tests again, to make sure they are stable (attempt 2) + - name: Run tests again (attempt 2) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test + run: pnpm test run - name: Verify data integrity run: pnpm run verify-data-integrity --no-bail diff --git a/.github/workflows/e2e-source-of-truth-api-tests.yaml b/.github/workflows/e2e-source-of-truth-api-tests.yaml index cb036f26c..99b66e51b 100644 --- a/.github/workflows/e2e-source-of-truth-api-tests.yaml +++ b/.github/workflows/e2e-source-of-truth-api-tests.yaml @@ -17,9 +17,13 @@ jobs: env: NODE_ENV: test STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING: yes + STACK_ACCESS_TOKEN_EXPIRATION_TIME: 30m STACK_OVERRIDE_SOURCE_OF_TRUTH: '{"type": "postgres", "connectionString": "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/source-of-truth-db?schema=sot-schema"}' STACK_TEST_SOURCE_OF_TRUTH: true STACK_DATABASE_CONNECTION_STRING: "postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/stackframe" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" strategy: matrix: @@ -95,6 +99,9 @@ jobs: - name: Wait on Svix run: pnpx wait-on tcp:localhost:8113 + + - name: Wait on QStash + run: pnpx wait-on tcp:localhost:8125 - name: Create source-of-truth database and schema run: | @@ -140,20 +147,29 @@ jobs: tail: true wait-for: 30s log-output-if: true + - name: Start run-cron-jobs in background + uses: JarvusInnovations/background-action@v1.0.7 + with: + run: pnpm -C apps/backend run run-cron-jobs --log-order=stream & + wait-on: | + http://localhost:8102 + tail: true + wait-for: 30s + log-output-if: true - name: Wait 10 seconds run: sleep 10 - name: Run tests - run: pnpm test + run: pnpm test run --exclude "**/external-db-sync*.test.ts" # external-db-sync does not support external sot - - name: Run tests again, to make sure they are stable (attempt 1) + - name: Run tests again (attempt 1) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test + run: pnpm test run --exclude "**/external-db-sync*.test.ts" - - name: Run tests again, to make sure they are stable (attempt 2) + - name: Run tests again (attempt 2) if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' - run: pnpm test + run: pnpm test run --exclude "**/external-db-sync*.test.ts" - name: Verify data integrity run: pnpm run verify-data-integrity --no-bail diff --git a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml index 6c1f64f45..65179046f 100644 --- a/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml +++ b/.github/workflows/restart-dev-and-test-with-custom-base-port.yaml @@ -19,6 +19,9 @@ jobs: runs-on: ubicloud-standard-16 env: NEXT_PUBLIC_STACK_PORT_PREFIX: "69" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: - uses: actions/checkout@v6 @@ -38,7 +41,7 @@ jobs: run: pnpm run restart-dev-environment - name: Run tests - run: pnpm run test --reporter=verbose + run: pnpm run test run --reporter=verbose - name: Print dev server logs run: cat dev-server.log.untracked.txt diff --git a/.github/workflows/restart-dev-and-test.yaml b/.github/workflows/restart-dev-and-test.yaml index 831148e4e..6d091edb3 100644 --- a/.github/workflows/restart-dev-and-test.yaml +++ b/.github/workflows/restart-dev-and-test.yaml @@ -17,6 +17,10 @@ env: jobs: restart-dev-and-test: runs-on: ubicloud-standard-16 + env: + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: - uses: actions/checkout@v6 @@ -36,7 +40,7 @@ jobs: run: pnpm run restart-dev-environment - name: Run tests - run: pnpm run test --reporter=verbose + run: pnpm run test run --reporter=verbose - name: Print dev server logs run: cat dev-server.log.untracked.txt diff --git a/.github/workflows/setup-tests-with-custom-base-port.yaml b/.github/workflows/setup-tests-with-custom-base-port.yaml index b3d15e503..0c9ac4a6c 100644 --- a/.github/workflows/setup-tests-with-custom-base-port.yaml +++ b/.github/workflows/setup-tests-with-custom-base-port.yaml @@ -19,6 +19,9 @@ jobs: runs-on: ubicloud-standard-16 env: NEXT_PUBLIC_STACK_PORT_PREFIX: "69" + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: - uses: actions/checkout@v6 @@ -46,4 +49,5 @@ jobs: tail: true wait-for: 120s log-output-if: true - - run: pnpm run test --reporter=verbose + - name: Run tests + run: pnpm run test run --reporter=verbose diff --git a/.github/workflows/setup-tests.yaml b/.github/workflows/setup-tests.yaml index d20748c93..66374120c 100644 --- a/.github/workflows/setup-tests.yaml +++ b/.github/workflows/setup-tests.yaml @@ -17,6 +17,10 @@ env: jobs: setup-tests: runs-on: ubicloud-standard-16 + env: + STACK_FORCE_EXTERNAL_DB_SYNC: "true" + STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS: "20000" + STACK_EXTERNAL_DB_SYNC_DIRECT: "false" steps: - uses: actions/checkout@v6 @@ -43,4 +47,5 @@ jobs: tail: true wait-for: 120s log-output-if: true - - run: pnpm run test --reporter=verbose + - name: Run tests + run: pnpm run test run --reporter=verbose diff --git a/apps/backend/.env.development b/apps/backend/.env.development index a11b38c8e..55d58dda4 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -35,8 +35,8 @@ STACK_DATABASE_REPLICATION_WAIT_STRATEGY=pg-stat-replication STACK_EMAIL_HOST=127.0.0.1 STACK_EMAIL_PORT=${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}29 STACK_EMAIL_SECURE=false -STACK_EMAIL_USERNAME=does not matter, ignored by Inbucket -STACK_EMAIL_PASSWORD=does not matter, ignored by Inbucket +STACK_EMAIL_USERNAME="does not matter, ignored by Inbucket" +STACK_EMAIL_PASSWORD="does not matter, ignored by Inbucket" STACK_EMAIL_SENDER=noreply@example.com STACK_ACCESS_TOKEN_EXPIRATION_TIME=60s @@ -50,7 +50,7 @@ STACK_ARTIFICIAL_DEVELOPMENT_DELAY_MS=500 STACK_ENABLE_HARDCODED_PASSKEY_CHALLENGE_FOR_TESTING=yes -STACK_INTEGRATION_CLIENTS_CONFIG=[{"client_id": "neon-local", "client_secret": "neon-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}, {"client_id": "custom-local", "client_secret": "custom-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}] +STACK_INTEGRATION_CLIENTS_CONFIG='[{"client_id": "neon-local", "client_secret": "neon-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}, {"client_id": "custom-local", "client_secret": "custom-local-secret", "id_token_signed_response_alg": "ES256", "redirect_uris": ["http://localhost:30000/api/v2/identity/authorize", "http://localhost:30000/api/v2/auth/authorize"]}]' CRON_SECRET=mock_cron_secret STACK_FREESTYLE_API_KEY=mock_stack_freestyle_key STACK_VERCEL_SANDBOX_TOKEN=vercel_sandbox_disabled_for_local_development diff --git a/apps/backend/package.json b/apps/backend/package.json index 8feb477d8..e5257b80d 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -10,7 +10,8 @@ "with-env": "dotenv -c --", "with-env:dev": "dotenv -c development --", "with-env:prod": "dotenv -c production --", - "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\"", + "with-env:test": "dotenv -c test --", + "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue,cron-jobs\" -k \"next dev --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 ${STACK_BACKEND_DEV_EXTRA_ARGS:-}\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\" \"pnpm run run-cron-jobs\"", "dev:inspect": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--inspect\" pnpm run dev", "dev:profile": "STACK_BACKEND_DEV_EXTRA_ARGS=\"--experimental-cpu-prof\" pnpm run dev", "build": "pnpm run codegen && next build", @@ -42,6 +43,8 @@ "codegen-docs:watch": "pnpm run with-env tsx watch --exclude '**/node_modules/**' --clear-screen=false scripts/generate-openapi-fumadocs.ts", "generate-keys": "pnpm run with-env tsx scripts/generate-keys.ts", "db-seed-script": "pnpm run db:seed", + "run-cron-jobs": "pnpm run with-env:dev tsx scripts/run-cron-jobs.ts", + "run-cron-jobs:test": "pnpm run with-env:test tsx scripts/run-cron-jobs.ts", "verify-data-integrity": "pnpm run with-env:dev tsx scripts/verify-data-integrity/index.ts", "run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts" }, diff --git a/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql b/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql new file mode 100644 index 000000000..e90bcf245 --- /dev/null +++ b/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql @@ -0,0 +1,105 @@ +-- Creates a global sequence starting at 1 with increment of 11 for tracking row changes. +-- This sequence is used to order data changes across all tables in the database. +CREATE SEQUENCE global_seq_id + AS BIGINT + START 1 + INCREMENT BY 11 + NO MINVALUE + NO MAXVALUE; + +-- SPLIT_STATEMENT_SENTINEL +-- Adds sequenceId column to ContactChannel and ProjectUser tables. +-- This column stores the sequence number from global_seq_id to track when each row was last modified. +ALTER TABLE "ContactChannel" ADD COLUMN "sequenceId" BIGINT; + +-- SPLIT_STATEMENT_SENTINEL +ALTER TABLE "ProjectUser" ADD COLUMN "sequenceId" BIGINT; + +-- SPLIT_STATEMENT_SENTINEL +-- Creates unique indexes on sequenceId columns to ensure no duplicate sequence IDs exist. +-- This guarantees each row has a unique position in the change sequence. +CREATE UNIQUE INDEX "ContactChannel_sequenceId_key" ON "ContactChannel"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE UNIQUE INDEX "ProjectUser_sequenceId_key" ON "ProjectUser"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates composite indexes on (tenancyId, sequenceId) for efficient sync-engine queries. +-- These allow fast lookups of rows by tenant ordered by sequence number. +CREATE INDEX "ProjectUser_tenancyId_sequenceId_idx" ON "ProjectUser"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "ContactChannel_tenancyId_sequenceId_idx" ON "ContactChannel"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates OutgoingRequest table to queue sync requests to external databases. +-- Each request stores the QStash options for making HTTP requests and tracks when fulfillment started. +CREATE TABLE "OutgoingRequest" ( + "id" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "deduplicationKey" TEXT, + "qstashOptions" JSONB NOT NULL, + "startedFulfillingAt" TIMESTAMP(3), + + CONSTRAINT "OutgoingRequest_pkey" PRIMARY KEY ("id"), + CONSTRAINT "OutgoingRequest_deduplicationKey_key" UNIQUE ("deduplicationKey") +); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "OutgoingRequest_startedFulfillingAt_deduplicationKey_idx" ON "OutgoingRequest"("startedFulfillingAt", "deduplicationKey"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates composite index on startedFulfillingAt and createdAt for efficient querying of pending requests in order. +-- This allows fast lookups of pending requests (WHERE startedFulfillingAt IS NULL) ordered by createdAt. +CREATE INDEX "OutgoingRequest_startedFulfillingAt_createdAt_idx" ON "OutgoingRequest"("startedFulfillingAt", "createdAt"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates DeletedRow table to log information about deleted rows from other tables. +-- Stores the primary key and full data of deleted rows so external databases can be notified of deletions. +CREATE TABLE "DeletedRow" ( + "id" UUID NOT NULL, + "tenancyId" UUID NOT NULL, + "tableName" TEXT NOT NULL, + "sequenceId" BIGINT, + "primaryKey" JSONB NOT NULL, + "data" JSONB, + "deletedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "startedFulfillingAt" TIMESTAMP(3), + + CONSTRAINT "DeletedRow_pkey" PRIMARY KEY ("id") +); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates indexes on DeletedRow table for efficient querying by sequence, table name, and tenant. +CREATE UNIQUE INDEX "DeletedRow_sequenceId_key" ON "DeletedRow"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "DeletedRow_tableName_idx" ON "DeletedRow"("tableName"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "DeletedRow_tenancyId_idx" ON "DeletedRow"("tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- Creates composite index for efficient querying of deleted rows by tenant and table, ordered by sequence. +CREATE INDEX "DeletedRow_tenancyId_tableName_sequenceId_idx" ON "DeletedRow"("tenancyId", "tableName", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- Adds shouldUpdateSequenceId flag to track which rows need their sequenceId updated. +ALTER TABLE "ProjectUser" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; + +-- SPLIT_STATEMENT_SENTINEL +ALTER TABLE "ContactChannel" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; + +-- SPLIT_STATEMENT_SENTINEL +ALTER TABLE "DeletedRow" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; + +-- SPLIT_STATEMENT_SENTINEL +-- Creates indexes on (shouldUpdateSequenceId, tenancyId) to quickly find rows that need updates +-- and support ORDER BY tenancyId for less fragmented updates. +CREATE INDEX "ProjectUser_shouldUpdateSequenceId_idx" ON "ProjectUser"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "ContactChannel_shouldUpdateSequenceId_idx" ON "ContactChannel"("shouldUpdateSequenceId", "tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +CREATE INDEX "DeletedRow_shouldUpdateSequenceId_idx" ON "DeletedRow"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql b/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql new file mode 100644 index 000000000..aa0a767c0 --- /dev/null +++ b/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql @@ -0,0 +1,24 @@ +-- DropIndex +DROP INDEX "ContactChannel_shouldUpdateSequenceId_idx"; + +-- DropIndex +DROP INDEX "DeletedRow_shouldUpdateSequenceId_idx"; + +-- DropIndex +DROP INDEX "ProjectUser_shouldUpdateSequenceId_idx"; + +-- CreateTable +CREATE TABLE "ExternalDbSyncMetadata" ( + "id" TEXT NOT NULL DEFAULT gen_random_uuid(), + "singleton" "BooleanTrue" NOT NULL DEFAULT 'TRUE', + "sequencerEnabled" BOOLEAN NOT NULL DEFAULT true, + "pollerEnabled" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "ExternalDbSyncMetadata_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "ExternalDbSyncMetadata_singleton_key" ON "ExternalDbSyncMetadata"("singleton"); + diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index ed724cc0e..e0cf9744d 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -94,6 +94,18 @@ model EnvironmentConfigOverride { @@id([projectId, branchId]) } +model ExternalDbSyncMetadata { + id String @id @default(dbgenerated("gen_random_uuid()")) + + singleton BooleanTrue @unique @default(TRUE) + + sequencerEnabled Boolean @default(true) + pollerEnabled Boolean @default(true) + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + model Team { tenancyId String @db.Uuid teamId String @default(uuid()) @db.Uuid @@ -190,6 +202,9 @@ model ProjectUser { updatedAt DateTime @updatedAt lastActiveAt DateTime @default(now()) + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + displayName String? serverMetadata Json? clientReadOnlyMetadata Json? @@ -227,6 +242,8 @@ model ProjectUser { @@index([tenancyId, displayName(sort: Desc)], name: "ProjectUser_displayName_desc") @@index([tenancyId, createdAt(sort: Asc)], name: "ProjectUser_createdAt_asc") @@index([tenancyId, createdAt(sort: Desc)], name: "ProjectUser_createdAt_desc") + @@index([tenancyId, sequenceId], name: "ProjectUser_tenancyId_sequenceId_idx") + // Partial index for external db sync backfill lives in migration SQL. } // This should be renamed to "OAuthAccount" as it is not always bound to a user @@ -273,6 +290,9 @@ model ContactChannel { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + type ContactChannelType isPrimary BooleanTrue? usedForAuth BooleanTrue? @@ -288,6 +308,8 @@ model ContactChannel { @@unique([tenancyId, projectUserId, type, value]) // only one contact channel per project with the same value and type can be used for auth @@unique([tenancyId, type, value, usedForAuth]) + @@index([tenancyId, sequenceId], name: "ContactChannel_tenancyId_sequenceId_idx") + // Partial index for external db sync backfill lives in migration SQL (WHERE shouldUpdateSequenceId = TRUE). } model AuthMethod { @@ -1058,3 +1080,38 @@ model SubscriptionInvoice { @@id([tenancyId, id]) @@unique([tenancyId, stripeInvoiceId]) } + +model OutgoingRequest { + id String @id @default(uuid()) @db.Uuid + + createdAt DateTime @default(now()) + + qstashOptions Json + startedFulfillingAt DateTime? + deduplicationKey String? + + @@unique([deduplicationKey]) + @@index([startedFulfillingAt, createdAt]) + @@index([startedFulfillingAt, deduplicationKey]) +} + +model DeletedRow { + id String @id @default(uuid()) @db.Uuid + tenancyId String @db.Uuid + tableName String + + sequenceId BigInt? @unique + shouldUpdateSequenceId Boolean @default(true) + + primaryKey Json + data Json? + + deletedAt DateTime @default(now()) + startedFulfillingAt DateTime? + + @@index([tableName]) + @@index([tenancyId]) + // composite index for efficient querying of deleted rows by tenant and table, ordered by sequence + @@index([tenancyId, tableName, sequenceId]) + // Partial index for external db sync backfill lives in migration SQL (WHERE shouldUpdateSequenceId = TRUE). +} diff --git a/apps/backend/scripts/db-migrations.ts b/apps/backend/scripts/db-migrations.ts index 14733a687..4f997ec7c 100644 --- a/apps/backend/scripts/db-migrations.ts +++ b/apps/backend/scripts/db-migrations.ts @@ -91,9 +91,9 @@ const generateMigrationFile = async () => { 'prisma', 'migrate', 'diff', - '--from-url', + '--from-config-datasource', diffUrl, - '--to-schema-datamodel', + '--to-schema', 'prisma/schema.prisma', '--script', ], diff --git a/apps/backend/scripts/db-migrations.tsup.config.ts b/apps/backend/scripts/db-migrations.tsup.config.ts index 72bb27ce9..5dda9ba65 100644 --- a/apps/backend/scripts/db-migrations.tsup.config.ts +++ b/apps/backend/scripts/db-migrations.tsup.config.ts @@ -12,6 +12,9 @@ const nodeBuiltins = builtinModules.flatMap((m) => [m, `node:${m}`]); // tsup config to build the self-hosting migration script so it can be // run in the Docker container with no extra dependencies. +type EsbuildPlugin = NonNullable[number]; +const basePlugin = createBasePlugin({}) as unknown as EsbuildPlugin; + export default defineConfig({ entry: ['scripts/db-migrations.ts'], format: ['esm'], @@ -32,7 +35,6 @@ const __filename = __fileURLToPath(import.meta.url); const __dirname = __dirname_fn(__filename); const require = __createRequire(import.meta.url);`, }, - esbuildPlugins: [ - createBasePlugin({}), - ], + // Cast to tsup's esbuild plugin type to avoid esbuild version mismatch in typecheck. + esbuildPlugins: [basePlugin], } satisfies Options); diff --git a/apps/backend/scripts/run-cron-jobs.ts b/apps/backend/scripts/run-cron-jobs.ts new file mode 100644 index 000000000..98b9680ce --- /dev/null +++ b/apps/backend/scripts/run-cron-jobs.ts @@ -0,0 +1,44 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError, StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { runAsynchronously, wait } from "@stackframe/stack-shared/dist/utils/promises"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; + +const endpoints = [ + "/api/latest/internal/external-db-sync/sequencer", + "/api/latest/internal/external-db-sync/poller", +]; + +async function main() { + console.log("Starting cron jobs..."); + const cronSecret = getEnvVariable('CRON_SECRET'); + + const baseUrl = `http://localhost:${getEnvVariable('NEXT_PUBLIC_STACK_PORT_PREFIX', '81')}02`; + + const run = async (endpoint: string) => { + console.log(`Running ${endpoint}...`); + const res = await fetch(`${baseUrl}${endpoint}`, { + headers: { 'Authorization': `Bearer ${cronSecret}` }, + }); + if (!res.ok) throw new StackAssertionError(`Failed to call ${endpoint}: ${res.status} ${res.statusText}\n${await res.text()}`, { res }); + console.log(`${endpoint} completed.`); + }; + + for (const endpoint of endpoints) { + runAsynchronously(async () => { + while (true) { + const runResult = await Result.fromPromise(run(endpoint)); + if (runResult.status === "error") { + captureError("run-cron-jobs", runResult.error); + } + // Vercel only guarantees minute-granularity for cron jobs, so we randomize the interval + await wait(Math.random() * 120_000); + } + }); + } +} + +// eslint-disable-next-line no-restricted-syntax +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/backend/src/app/api/latest/contact-channels/crud.tsx b/apps/backend/src/app/api/latest/contact-channels/crud.tsx index abc0edaa1..d5f3081f9 100644 --- a/apps/backend/src/app/api/latest/contact-channels/crud.tsx +++ b/apps/backend/src/app/api/latest/contact-channels/crud.tsx @@ -1,5 +1,6 @@ import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryById } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; +import { markProjectUserForExternalDbSync, recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { ensureContactChannelDoesNotExists, ensureContactChannelExists } from "@/lib/request-checks"; import { getPrismaClientForTenancy, retryTransaction } from "@/prisma-client"; import { createCrudHandlers } from "@/route-handlers/crud-handler"; @@ -121,6 +122,11 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl }); } + await markProjectUserForExternalDbSync(tx, { + tenancyId: auth.tenancy.id, + projectUserId: data.user_id, + }); + return await tx.contactChannel.findUnique({ where: { tenancyId_projectUserId_id: { @@ -188,7 +194,7 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl }); } - return await tx.contactChannel.update({ + const updated = await tx.contactChannel.update({ where: { tenancyId_projectUserId_id: { tenancyId: auth.tenancy.id, @@ -196,13 +202,20 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl id: params.contact_channel_id || throwErr("Missing contact channel id"), }, }, - data: { + data: withExternalDbSyncUpdate({ value: value, isVerified: data.is_verified ?? (value ? false : undefined), // if value is updated and is_verified is not provided, set to false usedForAuth: data.used_for_auth !== undefined ? (data.used_for_auth ? 'TRUE' : null) : undefined, isPrimary: data.is_primary !== undefined ? (data.is_primary ? 'TRUE' : null) : undefined, - }, + }), }); + + await markProjectUserForExternalDbSync(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + return updated; }); return contactChannelToCrud(updatedContactChannel); @@ -224,6 +237,13 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl contactChannelId: params.contact_channel_id || throwErr("Missing contact channel id"), }); + await recordExternalDbSyncDeletion(tx, { + tableName: "ContactChannel", + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + contactChannelId: params.contact_channel_id || throwErr("Missing contact channel id"), + }); + await tx.contactChannel.delete({ where: { tenancyId_projectUserId_id: { @@ -233,6 +253,11 @@ export const contactChannelsCrudHandlers = createLazyProxy(() => createCrudHandl }, }, }); + + await markProjectUserForExternalDbSync(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); }); }, onList: async ({ query, auth }) => { diff --git a/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx b/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx index 53d981fcf..fa2dfef55 100644 --- a/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx +++ b/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx @@ -1,4 +1,5 @@ import { sendEmailFromDefaultTemplate } from "@/lib/emails"; +import { markProjectUserForExternalDbSync, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { getPrismaClientForTenancy } from "@/prisma-client"; import { createVerificationCodeHandler } from "@/route-handlers/verification-code-handler"; @@ -68,9 +69,14 @@ export const contactChannelVerificationCodeHandler = createVerificationCodeHandl await prisma.contactChannel.update({ where: uniqueKeys, - data: { + data: withExternalDbSyncUpdate({ isVerified: true, - } + }), + }); + + await markProjectUserForExternalDbSync(prisma, { + tenancyId: tenancy.id, + projectUserId: data.user_id, }); return { diff --git a/apps/backend/src/app/api/latest/internal/config/override/[level]/route.tsx b/apps/backend/src/app/api/latest/internal/config/override/[level]/route.tsx index 019cce5bd..b9e0af9ef 100644 --- a/apps/backend/src/app/api/latest/internal/config/override/[level]/route.tsx +++ b/apps/backend/src/app/api/latest/internal/config/override/[level]/route.tsx @@ -1,4 +1,5 @@ import { getBranchConfigOverrideQuery, getEnvironmentConfigOverrideQuery, overrideBranchConfigOverride, overrideEnvironmentConfigOverride, setBranchConfigOverride, setBranchConfigOverrideSource, setEnvironmentConfigOverride } from "@/lib/config"; +import { enqueueExternalDbSync } from "@/lib/external-db-sync-queue"; import { globalPrismaClient, rawQuery } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { branchConfigSchema, environmentConfigSchema, getConfigOverrideErrors, migrateConfigOverride } from "@stackframe/stack-shared/dist/config/schema"; @@ -10,6 +11,19 @@ type BranchConfigSourceApi = yup.InferType; const levelSchema = yupString().oneOf(["branch", "environment"]).defined(); +function shouldEnqueueExternalDbSync(config: unknown): boolean { + if (!config || typeof config !== "object") return false; + const configRecord = config as Record; + if (Object.prototype.hasOwnProperty.call(configRecord, "dbSync.externalDatabases")) { + return true; + } + const dbSync = configRecord.dbSync; + if (dbSync && typeof dbSync === "object") { + return Object.prototype.hasOwnProperty.call(dbSync as Record, "externalDatabases"); + } + return false; +} + const levelConfigs = { branch: { schema: branchConfigSchema, @@ -165,6 +179,10 @@ export const PUT = createSmartRouteHandler({ source: req.body.source as BranchConfigSourceApi, }); + if (req.params.level === "environment" && shouldEnqueueExternalDbSync(parsedConfig)) { + await enqueueExternalDbSync(req.auth.tenancy.id); + } + return { statusCode: 200 as const, bodyType: "success" as const, @@ -202,10 +220,13 @@ export const PATCH = createSmartRouteHandler({ config: parsedConfig, }); + if (req.params.level === "environment" && shouldEnqueueExternalDbSync(parsedConfig)) { + await enqueueExternalDbSync(req.auth.tenancy.id); + } + return { statusCode: 200 as const, bodyType: "success" as const, }; }, }); - diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/fusebox/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/fusebox/route.ts new file mode 100644 index 000000000..1039e7fa6 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/fusebox/route.ts @@ -0,0 +1,98 @@ +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { + adaptSchema, + adminAuthTypeSchema, + yupBoolean, + yupNumber, + yupObject, + yupString, +} from "@stackframe/stack-shared/dist/schema-fields"; +import { KnownErrors } from "@stackframe/stack-shared"; +import { getExternalDbSyncFusebox, updateExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; + +const fuseboxResponseSchema = yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + sequencer_enabled: yupBoolean().defined(), + poller_enabled: yupBoolean().defined(), + }).defined(), +}); + +const fuseboxRequestSchema = yupObject({ + auth: yupObject({ + type: adminAuthTypeSchema, + tenancy: adaptSchema, + }).defined(), + body: yupObject({ + sequencer_enabled: yupBoolean().defined(), + poller_enabled: yupBoolean().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), +}); + +const fuseboxGetRequestSchema = yupObject({ + auth: yupObject({ + type: adminAuthTypeSchema, + tenancy: adaptSchema, + }).defined(), + method: yupString().oneOf(["GET"]).defined(), +}); + +function ensureInternalProject(projectId: string) { + if (projectId !== "internal") { + throw new KnownErrors.ExpectedInternalProject(); + } +} + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "Get external DB sync fusebox settings", + description: "Returns enablement flags for the external DB sync pipeline.", + tags: ["External DB Sync"], + hidden: true, + }, + request: fuseboxGetRequestSchema, + response: fuseboxResponseSchema, + handler: async ({ auth }) => { + ensureInternalProject(auth.tenancy.project.id); + const fusebox = await getExternalDbSyncFusebox(); + return { + statusCode: 200, + bodyType: "json" as const, + body: { + ok: true, + sequencer_enabled: fusebox.sequencerEnabled, + poller_enabled: fusebox.pollerEnabled, + }, + }; + }, +}); + +export const POST = createSmartRouteHandler({ + metadata: { + summary: "Update external DB sync fusebox settings", + description: "Updates enablement flags for the external DB sync pipeline.", + tags: ["External DB Sync"], + hidden: true, + }, + request: fuseboxRequestSchema, + response: fuseboxResponseSchema, + handler: async ({ auth, body }) => { + ensureInternalProject(auth.tenancy.project.id); + const fusebox = await updateExternalDbSyncFusebox({ + sequencerEnabled: body.sequencer_enabled, + pollerEnabled: body.poller_enabled, + }); + return { + statusCode: 200, + bodyType: "json" as const, + body: { + ok: true, + sequencer_enabled: fusebox.sequencerEnabled, + poller_enabled: fusebox.pollerEnabled, + }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts new file mode 100644 index 000000000..6e82f8bb4 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts @@ -0,0 +1,289 @@ +import type { OutgoingRequest } from "@/generated/prisma/client"; +import { getExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; +import { upstash } from "@/lib/upstash"; +import { globalPrismaClient, retryTransaction } from "@/prisma-client"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { traceSpan } from "@/utils/telemetry"; +import { + yupBoolean, + yupNumber, + yupObject, + yupString, + yupTuple, +} from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError, StackAssertionError, StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { wait } from "@stackframe/stack-shared/dist/utils/promises"; +import type { PublishBatchRequest } from "@upstash/qstash"; + +const DEFAULT_MAX_DURATION_MS = 3 * 60 * 1000; +const DIRECT_SYNC_ENV = "STACK_EXTERNAL_DB_SYNC_DIRECT"; +const POLLER_CLAIM_LIMIT_ENV = "STACK_EXTERNAL_DB_SYNC_POLL_CLAIM_LIMIT"; +const DEFAULT_POLL_CLAIM_LIMIT = 1000; + +function parseMaxDurationMs(value: string | undefined): number { + if (!value) return DEFAULT_MAX_DURATION_MS; + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StatusError(400, "maxDurationMs must be a positive integer"); + } + return parsed; +} + +function parseStopWhenIdle(value: string | undefined): boolean { + if (!value) return false; + if (value === "true") return true; + if (value === "false") return false; + throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'"); +} + +function directSyncEnabled(): boolean { + return getEnvVariable(DIRECT_SYNC_ENV, "") === "true"; +} + +function getPollerClaimLimit(): number { + const rawValue = getEnvVariable(POLLER_CLAIM_LIMIT_ENV, ""); + if (!rawValue) return DEFAULT_POLL_CLAIM_LIMIT; + const parsed = Number.parseInt(rawValue, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StackAssertionError( + `${POLLER_CLAIM_LIMIT_ENV} must be a positive integer. Received: ${JSON.stringify(rawValue)}` + ); + } + return parsed; +} + +function getLocalApiBaseUrl(): string { + const prefix = getEnvVariable("NEXT_PUBLIC_STACK_PORT_PREFIX", "81"); + return `http://localhost:${prefix}02`; +} + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "Poll outgoing requests and push to QStash", + description: + "Internal endpoint invoked by Vercel Cron to process pending outgoing requests.", + tags: ["External DB Sync"], + hidden: true, + }, + request: yupObject({ + auth: yupObject({}).nullable().optional(), + method: yupString().oneOf(["GET"]).defined(), + headers: yupObject({ + authorization: yupTuple([yupString().defined()]).defined(), + }).defined(), + query: yupObject({ + maxDurationMs: yupString().optional(), + stopWhenIdle: yupString().optional(), + }).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + requests_processed: yupNumber().defined(), + }).defined(), + }), + handler: async ({ headers, query }) => { + const authHeader = headers.authorization[0]; + if (authHeader !== `Bearer ${getEnvVariable("CRON_SECRET")}`) { + throw new StatusError(401, "Unauthorized"); + } + + return await traceSpan("external-db-sync.poller", async (span) => { + const startTime = performance.now(); + const maxDurationMs = parseMaxDurationMs(query.maxDurationMs); + const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle); + const pollIntervalMs = 50; + const staleClaimIntervalMinutes = 5; + const pollerClaimLimit = getPollerClaimLimit(); + + span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs); + span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle); + span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs); + span.setAttribute("stack.external-db-sync.poller-claim-limit", pollerClaimLimit); + span.setAttribute("stack.external-db-sync.direct-sync", directSyncEnabled()); + span.setAttribute("stack.external-db-sync.stale-claim-minutes", staleClaimIntervalMinutes); + + let totalRequestsProcessed = 0; + let iterationCount = 0; + + async function claimPendingRequests(): Promise { + return await traceSpan("external-db-sync.poller.claimPendingRequests", async (claimSpan) => { + const requests = await globalPrismaClient.$queryRaw` + UPDATE "OutgoingRequest" + SET "startedFulfillingAt" = NOW() + WHERE "id" IN ( + SELECT id + FROM "OutgoingRequest" + WHERE "startedFulfillingAt" IS NULL + LIMIT ${pollerClaimLimit} + FOR UPDATE SKIP LOCKED + ) + RETURNING *; + `; + claimSpan.setAttribute("stack.external-db-sync.claimed-count", requests.length); + return requests; + }); + } + + async function deleteOutgoingRequest(id: string): Promise { + await retryTransaction(globalPrismaClient, async (tx) => { + await tx.outgoingRequest.delete({ where: { id } }); + }); + } + + async function deleteOutgoingRequests(ids: string[]): Promise { + if (ids.length === 0) return; + await retryTransaction(globalPrismaClient, async (tx) => { + await tx.outgoingRequest.deleteMany({ where: { id: { in: ids } } }); + }); + } + async function processRequest(request: OutgoingRequest): Promise { + // Prisma JsonValue doesn't carry a precise shape for this JSON blob. + const options = request.qstashOptions as any; + const baseUrl = getEnvVariable("NEXT_PUBLIC_STACK_API_URL"); + + let fullUrl = new URL(options.url, baseUrl).toString(); + + // In dev/test, QStash runs in Docker so "localhost" won't work. + // Replace with "host.docker.internal" to reach the host machine. + if (getNodeEnvironment().includes("development") || getNodeEnvironment().includes("test")) { + const url = new URL(fullUrl); + if (url.hostname === "localhost" || url.hostname === "127.0.0.1") { + url.hostname = "host.docker.internal"; + fullUrl = url.toString(); + } + } + + await upstash.publishJSON({ + url: fullUrl, + body: options.body, + flowControl: options.flowControl, + }); + await deleteOutgoingRequest(request.id); + } + + type UpstashRequest = PublishBatchRequest; + + function buildUpstashRequest(request: OutgoingRequest): UpstashRequest { + // Prisma JsonValue doesn't carry a precise shape for this JSON blob. + const options = request.qstashOptions as any; + const baseUrl = getEnvVariable("NEXT_PUBLIC_STACK_API_URL"); + + let fullUrl = new URL(options.url, baseUrl).toString(); + + // In dev/test, QStash runs in Docker so "localhost" won't work. + // Replace with "host.docker.internal" to reach the host machine. + if (getNodeEnvironment().includes("development") || getNodeEnvironment().includes("test")) { + const url = new URL(fullUrl); + if (url.hostname === "localhost" || url.hostname === "127.0.0.1") { + url.hostname = "host.docker.internal"; + fullUrl = url.toString(); + } + } + + const flowControl = options.flowControl as UpstashRequest["flowControl"]; + + return { + url: fullUrl, + body: options.body, + ...(flowControl ? { flowControl } : {}), + }; + } + + async function processRequests(requests: OutgoingRequest[]): Promise { + return await traceSpan({ + description: "external-db-sync.poller.processRequests", + attributes: { + "stack.external-db-sync.pending-count": requests.length, + "stack.external-db-sync.direct-sync": directSyncEnabled(), + }, + }, async (processSpan) => { + let processed = 0; + + if (directSyncEnabled()) { + for (const request of requests) { + try { + await processRequest(request); + processed++; + } catch (error) { + processSpan.setAttribute("stack.external-db-sync.iteration-error", true); + captureError("poller-iteration-error", error); + } + } + processSpan.setAttribute("stack.external-db-sync.processed-count", processed); + return processed; + } + + if (requests.length === 0) { + processSpan.setAttribute("stack.external-db-sync.processed-count", 0); + return 0; + } + + try { + const batchPayload = requests.map(buildUpstashRequest); + await upstash.batchJSON(batchPayload); + await deleteOutgoingRequests(requests.map((request) => request.id)); + processSpan.setAttribute("stack.external-db-sync.processed-count", requests.length); + return requests.length; + } catch (error) { + processSpan.setAttribute("stack.external-db-sync.iteration-error", true); + captureError("poller-iteration-error", error); + processSpan.setAttribute("stack.external-db-sync.processed-count", 0); + return 0; + } + }); + } + + type PollerIterationResult = { + stopReason: "disabled" | "idle" | null, + processed: number, + }; + + while (performance.now() - startTime < maxDurationMs) { + const iterationResult = await traceSpan({ + description: "external-db-sync.poller.iteration", + attributes: { + "stack.external-db-sync.iteration": iterationCount + 1, + }, + }, async (iterationSpan) => { + const fusebox = await getExternalDbSyncFusebox(); + iterationSpan.setAttribute("stack.external-db-sync.poller-enabled", fusebox.pollerEnabled); + if (!fusebox.pollerEnabled) { + return { stopReason: "disabled", processed: 0 }; + } + + const pendingRequests = await claimPendingRequests(); + iterationSpan.setAttribute("stack.external-db-sync.pending-count", pendingRequests.length); + + if (stopWhenIdle && pendingRequests.length === 0) { + return { stopReason: "idle", processed: 0 }; + } + + const processed = await processRequests(pendingRequests); + iterationSpan.setAttribute("stack.external-db-sync.processed-count", processed); + return { stopReason: null, processed }; + }); + + iterationCount++; + totalRequestsProcessed += iterationResult.processed; + + await wait(pollIntervalMs); + } + + span.setAttribute("stack.external-db-sync.requests-processed", totalRequestsProcessed); + span.setAttribute("stack.external-db-sync.iterations", iterationCount); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { + ok: true, + requests_processed: totalRequestsProcessed, + }, + }; + }); + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts new file mode 100644 index 000000000..2a40b3e0e --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sequencer/route.ts @@ -0,0 +1,253 @@ +import { getExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; +import { enqueueExternalDbSyncBatch } from "@/lib/external-db-sync-queue"; +import { globalPrismaClient } from "@/prisma-client"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { traceSpan } from "@/utils/telemetry"; +import { + yupBoolean, + yupNumber, + yupObject, + yupString, + yupTuple, +} from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError, StackAssertionError, StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { wait } from "@stackframe/stack-shared/dist/utils/promises"; + +const DEFAULT_MAX_DURATION_MS = 3 * 60 * 1000; +const SEQUENCER_BATCH_SIZE_ENV = "STACK_EXTERNAL_DB_SYNC_SEQUENCER_BATCH_SIZE"; +const DEFAULT_BATCH_SIZE = 1000; + +function parseMaxDurationMs(value: string | undefined): number { + if (!value) return DEFAULT_MAX_DURATION_MS; + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StatusError(400, "maxDurationMs must be a positive integer"); + } + return parsed; +} + +function parseStopWhenIdle(value: string | undefined): boolean { + if (!value) return false; + if (value === "true") return true; + if (value === "false") return false; + throw new StatusError(400, "stopWhenIdle must be 'true' or 'false'"); +} + +function getSequencerBatchSize(): number { + const rawValue = getEnvVariable(SEQUENCER_BATCH_SIZE_ENV, ""); + if (!rawValue) return DEFAULT_BATCH_SIZE; + const parsed = Number.parseInt(rawValue, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StackAssertionError( + `${SEQUENCER_BATCH_SIZE_ENV} must be a positive integer. Received: ${JSON.stringify(rawValue)}` + ); + } + return parsed; +} + + +// Assigns sequence IDs to rows that need them and queues sync requests for affected tenants. +// Processes up to batchSize rows at a time from each table. +async function backfillSequenceIds(batchSize: number): Promise { + return await traceSpan({ + description: "external-db-sync.sequencer.backfill", + attributes: { + "stack.external-db-sync.batch-size": batchSize, + }, + }, async (span) => { + let didUpdate = false; + + const projectUserTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "projectUserId" + FROM "ProjectUser" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ProjectUser" pu + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE pu."tenancyId" = r."tenancyId" + AND pu."projectUserId" = r."projectUserId" + RETURNING pu."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.project-user-tenants", projectUserTenants.length); + + // Enqueue sync for all affected tenants in a single batch query + if (projectUserTenants.length > 0) { + await enqueueExternalDbSyncBatch(projectUserTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const contactChannelTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "tenancyId", "projectUserId", "id" + FROM "ContactChannel" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "ContactChannel" cc + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE cc."tenancyId" = r."tenancyId" + AND cc."projectUserId" = r."projectUserId" + AND cc."id" = r."id" + RETURNING cc."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.contact-channel-tenants", contactChannelTenants.length); + + if (contactChannelTenants.length > 0) { + await enqueueExternalDbSyncBatch(contactChannelTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + const deletedRowTenants = await globalPrismaClient.$queryRaw<{ tenancyId: string }[]>` + WITH rows_to_update AS ( + SELECT "id", "tenancyId" + FROM "DeletedRow" + WHERE "shouldUpdateSequenceId" = TRUE + ORDER BY "tenancyId" + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ), + updated_rows AS ( + UPDATE "DeletedRow" dr + SET "sequenceId" = nextval('global_seq_id'), + "shouldUpdateSequenceId" = FALSE + FROM rows_to_update r + WHERE dr."id" = r."id" + RETURNING dr."tenancyId" + ) + SELECT DISTINCT "tenancyId" FROM updated_rows + `; + + span.setAttribute("stack.external-db-sync.deleted-row-tenants", deletedRowTenants.length); + + if (deletedRowTenants.length > 0) { + await enqueueExternalDbSyncBatch(deletedRowTenants.map(t => t.tenancyId)); + didUpdate = true; + } + + span.setAttribute("stack.external-db-sync.did-update", didUpdate); + + return didUpdate; + }); +} + +// TODO: If we ever need to support non-hosted source-of-truth tenancies again, +// we'll need to implement a scalable way to iterate over them (pagination, etc.) +// instead of loading all tenancies into memory at once. + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "Run sequence ID backfill", + description: + "Internal endpoint invoked by Vercel Cron to backfill null sequence IDs.", + tags: ["External DB Sync"], + hidden: true, + }, + request: yupObject({ + auth: yupObject({}).nullable().optional(), + method: yupString().oneOf(["GET"]).defined(), + headers: yupObject({ + authorization: yupTuple([yupString().defined()]).defined(), + }).defined(), + query: yupObject({ + maxDurationMs: yupString().optional(), + stopWhenIdle: yupString().optional(), + }).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + iterations: yupNumber().defined(), + }).defined(), + }), + handler: async ({ headers, query }) => { + const authHeader = headers.authorization[0]; + if (authHeader !== `Bearer ${getEnvVariable("CRON_SECRET")}`) { + throw new StatusError(401, "Unauthorized"); + } + + return await traceSpan("external-db-sync.sequencer", async (span) => { + const startTime = performance.now(); + const maxDurationMs = parseMaxDurationMs(query.maxDurationMs); + const stopWhenIdle = parseStopWhenIdle(query.stopWhenIdle); + const pollIntervalMs = 50; + const batchSize = getSequencerBatchSize(); + + span.setAttribute("stack.external-db-sync.max-duration-ms", maxDurationMs); + span.setAttribute("stack.external-db-sync.stop-when-idle", stopWhenIdle); + span.setAttribute("stack.external-db-sync.poll-interval-ms", pollIntervalMs); + span.setAttribute("stack.external-db-sync.batch-size", batchSize); + + let iterations = 0; + + type SequencerIterationResult = { + stopReason: "disabled" | "idle" | null, + }; + + while (performance.now() - startTime < maxDurationMs) { + const iterationResult = await traceSpan({ + description: "external-db-sync.sequencer.iteration", + attributes: { + "stack.external-db-sync.iteration": iterations + 1, + }, + }, async (iterationSpan) => { + const fusebox = await getExternalDbSyncFusebox(); + iterationSpan.setAttribute("stack.external-db-sync.sequencer-enabled", fusebox.sequencerEnabled); + if (!fusebox.sequencerEnabled) { + return { stopReason: "disabled" }; + } + + try { + const didUpdate = await backfillSequenceIds(batchSize); + iterationSpan.setAttribute("stack.external-db-sync.did-update", didUpdate); + if (stopWhenIdle && !didUpdate) { + return { stopReason: "idle" }; + } + } catch (error) { + iterationSpan.setAttribute("stack.external-db-sync.iteration-error", true); + captureError( + `sequencer-iteration-error`, + error, + ); + } + + return { stopReason: null }; + }); + + iterations++; + await wait(pollIntervalMs); + } + + span.setAttribute("stack.external-db-sync.iterations", iterations); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { + ok: true, + iterations, + }, + }; + }); + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts new file mode 100644 index 000000000..cf1c89156 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/status/route.ts @@ -0,0 +1,817 @@ +import { globalPrismaClient } from "@/prisma-client"; +import { Prisma } from "@/generated/prisma/client"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import type { CompleteConfig } from "@stackframe/stack-shared/dist/config/schema"; +import { + adaptSchema, + adminAuthTypeSchema, + yupArray, + yupBoolean, + yupNumber, + yupObject, + yupString, +} from "@stackframe/stack-shared/dist/schema-fields"; +import { errorToNiceString, StackAssertionError, throwErr } from "@stackframe/stack-shared/dist/utils/errors"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { Client } from "pg"; +import { KnownErrors } from "@stackframe/stack-shared"; +import { traceSpan } from "@/utils/telemetry"; + +const STALE_CLAIM_INTERVAL_MINUTES = 5; + +const sequenceStatsSchema = yupObject({ + total: yupString().defined(), + pending: yupString().defined(), + null_sequence_id: yupString().defined(), + min_sequence_id: yupString().nullable().defined(), + max_sequence_id: yupString().nullable().defined(), +}); + +const deletedRowByTableSchema = yupObject({ + table_name: yupString().defined(), + total: yupString().defined(), + pending: yupString().defined(), + null_sequence_id: yupString().defined(), + min_sequence_id: yupString().nullable().defined(), + max_sequence_id: yupString().nullable().defined(), +}); + +const externalDbMetadataSchema = yupObject({ + mapping_name: yupString().defined(), + last_synced_sequence_id: yupString().defined(), + updated_at_millis: yupNumber().nullable().defined(), +}); + +const externalDbMappingStatusSchema = yupObject({ + mapping_id: yupString().defined(), + internal_max_sequence_id: yupString().nullable().defined(), + last_synced_sequence_id: yupString().nullable().defined(), + updated_at_millis: yupNumber().nullable().defined(), + backlog: yupString().nullable().defined(), +}); + +const externalDbSchema = yupObject({ + id: yupString().defined(), + type: yupString().defined(), + connection: yupObject({ + redacted: yupString().nullable().defined(), + host: yupString().nullable().defined(), + port: yupNumber().nullable().defined(), + database: yupString().nullable().defined(), + user: yupString().nullable().defined(), + }).defined(), + status: yupString().oneOf(["ok", "error"]).defined(), + error: yupString().nullable().defined(), + metadata: yupArray(externalDbMetadataSchema).defined(), + users_table: yupObject({ + exists: yupBoolean().defined(), + total_rows: yupString().nullable().defined(), + min_signed_up_at_millis: yupNumber().nullable().defined(), + max_signed_up_at_millis: yupNumber().nullable().defined(), + }).defined(), + mapping_status: yupArray(externalDbMappingStatusSchema).defined(), +}); + +const mappingSchema = yupObject({ + mapping_id: yupString().defined(), + internal_min_sequence_id: yupString().nullable().defined(), + internal_max_sequence_id: yupString().nullable().defined(), + internal_pending_count: yupString().defined(), +}); + +const globalSchema = yupObject({ + tenancies_total: yupString().defined(), + tenancies_with_db_sync: yupString().defined(), + sequencer: yupObject({ + project_users: sequenceStatsSchema.defined(), + contact_channels: sequenceStatsSchema.defined(), + deleted_rows: sequenceStatsSchema.shape({ + by_table: yupArray(deletedRowByTableSchema).defined(), + }).defined(), + }).defined(), + poller: yupObject({ + total: yupString().defined(), + pending: yupString().defined(), + in_flight: yupString().defined(), + stale: yupString().defined(), + oldest_created_at_millis: yupNumber().nullable().defined(), + newest_created_at_millis: yupNumber().nullable().defined(), + }).defined(), + sync_engine: yupObject({ + mappings: yupArray(mappingSchema).defined(), + }).defined(), +}); + +const responseSchema = yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + generated_at_millis: yupNumber().defined(), + global: globalSchema.nullable().defined(), + tenancy: yupObject({ + id: yupString().defined(), + project_id: yupString().defined(), + branch_id: yupString().defined(), + }).defined(), + sequencer: yupObject({ + project_users: sequenceStatsSchema.defined(), + contact_channels: sequenceStatsSchema.defined(), + deleted_rows: sequenceStatsSchema.shape({ + by_table: yupArray(deletedRowByTableSchema).defined(), + }).defined(), + }).defined(), + poller: yupObject({ + total: yupString().defined(), + pending: yupString().defined(), + in_flight: yupString().defined(), + stale: yupString().defined(), + oldest_created_at_millis: yupNumber().nullable().defined(), + newest_created_at_millis: yupNumber().nullable().defined(), + }).defined(), + sync_engine: yupObject({ + mappings: yupArray(mappingSchema).defined(), + external_databases: yupArray(externalDbSchema).defined(), + }).defined(), + }).defined(), +}); + +type SequenceStatsRow = { + total: unknown, + pending: unknown, + null_sequence_id: unknown, + min_sequence_id: unknown, + max_sequence_id: unknown, +}; + +type DeletedRowStatsRow = SequenceStatsRow & { + table_name: string, +}; + +type OutgoingStatsRow = { + total: unknown, + pending: unknown, + in_flight: unknown, + stale: unknown, + oldest_created_at: unknown, + newest_created_at: unknown, +}; + +type ExternalDbMetadataRow = { + mapping_name: string, + last_synced_sequence_id: unknown, + updated_at: unknown, +}; + +type UsersTableStatsRow = { + total_rows: unknown, + min_signed_up_at: unknown, + max_signed_up_at: unknown, +}; + +type CountRow = { + total: unknown, +}; + +type SequenceStats = ReturnType; +type DeletedRowSummary = SequenceStats & { table_name: string }; + +function toBigIntString(value: unknown): string | null { + if (value === null || value === undefined) return null; + if (typeof value === "bigint") return value.toString(); + if (typeof value === "number" && Number.isFinite(value)) return Math.trunc(value).toString(); + if (typeof value === "string") return value; + return null; +} + +function toBigIntStringOrThrow(value: unknown, label: string): string { + return toBigIntString(value) ?? throwErr(`Expected ${label} to be a bigint-compatible value.`, { value }); +} + +function toMillis(value: unknown): number | null { + if (value === null || value === undefined) return null; + if (value instanceof Date) return value.getTime(); + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed.getTime(); + } + return null; +} + +function addBigIntStrings(a: string | null | undefined, b: string | null | undefined): string { + const first = a ? BigInt(a) : 0n; + const second = b ? BigInt(b) : 0n; + return (first + second).toString(); +} + +function minBigIntString(values: Array): string | null { + let minValue: bigint | null = null; + for (const value of values) { + if (!value) continue; + const parsed = BigInt(value); + if (minValue === null || parsed < minValue) { + minValue = parsed; + } + } + return minValue === null ? null : minValue.toString(); +} + +function maxBigIntString(values: Array): string | null { + let maxValue: bigint | null = null; + for (const value of values) { + if (!value) continue; + const parsed = BigInt(value); + if (maxValue === null || parsed > maxValue) { + maxValue = parsed; + } + } + return maxValue === null ? null : maxValue.toString(); +} + +function buildMappingInternalStats( + projectUsersStats: SequenceStats, + deletedRowsByTable: DeletedRowSummary[], +) { + const deletedProjectUserStats = deletedRowsByTable.find((row) => row.table_name === "ProjectUser") ?? null; + + const mappingInternalStats = new Map(); + + const usersMappingMin = minBigIntString([ + projectUsersStats.min_sequence_id, + deletedProjectUserStats?.min_sequence_id, + ]); + const usersMappingMax = maxBigIntString([ + projectUsersStats.max_sequence_id, + deletedProjectUserStats?.max_sequence_id, + ]); + const usersMappingPending = addBigIntStrings( + projectUsersStats.pending, + deletedProjectUserStats?.pending, + ); + + mappingInternalStats.set("users", { + mapping_id: "users", + internal_min_sequence_id: usersMappingMin, + internal_max_sequence_id: usersMappingMax, + internal_pending_count: usersMappingPending, + }); + + const mappings = Array.from(mappingInternalStats.values()); + const mappingStatuses = mappings.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + })); + + return { mappings, mappingStatuses }; +} + +async function fetchInternalStats(tenancyId: string | null) { + const tenancyWhere = tenancyId + ? Prisma.sql`WHERE "tenancyId" = ${tenancyId}::uuid` + : Prisma.sql``; + + const projectUserStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ProjectUser" + ${tenancyWhere} + `).at(0) ?? throwErr("Project user stats query returned no rows."); + + const contactChannelStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "ContactChannel" + ${tenancyWhere} + `).at(0) ?? throwErr("Contact channel stats query returned no rows."); + + const deletedRowStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "DeletedRow" + ${tenancyWhere} + `).at(0) ?? throwErr("Deleted row stats query returned no rows."); + + const deletedRowsByTableRows = await globalPrismaClient.$queryRaw` + SELECT + "tableName" AS "table_name", + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "shouldUpdateSequenceId" = TRUE OR "sequenceId" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "sequenceId" IS NULL)::bigint AS "null_sequence_id", + MIN("sequenceId") AS "min_sequence_id", + MAX("sequenceId") AS "max_sequence_id" + FROM "DeletedRow" + ${tenancyWhere} + GROUP BY "tableName" + ORDER BY "tableName" ASC + `; + + const outgoingTenancyFilter = tenancyId + ? Prisma.sql`AND ("qstashOptions"->'body'->>'tenancyId') = ${tenancyId}` + : Prisma.sql``; + + const outgoingStatsRow = (await globalPrismaClient.$queryRaw` + SELECT + COUNT(*)::bigint AS "total", + COUNT(*) FILTER (WHERE "startedFulfillingAt" IS NULL)::bigint AS "pending", + COUNT(*) FILTER (WHERE "startedFulfillingAt" IS NOT NULL)::bigint AS "in_flight", + COUNT(*) FILTER ( + WHERE "startedFulfillingAt" < NOW() - (${STALE_CLAIM_INTERVAL_MINUTES} * INTERVAL '1 minute') + )::bigint AS "stale", + MIN("createdAt") AS "oldest_created_at", + MAX("createdAt") AS "newest_created_at" + FROM "OutgoingRequest" + WHERE ("qstashOptions"->>'url') = '/api/latest/internal/external-db-sync/sync-engine' + ${outgoingTenancyFilter} + `).at(0) ?? throwErr("Outgoing request stats query returned no rows."); + + const projectUsersStats = formatSequenceStats(projectUserStatsRow); + const contactChannelStats = formatSequenceStats(contactChannelStatsRow); + const deletedRowStats = formatSequenceStats(deletedRowStatsRow); + + const deletedRowsByTable = deletedRowsByTableRows.map((row) => ({ + table_name: row.table_name, + ...formatSequenceStats(row), + })); + + const { mappings, mappingStatuses } = buildMappingInternalStats(projectUsersStats, deletedRowsByTable); + + return { + projectUsersStats, + contactChannelStats, + deletedRowStats, + deletedRowsByTable, + outgoingStatsRow, + mappings, + mappingStatuses, + }; +} + +function formatPollerStats(outgoingStats: OutgoingStatsRow) { + return { + total: toBigIntStringOrThrow(outgoingStats.total, "outgoing total"), + pending: toBigIntStringOrThrow(outgoingStats.pending, "outgoing pending"), + in_flight: toBigIntStringOrThrow(outgoingStats.in_flight, "outgoing in_flight"), + stale: toBigIntStringOrThrow(outgoingStats.stale, "outgoing stale"), + oldest_created_at_millis: toMillis(outgoingStats.oldest_created_at), + newest_created_at_millis: toMillis(outgoingStats.newest_created_at), + }; +} + +function formatSequenceStats(row: SequenceStatsRow) { + return { + total: toBigIntStringOrThrow(row.total, "sequence stats total"), + pending: toBigIntStringOrThrow(row.pending, "sequence stats pending"), + null_sequence_id: toBigIntStringOrThrow(row.null_sequence_id, "sequence stats null_sequence_id"), + min_sequence_id: toBigIntString(row.min_sequence_id), + max_sequence_id: toBigIntString(row.max_sequence_id), + }; +} + +function formatError(error: unknown): string { + return errorToNiceString(error); +} + +function parseConnectionString(connectionString: string | null | undefined) { + if (!connectionString) { + return { + redacted: null, + host: null, + port: null, + database: null, + user: null, + }; + } + + const parsed = Result.fromThrowing(() => new URL(connectionString)); + if (parsed.status === "error") { + return { + redacted: null, + host: null, + port: null, + database: null, + user: null, + }; + } + + const url = parsed.data; + const user = url.username ? decodeURIComponent(url.username) : null; + const host = url.hostname || null; + const port = url.port ? Number.parseInt(url.port, 10) : null; + const database = url.pathname ? url.pathname.replace(/^\//, "") : null; + const redacted = `${url.protocol}//${url.username ? encodeURIComponent(url.username) : ""}${url.username ? ":" : ""}${url.password ? "***" : ""}${url.username ? "@" : ""}${url.hostname}${url.port ? ":" + url.port : ""}${url.pathname}${url.search}`; + + return { + redacted, + host, + port: Number.isFinite(port ?? NaN) ? port : null, + database, + user, + }; +} + +async function fetchExternalDatabaseStatus( + dbId: string, + dbConfig: CompleteConfig["dbSync"]["externalDatabases"][string], + mappingStatuses: Array<{ + mapping_id: string, + internal_max_sequence_id: string | null, + }>, +) { + const connection = parseConnectionString(dbConfig.connectionString ?? null); + + if (dbConfig.type !== "postgres") { + return { + id: dbId, + type: String(dbConfig.type), + connection, + status: "error" as const, + error: `Unsupported database type: ${String(dbConfig.type)}`, + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + + if (!dbConfig.connectionString) { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: "Missing connection string", + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + + const client = new Client({ connectionString: dbConfig.connectionString }); + const connectResult = await Result.fromPromise(client.connect()); + if (connectResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(connectResult.error), + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + + let metadata: ExternalDbMetadataRow[] = []; + let metadataExists = false; + let usersExists = false; + let usersStats: UsersTableStatsRow | null = null; + + try { + const metadataExistsResult = await Result.fromPromise(client.query<{ exists: boolean }>(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = '_stack_sync_metadata' + ) AS "exists"; + `)); + if (metadataExistsResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(metadataExistsResult.error), + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + metadataExists = metadataExistsResult.data.rows[0]?.exists === true; + + const usersExistsResult = await Result.fromPromise(client.query<{ exists: boolean }>(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'users' + ) AS "exists"; + `)); + if (usersExistsResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(usersExistsResult.error), + metadata: [], + users_table: { + exists: false, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + usersExists = usersExistsResult.data.rows[0]?.exists === true; + + if (metadataExists) { + const metadataResult = await Result.fromPromise(client.query(` + SELECT "mapping_name", "last_synced_sequence_id", "updated_at" + FROM "_stack_sync_metadata" + ORDER BY "mapping_name" ASC; + `)); + if (metadataResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(metadataResult.error), + metadata: [], + users_table: { + exists: usersExists, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + metadata = metadataResult.data.rows; + } + + if (usersExists) { + const usersStatsResult = await Result.fromPromise(client.query(` + SELECT + COUNT(*)::bigint AS "total_rows", + MIN("signed_up_at") AS "min_signed_up_at", + MAX("signed_up_at") AS "max_signed_up_at" + FROM "users"; + `)); + if (usersStatsResult.status === "error") { + return { + id: dbId, + type: dbConfig.type, + connection, + status: "error" as const, + error: formatError(usersStatsResult.error), + metadata: metadata.map((row) => ({ + mapping_name: row.mapping_name, + last_synced_sequence_id: toBigIntString(row.last_synced_sequence_id) ?? "-1", + updated_at_millis: toMillis(row.updated_at), + })), + users_table: { + exists: usersExists, + total_rows: null, + min_signed_up_at_millis: null, + max_signed_up_at_millis: null, + }, + mapping_status: mappingStatuses.map((mapping) => ({ + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: null, + updated_at_millis: null, + backlog: null, + })), + }; + } + usersStats = usersStatsResult.data.rows[0] ?? null; + } + } finally { + await Result.fromPromise(client.end()); + } + + const metadataMap = new Map(); + const formattedMetadata = metadata.map((row) => { + const lastSynced = toBigIntString(row.last_synced_sequence_id) ?? "-1"; + const updatedAt = toMillis(row.updated_at); + metadataMap.set(row.mapping_name, { last_synced_sequence_id: lastSynced, updated_at_millis: updatedAt }); + return { + mapping_name: row.mapping_name, + last_synced_sequence_id: lastSynced, + updated_at_millis: updatedAt, + }; + }); + + const mappingStatus = mappingStatuses.map((mapping) => { + const external = metadataMap.get(mapping.mapping_id); + const lastSynced = external?.last_synced_sequence_id ?? null; + const updatedAt = external?.updated_at_millis ?? null; + let backlog: string | null = null; + if (mapping.internal_max_sequence_id && lastSynced) { + backlog = (BigInt(mapping.internal_max_sequence_id) - BigInt(lastSynced)).toString(); + } + return { + mapping_id: mapping.mapping_id, + internal_max_sequence_id: mapping.internal_max_sequence_id, + last_synced_sequence_id: lastSynced, + updated_at_millis: updatedAt, + backlog, + }; + }); + + return { + id: dbId, + type: dbConfig.type, + connection, + status: "ok" as const, + error: null, + metadata: formattedMetadata, + users_table: { + exists: usersExists, + total_rows: toBigIntString(usersStats?.total_rows ?? null), + min_signed_up_at_millis: toMillis(usersStats?.min_signed_up_at ?? null), + max_signed_up_at_millis: toMillis(usersStats?.max_signed_up_at ?? null), + }, + mapping_status: mappingStatus, + }; +} + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "External DB sync status", + description: "Returns sequencing, queue, and external sync progress for the current tenancy. Optional global aggregate when scope=all.", + tags: ["External DB Sync"], + hidden: true, + }, + request: yupObject({ + auth: yupObject({ + type: adminAuthTypeSchema, + tenancy: adaptSchema, + }).defined(), + query: yupObject({ + scope: yupString().oneOf(["tenancy", "all"]).default("tenancy"), + }).defined(), + method: yupString().oneOf(["GET"]).defined(), + }), + response: responseSchema, + handler: async ({ auth, query }) => { + return await traceSpan({ + description: "external-db-sync.status", + attributes: { + "stack.external-db-sync.scope": query.scope, + "stack.external-db-sync.tenancy-id": auth.tenancy.id, + }, + }, async (span) => { + if (auth.tenancy.project.id !== "internal") { + throw new KnownErrors.ExpectedInternalProject(); + } + const tenancyId = auth.tenancy.id; + + const shouldIncludeGlobal = query.scope === "all"; + span.setAttribute("stack.external-db-sync.include-global", shouldIncludeGlobal); + + const currentStats = await traceSpan({ + description: "external-db-sync.status.fetchInternalStats", + attributes: { + "stack.external-db-sync.scope": shouldIncludeGlobal ? "all" : "tenancy", + }, + }, async () => shouldIncludeGlobal ? await fetchInternalStats(null) : await fetchInternalStats(tenancyId)); + + const globalStats = shouldIncludeGlobal ? currentStats : null; + const globalTenanciesCount = shouldIncludeGlobal + ? (await globalPrismaClient.$queryRaw` + SELECT COUNT(*)::bigint AS "total" + FROM "Tenancy" + `).at(0) ?? throwErr("Tenancy count query returned no rows.") + : null; + const globalDbSyncCount = shouldIncludeGlobal + ? (await globalPrismaClient.$queryRaw` + SELECT COUNT(*)::bigint AS "total" + FROM "EnvironmentConfigOverride" + WHERE ("config"->'dbSync'->'externalDatabases') IS NOT NULL + `).at(0) ?? throwErr("DB sync config count query returned no rows.") + : null; + + const externalDbStatuses = shouldIncludeGlobal + ? [] + : await traceSpan("external-db-sync.status.fetchExternalDatabaseStatuses", async (externalSpan) => { + const statuses = await Promise.all( + Object.entries( + auth.tenancy.config.dbSync.externalDatabases as CompleteConfig["dbSync"]["externalDatabases"], + ).map(([dbId, dbConfig]) => fetchExternalDatabaseStatus(dbId, dbConfig, currentStats.mappingStatuses)), + ); + externalSpan.setAttribute("stack.external-db-sync.external-db-count", statuses.length); + return statuses; + }); + + const outgoingStats = currentStats.outgoingStatsRow; + + return { + statusCode: 200 as const, + bodyType: "json" as const, + body: { + ok: true, + generated_at_millis: Date.now(), + global: shouldIncludeGlobal && globalStats && globalTenanciesCount && globalDbSyncCount ? { + tenancies_total: toBigIntStringOrThrow(globalTenanciesCount.total, "tenancies total"), + tenancies_with_db_sync: toBigIntStringOrThrow(globalDbSyncCount.total, "tenancies with db sync"), + sequencer: { + project_users: globalStats.projectUsersStats, + contact_channels: globalStats.contactChannelStats, + deleted_rows: { + ...globalStats.deletedRowStats, + by_table: globalStats.deletedRowsByTable, + }, + }, + poller: formatPollerStats(globalStats.outgoingStatsRow), + sync_engine: { + mappings: globalStats.mappings, + }, + } : null, + tenancy: { + id: tenancyId, + project_id: auth.tenancy.project.id, + branch_id: auth.tenancy.branchId, + }, + sequencer: { + project_users: currentStats.projectUsersStats, + contact_channels: currentStats.contactChannelStats, + deleted_rows: { + ...currentStats.deletedRowStats, + by_table: currentStats.deletedRowsByTable, + }, + }, + poller: formatPollerStats(outgoingStats), + sync_engine: { + mappings: currentStats.mappings, + external_databases: externalDbStatuses, + }, + }, + }; + }); + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/external-db-sync/sync-engine/route.tsx b/apps/backend/src/app/api/latest/internal/external-db-sync/sync-engine/route.tsx new file mode 100644 index 000000000..444cca370 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/external-db-sync/sync-engine/route.tsx @@ -0,0 +1,68 @@ +import { syncExternalDatabases } from "@/lib/external-db-sync"; +import { enqueueExternalDbSync } from "@/lib/external-db-sync-queue"; +import { getTenancy } from "@/lib/tenancies"; +import { ensureUpstashSignature } from "@/lib/upstash"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { getExternalDbSyncFusebox } from "@/lib/external-db-sync-metadata"; +import { yupNumber, yupObject, yupString, yupTuple } from "@stackframe/stack-shared/dist/schema-fields"; +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { traceSpan } from "@/utils/telemetry"; + +export const POST = createSmartRouteHandler({ + metadata: { + summary: "Sync engine webhook endpoint", + description: "Receives webhook from QStash to trigger external database sync for a tenant", + tags: ["External DB Sync"], + hidden: true, + }, + request: yupObject({ + headers: yupObject({ + "upstash-signature": yupTuple([yupString()]).defined(), + }).defined(), + body: yupObject({ + tenancyId: yupString().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["success"]).defined(), + }), + handler: async ({ body }, fullReq) => { + return await traceSpan({ + description: "external-db-sync.sync-engine", + attributes: { + "stack.external-db-sync.tenancy-id": body.tenancyId, + }, + }, async (span) => { + await ensureUpstashSignature(fullReq); + const { tenancyId } = body; + + const tenancy = await traceSpan("external-db-sync.sync-engine.loadTenancy", async (tenancySpan) => { + const foundTenancy = await getTenancy(tenancyId); + tenancySpan.setAttribute("stack.external-db-sync.tenancy-found", !!foundTenancy); + return foundTenancy; + }); + if (!tenancy) { + console.warn(`[sync-engine] Tenancy ${tenancyId} in queue but not found, assuming it was deleted.`); + throw new StatusError(400, `Tenancy ${tenancyId} not found.`); + } + + const needsResync = await traceSpan("external-db-sync.sync-engine.syncExternalDatabases", async (syncSpan) => { + const resync = await syncExternalDatabases(tenancy); + syncSpan.setAttribute("stack.external-db-sync.needs-resync", resync); + return resync; + }); + if (needsResync) { + await traceSpan("external-db-sync.sync-engine.enqueueResync", async () => { + await enqueueExternalDbSync(tenancy.id); + }); + } + + return { + statusCode: 200, + bodyType: "success", + }; + }); + }, +}); diff --git a/apps/backend/src/app/api/latest/users/crud.tsx b/apps/backend/src/app/api/latest/users/crud.tsx index 49f4641b0..d76b80e62 100644 --- a/apps/backend/src/app/api/latest/users/crud.tsx +++ b/apps/backend/src/app/api/latest/users/crud.tsx @@ -2,6 +2,7 @@ import { BooleanTrue, Prisma } from "@/generated/prisma/client"; import { getRenderedOrganizationConfigQuery, getRenderedProjectConfigQuery } from "@/lib/config"; import { demoteAllContactChannelsToNonPrimary, setContactChannelAsPrimaryByValue } from "@/lib/contact-channel"; import { normalizeEmail } from "@/lib/emails"; +import { recordExternalDbSyncContactChannelDeletionsForUser, recordExternalDbSyncDeletion, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { grantDefaultProjectPermissions } from "@/lib/permissions"; import { ensureTeamMembershipExists, ensureUserExists } from "@/lib/request-checks"; import { Tenancy } from "@/lib/tenancies"; @@ -934,9 +935,9 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC isPrimary: "TRUE", }, }, - data: { + data: withExternalDbSyncUpdate({ isVerified: data.primary_email_verified, - }, + }), }); } @@ -952,9 +953,9 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC isPrimary: "TRUE", }, }, - data: { + data: withExternalDbSyncUpdate({ usedForAuth: primaryEmailAuthEnabled ? BooleanTrue.TRUE : null, - }, + }), }); } @@ -1130,7 +1131,7 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC projectUserId: params.user_id, }, }, - data: { + data: withExternalDbSyncUpdate({ displayName: data.display_name === undefined ? undefined : (data.display_name || null), clientMetadata: data.client_metadata === null ? Prisma.JsonNull : data.client_metadata, clientReadOnlyMetadata: data.client_read_only_metadata === null ? Prisma.JsonNull : data.client_read_only_metadata, @@ -1142,7 +1143,7 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC restrictedByAdmin: data.restricted_by_admin ?? undefined, restrictedByAdminReason: restrictedByAdminReason, restrictedByAdminPrivateDetails: restrictedByAdminPrivateDetails, - }, + }), include: userFullInclude, }); @@ -1189,6 +1190,17 @@ export const usersCrudHandlers = createLazyProxy(() => createCrudHandlers(usersC }, }); + await recordExternalDbSyncDeletion(tx, { + tableName: "ProjectUser", + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + + await recordExternalDbSyncContactChannelDeletionsForUser(tx, { + tenancyId: auth.tenancy.id, + projectUserId: params.user_id, + }); + await tx.projectUser.delete({ where: { tenancyId_projectUserId: { diff --git a/apps/backend/src/lib/contact-channel.tsx b/apps/backend/src/lib/contact-channel.tsx index 7b90457a4..1a8836a1e 100644 --- a/apps/backend/src/lib/contact-channel.tsx +++ b/apps/backend/src/lib/contact-channel.tsx @@ -1,4 +1,5 @@ import { BooleanTrue, ContactChannelType } from "@/generated/prisma/client"; +import { markProjectUserForExternalDbSync, withExternalDbSyncUpdate } from "@/lib/external-db-sync"; import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; import { normalizeEmail } from "./emails"; import { PrismaTransaction } from "./types"; @@ -34,9 +35,13 @@ export async function demoteAllContactChannelsToNonPrimary( type: options.type, isPrimary: BooleanTrue.TRUE, }, - data: { + data: withExternalDbSyncUpdate({ isPrimary: null, - }, + }), + }); + await markProjectUserForExternalDbSync(tx, { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, }); } @@ -100,10 +105,14 @@ export async function setContactChannelAsPrimaryById( id: options.contactChannelId, }, }, - data: { + data: withExternalDbSyncUpdate({ isPrimary: BooleanTrue.TRUE, ...options.additionalUpdates, - }, + }), + }); + await markProjectUserForExternalDbSync(tx, { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, }); } @@ -141,10 +150,14 @@ export async function setContactChannelAsPrimaryByValue( value: options.value, }, }, - data: { + data: withExternalDbSyncUpdate({ isPrimary: BooleanTrue.TRUE, ...options.additionalUpdates, - }, + }), + }); + await markProjectUserForExternalDbSync(tx, { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, }); } diff --git a/apps/backend/src/lib/external-db-sync-metadata.ts b/apps/backend/src/lib/external-db-sync-metadata.ts new file mode 100644 index 000000000..dd64cb039 --- /dev/null +++ b/apps/backend/src/lib/external-db-sync-metadata.ts @@ -0,0 +1,32 @@ +import { BooleanTrue } from "@/generated/prisma/client"; +import { globalPrismaClient } from "@/prisma-client"; + +export type ExternalDbSyncFusebox = { + sequencerEnabled: boolean, + pollerEnabled: boolean, +}; + +const fuseboxSelect = { + sequencerEnabled: true, + pollerEnabled: true, +}; + +export async function getExternalDbSyncFusebox(): Promise { + return await globalPrismaClient.externalDbSyncMetadata.upsert({ + where: { singleton: BooleanTrue.TRUE }, + create: { singleton: BooleanTrue.TRUE }, + update: {}, + select: fuseboxSelect, + }); +} + +export async function updateExternalDbSyncFusebox( + updates: ExternalDbSyncFusebox, +): Promise { + return await globalPrismaClient.externalDbSyncMetadata.upsert({ + where: { singleton: BooleanTrue.TRUE }, + create: { singleton: BooleanTrue.TRUE, ...updates }, + update: updates, + select: fuseboxSelect, + }); +} diff --git a/apps/backend/src/lib/external-db-sync-queue.ts b/apps/backend/src/lib/external-db-sync-queue.ts new file mode 100644 index 000000000..ab666893e --- /dev/null +++ b/apps/backend/src/lib/external-db-sync-queue.ts @@ -0,0 +1,43 @@ +import { globalPrismaClient } from "@/prisma-client"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; + +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function assertUuid(value: unknown, label: string): asserts value is string { + if (typeof value !== "string" || value.trim().length === 0 || !UUID_REGEX.test(value)) { + throw new StackAssertionError(`${label} must be a valid UUID. Received: ${JSON.stringify(value)}`); + } +} + +// Queues a sync request for a specific tenant if one isn't already pending. +export async function enqueueExternalDbSync(tenancyId: string): Promise { + assertUuid(tenancyId, "tenancyId"); + await enqueueExternalDbSyncBatch([tenancyId]); +} + +// Queues sync requests for multiple tenants in a single query. +// Only inserts for tenants that don't already have a pending request. +export async function enqueueExternalDbSyncBatch(tenancyIds: string[]): Promise { + if (tenancyIds.length === 0) return; + + for (const id of tenancyIds) { + assertUuid(id, "tenancyId"); + } + + // Use unnest to pass array of UUIDs and insert all in one query + await globalPrismaClient.$executeRaw` + INSERT INTO "OutgoingRequest" ("id", "createdAt", "qstashOptions", "startedFulfillingAt", "deduplicationKey") + SELECT + gen_random_uuid(), + NOW(), + json_build_object( + 'url', '/api/latest/internal/external-db-sync/sync-engine', + 'body', json_build_object('tenancyId', t.tenancy_id), + 'flowControl', json_build_object('key', 'sentinel-sync-key', 'parallelism', 20) + ), + NULL, + 'sentinel-sync-key-' || t.tenancy_id + FROM unnest(${tenancyIds}::uuid[]) AS t(tenancy_id) + ON CONFLICT ("deduplicationKey") DO NOTHING + `; +} diff --git a/apps/backend/src/lib/external-db-sync.ts b/apps/backend/src/lib/external-db-sync.ts new file mode 100644 index 000000000..57f89127a --- /dev/null +++ b/apps/backend/src/lib/external-db-sync.ts @@ -0,0 +1,498 @@ +import { Tenancy } from "@/lib/tenancies"; +import type { PrismaTransaction } from "@/lib/types"; +import { getPrismaClientForTenancy, PrismaClientWithReplica } from "@/prisma-client"; +import { Prisma } from "@/generated/prisma/client"; +import { DEFAULT_DB_SYNC_MAPPINGS } from "@stackframe/stack-shared/dist/config/db-sync-mappings"; +import type { CompleteConfig } from "@stackframe/stack-shared/dist/config/schema"; +import { captureError, StackAssertionError, throwErr } from "@stackframe/stack-shared/dist/utils/errors"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { omit } from "@stackframe/stack-shared/dist/utils/objects"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { Client } from 'pg'; + +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const MAX_BATCHES_PER_MAPPING_ENV = "STACK_EXTERNAL_DB_SYNC_MAX_BATCHES_PER_MAPPING"; + +function assertNonEmptyString(value: unknown, label: string): asserts value is string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new StackAssertionError(`${label} must be a non-empty string.`); + } +} + +function assertUuid(value: unknown, label: string): asserts value is string { + assertNonEmptyString(value, label); + if (!UUID_REGEX.test(value)) { + throw new StackAssertionError(`${label} must be a valid UUID. Received: ${JSON.stringify(value)}`); + } +} + +type ExternalDbSyncClient = PrismaTransaction | PrismaClientWithReplica; + +type ExternalDbSyncTarget = + | { + tableName: "ProjectUser", + tenancyId: string, + projectUserId: string, + } + | { + tableName: "ContactChannel", + tenancyId: string, + projectUserId: string, + contactChannelId: string, + }; + +export function withExternalDbSyncUpdate(data: T): T & { shouldUpdateSequenceId: true } { + return { + ...data, + shouldUpdateSequenceId: true, + }; +} + +export async function markProjectUserForExternalDbSync( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + } +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + await tx.projectUser.update({ + where: { + tenancyId_projectUserId: { + tenancyId: options.tenancyId, + projectUserId: options.projectUserId, + }, + }, + data: { + shouldUpdateSequenceId: true, + }, + }); +} + +export async function recordExternalDbSyncDeletion( + tx: ExternalDbSyncClient, + target: ExternalDbSyncTarget, +): Promise { + assertUuid(target.tenancyId, "tenancyId"); + assertUuid(target.projectUserId, "projectUserId"); + + if (target.tableName === "ProjectUser") { + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ProjectUser', + jsonb_build_object('tenancyId', "tenancyId", 'projectUserId', "projectUserId"), + to_jsonb("ProjectUser".*), + NOW(), + TRUE + FROM "ProjectUser" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ProjectUser, got ${insertedCount}.` + ); + } + return; + } + + assertUuid(target.contactChannelId, "contactChannelId"); + const insertedCount = await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ContactChannel', + jsonb_build_object( + 'tenancyId', + "tenancyId", + 'projectUserId', + "projectUserId", + 'id', + "id" + ), + to_jsonb("ContactChannel".*), + NOW(), + TRUE + FROM "ContactChannel" + WHERE "tenancyId" = ${target.tenancyId}::uuid + AND "projectUserId" = ${target.projectUserId}::uuid + AND "id" = ${target.contactChannelId}::uuid + FOR UPDATE + `); + + if (insertedCount !== 1) { + throw new StackAssertionError( + `Expected to insert 1 DeletedRow entry for ContactChannel, got ${insertedCount}.` + ); + } +} + +export async function recordExternalDbSyncContactChannelDeletionsForUser( + tx: ExternalDbSyncClient, + options: { + tenancyId: string, + projectUserId: string, + }, +): Promise { + assertUuid(options.tenancyId, "tenancyId"); + assertUuid(options.projectUserId, "projectUserId"); + + await tx.$executeRaw(Prisma.sql` + INSERT INTO "DeletedRow" ( + "id", + "tenancyId", + "tableName", + "primaryKey", + "data", + "deletedAt", + "shouldUpdateSequenceId" + ) + SELECT + gen_random_uuid(), + "tenancyId", + 'ContactChannel', + jsonb_build_object( + 'tenancyId', + "tenancyId", + 'projectUserId', + "projectUserId", + 'id', + "id" + ), + to_jsonb("ContactChannel".*), + NOW(), + TRUE + FROM "ContactChannel" + WHERE "tenancyId" = ${options.tenancyId}::uuid + AND "projectUserId" = ${options.projectUserId}::uuid + FOR UPDATE + `); +} + +type PgErrorLike = { + code?: string, + constraint?: string, + message?: string, +}; + +function isDuplicateTypeError(error: unknown): error is PgErrorLike { + if (!error || typeof error !== "object") return false; + const pgError = error as PgErrorLike; + return pgError.code === "23505" && pgError.constraint === "pg_type_typname_nsp_index"; +} + +function isConcurrentUpdateError(error: unknown): error is PgErrorLike { + if (!error || typeof error !== "object") return false; + const pgError = error as PgErrorLike; + // "tuple concurrently updated" occurs when multiple transactions race to modify + // the same system catalog row (e.g., during concurrent CREATE TABLE IF NOT EXISTS) + return typeof pgError.message === "string" && pgError.message.includes("tuple concurrently updated"); +} + +function getMaxBatchesPerMapping(): number | null { + const rawValue = getEnvVariable(MAX_BATCHES_PER_MAPPING_ENV, ""); + if (!rawValue) return null; + const parsed = Number.parseInt(rawValue, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new StackAssertionError( + `${MAX_BATCHES_PER_MAPPING_ENV} must be a positive integer. Received: ${JSON.stringify(rawValue)}` + ); + } + return parsed; +} + +async function ensureExternalSchema( + externalClient: Client, + tableSchemaSql: string, + tableName: string, +) { + try { + await externalClient.query(tableSchemaSql); + } catch (error) { + // Concurrent CREATE TABLE can race and cause various errors: + // - duplicate type error (23505 on pg_type_typname_nsp_index) + // - tuple concurrently updated (system catalog row modified by another transaction) + // If the table now exists, we can safely continue. + if (!isDuplicateTypeError(error) && !isConcurrentUpdateError(error)) { + throw error; + } + + const existsResult = await externalClient.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = $1 + ); + `, [tableName]); + if (existsResult.rows[0]?.exists === true) { + return; + } + + throw new StackAssertionError( + `Schema creation error while creating table ${JSON.stringify(tableName)}, but table does not exist.` + ); + } +} + +async function pushRowsToExternalDb( + externalClient: Client, + tableName: string, + newRows: any[], + upsertQuery: string, + expectedTenancyId: string, + mappingId: string, +) { + assertNonEmptyString(tableName, "tableName"); + assertNonEmptyString(mappingId, "mappingId"); + assertUuid(expectedTenancyId, "expectedTenancyId"); + if (!Array.isArray(newRows)) { + throw new StackAssertionError(`newRows must be an array for table ${JSON.stringify(tableName)}.`); + } + if (newRows.length === 0) return; + // Just for our own sanity, make sure that we have the right number of positional parameters + // The last parameter is mapping_name for metadata tracking + const placeholderMatches = upsertQuery.match(/\$\d+/g) ?? throwErr(`Could not find any positional parameters ($1, $2, ...) in the update SQL query.`); + const expectedParamCount = Math.max(...placeholderMatches.map((m: string) => Number(m.slice(1)))); + const sampleRow = newRows[0]; + const orderedKeys = Object.keys(omit(sampleRow, ["tenancyId"])); + // +1 for mapping_name parameter which is appended + if (orderedKeys.length + 1 !== expectedParamCount) { + throw new StackAssertionError(` + Column count mismatch for table ${JSON.stringify(tableName)} + → upsertQuery expects ${expectedParamCount} parameters (last one should be mapping_name). + → internalDbFetchQuery returned ${orderedKeys.length} columns (excluding tenancyId) + 1 for mapping_name = ${orderedKeys.length + 1}. + Fix your SELECT column order or your SQL parameter order. + `); + } + + for (const row of newRows) { + const { tenancyId, ...rest } = row; + + // Validate that all rows belong to the expected tenant + if (tenancyId !== expectedTenancyId) { + throw new StackAssertionError( + `Row has unexpected tenancyId. Expected ${expectedTenancyId}, got ${tenancyId}. ` + + `This indicates a bug in the internalDbFetchQuery.` + ); + } + + const rowKeys = Object.keys(rest); + + const validShape = + rowKeys.length === orderedKeys.length && + rowKeys.every((k, i) => k === orderedKeys[i]); + + if (!validShape) { + throw new StackAssertionError( + ` Row shape mismatch for table "${tableName}".\n` + + `Expected column order: [${orderedKeys.join(", ")}]\n` + + `Received column order: [${rowKeys.join(", ")}]\n` + + `Your SELECT must be explicit, ordered, and NEVER use SELECT *.\n` + + `Fix the SELECT in internalDbFetchQuery immediately.` + ); + } + + // Append mapping_name as the last parameter for metadata tracking + await externalClient.query(upsertQuery, [...Object.values(rest), mappingId]); + } +} + + +async function syncMapping( + externalClient: Client, + mappingId: string, + mapping: typeof DEFAULT_DB_SYNC_MAPPINGS[keyof typeof DEFAULT_DB_SYNC_MAPPINGS], + internalPrisma: PrismaClientWithReplica, + dbId: string, + tenancyId: string, + dbType: 'postgres', +): Promise { + assertNonEmptyString(mappingId, "mappingId"); + assertNonEmptyString(mapping.targetTable, "mapping.targetTable"); + assertUuid(tenancyId, "tenancyId"); + const fetchQuery = mapping.internalDbFetchQuery; + const updateQuery = mapping.externalDbUpdateQueries[dbType]; + const tableName = mapping.targetTable; + assertNonEmptyString(fetchQuery, "internalDbFetchQuery"); + assertNonEmptyString(updateQuery, "externalDbUpdateQueries"); + if (!fetchQuery.includes("$1") || !fetchQuery.includes("$2")) { + throw new StackAssertionError( + `internalDbFetchQuery must reference $1 (tenancyId) and $2 (lastSequenceId). Mapping: ${mappingId}` + ); + } + + const tableSchema = mapping.targetTableSchemas[dbType]; + await ensureExternalSchema(externalClient, tableSchema, tableName); + + let lastSequenceId = -1; + const metadataResult = await externalClient.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = $1`, + [mappingId] + ); + if (metadataResult.rows.length > 0) { + lastSequenceId = Number(metadataResult.rows[0].last_synced_sequence_id); + } + if (!Number.isFinite(lastSequenceId)) { + throw new StackAssertionError( + `Invalid last_synced_sequence_id for mapping ${mappingId}: ${JSON.stringify(metadataResult.rows[0]?.last_synced_sequence_id)}` + ); + } + + const BATCH_LIMIT = 1000; + const maxBatchesPerMapping = getMaxBatchesPerMapping(); + let batchesProcessed = 0; + let throttled = false; + + while (true) { + assertUuid(tenancyId, "tenancyId"); + if (!Number.isFinite(lastSequenceId)) { + throw new StackAssertionError(`lastSequenceId must be a finite number for mapping ${mappingId}.`); + } + const rows = await internalPrisma.$replica().$queryRawUnsafe(fetchQuery, tenancyId, lastSequenceId); + + if (rows.length === 0) { + break; + } + + await pushRowsToExternalDb( + externalClient, + tableName, + rows, + updateQuery, + tenancyId, + mappingId, + ); + + let maxSeqInBatch = lastSequenceId; + for (const row of rows) { + const seq = row.sequence_id; + if (seq != null) { + const seqNum = typeof seq === 'bigint' ? Number(seq) : Number(seq); + if (seqNum > maxSeqInBatch) { + maxSeqInBatch = seqNum; + } + } + } + lastSequenceId = maxSeqInBatch; + + if (rows.length < BATCH_LIMIT) { + break; + } + + batchesProcessed++; + if (maxBatchesPerMapping !== null && batchesProcessed >= maxBatchesPerMapping) { + throttled = true; + break; + } + } + + return throttled; +} + + +async function syncDatabase( + dbId: string, + dbConfig: CompleteConfig["dbSync"]["externalDatabases"][string], + internalPrisma: PrismaClientWithReplica, + tenancyId: string, +): Promise { + assertNonEmptyString(dbId, "dbId"); + assertUuid(tenancyId, "tenancyId"); + const dbType = dbConfig.type; + if (dbType !== 'postgres') { + throw new StackAssertionError( + `Unsupported database type '${String(dbType)}' for external DB ${dbId}. Only 'postgres' is currently supported.` + ); + } + + if (!dbConfig.connectionString) { + throw new StackAssertionError( + `Invalid configuration for external DB ${dbId}: 'connectionString' is missing.` + ); + } + assertNonEmptyString(dbConfig.connectionString, `external DB ${dbId} connectionString`); + + const externalClient = new Client({ + connectionString: dbConfig.connectionString, + }); + + let needsResync = false; + const syncResult = await Result.fromPromise((async () => { + await externalClient.connect(); + + // Always use DEFAULT_DB_SYNC_MAPPINGS - users cannot customize mappings + // because internalDbFetchQuery runs against Stack Auth's internal DB + for (const [mappingId, mapping] of Object.entries(DEFAULT_DB_SYNC_MAPPINGS)) { + const mappingThrottled = await syncMapping( + externalClient, + mappingId, + mapping, + internalPrisma, + dbId, + tenancyId, + dbType, + ); + if (mappingThrottled) { + needsResync = true; + } + } + })()); + + const closeResult = await Result.fromPromise(externalClient.end()); + if (closeResult.status === "error") { + captureError(`external-db-sync-${dbId}-close`, closeResult.error); + } + + if (syncResult.status === "error") { + captureError(`external-db-sync-${dbId}`, syncResult.error); + return false; + } + + return needsResync; +} + + +export async function syncExternalDatabases(tenancy: Tenancy): Promise { + assertUuid(tenancy.id, "tenancy.id"); + const externalDatabases = tenancy.config.dbSync.externalDatabases; + const internalPrisma = await getPrismaClientForTenancy(tenancy); + let needsResync = false; + + for (const [dbId, dbConfig] of Object.entries(externalDatabases)) { + try { + const databaseThrottled = await syncDatabase(dbId, dbConfig, internalPrisma, tenancy.id); + if (databaseThrottled) { + needsResync = true; + } + } catch (error) { + // Log the error but continue syncing other databases + // This ensures one bad database config doesn't block successful syncs to other databases + captureError(`external-db-sync-${dbId}`, error); + } + } + + return needsResync; +} diff --git a/apps/backend/src/lib/tokens.tsx b/apps/backend/src/lib/tokens.tsx index deda644dd..0f5f96eea 100644 --- a/apps/backend/src/lib/tokens.tsx +++ b/apps/backend/src/lib/tokens.tsx @@ -1,5 +1,6 @@ import { usersCrudHandlers } from '@/app/api/latest/users/crud'; import { getPrismaClientForTenancy, globalPrismaClient } from '@/prisma-client'; +import { withExternalDbSyncUpdate } from '@/lib/external-db-sync'; import { KnownErrors } from '@stackframe/stack-shared'; import { yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; import { AccessTokenPayload } from '@stackframe/stack-shared/dist/sessions'; @@ -244,9 +245,9 @@ export async function generateAccessTokenFromRefreshTokenIfValid(options: Refres projectUserId: options.refreshTokenObj.projectUserId, }, }, - data: { + data: withExternalDbSyncUpdate({ lastActiveAt: now, - }, + }), }), globalPrismaClient.projectUserRefreshToken.update({ where: { diff --git a/apps/backend/src/proxy.tsx b/apps/backend/src/proxy.tsx index 858ec1b9f..094340ff7 100644 --- a/apps/backend/src/proxy.tsx +++ b/apps/backend/src/proxy.tsx @@ -78,7 +78,7 @@ export async function proxy(request: NextRequest) { } : undefined; // ensure our clients can handle 429 responses - if (isApiRequest && !request.headers.get('x-stack-disable-artificial-development-delay') && getNodeEnvironment() === 'development' && request.method !== 'OPTIONS' && !request.url.includes(".well-known")) { + if (isApiRequest && !request.headers.get('x-stack-disable-artificial-development-delay') && getNodeEnvironment() === 'development' && request.method !== 'OPTIONS' && !request.url.includes(".well-known") && !request.url.includes("/api/latest/internal/external-db-sync/")) { const now = Date.now(); while (devRateLimitTimestamps.length > 0 && now - devRateLimitTimestamps[0] > DEV_RATE_LIMIT_WINDOW_MS) { devRateLimitTimestamps.shift(); diff --git a/apps/backend/vercel.json b/apps/backend/vercel.json index c90d9cdc3..f40117b3e 100644 --- a/apps/backend/vercel.json +++ b/apps/backend/vercel.json @@ -4,6 +4,14 @@ { "path": "/api/latest/internal/email-queue-step", "schedule": "* * * * *" + }, + { + "path": "/api/latest/internal/external-db-sync/poller", + "schedule": "* * * * *" + }, + { + "path": "/api/latest/internal/external-db-sync/sequencer", + "schedule": "* * * * *" } ], "github": { diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx new file mode 100644 index 000000000..3a68d8d9d --- /dev/null +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page-client.tsx @@ -0,0 +1,732 @@ +"use client"; + +import { PageLayout } from "../page-layout"; +import { useAdminApp } from "../use-admin-app"; +import { + Alert, + Button, + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, + Skeleton, + Switch, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, + Typography, +} from "@/components/ui"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { runAsynchronously, runAsynchronouslyWithAlert } from "@stackframe/stack-shared/dist/utils/promises"; +import { urlString } from "@stackframe/stack-shared/dist/utils/urls"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { notFound } from "next/navigation"; + +const stackAppInternalsSymbol = Symbol.for("StackAuth--DO-NOT-USE-OR-YOU-WILL-BE-FIRED--StackAppInternals"); +const AUTO_REFRESH_INTERVAL_MS = 5000; + +type SequenceStats = { + total: string, + pending: string, + null_sequence_id: string, + min_sequence_id: string | null, + max_sequence_id: string | null, +}; + +type DeletedRowStats = SequenceStats & { + by_table: Array, +}; + +type PollerStats = { + total: string, + pending: string, + in_flight: string, + stale: string, + oldest_created_at_millis: number | null, + newest_created_at_millis: number | null, +}; + +type MappingStats = { + mapping_id: string, + internal_min_sequence_id: string | null, + internal_max_sequence_id: string | null, + internal_pending_count: string, +}; + +type ExternalDbMetadata = { + mapping_name: string, + last_synced_sequence_id: string, + updated_at_millis: number | null, +}; + +type ExternalDbMappingStatus = { + mapping_id: string, + internal_max_sequence_id: string | null, + last_synced_sequence_id: string | null, + updated_at_millis: number | null, + backlog: string | null, +}; + +type ExternalDbSyncStatus = { + ok: true, + generated_at_millis: number, + global: { + tenancies_total: string, + tenancies_with_db_sync: string, + sequencer: { + project_users: SequenceStats, + contact_channels: SequenceStats, + deleted_rows: DeletedRowStats, + }, + poller: PollerStats, + sync_engine: { + mappings: MappingStats[], + }, + } | null, + tenancy: { + id: string, + project_id: string, + branch_id: string, + }, + sequencer: { + project_users: SequenceStats, + contact_channels: SequenceStats, + deleted_rows: DeletedRowStats, + }, + poller: PollerStats, + sync_engine: { + mappings: MappingStats[], + external_databases: Array<{ + id: string, + type: string, + connection: { + redacted: string | null, + host: string | null, + port: number | null, + database: string | null, + user: string | null, + }, + status: "ok" | "error", + error: string | null, + metadata: ExternalDbMetadata[], + users_table: { + exists: boolean, + total_rows: string | null, + min_signed_up_at_millis: number | null, + max_signed_up_at_millis: number | null, + }, + mapping_status: ExternalDbMappingStatus[], + }>, + }, +}; + +type ExternalDbSyncFusebox = { + sequencerEnabled: boolean, + pollerEnabled: boolean, +}; + +type ExternalDbSyncFuseboxResponse = { + ok: true, + sequencer_enabled: boolean, + poller_enabled: boolean, +}; + +type AdminAppInternals = { + sendRequest: (path: string, requestOptions: RequestInit, requestType?: "client" | "server" | "admin") => Promise, +}; + +type AdminAppWithInternals = ReturnType & { + [stackAppInternalsSymbol]: AdminAppInternals, +}; + +function formatBigInt(value: string | null) { + if (value === null) return "—"; + if (value.length > 15) return value; + const asNumber = Number(value); + return Number.isFinite(asNumber) ? new Intl.NumberFormat().format(asNumber) : value; +} + +function formatMillis(value: number | null) { + if (!value) return "—"; + return new Date(value).toLocaleString(); +} + +function sumBigIntStrings(values: Array) { + let total = BigInt(0); + for (const value of values) { + if (!value) continue; + if (!/^[-]?\d+$/.test(value)) continue; + total += BigInt(value); + } + return total.toString(); +} + +function parseBigIntString(value: string | null | undefined) { + if (value === null || value === undefined) return null; + if (!/^[-]?\d+$/.test(value)) return null; + return BigInt(value); +} + +const BIGINT_ZERO = BigInt(0); +const BIGINT_HUNDRED = BigInt(100); +const BIGINT_THOUSAND = BigInt(1000); + +function formatThroughput(value: bigint | number | null) { + if (value === null) return "—"; + if (typeof value === "number") { + if (!Number.isFinite(value)) return "—"; + if (value === 0) return "0/s"; + const sign = value > 0 ? "+" : ""; + const abs = Math.abs(value); + const display = abs >= 100 ? abs.toFixed(0) : abs >= 10 ? abs.toFixed(1) : abs.toFixed(2); + return `${sign}${display}/s`; + } + if (value === BIGINT_ZERO) return "0/s"; + const sign = value > BIGINT_ZERO ? "+" : ""; + const abs = value > BIGINT_ZERO ? value : -value; + const intPart = abs / BIGINT_HUNDRED; + const fracPart = abs % BIGINT_HUNDRED; + const intDisplay = new Intl.NumberFormat().format(intPart); + const fracDisplay = fracPart.toString().padStart(2, "0"); + const display = intPart === BIGINT_ZERO ? `0.${fracDisplay}` : `${intDisplay}.${fracDisplay}`; + return `${sign}${display}/s`; +} + +function calculateThroughputScaled(prev: bigint | null, current: bigint | null, deltaMillis: number) { + if (prev === null || current === null) return null; + if (deltaMillis <= 0) return null; + const deltaMillisBigInt = BigInt(deltaMillis); + return (prev - current) * BIGINT_THOUSAND * BIGINT_HUNDRED / deltaMillisBigInt; +} + +function DataValue(props: { value: string | null | undefined, loading: boolean }) { + if (props.loading) { + return ; + } + return {formatBigInt(props.value ?? null)}; +} + +function DataDate(props: { value: number | null | undefined, loading: boolean }) { + if (props.loading) { + return ; + } + return {formatMillis(props.value ?? null)}; +} + +export default function PageClient() { + const adminApp = useAdminApp() as AdminAppWithInternals; + const [status, setStatus] = useState(null); + const [fusebox, setFusebox] = useState(null); + const [savedFusebox, setSavedFusebox] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(false); + const [autoRefresh, setAutoRefresh] = useState(true); + const [savingFusebox, setSavingFusebox] = useState(false); + const inFlightRef = useRef(false); + const summarySamplesRef = useRef>([]); + + const loadStatus = useCallback(async () => { + if (inFlightRef.current) return; + inFlightRef.current = true; + setLoading(true); + + const result = await Result.fromPromise((async () => { + const response = await adminApp[stackAppInternalsSymbol].sendRequest( + "/internal/external-db-sync/status?scope=all", + { method: "GET" }, + "admin", + ); + const body = await response.json(); + if (!response.ok) { + const message = typeof body?.error === "string" ? body.error : "Failed to load external DB sync status."; + throw new Error(message); + } + return body as ExternalDbSyncStatus; + })()); + + if (result.status === "error") { + const message = result.error instanceof Error ? result.error.message : String(result.error); + setError(message); + setLoading(false); + inFlightRef.current = false; + return; + } + + setStatus(result.data); + setError(null); + setLoading(false); + inFlightRef.current = false; + }, [adminApp]); + + const loadFusebox = useCallback(async () => { + const result = await Result.fromPromise((async () => { + const response = await adminApp[stackAppInternalsSymbol].sendRequest( + urlString`/internal/external-db-sync/fusebox`, + { method: "GET" }, + "admin", + ); + const body = await response.json(); + if (!response.ok) { + const message = typeof body?.error === "string" ? body.error : "Failed to load external DB sync fusebox."; + throw new Error(message); + } + return body as ExternalDbSyncFuseboxResponse; + })()); + + if (result.status === "error") { + const message = result.error instanceof Error ? result.error.message : String(result.error); + setError(message); + return; + } + + const nextFusebox = { + sequencerEnabled: result.data.sequencer_enabled, + pollerEnabled: result.data.poller_enabled, + }; + setFusebox(nextFusebox); + setSavedFusebox(nextFusebox); + setError(null); + }, [adminApp]); + + const saveFusebox = useCallback(async () => { + if (!fusebox) return; + setSavingFusebox(true); + const result = await Result.fromPromise((async () => { + const response = await adminApp[stackAppInternalsSymbol].sendRequest( + urlString`/internal/external-db-sync/fusebox`, + { + method: "POST", + body: JSON.stringify({ + sequencer_enabled: fusebox.sequencerEnabled, + poller_enabled: fusebox.pollerEnabled, + }), + headers: { "content-type": "application/json" }, + }, + "admin", + ); + const body = await response.json(); + if (!response.ok) { + const message = typeof body?.error === "string" ? body.error : "Failed to update external DB sync fusebox."; + throw new Error(message); + } + return body as ExternalDbSyncFuseboxResponse; + })()); + setSavingFusebox(false); + + if (result.status === "error") { + const message = result.error instanceof Error ? result.error.message : String(result.error); + setError(message); + return; + } + + const nextFusebox = { + sequencerEnabled: result.data.sequencer_enabled, + pollerEnabled: result.data.poller_enabled, + }; + setFusebox(nextFusebox); + setSavedFusebox(nextFusebox); + setError(null); + }, [adminApp, fusebox]); + + const refreshWithAlert = useCallback(() => { + runAsynchronouslyWithAlert(loadStatus); + }, [loadStatus]); + + useEffect(() => { + runAsynchronously(loadStatus); + }, [loadStatus]); + + useEffect(() => { + runAsynchronously(loadFusebox); + }, [loadFusebox]); + + useEffect(() => { + if (!autoRefresh) return undefined; + const interval = setInterval(() => { + runAsynchronously(loadStatus); + }, AUTO_REFRESH_INTERVAL_MS); + return () => clearInterval(interval); + }, [autoRefresh, loadStatus]); + + const summaryStats = useMemo(() => { + if (!status) return null; + const summarySource = status.global ?? status; + const sequencerPending = sumBigIntStrings([ + summarySource.sequencer.project_users.pending, + summarySource.sequencer.contact_channels.pending, + summarySource.sequencer.deleted_rows.pending, + ]); + const mappingPending = sumBigIntStrings( + summarySource.sync_engine.mappings.map((mapping) => mapping.internal_pending_count), + ); + + return { + sequencerPending, + pollerPending: summarySource.poller.pending, + mappingPending, + isGlobal: Boolean(status.global), + }; + }, [status]); + + const throughputStats = useMemo(() => { + if (!status || !summaryStats) return null; + const currentSample = { + timestampMillis: status.generated_at_millis, + sequencerPending: summaryStats.sequencerPending, + pollerPending: summaryStats.pollerPending, + mappingPending: summaryStats.mappingPending, + }; + const samples = summarySamplesRef.current; + const samplesWithCurrent = samples.length === 0 || samples[samples.length - 1].timestampMillis !== currentSample.timestampMillis + ? [...samples, currentSample] + : samples; + const windowStart = status.generated_at_millis - 20000; + const windowedSamples = samplesWithCurrent.filter((sample) => sample.timestampMillis >= windowStart); + if (windowedSamples.length < 2) return null; + const oldest = windowedSamples[0]; + const deltaMillis = status.generated_at_millis - oldest.timestampMillis; + if (deltaMillis <= 0) return null; + + return { + sequencer: calculateThroughputScaled( + parseBigIntString(oldest.sequencerPending), + parseBigIntString(summaryStats.sequencerPending), + deltaMillis, + ), + poller: calculateThroughputScaled( + parseBigIntString(oldest.pollerPending), + parseBigIntString(summaryStats.pollerPending), + deltaMillis, + ), + mapping: calculateThroughputScaled( + parseBigIntString(oldest.mappingPending), + parseBigIntString(summaryStats.mappingPending), + deltaMillis, + ), + }; + }, [status, summaryStats]); + + useEffect(() => { + if (!status || !summaryStats) return; + const nextSamples = [...summarySamplesRef.current, { + timestampMillis: status.generated_at_millis, + sequencerPending: summaryStats.sequencerPending, + pollerPending: summaryStats.pollerPending, + mappingPending: summaryStats.mappingPending, + }]; + const windowStart = status.generated_at_millis - 20000; + summarySamplesRef.current = nextSamples.filter((sample) => sample.timestampMillis >= windowStart); + }, [status, summaryStats]); + + const loadingState = loading && !status; + const globalStatus = status?.global ?? null; + const deletedRowsByTable = status?.sequencer.deleted_rows.by_table ?? []; + const mappingRows = status?.sync_engine.mappings ?? []; + const fuseboxDirty = useMemo(() => { + if (!fusebox || !savedFusebox) return false; + return fusebox.sequencerEnabled !== savedFusebox.sequencerEnabled + || fusebox.pollerEnabled !== savedFusebox.pollerEnabled; + }, [fusebox, savedFusebox]); + + if (adminApp.projectId !== "internal") { + return notFound(); + } + + return ( + +
+ + Auto refresh +
+ + + } + fillWidth + > + {error && {error}} + +
+ Scope: {globalStatus ? "All tenancies" : "Current tenancy"} + {globalStatus && ( + <> + Tenancies: {formatBigInt(globalStatus.tenancies_total)} + DB sync configs: {formatBigInt(globalStatus.tenancies_with_db_sync)} + + )} + Last updated: {status ? formatMillis(status.generated_at_millis) : "—"} +
+ + +
+ + + Sequencer pending rows + + {loadingState ? : formatBigInt(summaryStats?.sequencerPending ?? null)} + + + +
ProjectUser + ContactChannel + DeletedRow rows waiting for sequence IDs.
+
+ Throughput + {loadingState ? "—" : formatThroughput(throughputStats?.sequencer ?? null)} +
+ + + + + + Outgoing sync requests + + {loadingState ? : formatBigInt(summaryStats?.pollerPending ?? null)} + + + +
Requests still queued for the poller to dispatch.
+
+ Throughput + {loadingState ? "—" : formatThroughput(throughputStats?.poller ?? null)} +
+ + + + + + Mapping pending rows + + {loadingState ? : formatBigInt(summaryStats?.mappingPending ?? null)} + + + +
Pending internal rows waiting for sync across mappings.
+
+ Throughput + {loadingState ? "—" : formatThroughput(throughputStats?.mapping ?? null)} +
+ + +
+ +
+ + + Sequencer + Rows awaiting sequence ID backfill per table. + + + + + + Table + Total + Pending + Null Seq + Min Seq + Max Seq + + + + + ProjectUser + + + + + + + + ContactChannel + + + + + + + + DeletedRow + + + + + + + +
+ +
+ + Deleted rows by table + + + + + Table + Total + Pending + Null Seq + Min Seq + Max Seq + + + + {deletedRowsByTable.map((row) => ( + + {row.table_name} + + + + + + + ))} + {!loadingState && deletedRowsByTable.length === 0 && ( + + + No deleted rows recorded yet. + + + )} + +
+
+ + + + + + Poller + OutgoingRequest queue and processing overview. + + + + + + Total + Pending + In Flight + Stale + + + + + + + + + + +
+ +
+
+ Oldest request + +
+
+ Newest request + +
+
+ + +
+ + + + Sync Engine + Internal mapping checkpoints before external sync. + + + + + + Mapping + Min Seq + Max Seq + Pending Rows + + + + {mappingRows.map((mapping) => ( + + {mapping.mapping_id} + + + + + ))} + {!loadingState && mappingRows.length === 0 && ( + + + No mappings configured. + + + )} + +
+ + + + + + Fusebox + + + {!fusebox ? ( +
+ + + + +
+ ) : ( + <> +
+
+ Sequencer + Assigns sequence IDs and queues sync work. +
+ setFusebox((current) => current ? { ...current, sequencerEnabled: checked } : current)} + /> +
+
+
+ Poller + Dispatches queued sync jobs to QStash. +
+ setFusebox((current) => current ? { ...current, pollerEnabled: checked } : current)} + /> +
+ +
+ +
+ + )} + + + + + ); +} diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page.tsx b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page.tsx new file mode 100644 index 000000000..60d3832f9 --- /dev/null +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/external-db-sync/page.tsx @@ -0,0 +1,9 @@ +import PageClient from "./page-client"; + +export const metadata = { + title: "External DB Sync", +}; + +export default function Page() { + return ; +} diff --git a/apps/e2e/.env.development b/apps/e2e/.env.development index 331666f8c..42b681a54 100644 --- a/apps/e2e/.env.development +++ b/apps/e2e/.env.development @@ -10,3 +10,5 @@ STACK_INBUCKET_API_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}05 STACK_SVIX_SERVER_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}13 STACK_EMAIL_MONITOR_SECRET_TOKEN=this-secret-token-is-for-local-development-only + +CRON_SECRET=mock_cron_secret diff --git a/apps/e2e/package.json b/apps/e2e/package.json index 2993b8cb1..29481d0f2 100644 --- a/apps/e2e/package.json +++ b/apps/e2e/package.json @@ -20,8 +20,10 @@ "js-beautify": "^1.15.4" }, "devDependencies": { + "@types/pg": "^8.15.6", "@types/js-beautify": "^1.14.3", - "jose": "^5.6.3" + "jose": "^5.6.3", + "pg": "^8.16.3" }, "packageManager": "pnpm@10.23.0" } diff --git a/apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts index 3e0d88b4b..5766ecef3 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts @@ -100,8 +100,8 @@ it("creates sessions that expire", async ({ expect }) => { await Auth.expectToBeSignedIn(); } finally { const timeSinceBeginDate = new Date().getTime() - beginDate.getTime(); - if (timeSinceBeginDate > 4_000) { - throw new StackAssertionError(`Timeout error: Requests were too slow (${timeSinceBeginDate}ms > 4000ms); try again or try to understand why they were slow.`); + if (timeSinceBeginDate > 6_000) { + throw new StackAssertionError(`Timeout error: Requests were too slow (${timeSinceBeginDate}ms > 6000ms); try again or try to understand why they were slow.`); } } await waitPromise; diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-advanced.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-advanced.test.ts new file mode 100644 index 000000000..34b31f601 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-advanced.test.ts @@ -0,0 +1,1124 @@ +import { Client } from 'pg'; +import { afterAll, beforeAll, describe, expect } from 'vitest'; +import { test } from '../../../../helpers'; +import { InternalApiKey, User, backendContext, niceBackendFetch } from '../../../backend-helpers'; +import { + HIGH_VOLUME_TIMEOUT, + POSTGRES_HOST, + POSTGRES_PASSWORD, + POSTGRES_USER, + TEST_TIMEOUT, + TestDbManager, + createProjectWithExternalDb as createProjectWithExternalDbRaw, + verifyNotInExternalDb, + waitForCondition, + waitForSyncedData, + waitForSyncedDeletion, + waitForTable +} from './external-db-sync-utils'; + +const COMPLEX_SEQUENCE_TIMEOUT = TEST_TIMEOUT * 2 + 30_000; + +describe.sequential('External DB Sync - Advanced Tests', () => { + let dbManager: TestDbManager; + const createProjectWithExternalDb = ( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string } + ) => { + return createProjectWithExternalDbRaw( + externalDatabases, + projectOptions, + { projectTracker: dbManager.createdProjects } + ); + }; + + beforeAll(async () => { + dbManager = new TestDbManager(); + await dbManager.init(); + }); + + afterAll(async () => { + await dbManager.cleanup(); + }); + + /** + * What it does: + * - Creates two separate projects with different external DB lists, one user per project, and triggers sync. + * - Queries every database to confirm each tenant’s user only appears in its own configured targets. + * + * Why it matters: + * - Prevents tenant data leakage by proving cross-project isolation at the sync layer. + */ + test('Multi-Tenant Isolation: User 1 -> 2 DBs, User 2 -> 3 DBs', async () => { + await InternalApiKey.createAndSetProjectKeys(); + + const db_a1 = await dbManager.createDatabase('tenant_a_db1'); + const db_a2 = await dbManager.createDatabase('tenant_a_db2'); + const db_b1 = await dbManager.createDatabase('tenant_b_db1'); + const db_b2 = await dbManager.createDatabase('tenant_b_db2'); + const db_b3 = await dbManager.createDatabase('tenant_b_db3'); + + await createProjectWithExternalDb({ + main_a1: { + type: 'postgres', + connectionString: db_a1, + }, + main_a2: { + type: 'postgres', + connectionString: db_a2, + } + }); + + const userA = await User.create({ primary_email: 'user-a@example.com' }); + await niceBackendFetch(`/api/v1/users/${userA.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User A' } + }); + + await createProjectWithExternalDb({ + main_b1: { + type: 'postgres', + connectionString: db_b1, + }, + main_b2: { + type: 'postgres', + connectionString: db_b2, + }, + main_b3: { + type: 'postgres', + connectionString: db_b3, + } + }); + + const userB = await User.create({ primary_email: 'user-b@example.com' }); + await niceBackendFetch(`/api/v1/users/${userB.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User B' } + }); + + const clientA1 = dbManager.getClient('tenant_a_db1'); + const clientA2 = dbManager.getClient('tenant_a_db2'); + const clientB1 = dbManager.getClient('tenant_b_db1'); + const clientB2 = dbManager.getClient('tenant_b_db2'); + const clientB3 = dbManager.getClient('tenant_b_db3'); + + await waitForCondition( + async () => { + try { + const res1 = await clientA1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + const res2 = await clientA2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + return res1.rows.length === 1 && res2.rows.length === 1; + } catch (err: any) { + if (err.code === '42P01') return false; + throw err; + } + }, + { description: 'User A to appear in both Project A databases', timeoutMs: 120000 } + ); + + await waitForCondition( + async () => { + try { + const res1 = await clientB1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + const res2 = await clientB2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + const res3 = await clientB3.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + return res1.rows.length === 1 && res2.rows.length === 1 && res3.rows.length === 1; + } catch (err: any) { + if (err.code === '42P01') return false; + throw err; + } + }, + { description: 'User B to appear in all three Project B databases', timeoutMs: 120000 } + ); + + const resA1 = await clientA1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resA1.rows.length).toBe(1); + expect(resA1.rows[0].display_name).toBe('User A'); + + const resA2 = await clientA2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resA2.rows.length).toBe(1); + expect(resA2.rows[0].display_name).toBe('User A'); + + const resB1_A = await clientB1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resB1_A.rows.length).toBe(0); + + const resB2_A = await clientB2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resB2_A.rows.length).toBe(0); + + const resB3_A = await clientB3.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-a@example.com']); + expect(resB3_A.rows.length).toBe(0); + + const resB1 = await clientB1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resB1.rows.length).toBe(1); + expect(resB1.rows[0].display_name).toBe('User B'); + + const resB2 = await clientB2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resB2.rows.length).toBe(1); + expect(resB2.rows[0].display_name).toBe('User B'); + + const resB3 = await clientB3.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resB3.rows.length).toBe(1); + expect(resB3.rows[0].display_name).toBe('User B'); + + const resA1_B = await clientA1.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resA1_B.rows.length).toBe(0); + + const resA2_B = await clientA2.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user-b@example.com']); + expect(resA2_B.rows.length).toBe(0); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Syncs three baseline users to capture their sequence ordering, then exports a fourth user. + * - Compares sequenceIds to ensure the newest export exceeds the previous maximum. + * + * Why it matters: + * - Verifies metadata table tracks progress correctly for incremental sync. + */ + test('Metadata Tracking: Verify sync progress is tracked in metadata table', async () => { + const dbName = 'metadata_tracking_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user1 = await User.create({ primary_email: 'seq1@example.com' }); + const user2 = await User.create({ primary_email: 'seq2@example.com' }); + const user3 = await User.create({ primary_email: 'seq3@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 1' } + }); + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 2' } + }); + await niceBackendFetch(`/api/v1/users/${user3.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 3' } + }); + + await waitForTable(client, 'users'); + + await waitForCondition( + async () => { + const res = await client.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(res.rows[0].count) === 3; + }, + { description: 'all 3 users to be synced' } + ); + + const res1 = await client.query(`SELECT * FROM "users" ORDER BY "primary_email"`); + expect(res1.rows.length).toBe(3); + + // Check metadata table tracks progress + const metadata1 = await client.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = 'users'` + ); + expect(metadata1.rows.length).toBe(1); + const seq1 = Number(metadata1.rows[0].last_synced_sequence_id); + expect(seq1).toBeGreaterThan(0); + + const user4 = await User.create({ primary_email: 'seq4@example.com' }); + await niceBackendFetch(`/api/v1/users/${user4.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 4' } + }); + + await waitForSyncedData(client, 'seq4@example.com', 'User 4'); + const res2 = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['seq4@example.com']); + expect(res2.rows.length).toBe(1); + + // Metadata should have advanced + const metadata2 = await client.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = 'users'` + ); + const seq2 = Number(metadata2.rows[0].last_synced_sequence_id); + expect(seq2).toBeGreaterThan(seq1); + + const finalRes = await client.query(`SELECT COUNT(*) as count FROM "users"`); + expect(parseInt(finalRes.rows[0].count)).toBe(4); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Exports a single user, then syncs again after adding a second user. + * - Ensures the first user's data stays untouched and both users exist. + * + * Why it matters: + * - Confirms repeated sync runs don't duplicate or rewrite already exported rows. + */ + test('Idempotency & Resume: Multiple syncs should not duplicate', async () => { + const dbName = 'idempotency_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const user1 = await User.create({ primary_email: 'user1@example.com' }); + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 1' } + }); + + const client = dbManager.getClient(dbName); + + await waitForSyncedData(client, 'user1@example.com', 'User 1'); + + let res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user1@example.com']); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('User 1'); + const user1Id = res.rows[0].id; + + const user2 = await User.create({ primary_email: 'user2@example.com' }); + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 2' } + }); + + await waitForSyncedData(client, 'user2@example.com', 'User 2'); + + const user1Row = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user1@example.com']); + const user2Row = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['user2@example.com']); + + expect(user1Row.rows.length).toBe(1); + expect(user2Row.rows.length).toBe(1); + expect(user1Row.rows[0].display_name).toBe('User 1'); + expect(user2Row.rows[0].display_name).toBe('User 2'); + // User 1's ID should be unchanged + expect(user1Row.rows[0].id).toBe(user1Id); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Exports a user whose display name contains quotes, emoji, and non-Latin characters. + * - Queries users to confirm the string survives unchanged. + * + * Why it matters: + * - Ensures text encoding and escaping don’t corrupt data during sync. + */ + test('Special Characters: Emojis, quotes, international symbols', async () => { + const dbName = 'special_chars_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const specialName = "O'Connor 🚀 用户 \"Test\""; + const user = await User.create({ primary_email: 'special@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: specialName } + }); + + await waitForSyncedData(dbManager.getClient(dbName), 'special@example.com', specialName); + + const client = dbManager.getClient(dbName); + const res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['special@example.com']); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe(specialName); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates 200 users directly in the internal database using SQL (much faster than API). + * - Waits for all of them to sync to the external database. + * + * Why it matters: + * - Exercises batching code paths to ensure high volumes eventually flush completely. + */ + test('High Volume: 200+ users to test batching', async () => { + const dbName = 'high_volume_test'; + const externalConnectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString: externalConnectionString, + } + }); + + const projectKeys = backendContext.value.projectKeys; + if (projectKeys === "no-project") throw new Error("No project keys found"); + const projectId = projectKeys.projectId; + const externalClient = dbManager.getClient(dbName); + + // Connect to internal database to insert users directly + const internalClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/stackframe`, + }); + await internalClient.connect(); + + const userCount = 200; + + try { + // Get the tenancy ID for this project + const tenancyRes = await internalClient.query( + `SELECT id FROM "Tenancy" WHERE "projectId" = $1 AND "branchId" = 'main' LIMIT 1`, + [projectId] + ); + if (tenancyRes.rows.length === 0) { + throw new Error(`Tenancy not found for project ${projectId}`); + } + const tenancyId = tenancyRes.rows[0].id; + + // Insert all 200 users in a single batch + await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + gs AS idx, + now() AS ts + FROM generate_series(1, $3::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'HV User ' || idx, + ts, + ts, + false + FROM generated + RETURNING "tenancyId", "projectUserId" + ) + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'hv-user-' || g.idx || '@test.example.com', + g.ts, + g.ts + FROM generated g + `, [tenancyId, projectId, userCount]); + + await waitForTable(externalClient, 'users'); + + await waitForCondition( + async () => { + const res = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(res.rows[0].count) >= userCount; + }, + { description: `all ${userCount} users to be synced`, timeoutMs: 120000 } + ); + + const res = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + const finalCount = parseInt(res.rows[0].count); + expect(finalCount).toBeGreaterThanOrEqual(userCount); + } finally { + await internalClient.end(); + } + }, HIGH_VOLUME_TIMEOUT); + + /** + * What it does: + * - Starts with three users, then mixes updates, deletes, and inserts before re-syncing. + * - Validates the external table reflects the final expected set. + * + * Why it matters: + * - Proves sequencing rules handle interleaved operations correctly. + */ + test('Complex Sequence: Multiple operations in different orders', async () => { + const dbName = 'complex_sequence_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const user1 = await User.create({ primary_email: 'seq1@example.com' }); + const user2 = await User.create({ primary_email: 'seq2@example.com' }); + const user3 = await User.create({ primary_email: 'seq3@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 1' } + }); + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 2' } + }); + await niceBackendFetch(`/api/v1/users/${user3.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 3' } + }); + + const client = dbManager.getClient(dbName); + + await waitForCondition( + async () => { + try { + const res = await client.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(res.rows[0].count) === 3; + } catch (err: any) { + if (err.code === '42P01') return false; + throw err; + } + }, + { description: 'initial 3 users sync', timeoutMs: 120000 } + ); + + let res = await client.query(`SELECT COUNT(*) as count FROM "users"`); + expect(parseInt(res.rows[0].count)).toBe(3); + + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 2 Updated' } + }); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + const user4 = await User.create({ primary_email: 'seq4@example.com' }); + await niceBackendFetch(`/api/v1/users/${user4.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'User 4' } + }); + + await waitForCondition( + async () => { + try { + const res = await client.query(`SELECT * FROM "users" ORDER BY "primary_email"`); + if (res.rows.length !== 3) return false; + + const emails = res.rows.map(r => r.primary_email); + if (emails.includes('seq1@example.com')) return false; + if (!emails.includes('seq2@example.com')) return false; + if (!emails.includes('seq3@example.com')) return false; + if (!emails.includes('seq4@example.com')) return false; + + const user2Row = res.rows.find(r => r.primary_email === 'seq2@example.com'); + return user2Row.display_name === 'User 2 Updated'; + } catch (err: any) { + if (err.code === '42P01') return false; + throw err; + } + }, + { description: 'final sync state correct', timeoutMs: 120000 } + ); + + res = await client.query(`SELECT * FROM "users" ORDER BY "primary_email"`); + expect(res.rows.length).toBe(3); + + const emails = res.rows.map(r => r.primary_email); + expect(emails).not.toContain('seq1@example.com'); + expect(emails).toContain('seq2@example.com'); + expect(emails).toContain('seq3@example.com'); + expect(emails).toContain('seq4@example.com'); + + const user2Row = res.rows.find(r => r.primary_email === 'seq2@example.com'); + expect(user2Row.display_name).toBe('User 2 Updated'); + }, COMPLEX_SEQUENCE_TIMEOUT); + + /** + * What it does: + * - Creates a readonly database role, grants SELECT on users, and tests SELECT/INSERT/UPDATE/DELETE commands. + * - Expects reads to succeed while writes fail. + * + * Why it matters: + * - Protects external tables from being mutated by consumers using readonly credentials. + */ + test('External write protection: readonly client cannot modify users', async () => { + const dbName = 'write_protection_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const superClient = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'write-protect@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Write Protect User' }, + }); + await waitForTable(superClient, 'users'); + await waitForSyncedData(superClient, 'write-protect@example.com', 'Write Protect User'); + + const readonlyUser = 'readonly_partialusers'; + const readonlyPassword = 'readonly_password'; + await superClient.query(`DO $$ +BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${readonlyUser}') THEN + CREATE ROLE ${readonlyUser} LOGIN PASSWORD '${readonlyPassword}'; + END IF; +END +$$;`); + + const url = new URL(connectionString); + url.username = readonlyUser; + url.password = readonlyPassword; + const readonlyClient = new Client({ connectionString: url.toString() }); + await readonlyClient.connect(); + + try { + const selectRes = await readonlyClient.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['write-protect@example.com'], + ); + expect(selectRes.rows.length).toBe(1); + await expect( + readonlyClient.query( + `INSERT INTO "users" ("id", "primary_email") VALUES (gen_random_uuid(), $1)`, + ['should-not-insert@example.com'], + ), + ).rejects.toThrow(); + + await expect( + readonlyClient.query( + `UPDATE "users" SET "display_name" = 'Hacked' WHERE "primary_email" = $1`, + ['write-protect@example.com'], + ), + ).rejects.toThrow(); + + await expect( + readonlyClient.query( + `DELETE FROM "users" WHERE "primary_email" = $1`, + ['write-protect@example.com'], + ), + ).rejects.toThrow(); + } finally { + await readonlyClient.end(); + } + }, TEST_TIMEOUT); + + /** + * What it does: + * - Patches the same user three times without syncing, then syncs once. + * - Checks users to confirm only the final name persists. + * + * Why it matters: + * - Verifies we export the latest snapshot instead of intermediate states. + */ + test('Multiple updates before sync: last update wins', async () => { + const dbName = 'multi_update_before_sync_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'multi-update@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Name v1' }, + }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Name v2' }, + }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Name v3' }, + }); + + await waitForTable(client, 'users'); + await waitForSyncedData(client, 'multi-update@example.com', 'Name v3'); + + const row = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['multi-update@example.com'], + ); + expect(row.rows.length).toBe(1); + expect(row.rows[0].display_name).toBe('Name v3'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates then deletes a user before the first sync happens. + * - Runs sync and checks that users never receives the email. + * + * Why it matters: + * - Ensures we don’t leak records that were deleted before the initial export cycle. + */ + test('Delete before first sync: row is never exported', async () => { + const dbName = 'delete_before_first_sync_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'delete-before-sync@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'To Be Deleted' }, + }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForTable(client, 'users'); + + await waitForCondition( + async () => { + const res = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['delete-before-sync@example.com'], + ); + return res.rows.length === 0; + }, + { description: 'deleted user should never appear', timeoutMs: 120000 } + ); + + const res = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['delete-before-sync@example.com'], + ); + expect(res.rows.length).toBe(0); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Syncs a user, deletes it, recreates the same email, and syncs again. + * - Compares IDs and sequenceIds to confirm the new row is distinct and persistent. + * + * Why it matters: + * - Proves a previous delete doesn’t block future users with the same email. + */ + test('Re-create same email after delete exports fresh contact channel', async () => { + const dbName = 'recreate_email_after_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + const email = 'recreate-after-delete@example.com'; + + const firstUser = await User.create({ primary_email: email }); + await niceBackendFetch(`/api/v1/users/${firstUser.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Original Export' }, + }); + + await waitForSyncedData(client, email, 'Original Export'); + + let res = await client.query( + `SELECT "id" FROM "users" WHERE "primary_email" = $1`, + [email], + ); + expect(res.rows.length).toBe(1); + const firstId = res.rows[0].id; + + await niceBackendFetch(`/api/v1/users/${firstUser.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedDeletion(client, email); + await verifyNotInExternalDb(client, email); + + const secondUser = await User.create({ primary_email: email }); + await niceBackendFetch(`/api/v1/users/${secondUser.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Recreated Export' }, + }); + + await waitForSyncedData(client, email, 'Recreated Export'); + + res = await client.query( + `SELECT "id", "display_name" FROM "users" WHERE "primary_email" = $1`, + [email], + ); + expect(res.rows.length).toBe(1); + + const recreatedRow = res.rows[0]; + expect(recreatedRow.display_name).toBe('Recreated Export'); + expect(recreatedRow.id).not.toBe(firstId); + + await waitForCondition( + async () => { + const followUp = await client.query( + `SELECT "display_name" FROM "users" WHERE "primary_email" = $1`, + [email], + ); + return followUp.rows.length === 1 && followUp.rows[0].display_name === 'Recreated Export'; + }, + { description: 'recreated row persists after extra sync', timeoutMs: 120000 }, + ); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Performs a complex sequence: create → update → update → delete → create (same email) → update + * - Syncs after each phase and verifies the external DB reflects the correct state. + * + * Why it matters: + * - Proves the sync engine handles rapid lifecycle transitions on the same email correctly. + */ + test('Complex lifecycle: create → update → update → delete → create → update', async () => { + const dbName = 'complex_lifecycle_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + const email = 'lifecycle-test@example.com'; + + const user1 = await User.create({ primary_email: email }); + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Initial Name' }, + }); + + await waitForSyncedData(client, email, 'Initial Name'); + + let res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Initial Name'); + const firstId = res.rows[0].id; + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Once' }, + }); + + await waitForSyncedData(client, email, 'Updated Once'); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Updated Once'); + expect(res.rows[0].id).toBe(firstId); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Twice' }, + }); + + await waitForSyncedData(client, email, 'Updated Twice'); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Updated Twice'); + expect(res.rows[0].id).toBe(firstId); + + await niceBackendFetch(`/api/v1/users/${user1.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedDeletion(client, email); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(0); + + const user2 = await User.create({ primary_email: email }); + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Recreated User' }, + }); + + await waitForSyncedData(client, email, 'Recreated User'); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Recreated User'); + expect(res.rows[0].id).not.toBe(firstId); + const newId = res.rows[0].id; + + await niceBackendFetch(`/api/v1/users/${user2.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Final Name' }, + }); + + await waitForSyncedData(client, email, 'Final Name'); + + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Final Name'); + expect(res.rows[0].id).toBe(newId); + }, COMPLEX_SEQUENCE_TIMEOUT); + + /** + * What it does: + * - Exports 50 users, deletes 10, inserts 10 replacements, and syncs again. + * - Validates the final users dataset contains the remaining 40 originals plus 10 replacements (total 50). + * + * Why it matters: + * - Proves high-volume batches stay accurate even when deletes and inserts interleave. + */ + test('High volume with deletes interleaved retains the expected dataset', async () => { + const dbName = 'high_volume_delete_mix_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const projectKeys = backendContext.value.projectKeys; + if (projectKeys === "no-project") throw new Error("No project keys found"); + const projectId = projectKeys.projectId; + + const externalClient = dbManager.getClient(dbName); + const initialUserCount = 50; + const deletions = 10; + const replacements = 10; + + // Connect to internal database to insert users directly + const internalClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/stackframe`, + }); + await internalClient.connect(); + + let initialUsers: { projectUserId: string, email: string }[] = []; + + try { + // Get the tenancy ID for this project + const tenancyRes = await internalClient.query( + `SELECT id FROM "Tenancy" WHERE "projectId" = $1 AND "branchId" = 'main' LIMIT 1`, + [projectId] + ); + if (tenancyRes.rows.length === 0) { + throw new Error(`Tenancy not found for project ${projectId}`); + } + const tenancyId = tenancyRes.rows[0].id; + const testRunId = `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + // Insert initial users and get their IDs back + const insertResult = await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + gs AS idx, + now() AS ts + FROM generate_series(1, $3::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'Interleave User ' || idx, + ts, + ts, + false + FROM generated + RETURNING "projectUserId" + ), + insert_contacts AS ( + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'interleave-' || g.idx || '-' || $4 || '@example.com', + g.ts, + g.ts + FROM generated g + RETURNING "projectUserId", "value" AS email + ) + SELECT "projectUserId"::text, email FROM insert_contacts ORDER BY email + `, [tenancyId, projectId, initialUserCount, testRunId]); + + initialUsers = insertResult.rows.map(row => ({ + email: row.email, + projectUserId: row.projectUserId, + })); + + await waitForTable(externalClient, 'users'); + + await waitForCondition( + async () => { + const countRes = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(countRes.rows[0].count) === initialUserCount; + }, + { description: 'initial batch exported', timeoutMs: 60000 }, + ); + + // Delete first 10 users + const deletedUsers = initialUsers.slice(0, deletions); + for (const entry of deletedUsers) { + await niceBackendFetch(`/api/v1/users/${entry.projectUserId}`, { + accessType: 'admin', + method: 'DELETE', + }); + } + await waitForCondition( + async () => { + const countRes = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(countRes.rows[0].count) === (initialUserCount - deletions); + }, + { description: 'deletions synced to external DB', timeoutMs: 180000 }, + ); + + // Insert replacement users via direct SQL + const replacementResult = await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + gs AS idx, + now() AS ts + FROM generate_series(1, $3::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'Replacement ' || idx, + ts, + ts, + false + FROM generated + RETURNING "projectUserId" + ), + insert_contacts AS ( + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'interleave-replacement-' || g.idx || '-' || $4 || '@example.com', + g.ts, + g.ts + FROM generated g + RETURNING "value" AS email + ) + SELECT email FROM insert_contacts + `, [tenancyId, projectId, replacements, testRunId]); + + const replacementEmails = replacementResult.rows.map(row => row.email); + + const expectedFinalCount = initialUserCount - deletions + replacements; + await waitForCondition( + async () => { + const countRes = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + return parseInt(countRes.rows[0].count) === expectedFinalCount; + }, + { description: 'final mixed batch exported', timeoutMs: 180000 }, + ); + + const finalRows = await externalClient.query(`SELECT "primary_email" FROM "users"`); + const finalEmails = new Set(finalRows.rows.map((row) => row.primary_email)); + expect(finalEmails.size).toBe(expectedFinalCount); + + for (const deleted of deletedUsers) { + expect(finalEmails.has(deleted.email)).toBe(false); + } + for (const survivor of initialUsers.slice(deletions)) { + expect(finalEmails.has(survivor.email)).toBe(true); + } + for (const replacement of replacementEmails) { + expect(finalEmails.has(replacement)).toBe(true); + } + } finally { + await internalClient.end(); + } + }, HIGH_VOLUME_TIMEOUT); +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts new file mode 100644 index 000000000..acc149707 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts @@ -0,0 +1,489 @@ +import { afterAll, beforeAll, describe, expect } from 'vitest'; +import { test } from '../../../../helpers'; +import { User, niceBackendFetch } from '../../../backend-helpers'; +import { + TEST_TIMEOUT, + TestDbManager, + createProjectWithExternalDb as createProjectWithExternalDbRaw, + verifyInExternalDb, + verifyNotInExternalDb, + waitForCondition, + waitForSyncedData, + waitForSyncedDeletion, + waitForTable +} from './external-db-sync-utils'; + +// Run tests sequentially to avoid concurrency issues with shared backend state +describe.sequential('External DB Sync - Basic Tests', () => { + let dbManager: TestDbManager; + const createProjectWithExternalDb = ( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string } + ) => { + return createProjectWithExternalDbRaw( + externalDatabases, + projectOptions, + { projectTracker: dbManager.createdProjects } + ); + }; + + beforeAll(async () => { + dbManager = new TestDbManager(); + await dbManager.init(); + }); + + afterAll(async () => { + await dbManager.cleanup(); + }); + + /** + * What it does: + * - Creates a user, patches the display name, and triggers the sync once. + * - Checks the users table for a matching row only after the sync completes. + * + * Why it matters: + * - Ensures inserts never appear externally until the sync pipeline runs. + */ + test('Insert: New user is synced to external DB', async () => { + const dbName = 'insert_only_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'insert-only@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Insert Only User' } + }); + + await waitForSyncedData(client, 'insert-only@example.com', 'Insert Only User'); + + await verifyInExternalDb(client, 'insert-only@example.com', 'Insert Only User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Exports a baseline row, mutates the display name, runs another sync, and reads users table. + * - Compares the stored display name to guarantee it reflects the latest mutation. + * + * Why it matters: + * - Proves updates propagate to the external DB instead of leaving stale data. + */ + test('Update: Existing user changes are reflected in external DB', async () => { + const dbName = 'update_only_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'update-only@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Before Update' } + }); + + await waitForSyncedData(client, 'update-only@example.com', 'Before Update'); + + await verifyInExternalDb(client, 'update-only@example.com', 'Before Update'); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'After Update' } + }); + + await waitForSyncedData(client, 'update-only@example.com', 'After Update'); + + await verifyInExternalDb(client, 'update-only@example.com', 'After Update'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Syncs a user into the users table, deletes the user internally, and waits for the deletion helper. + * - Queries users table to ensure the row disappears. + * + * Why it matters: + * - Validates deletion events propagate and prevent orphaned rows in external DBs. + */ + test('Delete: Deleted user is removed from external DB', async () => { + const dbName = 'delete_only_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + display_name: '🗑️ Delete Test Project', + description: 'Testing deletion sync to external database' + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'delete-only@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Delete Only User' } + }); + + await waitForSyncedData(client, 'delete-only@example.com', 'Delete Only User'); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + const deletedUserResponse = await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'GET', + }); + expect(deletedUserResponse.status).toBe(404); + + await waitForSyncedDeletion(client, 'delete-only@example.com'); + await verifyNotInExternalDb(client, 'delete-only@example.com'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user while verifying the users table is absent before sync. + * - Triggers sync, waits for table creation, and confirms the row appears afterward. + * + * Why it matters: + * - Demonstrates that syncs control both table provisioning and data export timing. + */ + test('Sync Mechanism Verification: Data appears ONLY after sync', async () => { + const dbName = 'sync_verification_test'; + const connectionString = await dbManager.createDatabase(dbName); + + const client = dbManager.getClient(dbName); + + // Verify the fresh database has no users table BEFORE we configure sync + const tableCheckBefore = await client.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'users' + ); + `); + expect(tableCheckBefore.rows[0].exists).toBe(false); + + // Now configure the external DB - this will trigger sync + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }, { + display_name: '🔄 Sync Verification Test Project', + description: 'Testing that data only appears after sync is triggered' + }); + + const user = await User.create({ primary_email: 'sync-verify@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Sync Verify User' } + }); + + // Wait for sync to create the table and populate data + await waitForTable(client, 'users'); + + await waitForSyncedData(client, 'sync-verify@example.com', 'Sync Verify User'); + await verifyInExternalDb(client, 'sync-verify@example.com', 'Sync Verify User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Runs create, update, and delete actions in order while syncing between each step. + * - Verifies the users table reflects each intermediate state. + * + * Why it matters: + * - Confirms the sync handles the entire lifecycle without leaving stale records. + */ + test('Full CRUD Lifecycle: Create, Update, Delete', async () => { + const dbName = 'crud_lifecycle_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'crud-test@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Original Name' } + }); + + await waitForSyncedData(client, 'crud-test@example.com', 'Original Name'); + + await verifyInExternalDb(client, 'crud-test@example.com', 'Original Name'); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Name' } + }); + + await waitForSyncedData(client, 'crud-test@example.com', 'Updated Name'); + await verifyInExternalDb(client, 'crud-test@example.com', 'Updated Name'); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForSyncedDeletion(client, 'crud-test@example.com'); + + await verifyNotInExternalDb(client, 'crud-test@example.com'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Syncs a user into an empty database to trigger table auto-creation. + * - Queries `information_schema` and users table to confirm the table and row exist. + * + * Why it matters: + * - Ensures mappings can provision their own schema without manual migrations. + */ + test('Automatic Table Creation', async () => { + const dbName = 'auto_table_creation_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const user = await User.create({ primary_email: 'auto-create@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Auto Create User' } + }); + + const client = dbManager.getClient(dbName); + + await waitForSyncedData(client, 'auto-create@example.com', 'Auto Create User'); + + const tableCheck = await client.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'users' + ); + `); + expect(tableCheck.rows[0].exists).toBe(true); + await verifyInExternalDb(client, 'auto-create@example.com', 'Auto Create User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Configures one valid and one invalid external DB mapping for the same project. + * - Runs sync and verifies the healthy DB still receives the exported row. + * + * Why it matters: + * - Shows a failing database connection does not block successful targets. + */ + test('Resilience: One bad DB should not crash the sync', async () => { + const goodDbName = 'resilience_good_db'; + const goodConnectionString = await dbManager.createDatabase(goodDbName); + const badConnectionString = 'postgresql://invalid:invalid@invalid:5432/invalid'; + + await createProjectWithExternalDb({ + good_db: { + type: 'postgres', + connectionString: goodConnectionString, + }, + bad_db: { + type: 'postgres', + connectionString: badConnectionString, + } + }); + + const user = await User.create({ primary_email: 'resilience@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Resilience User' } + }); + + await waitForSyncedData(dbManager.getClient(goodDbName), 'resilience@example.com', 'Resilience User'); + + const client = dbManager.getClient(goodDbName); + const res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['resilience@example.com']); + expect(res.rows.length).toBe(1); + expect(res.rows[0].display_name).toBe('Resilience User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user with a primary email and adds a secondary email. + * - Verifies only one user row exists (the new schema is user-centric, not channel-centric). + * - Confirms the primary_email field contains the primary email. + * + * Why it matters: + * - Validates that the new user-centric schema syncs users, not individual contact channels. + */ + test('User with multiple emails: Only one row synced with primary email', async () => { + const dbName = 'multi_email_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'primary@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Multi Email User' } + }); + + // Add a secondary email + const secondEmailResponse = await niceBackendFetch(`/api/v1/contact-channels`, { + accessType: 'admin', + method: 'POST', + body: { + user_id: user.userId, + type: 'email', + value: 'secondary@example.com', + is_verified: false, + used_for_auth: false, + } + }); + expect(secondEmailResponse.status).toBe(201); + + await waitForSyncedData(client, 'primary@example.com', 'Multi Email User'); + + // Should only have ONE row per user (the new schema is user-centric) + const allRows = await client.query(`SELECT * FROM "users"`); + expect(allRows.rows.length).toBe(1); + + // The row should have the primary email + expect(allRows.rows[0].primary_email).toBe('primary@example.com'); + expect(allRows.rows[0].display_name).toBe('Multi Email User'); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Creates a user, updates it multiple times, verifies each update is reflected. + * - Checks that the metadata table tracks the last synced sequence_id. + * + * Why it matters: + * - Demonstrates that updates are properly synced and metadata tracking works. + */ + test('Updates are synced correctly and metadata tracks progress', async () => { + const dbName = 'update_tracking_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + } + }); + + const client = dbManager.getClient(dbName); + + const user = await User.create({ primary_email: 'update-test@example.com' }); + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Original Name' } + }); + + await waitForSyncedData(client, 'update-test@example.com', 'Original Name'); + await verifyInExternalDb(client, 'update-test@example.com', 'Original Name'); + + // Check metadata table exists and has a positive sequence_id + const metadata1 = await client.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = 'users'` + ); + expect(metadata1.rows.length).toBe(1); + const seq1 = Number(metadata1.rows[0].last_synced_sequence_id); + expect(seq1).toBeGreaterThan(0); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Updated Name' } + }); + + await waitForSyncedData(client, 'update-test@example.com', 'Updated Name'); + await verifyInExternalDb(client, 'update-test@example.com', 'Updated Name'); + + // Metadata should have advanced + const metadata2 = await client.query( + `SELECT "last_synced_sequence_id" FROM "_stack_sync_metadata" WHERE "mapping_name" = 'users'` + ); + const seq2 = Number(metadata2.rows[0].last_synced_sequence_id); + expect(seq2).toBeGreaterThan(seq1); + }, TEST_TIMEOUT); + + + /** + * What it does: + * - Reads the external DB sync fusebox settings. + * - Writes the same values back to confirm the update endpoint. + * + * Why it matters: + * - Ensures internal fusebox controls are reachable and validated. + */ + test('Fusebox endpoint returns and accepts enablement flags', async () => { + const getResponse = await niceBackendFetch('/api/latest/internal/external-db-sync/fusebox', { + accessType: 'admin', + }); + + expect(getResponse.status).toBe(200); + expect(getResponse.body).toMatchObject({ + ok: true, + sequencer_enabled: expect.any(Boolean), + poller_enabled: expect.any(Boolean), + }); + + const postResponse = await niceBackendFetch('/api/latest/internal/external-db-sync/fusebox', { + accessType: 'admin', + method: 'POST', + body: { + sequencer_enabled: getResponse.body.sequencer_enabled, + poller_enabled: getResponse.body.poller_enabled, + }, + }); + + expect(postResponse.status).toBe(200); + expect(postResponse.body).toMatchObject({ + ok: true, + sequencer_enabled: getResponse.body.sequencer_enabled, + poller_enabled: getResponse.body.poller_enabled, + }); + }, TEST_TIMEOUT); +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-high-volume.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-high-volume.test.ts new file mode 100644 index 000000000..fbd71db00 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-high-volume.test.ts @@ -0,0 +1,187 @@ +import { Client } from 'pg'; +import { afterAll, beforeAll, describe, expect } from 'vitest'; +import { test } from '../../../../helpers'; +import { backendContext } from '../../../backend-helpers'; +import { + HIGH_VOLUME_TIMEOUT, + POSTGRES_HOST, + POSTGRES_PASSWORD, + POSTGRES_USER, + TestDbManager, + createProjectWithExternalDb as createProjectWithExternalDbRaw, + waitForCondition, + waitForTable, +} from './external-db-sync-utils'; + +// Run tests sequentially to avoid concurrency issues with shared backend state +describe.sequential('External DB Sync - High Volume Tests', () => { + let dbManager: TestDbManager; + const createProjectWithExternalDb = ( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string } + ) => { + return createProjectWithExternalDbRaw( + externalDatabases, + projectOptions, + { projectTracker: dbManager.createdProjects } + ); + }; + + beforeAll(async () => { + dbManager = new TestDbManager(); + await dbManager.init(); + }); + + afterAll(async () => { + await dbManager.cleanup(); + }, 60000); // 60 second timeout for cleanup + + /** + * What it does: + * - Creates 1500 users directly in the internal database using SQL (much faster than API) + * - Waits for all of them to sync to the external database + * + * Why it matters: + * - Ensures that when more than 1000 rows accumulate (e.g., external DB was down), + * the sync process loops and syncs all rows, not just the first 1000. + * - This tests the pagination logic in syncMapping() + */ + test('High Volume: Syncs more than 1000 users', async () => { + const dbName = 'high_volume_test'; + const externalConnectionString = await dbManager.createDatabase(dbName); + + // Create project with external DB config (this also tracks for cleanup) + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString: externalConnectionString, + } + }); + + const projectKeys = backendContext.value.projectKeys; + if (projectKeys === "no-project") throw new Error("No project keys found"); + const projectId = projectKeys.projectId; + const externalClient = dbManager.getClient(dbName); + + // Connect to internal database to insert users directly + const internalClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/stackframe`, + }); + await internalClient.connect(); + + const userCount = 1500; + console.log(`Inserting ${userCount} users directly into internal database...`); + + try { + // First, get the tenancy ID for this project + const tenancyRes = await internalClient.query( + `SELECT id FROM "Tenancy" WHERE "projectId" = $1 AND "branchId" = 'main' LIMIT 1`, + [projectId] + ); + if (tenancyRes.rows.length === 0) { + throw new Error(`Tenancy not found for project ${projectId}`); + } + const tenancyId = tenancyRes.rows[0].id; + console.log(`Found tenancy ID: ${tenancyId}`); + + // Insert users in batches using SQL + // This mimics what the users/crud.tsx does but without password hashing + const batchSize = 500; + for (let batch = 0; batch < userCount; batch += batchSize) { + const batchCount = Math.min(batchSize, userCount - batch); + const startIdx = batch + 1; + + await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + (gs + $3::int - 1) AS idx, + now() AS ts + FROM generate_series(1, $4::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'HV User ' || idx, + ts, + ts, + false + FROM generated + RETURNING "tenancyId", "projectUserId" + ) + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'hv-user-' || g.idx || '@test.example.com', + g.ts, + g.ts + FROM generated g + `, [tenancyId, projectId, startIdx, batchCount]); + + console.log(`Inserted batch ${batch / batchSize + 1}: users ${startIdx} to ${startIdx + batchCount - 1}`); + } + + // Verify users were actually inserted + const verifyRes = await internalClient.query( + `SELECT COUNT(*) as count FROM "ProjectUser" WHERE "tenancyId" = $1::uuid`, + [tenancyId] + ); + console.log(`Verified ${verifyRes.rows[0].count} users in internal DB`); + + console.log(`Waiting for sync...`); + + await waitForTable(externalClient, 'users'); + + // Wait for all users to appear in the external DB + await waitForCondition( + async () => { + const res = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + const count = parseInt(res.rows[0].count, 10); + console.log(`Synced ${count}/${userCount} users`); + return count >= userCount; + }, + { + description: `all ${userCount} users to sync to external DB`, + timeoutMs: 480000, // 8 minutes + intervalMs: 5000, // Check every 5 seconds + } + ); + + // Verify the final count + const finalRes = await externalClient.query(`SELECT COUNT(*) as count FROM "users"`); + const finalCount = parseInt(finalRes.rows[0].count, 10); + expect(finalCount).toBeGreaterThanOrEqual(userCount); + + // Spot-check a few specific users exist + const firstUser = await externalClient.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['hv-user-1@test.example.com']); + expect(firstUser.rows).toHaveLength(1); + + const middleUser = await externalClient.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, ['hv-user-750@test.example.com']); + expect(middleUser.rows).toHaveLength(1); + + const lastUser = await externalClient.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [`hv-user-${userCount}@test.example.com`]); + expect(lastUser.rows).toHaveLength(1); + + console.log(`Successfully synced all ${userCount} users!`); + } finally { + await internalClient.end(); + } + }, HIGH_VOLUME_TIMEOUT); +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts new file mode 100644 index 000000000..2cb93a491 --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts @@ -0,0 +1,410 @@ +import { Client } from 'pg'; +import { afterAll, beforeAll, describe, expect } from 'vitest'; +import { test } from '../../../../helpers'; +import { User, backendContext, niceBackendFetch } from '../../../backend-helpers'; +import { + HIGH_VOLUME_TIMEOUT, + POSTGRES_HOST, + POSTGRES_PASSWORD, + POSTGRES_USER, + TEST_TIMEOUT, + TestDbManager, + createProjectWithExternalDb as createProjectWithExternalDbRaw, + forceExternalDbSync, + waitForCondition, + waitForSyncedDeletion, + waitForTable +} from './external-db-sync-utils'; + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +describe.sequential('External DB Sync - Race Condition Tests', () => { + let dbManager: TestDbManager; + const createProjectWithExternalDb = ( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string } + ) => { + return createProjectWithExternalDbRaw( + externalDatabases, + projectOptions, + { projectTracker: dbManager.createdProjects } + ); + }; + + beforeAll(async () => { + dbManager = new TestDbManager(); + await dbManager.init(); + }); + + afterAll(async () => { + await dbManager.cleanup(); + }); + + /** + * What it does: + * - Updates a user, triggers two sync cycles concurrently, and waits for users table to show the last value. + * - Confirms only a single row exists with the final display name. + * + * Why it matters: + * - Demonstrates overlapping pollers remain idempotent instead of duplicating or reverting data. + */ + test('Concurrent sync triggers produce a single consistent export', async () => { + const dbName = 'race_parallel_sync_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + const user = await User.create({ primary_email: 'parallel-sync@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Initial Name' }, + }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Final Name' }, + }); + + await waitForTable(client, 'users'); + + await waitForCondition( + async () => { + const res = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['parallel-sync@example.com'], + ); + return res.rows.length === 1 && res.rows[0].display_name === 'Final Name'; + }, + { description: 'sync to converge on final state', timeoutMs: 90000 }, + ); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Issues a final update, deletes the user immediately afterward, and runs the deletion helper. + * - Confirms users table has zero rows for that value. + * + * Why it matters: + * - Shows delete events win over closely preceding updates, preventing stale data resurrection. + */ + test('Immediate delete after update removes the contact channel', async () => { + const dbName = 'race_update_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const client = dbManager.getClient(dbName); + const user = await User.create({ primary_email: 'update-delete@example.com' }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Before Delete' }, + }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'PATCH', + body: { display_name: 'Should Be Deleted' }, + }); + + await niceBackendFetch(`/api/v1/users/${user.userId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForTable(client, 'users'); + await waitForSyncedDeletion(client, 'update-delete@example.com'); + + const res = await client.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + ['update-delete@example.com'], + ); + expect(res.rows.length).toBe(0); + }, TEST_TIMEOUT); + + /** + * What it does: + * - Exports 300 users (forcing multi-page fetches), deletes a low-sequence contact channel, and syncs again. + * - Checks the deleted row is gone and the total count drops by exactly one. + * + * Why it matters: + * - Prevents pagination LIMIT boundaries from causing delete events to be skipped. + */ + test('Deletes near pagination boundaries are honored', async () => { + const dbName = 'race_pagination_delete_test'; + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const projectKeys = backendContext.value.projectKeys; + if (projectKeys === "no-project") throw new Error("No project keys found"); + const projectId = projectKeys.projectId; + + const externalClient = dbManager.getClient(dbName); + const totalUsers = 300; + + // Connect to internal database to insert users directly + const internalClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/stackframe`, + }); + await internalClient.connect(); + + let users: { email: string, projectUserId: string }[] = []; + + try { + // Get the tenancy ID for this project + const tenancyRes = await internalClient.query( + `SELECT id FROM "Tenancy" WHERE "projectId" = $1 AND "branchId" = 'main' LIMIT 1`, + [projectId] + ); + if (tenancyRes.rows.length === 0) { + throw new Error(`Tenancy not found for project ${projectId}`); + } + const tenancyId = tenancyRes.rows[0].id; + + // Insert all users and get their IDs back + const insertResult = await internalClient.query(` + WITH generated AS ( + SELECT + $1::uuid AS tenancy_id, + $2::uuid AS project_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS contact_id, + gs AS idx, + now() AS ts + FROM generate_series(1, $3::int) AS gs + ), + insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", + "displayName", "createdAt", "updatedAt", "isAnonymous") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'Paged User ' || idx, + ts, + ts, + false + FROM generated + RETURNING "projectUserId" + ), + insert_contacts AS ( + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", + "isVerified", "value", "createdAt", "updatedAt") + SELECT + g.tenancy_id, + g.project_user_id, + g.contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'page-user-' || g.idx || '@example.com', + g.ts, + g.ts + FROM generated g + RETURNING "projectUserId", "value" AS email + ) + SELECT "projectUserId"::text, email FROM insert_contacts ORDER BY email + `, [tenancyId, projectId, totalUsers]); + + users = insertResult.rows.map(row => ({ + email: row.email, + projectUserId: row.projectUserId, + })); + + await waitForTable(externalClient, 'users'); + + await waitForCondition( + async () => { + const res = await externalClient.query(`SELECT COUNT(*) AS count FROM "users"`); + return parseInt(res.rows[0].count, 10) === totalUsers; + }, + { description: 'initial >300 users exported', timeoutMs: 120000 }, + ); + + // Delete user at index 1 (low sequence ID) + const deletedUser = users[1]; + await niceBackendFetch(`/api/v1/users/${deletedUser.projectUserId}`, { + accessType: 'admin', + method: 'DELETE', + }); + + await waitForCondition( + async () => { + const res = await externalClient.query(`SELECT COUNT(*) AS count FROM "users"`); + return parseInt(res.rows[0].count, 10) === totalUsers - 1; + }, + { description: 'pagination delete reflected', timeoutMs: 180000 }, + ); + + const deletedRow = await externalClient.query( + `SELECT * FROM "users" WHERE "primary_email" = $1`, + [deletedUser.email], + ); + expect(deletedRow.rows.length).toBe(0); + } finally { + await internalClient.end(); + } + }, HIGH_VOLUME_TIMEOUT); + + /** + * What it does: + * - Creates overlapping database transactions that update the same row + * - Commits them at different times while sync is happening + * - Verifies that the highest sequence ID wins in the external DB + * + * Why it matters: + * - Proves true database-level race conditions are handled correctly + * - Tests that sync captures all committed changes eventually + */ + describe('Race conditions with overlapping transactions', () => { + const LOCAL_TEST_TIMEOUT = TEST_TIMEOUT + 60_000; // Must cover baseline sync + fallback sleep on slow CI + + async function setupExternalDbWithBaseline(dbName: string) { + const connectionString = await dbManager.createDatabase(dbName); + + await createProjectWithExternalDb({ + main: { + type: 'postgres', + connectionString, + }, + }); + + const externalClient = dbManager.getClient(dbName); + const user = await User.create({ primary_email: `${dbName}@example.com` }); + + // Make sure the users row exists + await waitForTable(externalClient, 'users'); + + await waitForCondition( + async () => { + const res = await externalClient.query<{ + display_name: string | null, + }>( + ` + SELECT "display_name" + FROM "users" + WHERE "primary_email" = $1 + `, + [`${dbName}@example.com`], + ); + return res.rows.length === 1; + }, + { description: `baseline row for ${dbName}`, timeoutMs: 60000 }, + ); + + const baseline = await externalClient.query<{ + display_name: string | null, + }>( + ` + SELECT "display_name" + FROM "users" + WHERE "primary_email" = $1 + `, + [`${dbName}@example.com`], + ); + + if (baseline.rows.length !== 1) { + throw new Error(`Expected baseline row for ${dbName}, got ${baseline.rows.length}`); + } + + const baselineRow = baseline.rows[0]; + const baselineDisplayName = baselineRow.display_name; + + return { + externalClient, + user, + baselineDisplayName, + }; + } + + function makeInternalDbUrl() { + const portPrefix = process.env.NEXT_PUBLIC_STACK_PORT_PREFIX || '81'; + return `postgres://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:${portPrefix}28/stackframe`; + } + + /** + * Scenario 1: + * Poller runs while a transaction is in-flight and uncommitted. + * Only the baseline committed value should be visible. + * + */ + test( + 'Poller ignores uncommitted overlapping updates', + async () => { + const dbName = 'race_uncommitted_poll_test'; + const { externalClient, user, baselineDisplayName } = + await setupExternalDbWithBaseline(dbName); + + const internalDbUrl = makeInternalDbUrl(); + const internalClient = new Client({ connectionString: internalDbUrl }); + + await internalClient.connect(); + + try { + await internalClient.query('BEGIN'); + await internalClient.query( + ` + UPDATE "ProjectUser" + SET "displayName" = 'Transaction 1', "updatedAt" = NOW() + WHERE "projectUserId" = $1 + `, + [user.userId], + ); + + const forced = await forceExternalDbSync(); + if (!forced) { + await sleep(70000); + } + + const during = await externalClient.query<{ + display_name: string | null, + }>( + ` + SELECT "display_name" + FROM "users" + WHERE "primary_email" = $1 + `, + [`${dbName}@example.com`], + ); + + expect(during.rows.length).toBe(1); + const row = during.rows[0]; + + // Uncommitted transaction should not be visible + expect(row.display_name).not.toBe('Transaction 1'); + expect(row.display_name).toBe(baselineDisplayName); + + await internalClient.query('ROLLBACK'); + } finally { + await internalClient.end(); + } + }, + LOCAL_TEST_TIMEOUT, + ); + }); +}); diff --git a/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts new file mode 100644 index 000000000..2e588e61f --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts @@ -0,0 +1,387 @@ +import { Client, ClientConfig } from 'pg'; +import { expect } from 'vitest'; +import { niceFetch, STACK_BACKEND_BASE_URL } from '../../../../helpers'; +import { InternalApiKey, Project } from '../../../backend-helpers'; + + +const PORT_PREFIX = process.env.NEXT_PUBLIC_STACK_PORT_PREFIX || '81'; +export const POSTGRES_HOST = process.env.EXTERNAL_DB_TEST_HOST || `localhost:${PORT_PREFIX}28`; +export const POSTGRES_USER = process.env.EXTERNAL_DB_TEST_USER || 'postgres'; +export const POSTGRES_PASSWORD = process.env.EXTERNAL_DB_TEST_PASSWORD || 'PASSWORD-PLACEHOLDER--uqfEC1hmmv'; +export const TEST_TIMEOUT = 240000; +export const HIGH_VOLUME_TIMEOUT = 600000; // 10 minutes for 1500+ users +const SHOULD_FORCE_EXTERNAL_DB_SYNC = process.env.STACK_FORCE_EXTERNAL_DB_SYNC === 'true'; +const FORCE_SYNC_MAX_DURATION_MS = (() => { + const raw = process.env.STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS; + if (!raw) return 5000; + const parsed = Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error('STACK_EXTERNAL_DB_SYNC_MAX_DURATION_MS must be a positive integer'); + } + return parsed; +})(); +const FORCE_SYNC_INTERVAL_MS = 2000; +let lastForcedSyncAt = -Infinity; + +// Connection settings to prevent connection leaks +const CLIENT_CONFIG: Partial = { + // Timeout for connecting (10 seconds) + connectionTimeoutMillis: 10000, + // Timeout for queries (30 seconds) + query_timeout: 30000, + // Timeout for idle connections (60 seconds) + idle_in_transaction_session_timeout: 60000, +}; + +// Track all projects created with external DB configs for cleanup +export type ProjectContext = { + projectId: string, + superSecretAdminKey: string, +}; + +/** + * Helper class to manage external test databases + */ +export class TestDbManager { + private setupClient: Client | null = null; + private databases: Map = new Map(); + private databaseNames: Set = new Set(); + public readonly createdProjects: ProjectContext[] = []; + + async init() { + this.setupClient = new Client({ + connectionString: `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/postgres`, + ...CLIENT_CONFIG, + }); + await this.setupClient.connect(); + } + + async createDatabase(dbName: string): Promise { + if (!this.setupClient) throw new Error('TestDbManager not initialized'); + + const uniqueDbName = `${dbName}_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + await this.setupClient.query(`CREATE DATABASE "${uniqueDbName}"`); + const connectionString = `postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${uniqueDbName}`; + const client = new Client({ + connectionString, + ...CLIENT_CONFIG, + }); + await client.connect(); + + this.databases.set(dbName, client); + this.databaseNames.add(uniqueDbName); + return connectionString; + } + + getClient(dbName: string): Client { + const client = this.databases.get(dbName); + if (!client) throw new Error(`Database ${dbName} not found`); + return client; + } + + async cleanup() { + // First, clean up all project configs to stop the sync cron from trying to connect + await cleanupProjectConfigs(this.createdProjects); + this.createdProjects.length = 0; + + // Close all tracked database clients + const closePromises = Array.from(this.databases.values()).map(async (client) => { + try { + await Promise.race([ + client.end(), + new Promise((_, reject) => setTimeout(() => reject(new Error('Client close timeout')), 5000)), + ]); + } catch (err) { + // Ignore errors when closing clients - they may already be closed or timed out + } + }); + await Promise.all(closePromises); + this.databases.clear(); + + if (this.setupClient) { + // Terminate all connections and drop databases + for (const dbName of this.databaseNames) { + try { + // Forcefully terminate ALL connections to this database + await this.setupClient.query(` + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = $1 + AND pid <> pg_backend_pid() + `, [dbName]); + + // Small delay to ensure connections are terminated + await new Promise(r => setTimeout(r, 100)); + + await this.setupClient.query(`DROP DATABASE IF EXISTS "${dbName}"`); + } catch (err) { + console.warn(`Failed to drop database ${dbName}:`, err); + } + } + this.databaseNames.clear(); + + try { + await this.setupClient.end(); + } catch (err) { + // Ignore errors when closing setup client + } + this.setupClient = null; + } + } +} + + +/** + * Wait for a condition to be true by polling, with timeout + */ +export async function waitForCondition( + checkFn: () => Promise, + options: { timeoutMs?: number, intervalMs?: number, description?: string } = {} +): Promise { + const { timeoutMs = 10000, intervalMs = 100, description = 'condition' } = options; + const startTime = performance.now(); + + while (performance.now() - startTime < timeoutMs) { + try { + await maybeForceExternalDbSync(); + if (await checkFn()) { + return; + } + } catch (err: any) { + // If the error is a connection error, wait and retry + if (err?.code === '57P01' || err?.code === '08006' || err?.code === '53300') { + // Connection terminated, connection failure, or too many clients + await new Promise(r => setTimeout(r, intervalMs)); + continue; + } + throw err; + } + await new Promise(r => setTimeout(r, intervalMs)); + } + + throw new Error(`Timeout waiting for ${description} after ${timeoutMs}ms`); +} + +export async function forceExternalDbSync(): Promise { + if (!SHOULD_FORCE_EXTERNAL_DB_SYNC) return false; + + const cronSecret = process.env.CRON_SECRET; + if (!cronSecret) { + throw new Error('CRON_SECRET is required when STACK_FORCE_EXTERNAL_DB_SYNC=true'); + } + + lastForcedSyncAt = performance.now(); + + await niceFetch(new URL('/api/latest/internal/external-db-sync/sequencer', STACK_BACKEND_BASE_URL), { + query: { + maxDurationMs: String(FORCE_SYNC_MAX_DURATION_MS), + stopWhenIdle: "true", + }, + headers: { + Authorization: `Bearer ${cronSecret}`, + }, + }); + await niceFetch(new URL('/api/latest/internal/external-db-sync/poller', STACK_BACKEND_BASE_URL), { + query: { + maxDurationMs: String(FORCE_SYNC_MAX_DURATION_MS), + stopWhenIdle: "true", + }, + headers: { + Authorization: `Bearer ${cronSecret}`, + }, + }); + return true; +} + +async function maybeForceExternalDbSync() { + if (!SHOULD_FORCE_EXTERNAL_DB_SYNC) return; + + const now = performance.now(); + if (now - lastForcedSyncAt < FORCE_SYNC_INTERVAL_MS) return; + + await forceExternalDbSync(); +} + +/** + * Wait for data to appear in external DB (relies on automatic cron job) + */ +export async function waitForSyncedData(client: Client, email: string, expectedName?: string) { + + await waitForCondition( + async () => { + let res; + try { + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + } catch (err: any) { + if (err && err.code === '42P01') { + return false; + } + throw err; + } + if (res.rows.length === 0) { + return false; + } + if (expectedName && res.rows[0].display_name !== expectedName) { + return false; + } + return true; + }, + { + description: `data for ${email} to appear in external DB`, + timeoutMs: 120000, + intervalMs: 500, + } + ); +} + +/** + * Wait for data to be removed from external DB (relies on automatic cron job) + */ +export async function waitForSyncedDeletion(client: Client, email: string) { + await waitForCondition( + async () => { + let res; + try { + res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + } catch (err: any) { + if (err && err.code === '42P01') { + return false; + } + throw err; + } + return res.rows.length === 0; + }, + { + description: `data for ${email} to be removed from external DB`, + timeoutMs: 120000, + intervalMs: 500, + } + ); +} + +/** + * Wait for table to be created (relies on automatic cron job) + */ +export async function waitForTable(client: Client, tableName: string) { + await waitForCondition( + async () => { + const res = await client.query(` + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = $1 + ); + `, [tableName]); + const exists = res.rows[0].exists; + return exists; + }, + { + description: `table ${tableName} to be created`, + timeoutMs: 120000, + intervalMs: 500, + } + ); +} + +/** + * Helper to verify data does NOT exist in external DB + */ +export async function verifyNotInExternalDb(client: Client, email: string) { + const res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(0); +} + +/** + * Helper to verify data DOES exist in external DB + */ +export async function verifyInExternalDb(client: Client, email: string, expectedName?: string) { + const res = await client.query(`SELECT * FROM "users" WHERE "primary_email" = $1`, [email]); + expect(res.rows.length).toBe(1); + if (expectedName) { + expect(res.rows[0].display_name).toBe(expectedName); + } + return res.rows[0]; +} + +/** + * Helper to count total users in external DB + */ +export async function countUsersInExternalDb(client: Client): Promise { + try { + const res = await client.query(`SELECT COUNT(*) FROM "users"`); + return parseInt(res.rows[0].count, 10); + } catch (err: any) { + if (err && err.code === '42P01') { + return 0; + } + throw err; + } +} + +/** + * Helper to create a project and update its config with external DB settings. + * Tracks the project for cleanup later. + */ +export async function createProjectWithExternalDb( + externalDatabases: any, + projectOptions?: { display_name?: string, description?: string }, + options?: { projectTracker?: ProjectContext[] } +) { + const project = await Project.createAndSwitch(projectOptions); + const { projectKeys } = await InternalApiKey.createAndSetProjectKeys(project.adminAccessToken); + if (!projectKeys.superSecretAdminKey) { + throw new Error('Expected super secret admin key to be present for external DB sync tests.'); + } + await Project.updateConfig({ + "dbSync.externalDatabases": externalDatabases + }); + + // Track this project for cleanup + if (options?.projectTracker) { + options.projectTracker.push({ + projectId: project.projectId, + superSecretAdminKey: projectKeys.superSecretAdminKey, + }); + } + + return project; +} + +/** + * Helper to remove external DB config from current project + */ +export async function cleanupProjectExternalDb() { + await Project.updateConfig({ + "dbSync.externalDatabases": {} + }); +} + +/** + * Clean up external DB configs for all tracked projects. + * This prevents the sync cron from trying to connect to deleted databases. + * + * Note: This function makes direct HTTP calls instead of using backendContext + * because it runs in afterAll, which is outside the test context. + */ +export async function cleanupProjectConfigs(projects: ProjectContext[]) { + for (const project of projects) { + try { + // Make direct HTTP call to clear the external DB config + await niceFetch(new URL('/api/latest/internal/config/override', STACK_BACKEND_BASE_URL), { + method: 'PATCH', + headers: { + 'Content-Type': 'application/json', + 'x-stack-access-type': 'admin', + 'x-stack-project-id': project.projectId, + 'x-stack-super-secret-admin-key': project.superSecretAdminKey, + }, + body: JSON.stringify({ + config_override_string: JSON.stringify({ "dbSync.externalDatabases": {} }) + }), + }); + } catch (err) { + // Ignore errors - project might have been deleted or config update might fail + console.warn(`Failed to cleanup project ${project.projectId}:`, err); + } + } +} diff --git a/apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql b/apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql new file mode 100644 index 000000000..269d06179 --- /dev/null +++ b/apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql @@ -0,0 +1,262 @@ +--set -a; source apps/backend/.env.development; set +a; psql "$STACK_DATABASE_CONNECTION_STRING" -v ON_ERROR_STOP=1 -f apps/e2e/tests/backend/performance/mock-external-db-sync-projects.sql + +BEGIN; + +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- NOTE: +-- - This script is intentionally heavy (1,000,000 projects + 3,000,000 users). +-- - Update BOTH settings blocks if you need a different external DB connection string. +-- - The external DB should be reachable from the backend (default uses docker postgres on port 8128). + +-- ===================================================================================== +-- 1) One million projects, one user each +-- ===================================================================================== +WITH settings AS ( + SELECT + 'postgresql://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/loadtest'::text AS external_connection_string, + 1000000::int AS project_count +), +config AS ( + SELECT jsonb_build_object( + 'dbSync', + jsonb_build_object( + 'externalDatabases', + jsonb_build_object( + 'main', + jsonb_build_object( + 'type', 'postgres', + 'connectionString', external_connection_string + ) + ) + ) + ) AS config_json + FROM settings +), +small_projects AS ( + SELECT + gen_random_uuid() AS project_id, + gen_random_uuid() AS tenancy_id, + gen_random_uuid() AS project_user_id, + gen_random_uuid() AS auth_method_id, + gen_random_uuid() AS contact_id, + gs AS idx, + lpad(gs::text, 7, '0') AS padded_idx, + now() AS ts + FROM settings + CROSS JOIN generate_series(1, settings.project_count) AS gs +), +insert_projects AS ( + INSERT INTO "Project" ("id", "displayName", "description", "isProductionMode", "ownerTeamId", "createdAt", "updatedAt") + SELECT + project_id, + 'External DB Sync Project ' || padded_idx, + 'External DB sync load test project', + FALSE, + NULL, + ts, + ts + FROM small_projects + RETURNING "id" +), +insert_tenancies AS ( + INSERT INTO "Tenancy" ("id", "projectId", "branchId", "organizationId", "hasNoOrganization", "createdAt", "updatedAt") + SELECT + tenancy_id, + project_id, + 'main', + NULL, + 'TRUE'::"BooleanTrue", + ts, + ts + FROM small_projects + RETURNING "id" +), +insert_env_config AS ( + INSERT INTO "EnvironmentConfigOverride" ("projectId", "branchId", "config", "createdAt", "updatedAt") + SELECT + project_id, + 'main', + (SELECT config_json FROM config), + ts, + ts + FROM small_projects + ON CONFLICT ("projectId", "branchId") DO UPDATE SET + "config" = EXCLUDED."config", + "updatedAt" = EXCLUDED."updatedAt" + RETURNING "projectId" +), +insert_users AS ( + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", "displayName", "projectId", "createdAt", "updatedAt") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'External Sync User ' || padded_idx, + project_id, + ts, + ts + FROM small_projects + RETURNING "tenancyId", "projectUserId" +), +insert_contacts AS ( + INSERT INTO "ContactChannel" + ("tenancyId", "projectUserId", "id", "type", "isPrimary", "usedForAuth", "isVerified", "value", "createdAt", "updatedAt") + SELECT + tenancy_id, + project_user_id, + contact_id, + 'EMAIL', + 'TRUE'::"BooleanTrue", + 'TRUE'::"BooleanTrue", + false, + 'external-sync-user-' || padded_idx || '@load.local', + ts, + ts + FROM small_projects + RETURNING "tenancyId", "projectUserId" +), +insert_auth_methods AS ( + INSERT INTO "AuthMethod" + ("tenancyId", "id", "projectUserId", "createdAt", "updatedAt") + SELECT + tenancy_id, + auth_method_id, + project_user_id, + ts, + ts + FROM small_projects + RETURNING "tenancyId", "id", "projectUserId" +) +INSERT INTO "PasswordAuthMethod" + ("tenancyId", "authMethodId", "projectUserId", "passwordHash", "createdAt", "updatedAt") +SELECT + tenancy_id, + auth_method_id, + project_user_id, + '$2a$13$TVyY/gpw9Db/w1fBeJkCgeNg2Rae2JfNqrPnSACtj.ufAO5cVF13.', + ts, + ts +FROM small_projects; + +COMMIT; + +BEGIN; + +-- ===================================================================================== +-- 2) Three projects, one million users each +-- ===================================================================================== +SET LOCAL synchronous_commit = off; + +CREATE TEMP TABLE tmp_large_projects AS +SELECT + gen_random_uuid() AS project_id, + gen_random_uuid() AS tenancy_id, + gs AS project_idx, + lpad(gs::text, 2, '0') AS padded_project_idx, + now() AS ts +FROM generate_series(1, 3) AS gs; + +INSERT INTO "Project" ("id", "displayName", "description", "isProductionMode", "ownerTeamId", "createdAt", "updatedAt") +SELECT + project_id, + 'External DB Sync Mega Project ' || padded_project_idx, + 'External DB sync load test project (mega)', + FALSE, + NULL, + ts, + ts +FROM tmp_large_projects; + +INSERT INTO "Tenancy" ("id", "projectId", "branchId", "organizationId", "hasNoOrganization", "createdAt", "updatedAt") +SELECT + tenancy_id, + project_id, + 'main', + NULL, + 'TRUE'::"BooleanTrue", + ts, + ts +FROM tmp_large_projects; + +WITH settings AS ( + SELECT + 'postgresql://postgres:PASSWORD-PLACEHOLDER--uqfEC1hmmv@localhost:8128/loadtest'::text AS external_connection_string +), +config AS ( + SELECT jsonb_build_object( + 'dbSync', + jsonb_build_object( + 'externalDatabases', + jsonb_build_object( + 'main', + jsonb_build_object( + 'type', 'postgres', + 'connectionString', external_connection_string + ) + ) + ) + ) AS config_json + FROM settings +) +INSERT INTO "EnvironmentConfigOverride" ("projectId", "branchId", "config", "createdAt", "updatedAt") +SELECT + project_id, + 'main', + (SELECT config_json FROM config), + ts, + ts +FROM tmp_large_projects +ON CONFLICT ("projectId", "branchId") DO UPDATE SET + "config" = EXCLUDED."config", + "updatedAt" = EXCLUDED."updatedAt"; + +-- ALTER TABLE "ProjectUser" DISABLE TRIGGER project_user_insert_trigger; + +DO $$ +DECLARE + users_per_project int := 1000000; + batch_size int := 10000; + batch_start int := 1; + batch_end int; +BEGIN + WHILE batch_start <= users_per_project LOOP + batch_end := LEAST(batch_start + batch_size - 1, users_per_project); + + WITH mega_users AS ( + SELECT + lp.project_id, + lp.tenancy_id, + lp.project_idx, + lp.padded_project_idx, + gs AS user_idx, + lpad(gs::text, 7, '0') AS padded_user_idx, + gen_random_uuid() AS project_user_id, + lp.ts AS ts + FROM tmp_large_projects lp + CROSS JOIN generate_series(batch_start, batch_end) AS gs + ) + INSERT INTO "ProjectUser" + ("tenancyId", "projectUserId", "mirroredProjectId", "mirroredBranchId", "displayName", "projectId", "createdAt", "updatedAt") + SELECT + tenancy_id, + project_user_id, + project_id, + 'main', + 'Mega User ' || padded_project_idx || '-' || padded_user_idx, + project_id, + ts, + ts + FROM mega_users; + + RAISE NOTICE 'Inserted users %-% of % per project', batch_start, batch_end, users_per_project; + + batch_start := batch_end + 1; + END LOOP; +END $$; + +-- ALTER TABLE "ProjectUser" ENABLE TRIGGER project_user_insert_trigger; + +COMMIT; diff --git a/apps/e2e/tests/global-setup.ts b/apps/e2e/tests/global-setup.ts index f15e5b472..99be2b1a8 100644 --- a/apps/e2e/tests/global-setup.ts +++ b/apps/e2e/tests/global-setup.ts @@ -4,6 +4,8 @@ import path from "path"; export default function globalSetup() { dotenv.config({ path: [ + ".env.test.local", + ".env.test", ".env.development.local", ".env.local", ".env.development", diff --git a/apps/e2e/tests/helpers.ts b/apps/e2e/tests/helpers.ts index 2ff135a36..bff855132 100644 --- a/apps/e2e/tests/helpers.ts +++ b/apps/e2e/tests/helpers.ts @@ -241,7 +241,7 @@ export class Mailbox { }; this.waitForMessagesWithSubjectCount = async (subject: string, minCount: number, options?: { noBody?: boolean }) => { - const maxRetries = 20; + const maxRetries = 25; let messages: MailboxMessage[] = []; for (let i = 0; i < maxRetries; i++) { messages = await this.fetchMessages(options); diff --git a/docker/dependencies/docker.compose.yaml b/docker/dependencies/docker.compose.yaml index b58ff352e..3cebc981a 100644 --- a/docker/dependencies/docker.compose.yaml +++ b/docker/dependencies/docker.compose.yaml @@ -12,6 +12,8 @@ services: POSTGRES_DB: stackframe POSTGRES_DELAY_MS: ${POSTGRES_DELAY_MS:-0} POSTGRES_INITDB_ARGS: --nosync + # Increase max_connections for E2E tests that create many databases + command: postgres -c max_connections=500 ports: - "${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}28:5432" volumes: diff --git a/docker/dev-postgres-replica/entrypoint.sh b/docker/dev-postgres-replica/entrypoint.sh index 748c2ccd0..d96a4dc64 100644 --- a/docker/dev-postgres-replica/entrypoint.sh +++ b/docker/dev-postgres-replica/entrypoint.sh @@ -42,6 +42,10 @@ if [ -z "$(ls -A ${PGDATA} 2>/dev/null)" ]; then primary_conninfo = 'host=${PRIMARY_HOST} port=${PRIMARY_PORT} user=${REPLICATOR_USER} password=${REPLICATOR_PASSWORD}' recovery_min_apply_delay = ${RECOVERY_MIN_APPLY_DELAY} hot_standby = on + +# pg_stat_statements for query stats +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.track = all EOF # Create standby.signal to indicate this is a standby diff --git a/packages/stack-shared/src/config/db-sync-mappings.ts b/packages/stack-shared/src/config/db-sync-mappings.ts new file mode 100644 index 000000000..e73113590 --- /dev/null +++ b/packages/stack-shared/src/config/db-sync-mappings.ts @@ -0,0 +1,165 @@ +export const DEFAULT_DB_SYNC_MAPPINGS = { + "users": { + sourceTables: { "ProjectUser": "ProjectUser" }, + targetTable: "users", + targetTableSchemas: { + postgres: ` + CREATE TABLE IF NOT EXISTS "users" ( + "id" uuid PRIMARY KEY NOT NULL, + "display_name" text, + "profile_image_url" text, + "primary_email" text, + "primary_email_verified" boolean NOT NULL DEFAULT false, + "signed_up_at" timestamp without time zone NOT NULL, + "client_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "client_read_only_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "server_metadata" jsonb NOT NULL DEFAULT '{}'::jsonb, + "is_anonymous" boolean NOT NULL DEFAULT false + ); + REVOKE ALL ON "users" FROM PUBLIC; + GRANT SELECT ON "users" TO PUBLIC; + + CREATE TABLE IF NOT EXISTS "_stack_sync_metadata" ( + "mapping_name" text PRIMARY KEY NOT NULL, + "last_synced_sequence_id" bigint NOT NULL DEFAULT -1, + "updated_at" timestamp without time zone NOT NULL DEFAULT now() + ); + `.trim(), + }, + internalDbFetchQuery: ` + SELECT * + FROM ( + SELECT + "ProjectUser"."projectUserId" AS "id", + "ProjectUser"."displayName" AS "display_name", + "ProjectUser"."profileImageUrl" AS "profile_image_url", + ( + SELECT "ContactChannel"."value" + FROM "ContactChannel" + WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" + AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" + AND "ContactChannel"."type" = 'EMAIL' + AND "ContactChannel"."isPrimary" = 'TRUE' + LIMIT 1 + ) AS "primary_email", + COALESCE( + ( + SELECT "ContactChannel"."isVerified" + FROM "ContactChannel" + WHERE "ContactChannel"."projectUserId" = "ProjectUser"."projectUserId" + AND "ContactChannel"."tenancyId" = "ProjectUser"."tenancyId" + AND "ContactChannel"."type" = 'EMAIL' + AND "ContactChannel"."isPrimary" = 'TRUE' + LIMIT 1 + ), + false + ) AS "primary_email_verified", + "ProjectUser"."createdAt" AS "signed_up_at", + COALESCE("ProjectUser"."clientMetadata", '{}'::jsonb) AS "client_metadata", + COALESCE("ProjectUser"."clientReadOnlyMetadata", '{}'::jsonb) AS "client_read_only_metadata", + COALESCE("ProjectUser"."serverMetadata", '{}'::jsonb) AS "server_metadata", + "ProjectUser"."isAnonymous" AS "is_anonymous", + "ProjectUser"."sequenceId" AS "sequence_id", + "ProjectUser"."tenancyId", + false AS "is_deleted" + FROM "ProjectUser" + WHERE "ProjectUser"."tenancyId" = $1::uuid + + UNION ALL + + SELECT + ("DeletedRow"."primaryKey"->>'projectUserId')::uuid AS "id", + NULL::text AS "display_name", + NULL::text AS "profile_image_url", + NULL::text AS "primary_email", + false AS "primary_email_verified", + "DeletedRow"."deletedAt"::timestamp without time zone AS "signed_up_at", + '{}'::jsonb AS "client_metadata", + '{}'::jsonb AS "client_read_only_metadata", + '{}'::jsonb AS "server_metadata", + false AS "is_anonymous", + "DeletedRow"."sequenceId" AS "sequence_id", + "DeletedRow"."tenancyId", + true AS "is_deleted" + FROM "DeletedRow" + WHERE + "DeletedRow"."tenancyId" = $1::uuid + AND "DeletedRow"."tableName" = 'ProjectUser' + ) AS "_src" + WHERE "sequence_id" IS NOT NULL + AND "sequence_id" > $2::bigint + ORDER BY "sequence_id" ASC + LIMIT 1000 + `.trim(), + // Last parameter = mapping_name (for metadata tracking) + externalDbUpdateQueries: { + postgres: ` + WITH params AS ( + SELECT + $1::uuid AS "id", + $2::text AS "display_name", + $3::text AS "profile_image_url", + $4::text AS "primary_email", + $5::boolean AS "primary_email_verified", + $6::timestamp without time zone AS "signed_up_at", + $7::jsonb AS "client_metadata", + $8::jsonb AS "client_read_only_metadata", + $9::jsonb AS "server_metadata", + $10::boolean AS "is_anonymous", + $11::bigint AS "sequence_id", + $12::boolean AS "is_deleted", + $13::text AS "mapping_name" + ), + deleted AS ( + DELETE FROM "users" u + USING params p + WHERE p."is_deleted" = true AND u."id" = p."id" + RETURNING 1 + ), + upserted AS ( + INSERT INTO "users" ( + "id", + "display_name", + "profile_image_url", + "primary_email", + "primary_email_verified", + "signed_up_at", + "client_metadata", + "client_read_only_metadata", + "server_metadata", + "is_anonymous" + ) + SELECT + p."id", + p."display_name", + p."profile_image_url", + p."primary_email", + p."primary_email_verified", + p."signed_up_at", + p."client_metadata", + p."client_read_only_metadata", + p."server_metadata", + p."is_anonymous" + FROM params p + WHERE p."is_deleted" = false + ON CONFLICT ("id") DO UPDATE SET + "display_name" = EXCLUDED."display_name", + "profile_image_url" = EXCLUDED."profile_image_url", + "primary_email" = EXCLUDED."primary_email", + "primary_email_verified" = EXCLUDED."primary_email_verified", + "signed_up_at" = EXCLUDED."signed_up_at", + "client_metadata" = EXCLUDED."client_metadata", + "client_read_only_metadata" = EXCLUDED."client_read_only_metadata", + "server_metadata" = EXCLUDED."server_metadata", + "is_anonymous" = EXCLUDED."is_anonymous" + RETURNING 1 + ) + INSERT INTO "_stack_sync_metadata" ("mapping_name", "last_synced_sequence_id", "updated_at") + SELECT p."mapping_name", p."sequence_id", now() FROM params p + ON CONFLICT ("mapping_name") DO UPDATE SET + "last_synced_sequence_id" = GREATEST("_stack_sync_metadata"."last_synced_sequence_id", EXCLUDED."last_synced_sequence_id"), + "updated_at" = now(); + `.trim(), + }, + }, +} as const; diff --git a/packages/stack-shared/src/config/schema-fuzzer.test.ts b/packages/stack-shared/src/config/schema-fuzzer.test.ts index 7b2e4ba3f..30ce228b9 100644 --- a/packages/stack-shared/src/config/schema-fuzzer.test.ts +++ b/packages/stack-shared/src/config/schema-fuzzer.test.ts @@ -62,6 +62,17 @@ const branchSchemaFuzzerConfig = [{ }], signUpRulesDefaultAction: ["allow", "reject"], }], + dbSync: [{ + externalDatabases: [{ + "some-external-db-id": [{ + type: ["postgres"] as const, + connectionString: [ + "postgres://user:password@host:port/database", + "some-connection-string", + ], + }], + }], + }], dataVault: [{ stores: [{ "some-store-id": [{ diff --git a/packages/stack-shared/src/config/schema.ts b/packages/stack-shared/src/config/schema.ts index 032919afa..494dea24d 100644 --- a/packages/stack-shared/src/config/schema.ts +++ b/packages/stack-shared/src/config/schema.ts @@ -235,6 +235,16 @@ export const branchConfigSchema = canNoLongerBeOverridden(projectConfigSchema, [ payments: branchPaymentsSchema, + dbSync: yupObject({ + externalDatabases: yupRecord( + userSpecifiedIdSchema("externalDatabaseId"), + yupObject({ + type: yupString().oneOf(['postgres']).defined(), + connectionString: yupString().defined(), + }) + ), + }), + dataVault: yupObject({ stores: yupRecord( userSpecifiedIdSchema("storeId"), @@ -637,6 +647,14 @@ const organizationConfigDefaults = { } as const) }, + + dbSync: { + externalDatabases: (key: string) => ({ + type: undefined, + connectionString: undefined, + }), + }, + dataVault: { stores: (key: string) => ({ displayName: "Unnamed Vault", @@ -1001,7 +1019,7 @@ export async function getConfigOverrideErrors(schema: T // This is how the implementation would look like, but we don't support arrays in config JSON files (besides tuples) // const arraySchema = schema as yup.ArraySchema; // const innerType = arraySchema.innerType; - // return yupArray(innerType ? getRestrictedSchema(path + ".[]", innerType as any) : undefined); + // return yupArray(innerType ? getRestrictedSchema(path + ".[]", innerType as any) : undefined()); } case "tuple": { return yupTuple(schemaInfo.items.map((s, index) => getRestrictedSchema(path + `[${index}]`, s)) as any); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 21554297e..64bc66623 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -182,7 +182,7 @@ importers: version: 1.2.1(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@sentry/nextjs': specifier: ^10.11.0 - version: 10.11.0(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.24.2)) + version: 10.11.0(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)) '@simplewebauthn/server': specifier: ^11.0.0 version: 11.0.0(encoding@0.1.13) @@ -233,7 +233,7 @@ importers: version: 1.0.6 next: specifier: 16.1.5 - version: 16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + version: 16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) nodemailer: specifier: ^6.9.10 version: 6.9.13 @@ -333,7 +333,7 @@ importers: version: 5.0.7 tsup: specifier: ^8.3.0 - version: 8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(typescript@5.8.3)(yaml@2.4.5) + version: 8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(typescript@5.8.3)(yaml@2.4.5) tsx: specifier: ^4.7.2 version: 4.15.5 @@ -672,9 +672,15 @@ importers: '@types/js-beautify': specifier: ^1.14.3 version: 1.14.3 + '@types/pg': + specifier: ^8.15.6 + version: 8.16.0 jose: specifier: ^5.6.3 version: 5.6.3 + pg: + specifier: ^8.16.3 + version: 8.16.3 apps/mock-oauth-server: dependencies: @@ -1820,7 +1826,7 @@ importers: version: 3.4.14 tsup: specifier: ^8.0.2 - version: 8.1.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(postcss@8.4.47)(typescript@5.8.3) + version: 8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0) packages/stack-sc: dependencies: @@ -2284,7 +2290,6 @@ packages: '@assistant-ui/react-edge@0.2.12': resolution: {integrity: sha512-95Y912lW8ASMT52qZd6ZHRiF+T7WxbeJ1yb2z/I0lCKegPt0q3spGy92YnO7mwz0uJaNjqu4/oZZybYfeIDzJg==} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. peerDependencies: '@assistant-ui/react': '*' '@types/react': ^18.2.0 @@ -2583,10 +2588,6 @@ packages: resolution: {integrity: sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==} engines: {node: '>=6.9.0'} - '@babel/core@7.28.0': - resolution: {integrity: sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==} - engines: {node: '>=6.9.0'} - '@babel/core@7.28.5': resolution: {integrity: sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==} engines: {node: '>=6.9.0'} @@ -2599,10 +2600,6 @@ packages: resolution: {integrity: sha512-kEWdzjOAUMW4hAyrzJ0ZaTOu9OmpyDIQicIh0zg0EEcEkYXZb2TjtBhnHi2ViX7PKwZqF4xwqfAm299/QMP3lg==} engines: {node: '>=6.9.0'} - '@babel/generator@7.28.0': - resolution: {integrity: sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==} - engines: {node: '>=6.9.0'} - '@babel/generator@7.28.5': resolution: {integrity: sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==} engines: {node: '>=6.9.0'} @@ -2651,12 +2648,6 @@ packages: peerDependencies: '@babel/core': ^7.0.0 - '@babel/helper-module-transforms@7.27.3': - resolution: {integrity: sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0 - '@babel/helper-module-transforms@7.28.3': resolution: {integrity: sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==} engines: {node: '>=6.9.0'} @@ -2667,10 +2658,6 @@ packages: resolution: {integrity: sha512-jKiTsW2xmWwxT1ixIdfXUZp+P5yURx2suzLZr5Hi64rURpDYdMW0pv+Uf17EYk2Rd428Lx4tLsnjGJzYKDM/6A==} engines: {node: '>=6.9.0'} - '@babel/helper-plugin-utils@7.24.8': - resolution: {integrity: sha512-FFWx5142D8h2Mgr/iPVGH5G7w6jDn4jUSpZTyDnQO0Yn7Ks2Kuz6Pci8H6MPCoUJegd/UZQ3tAvfLCxQSnWWwg==} - engines: {node: '>=6.9.0'} - '@babel/helper-plugin-utils@7.27.1': resolution: {integrity: sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==} engines: {node: '>=6.9.0'} @@ -2725,10 +2712,6 @@ packages: resolution: {integrity: sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==} engines: {node: '>=6.9.0'} - '@babel/helpers@7.27.6': - resolution: {integrity: sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==} - engines: {node: '>=6.9.0'} - '@babel/helpers@7.28.4': resolution: {integrity: sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w==} engines: {node: '>=6.9.0'} @@ -2839,10 +2822,6 @@ packages: resolution: {integrity: sha512-ZYW7L+pL8ahU5fXmNbPF+iZFHCv5scFak7MZ9bwaRPLUhHh7QQEMjZUg0HevihoqCM5iSYHN61EyCoZvqC+bxg==} engines: {node: '>=6.9.0'} - '@babel/traverse@7.28.0': - resolution: {integrity: sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==} - engines: {node: '>=6.9.0'} - '@babel/traverse@7.28.5': resolution: {integrity: sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==} engines: {node: '>=6.9.0'} @@ -2863,10 +2842,6 @@ packages: resolution: {integrity: sha512-jYnje+JyZG5YThjHiF28oT4SIZLnYOcSBb6+SDaFIyzDVSkXQmQQYclJ2R+YxcdmK0AX6x1E5OQNtuh3jHDrUg==} engines: {node: '>=6.9.0'} - '@babel/types@7.28.1': - resolution: {integrity: sha512-x0LvFTekgSX+83TI28Y9wYPUfzrnl2aT5+5QLnO6v7mSJYtEEevuDRN0F0uSHRk1G1IWZC43o00Y0xDDrpBGPQ==} - engines: {node: '>=6.9.0'} - '@babel/types@7.28.5': resolution: {integrity: sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==} engines: {node: '>=6.9.0'} @@ -4639,16 +4614,9 @@ packages: resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - '@jridgewell/gen-mapping@0.3.12': - resolution: {integrity: sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg==} - '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} - '@jridgewell/gen-mapping@0.3.5': - resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==} - engines: {node: '>=6.0.0'} - '@jridgewell/remapping@2.3.5': resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} @@ -4656,10 +4624,6 @@ packages: resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} engines: {node: '>=6.0.0'} - '@jridgewell/set-array@1.2.1': - resolution: {integrity: sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==} - engines: {node: '>=6.0.0'} - '@jridgewell/source-map@0.3.11': resolution: {integrity: sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==} @@ -4672,9 +4636,6 @@ packages: '@jridgewell/trace-mapping@0.3.25': resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} - '@jridgewell/trace-mapping@0.3.29': - resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} - '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} @@ -7317,11 +7278,6 @@ packages: rollup: optional: true - '@rollup/rollup-android-arm-eabi@4.18.0': - resolution: {integrity: sha512-Tya6xypR10giZV1XzxmH5wr25VcZSncG0pZIjfePT0OVBvqNEurzValetGNarVrGiq66EBVAFn15iYX4w6FKgQ==} - cpu: [arm] - os: [android] - '@rollup/rollup-android-arm-eabi@4.24.4': resolution: {integrity: sha512-jfUJrFct/hTA0XDM5p/htWKoNNTbDLY0KRwEt6pyOA6k2fmk0WVwl65PdUdJZgzGEHWx+49LilkcSaumQRyNQw==} cpu: [arm] @@ -7337,11 +7293,6 @@ packages: cpu: [arm] os: [android] - '@rollup/rollup-android-arm64@4.18.0': - resolution: {integrity: sha512-avCea0RAP03lTsDhEyfy+hpfr85KfyTctMADqHVhLAF3MlIkq83CP8UfAHUssgXTYd+6er6PaAhx/QGv4L1EiA==} - cpu: [arm64] - os: [android] - '@rollup/rollup-android-arm64@4.24.4': resolution: {integrity: sha512-j4nrEO6nHU1nZUuCfRKoCcvh7PIywQPUCBa2UsootTHvTHIoIu2BzueInGJhhvQO/2FTRdNYpf63xsgEqH9IhA==} cpu: [arm64] @@ -7357,11 +7308,6 @@ packages: cpu: [arm64] os: [android] - '@rollup/rollup-darwin-arm64@4.18.0': - resolution: {integrity: sha512-IWfdwU7KDSm07Ty0PuA/W2JYoZ4iTj3TUQjkVsO/6U+4I1jN5lcR71ZEvRh52sDOERdnNhhHU57UITXz5jC1/w==} - cpu: [arm64] - os: [darwin] - '@rollup/rollup-darwin-arm64@4.24.4': resolution: {integrity: sha512-GmU/QgGtBTeraKyldC7cDVVvAJEOr3dFLKneez/n7BvX57UdhOqDsVwzU7UOnYA7AAOt+Xb26lk79PldDHgMIQ==} cpu: [arm64] @@ -7377,11 +7323,6 @@ packages: cpu: [arm64] os: [darwin] - '@rollup/rollup-darwin-x64@4.18.0': - resolution: {integrity: sha512-n2LMsUz7Ynu7DoQrSQkBf8iNrjOGyPLrdSg802vk6XT3FtsgX6JbE8IHRvposskFm9SNxzkLYGSq9QdpLYpRNA==} - cpu: [x64] - os: [darwin] - '@rollup/rollup-darwin-x64@4.24.4': resolution: {integrity: sha512-N6oDBiZCBKlwYcsEPXGDE4g9RoxZLK6vT98M8111cW7VsVJFpNEqvJeIPfsCzbf0XEakPslh72X0gnlMi4Ddgg==} cpu: [x64] @@ -7427,12 +7368,6 @@ packages: cpu: [x64] os: [freebsd] - '@rollup/rollup-linux-arm-gnueabihf@4.18.0': - resolution: {integrity: sha512-C/zbRYRXFjWvz9Z4haRxcTdnkPt1BtCkz+7RtBSuNmKzMzp3ZxdM28Mpccn6pt28/UWUCTXa+b0Mx1k3g6NOMA==} - cpu: [arm] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-arm-gnueabihf@4.24.4': resolution: {integrity: sha512-10ICosOwYChROdQoQo589N5idQIisxjaFE/PAnX2i0Zr84mY0k9zul1ArH0rnJ/fpgiqfu13TFZR5A5YJLOYZA==} cpu: [arm] @@ -7451,12 +7386,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-arm-musleabihf@4.18.0': - resolution: {integrity: sha512-l3m9ewPgjQSXrUMHg93vt0hYCGnrMOcUpTz6FLtbwljo2HluS4zTXFy2571YQbisTnfTKPZ01u/ukJdQTLGh9A==} - cpu: [arm] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-arm-musleabihf@4.24.4': resolution: {integrity: sha512-ySAfWs69LYC7QhRDZNKqNhz2UKN8LDfbKSMAEtoEI0jitwfAG2iZwVqGACJT+kfYvvz3/JgsLlcBP+WWoKCLcw==} cpu: [arm] @@ -7475,12 +7404,6 @@ packages: os: [linux] libc: [musl] - '@rollup/rollup-linux-arm64-gnu@4.18.0': - resolution: {integrity: sha512-rJ5D47d8WD7J+7STKdCUAgmQk49xuFrRi9pZkWoRD1UeSMakbcepWXPF8ycChBoAqs1pb2wzvbY6Q33WmN2ftw==} - cpu: [arm64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-arm64-gnu@4.24.4': resolution: {integrity: sha512-uHYJ0HNOI6pGEeZ/5mgm5arNVTI0nLlmrbdph+pGXpC9tFHFDQmDMOEqkmUObRfosJqpU8RliYoGz06qSdtcjg==} cpu: [arm64] @@ -7499,12 +7422,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-arm64-musl@4.18.0': - resolution: {integrity: sha512-be6Yx37b24ZwxQ+wOQXXLZqpq4jTckJhtGlWGZs68TgdKXJgw54lUUoFYrg6Zs/kjzAQwEwYbp8JxZVzZLRepQ==} - cpu: [arm64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-arm64-musl@4.24.4': resolution: {integrity: sha512-38yiWLemQf7aLHDgTg85fh3hW9stJ0Muk7+s6tIkSUOMmi4Xbv5pH/5Bofnsb6spIwD5FJiR+jg71f0CH5OzoA==} cpu: [arm64] @@ -7535,12 +7452,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-powerpc64le-gnu@4.18.0': - resolution: {integrity: sha512-hNVMQK+qrA9Todu9+wqrXOHxFiD5YmdEi3paj6vP02Kx1hjd2LLYR2eaN7DsEshg09+9uzWi2W18MJDlG0cxJA==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-powerpc64le-gnu@4.24.4': resolution: {integrity: sha512-q73XUPnkwt9ZNF2xRS4fvneSuaHw2BXuV5rI4cw0fWYVIWIBeDZX7c7FWhFQPNTnE24172K30I+dViWRVD9TwA==} cpu: [ppc64] @@ -7559,12 +7470,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-riscv64-gnu@4.18.0': - resolution: {integrity: sha512-ROCM7i+m1NfdrsmvwSzoxp9HFtmKGHEqu5NNDiZWQtXLA8S5HBCkVvKAxJ8U+CVctHwV2Gb5VUaK7UAkzhDjlg==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-riscv64-gnu@4.24.4': resolution: {integrity: sha512-Aie/TbmQi6UXokJqDZdmTJuZBCU3QBDA8oTKRGtd4ABi/nHgXICulfg1KI6n9/koDsiDbvHAiQO3YAUNa/7BCw==} cpu: [riscv64] @@ -7589,12 +7494,6 @@ packages: os: [linux] libc: [musl] - '@rollup/rollup-linux-s390x-gnu@4.18.0': - resolution: {integrity: sha512-0UyyRHyDN42QL+NbqevXIIUnKA47A+45WyasO+y2bGJ1mhQrfrtXUpTxCOrfxCR4esV3/RLYyucGVPiUsO8xjg==} - cpu: [s390x] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-s390x-gnu@4.24.4': resolution: {integrity: sha512-P8MPErVO/y8ohWSP9JY7lLQ8+YMHfTI4bAdtCi3pC2hTeqFJco2jYspzOzTUB8hwUWIIu1xwOrJE11nP+0JFAQ==} cpu: [s390x] @@ -7613,12 +7512,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-x64-gnu@4.18.0': - resolution: {integrity: sha512-xuglR2rBVHA5UsI8h8UbX4VJ470PtGCf5Vpswh7p2ukaqBGFTnsfzxUBetoWBWymHMxbIG0Cmx7Y9qDZzr648w==} - cpu: [x64] - os: [linux] - libc: [glibc] - '@rollup/rollup-linux-x64-gnu@4.24.4': resolution: {integrity: sha512-K03TljaaoPK5FOyNMZAAEmhlyO49LaE4qCsr0lYHUKyb6QacTNF9pnfPpXnFlFD3TXuFbFbz7tJ51FujUXkXYA==} cpu: [x64] @@ -7637,12 +7530,6 @@ packages: os: [linux] libc: [glibc] - '@rollup/rollup-linux-x64-musl@4.18.0': - resolution: {integrity: sha512-LKaqQL9osY/ir2geuLVvRRs+utWUNilzdE90TpyoX0eNqPzWjRm14oMEE+YLve4k/NAqCdPkGYDaDF5Sw+xBfg==} - cpu: [x64] - os: [linux] - libc: [musl] - '@rollup/rollup-linux-x64-musl@4.24.4': resolution: {integrity: sha512-VJYl4xSl/wqG2D5xTYncVWW+26ICV4wubwN9Gs5NrqhJtayikwCXzPL8GDsLnaLU3WwhQ8W02IinYSFJfyo34Q==} cpu: [x64] @@ -7666,11 +7553,6 @@ packages: cpu: [arm64] os: [openharmony] - '@rollup/rollup-win32-arm64-msvc@4.18.0': - resolution: {integrity: sha512-7J6TkZQFGo9qBKH0pk2cEVSRhJbL6MtfWxth7Y5YmZs57Pi+4x6c2dStAUvaQkHQLnEQv1jzBUW43GvZW8OFqA==} - cpu: [arm64] - os: [win32] - '@rollup/rollup-win32-arm64-msvc@4.24.4': resolution: {integrity: sha512-ku2GvtPwQfCqoPFIJCqZ8o7bJcj+Y54cZSr43hHca6jLwAiCbZdBUOrqE6y29QFajNAzzpIOwsckaTFmN6/8TA==} cpu: [arm64] @@ -7686,11 +7568,6 @@ packages: cpu: [arm64] os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.18.0': - resolution: {integrity: sha512-Txjh+IxBPbkUB9+SXZMpv+b/vnTEtFyfWZgJ6iyCmt2tdx0OF5WhFowLmnh8ENGNpfUlUZkdI//4IEmhwPieNg==} - cpu: [ia32] - os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.24.4': resolution: {integrity: sha512-V3nCe+eTt/W6UYNr/wGvO1fLpHUrnlirlypZfKCT1fG6hWfqhPgQV/K/mRBXBpxc0eKLIF18pIOFVPh0mqHjlg==} cpu: [ia32] @@ -7706,11 +7583,6 @@ packages: cpu: [ia32] os: [win32] - '@rollup/rollup-win32-x64-msvc@4.18.0': - resolution: {integrity: sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g==} - cpu: [x64] - os: [win32] - '@rollup/rollup-win32-x64-msvc@4.24.4': resolution: {integrity: sha512-LTw1Dfd0mBIEqUVCxbvTE/LLo+9ZxVC9k99v1v4ahg9Aak6FpqOfNu5kRkeTAn0wphoC4JU7No1/rL+bBCEwhg==} cpu: [x64] @@ -7940,7 +7812,6 @@ packages: '@simplewebauthn/types@11.0.0': resolution: {integrity: sha512-b2o0wC5u2rWts31dTgBkAtSNKGX0cvL6h8QedNsKmj8O4QoLFQFR3DBVBUlpyVEhYKA+mXGUaXbcOc4JdQ3HzA==} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. '@sinclair/typebox@0.27.8': resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} @@ -8650,9 +8521,6 @@ packages: '@types/estree-jsx@1.0.5': resolution: {integrity: sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==} - '@types/estree@1.0.5': - resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} - '@types/estree@1.0.6': resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==} @@ -9568,12 +9436,6 @@ packages: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} engines: {node: '>=18'} - bundle-require@4.2.1: - resolution: {integrity: sha512-7Q/6vkyYAwOmQNRw75x+4yRtZCZJXUDmHHlFdkiV0wgv/reNjtJwpu1jPJ0w2kbEpIM0uoKI3S4/f39dU7AjSA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - peerDependencies: - esbuild: '>=0.17' - bundle-require@5.0.0: resolution: {integrity: sha512-GuziW3fSSmopcx4KRymQEJVbZUfqlCqcq7dvs6TYwKRZiegK/2buMxQTPs6MGlNv50wms1699qYO54R8XfRX4w==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -9992,10 +9854,6 @@ packages: resolution: {integrity: sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==} engines: {node: ^14.18.0 || >=16.10.0} - consola@3.4.0: - resolution: {integrity: sha512-EiPU8G6dQG0GFHNR8ljnZFki/8a+cQwEQ+7wpxdChl02Q8HXlwEZWD5lqAF8vC2sEC3Tehr8hy7vErz88LHyUA==} - engines: {node: ^14.18.0 || >=16.10.0} - consola@3.4.2: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} @@ -10348,15 +10206,6 @@ packages: supports-color: optional: true - debug@4.3.5: - resolution: {integrity: sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - debug@4.3.7: resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} engines: {node: '>=6.0'} @@ -11228,22 +11077,6 @@ packages: picomatch: optional: true - fdir@6.4.3: - resolution: {integrity: sha512-PMXmW2y1hDDfTSRc9gaXIuCCRpuoz3Kaz8cUelp3smouvfT632ozg2vrT6lJsHKKOF59YLbOGfAWGUcKEfRMQw==} - peerDependencies: - picomatch: ^3 || ^4 - peerDependenciesMeta: - picomatch: - optional: true - - fdir@6.4.6: - resolution: {integrity: sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==} - peerDependencies: - picomatch: ^3 || ^4 - peerDependenciesMeta: - picomatch: - optional: true - fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -12410,7 +12243,6 @@ packages: keygrip@1.1.0: resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} engines: {node: '>= 0.6'} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -13069,9 +12901,6 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} - ms@2.1.2: - resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -14521,11 +14350,6 @@ packages: robust-predicates@3.0.2: resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} - rollup@4.18.0: - resolution: {integrity: sha512-QmJz14PX3rzbJCN1SG4Xe/bAAX2a6NpCP8ab2vfu2GiUr8AQcr2nCV/oEO3yneFarB67zk8ShlIyWb2LGTb3Sg==} - engines: {node: '>=18.0.0', npm: '>=8.0.0'} - hasBin: true - rollup@4.24.4: resolution: {integrity: sha512-vGorVWIsWfX3xbcyAS+I047kFKapHYivmkaT63Smj77XwvLSJos6M1xGqZnBPFQFBRZDOcG1QnYEIxAvTr/HjA==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -15225,10 +15049,6 @@ packages: resolution: {integrity: sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==} engines: {node: '>=12.0.0'} - tinyglobby@0.2.12: - resolution: {integrity: sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==} - engines: {node: '>=12.0.0'} - tinyglobby@0.2.15: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} @@ -15330,25 +15150,6 @@ packages: resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} engines: {node: '>=0.6.x'} - tsup@8.1.0: - resolution: {integrity: sha512-UFdfCAXukax+U6KzeTNO2kAARHcWxmKsnvSPXUcfA1D+kU05XDccCrkffCQpFaWDsZfV0jMyTsxU39VfCp6EOg==} - engines: {node: '>=18'} - hasBin: true - peerDependencies: - '@microsoft/api-extractor': ^7.36.0 - '@swc/core': ^1 - postcss: ^8.4.12 - typescript: '>=4.5.0' - peerDependenciesMeta: - '@microsoft/api-extractor': - optional: true - '@swc/core': - optional: true - postcss: - optional: true - typescript: - optional: true - tsup@8.3.5: resolution: {integrity: sha512-Tunf6r6m6tnZsG9GYWndg0z8dEV7fD733VBFzFJ5Vcm1FtlXB8xBD/rtrBi2a3YKEV7hHtxiZtW5EAVADoe1pA==} engines: {node: '>=18'} @@ -15946,10 +15747,6 @@ packages: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} - webpack-sources@3.2.3: - resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==} - engines: {node: '>=10.13.0'} - webpack-sources@3.3.3: resolution: {integrity: sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg==} engines: {node: '>=10.13.0'} @@ -16308,7 +16105,7 @@ snapshots: '@ampproject/remapping@2.3.0': dependencies: - '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.25 '@antfu/install-pkg@1.1.0': @@ -17223,26 +17020,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/core@7.28.0': - dependencies: - '@ampproject/remapping': 2.3.0 - '@babel/code-frame': 7.27.1 - '@babel/generator': 7.28.0 - '@babel/helper-compilation-targets': 7.27.2 - '@babel/helper-module-transforms': 7.27.3(@babel/core@7.28.0) - '@babel/helpers': 7.27.6 - '@babel/parser': 7.28.0 - '@babel/template': 7.27.2 - '@babel/traverse': 7.28.0 - '@babel/types': 7.28.1 - convert-source-map: 2.0.0 - debug: 4.4.1 - gensync: 1.0.0-beta.2 - json5: 2.2.3 - semver: 6.3.1 - transitivePeerDependencies: - - supports-color - '@babel/core@7.28.5': dependencies: '@babel/code-frame': 7.27.1 @@ -17267,7 +17044,7 @@ snapshots: dependencies: '@babel/parser': 7.26.2 '@babel/types': 7.26.0 - '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.25 jsesc: 3.0.2 @@ -17276,15 +17053,7 @@ snapshots: '@babel/parser': 7.28.5 '@babel/types': 7.28.5 '@jridgewell/gen-mapping': 0.3.13 - '@jridgewell/trace-mapping': 0.3.25 - jsesc: 3.0.2 - - '@babel/generator@7.28.0': - dependencies: - '@babel/parser': 7.28.5 - '@babel/types': 7.28.5 - '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.0.2 '@babel/generator@7.28.5': @@ -17323,7 +17092,7 @@ snapshots: '@babel/helper-optimise-call-expression': 7.24.7 '@babel/helper-replace-supers': 7.25.0(@babel/core@7.28.5) '@babel/helper-skip-transparent-expression-wrappers': 7.24.7 - '@babel/traverse': 7.26.9 + '@babel/traverse': 7.28.5 semver: 6.3.1 transitivePeerDependencies: - supports-color @@ -17367,15 +17136,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.27.3(@babel/core@7.28.0)': - dependencies: - '@babel/core': 7.28.0 - '@babel/helper-module-imports': 7.27.1 - '@babel/helper-validator-identifier': 7.28.5 - '@babel/traverse': 7.28.5 - transitivePeerDependencies: - - supports-color - '@babel/helper-module-transforms@7.28.3(@babel/core@7.28.5)': dependencies: '@babel/core': 7.28.5 @@ -17389,8 +17149,6 @@ snapshots: dependencies: '@babel/types': 7.28.5 - '@babel/helper-plugin-utils@7.24.8': {} - '@babel/helper-plugin-utils@7.27.1': {} '@babel/helper-replace-supers@7.25.0(@babel/core@7.28.5)': @@ -17432,11 +17190,6 @@ snapshots: '@babel/template': 7.25.9 '@babel/types': 7.26.0 - '@babel/helpers@7.27.6': - dependencies: - '@babel/template': 7.27.2 - '@babel/types': 7.28.5 - '@babel/helpers@7.28.4': dependencies: '@babel/template': 7.27.2 @@ -17474,7 +17227,7 @@ snapshots: '@babel/core': 7.28.5 '@babel/helper-annotate-as-pure': 7.24.7 '@babel/helper-create-class-features-plugin': 7.25.0(@babel/core@7.28.5) - '@babel/helper-plugin-utils': 7.24.8 + '@babel/helper-plugin-utils': 7.27.1 '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.28.5) transitivePeerDependencies: - supports-color @@ -17557,7 +17310,7 @@ snapshots: '@babel/parser': 7.26.9 '@babel/template': 7.26.9 '@babel/types': 7.26.9 - debug: 4.4.1 + debug: 4.4.3 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -17574,18 +17327,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@babel/traverse@7.28.0': - dependencies: - '@babel/code-frame': 7.27.1 - '@babel/generator': 7.28.5 - '@babel/helper-globals': 7.28.0 - '@babel/parser': 7.28.5 - '@babel/template': 7.27.2 - '@babel/types': 7.28.5 - debug: 4.4.3 - transitivePeerDependencies: - - supports-color - '@babel/traverse@7.28.5': dependencies: '@babel/code-frame': 7.27.1 @@ -17619,11 +17360,6 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.27.1 - '@babel/types@7.28.1': - dependencies: - '@babel/helper-string-parser': 7.27.1 - '@babel/helper-validator-identifier': 7.28.5 - '@babel/types@7.28.5': dependencies: '@babel/helper-string-parser': 7.27.1 @@ -18641,7 +18377,7 @@ snapshots: '@eslint/eslintrc@1.4.1': dependencies: ajv: 6.12.6 - debug: 4.4.0 + debug: 4.4.3 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.2 @@ -18820,7 +18556,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.4.0 + debug: 4.4.3 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -18840,7 +18576,7 @@ snapshots: '@antfu/install-pkg': 1.1.0 '@antfu/utils': 8.1.1 '@iconify/types': 2.0.0 - debug: 4.4.1 + debug: 4.4.3 globals: 15.15.0 kolorist: 1.8.0 local-pkg: 1.1.1 @@ -18961,22 +18697,11 @@ snapshots: dependencies: '@sinclair/typebox': 0.27.8 - '@jridgewell/gen-mapping@0.3.12': - dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 - '@jridgewell/trace-mapping': 0.3.29 - '@jridgewell/gen-mapping@0.3.13': dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping': 0.3.31 - '@jridgewell/gen-mapping@0.3.5': - dependencies: - '@jridgewell/set-array': 1.2.1 - '@jridgewell/sourcemap-codec': 1.5.0 - '@jridgewell/trace-mapping': 0.3.25 - '@jridgewell/remapping@2.3.5': dependencies: '@jridgewell/gen-mapping': 0.3.13 @@ -18984,8 +18709,6 @@ snapshots: '@jridgewell/resolve-uri@3.1.2': {} - '@jridgewell/set-array@1.2.1': {} - '@jridgewell/source-map@0.3.11': dependencies: '@jridgewell/gen-mapping': 0.3.13 @@ -18998,12 +18721,7 @@ snapshots: '@jridgewell/trace-mapping@0.3.25': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 - - '@jridgewell/trace-mapping@0.3.29': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping@0.3.31': dependencies: @@ -19020,7 +18738,7 @@ snapshots: '@koa/router@12.0.1': dependencies: - debug: 4.4.0 + debug: 4.4.3 http-errors: 2.0.0 koa-compose: 4.1.0 methods: 1.1.2 @@ -23091,10 +22809,10 @@ snapshots: '@rollup/pluginutils': 5.1.0(rollup@4.50.1) commondir: 1.0.1 estree-walker: 2.0.2 - fdir: 6.4.6(picomatch@4.0.2) + fdir: 6.5.0(picomatch@4.0.3) is-reference: 1.2.1 magic-string: 0.30.17 - picomatch: 4.0.2 + picomatch: 4.0.3 optionalDependencies: rollup: 4.50.1 @@ -23106,9 +22824,6 @@ snapshots: optionalDependencies: rollup: 4.50.1 - '@rollup/rollup-android-arm-eabi@4.18.0': - optional: true - '@rollup/rollup-android-arm-eabi@4.24.4': optional: true @@ -23118,9 +22833,6 @@ snapshots: '@rollup/rollup-android-arm-eabi@4.50.1': optional: true - '@rollup/rollup-android-arm64@4.18.0': - optional: true - '@rollup/rollup-android-arm64@4.24.4': optional: true @@ -23130,9 +22842,6 @@ snapshots: '@rollup/rollup-android-arm64@4.50.1': optional: true - '@rollup/rollup-darwin-arm64@4.18.0': - optional: true - '@rollup/rollup-darwin-arm64@4.24.4': optional: true @@ -23142,9 +22851,6 @@ snapshots: '@rollup/rollup-darwin-arm64@4.50.1': optional: true - '@rollup/rollup-darwin-x64@4.18.0': - optional: true - '@rollup/rollup-darwin-x64@4.24.4': optional: true @@ -23172,9 +22878,6 @@ snapshots: '@rollup/rollup-freebsd-x64@4.50.1': optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.18.0': - optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.24.4': optional: true @@ -23184,9 +22887,6 @@ snapshots: '@rollup/rollup-linux-arm-gnueabihf@4.50.1': optional: true - '@rollup/rollup-linux-arm-musleabihf@4.18.0': - optional: true - '@rollup/rollup-linux-arm-musleabihf@4.24.4': optional: true @@ -23196,9 +22896,6 @@ snapshots: '@rollup/rollup-linux-arm-musleabihf@4.50.1': optional: true - '@rollup/rollup-linux-arm64-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-arm64-gnu@4.24.4': optional: true @@ -23208,9 +22905,6 @@ snapshots: '@rollup/rollup-linux-arm64-gnu@4.50.1': optional: true - '@rollup/rollup-linux-arm64-musl@4.18.0': - optional: true - '@rollup/rollup-linux-arm64-musl@4.24.4': optional: true @@ -23226,9 +22920,6 @@ snapshots: '@rollup/rollup-linux-loongarch64-gnu@4.50.1': optional: true - '@rollup/rollup-linux-powerpc64le-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-powerpc64le-gnu@4.24.4': optional: true @@ -23238,9 +22929,6 @@ snapshots: '@rollup/rollup-linux-ppc64-gnu@4.50.1': optional: true - '@rollup/rollup-linux-riscv64-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-riscv64-gnu@4.24.4': optional: true @@ -23253,9 +22941,6 @@ snapshots: '@rollup/rollup-linux-riscv64-musl@4.50.1': optional: true - '@rollup/rollup-linux-s390x-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-s390x-gnu@4.24.4': optional: true @@ -23265,9 +22950,6 @@ snapshots: '@rollup/rollup-linux-s390x-gnu@4.50.1': optional: true - '@rollup/rollup-linux-x64-gnu@4.18.0': - optional: true - '@rollup/rollup-linux-x64-gnu@4.24.4': optional: true @@ -23277,9 +22959,6 @@ snapshots: '@rollup/rollup-linux-x64-gnu@4.50.1': optional: true - '@rollup/rollup-linux-x64-musl@4.18.0': - optional: true - '@rollup/rollup-linux-x64-musl@4.24.4': optional: true @@ -23292,9 +22971,6 @@ snapshots: '@rollup/rollup-openharmony-arm64@4.50.1': optional: true - '@rollup/rollup-win32-arm64-msvc@4.18.0': - optional: true - '@rollup/rollup-win32-arm64-msvc@4.24.4': optional: true @@ -23304,9 +22980,6 @@ snapshots: '@rollup/rollup-win32-arm64-msvc@4.50.1': optional: true - '@rollup/rollup-win32-ia32-msvc@4.18.0': - optional: true - '@rollup/rollup-win32-ia32-msvc@4.24.4': optional: true @@ -23316,9 +22989,6 @@ snapshots: '@rollup/rollup-win32-ia32-msvc@4.50.1': optional: true - '@rollup/rollup-win32-x64-msvc@4.18.0': - optional: true - '@rollup/rollup-win32-x64-msvc@4.24.4': optional: true @@ -23378,7 +23048,7 @@ snapshots: '@sentry/bundler-plugin-core@4.3.0(encoding@0.1.13)': dependencies: - '@babel/core': 7.28.0 + '@babel/core': 7.28.5 '@sentry/babel-plugin-component-annotate': 4.3.0 '@sentry/cli': 2.53.0(encoding@0.1.13) dotenv: 16.6.1 @@ -23436,7 +23106,7 @@ snapshots: '@sentry/core@10.11.0': {} - '@sentry/nextjs@10.11.0(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.24.2))': + '@sentry/nextjs@10.11.0(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(encoding@0.1.13)(next@16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11))': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/semantic-conventions': 1.37.0 @@ -23448,9 +23118,9 @@ snapshots: '@sentry/opentelemetry': 10.11.0(@opentelemetry/api@1.9.0)(@opentelemetry/context-async-hooks@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/core@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.37.0) '@sentry/react': 10.11.0(react@19.2.3) '@sentry/vercel-edge': 10.11.0 - '@sentry/webpack-plugin': 4.3.0(encoding@0.1.13)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.24.2)) + '@sentry/webpack-plugin': 4.3.0(encoding@0.1.13)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)) chalk: 3.0.0 - next: 16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + next: 16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) resolve: 1.22.8 rollup: 4.50.1 stacktrace-parser: 0.1.11 @@ -23627,6 +23297,16 @@ snapshots: - encoding - supports-color + '@sentry/webpack-plugin@4.3.0(encoding@0.1.13)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11))': + dependencies: + '@sentry/bundler-plugin-core': 4.3.0(encoding@0.1.13) + unplugin: 1.0.1 + uuid: 9.0.1 + webpack: 5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11) + transitivePeerDependencies: + - encoding + - supports-color + '@shikijs/core@3.14.0': dependencies: '@shikijs/types': 3.14.0 @@ -24602,8 +24282,6 @@ snapshots: dependencies: '@types/estree': 1.0.8 - '@types/estree@1.0.5': {} - '@types/estree@1.0.6': {} '@types/estree@1.0.8': {} @@ -25340,7 +25018,7 @@ snapshots: agent-base@7.1.1: dependencies: - debug: 4.4.1 + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -25745,7 +25423,7 @@ snapshots: dependencies: bytes: 3.1.2 content-type: 1.0.5 - debug: 4.4.1 + debug: 4.4.3 http-errors: 2.0.0 iconv-lite: 0.6.3 on-finished: 2.4.1 @@ -25815,7 +25493,7 @@ snapshots: browserslist@4.23.1: dependencies: - caniuse-lite: 1.0.30001751 + caniuse-lite: 1.0.30001696 electron-to-chromium: 1.4.803 node-releases: 2.0.14 update-browserslist-db: 1.0.16(browserslist@4.23.1) @@ -25851,19 +25529,14 @@ snapshots: dependencies: run-applescript: 7.0.0 - bundle-require@4.2.1(esbuild@0.21.5): - dependencies: - esbuild: 0.21.5 - load-tsconfig: 0.2.5 - bundle-require@5.0.0(esbuild@0.24.2): dependencies: esbuild: 0.24.2 load-tsconfig: 0.2.5 - bundle-require@5.1.0(esbuild@0.25.3): + bundle-require@5.1.0(esbuild@0.25.11): dependencies: - esbuild: 0.25.3 + esbuild: 0.25.11 load-tsconfig: 0.2.5 busboy@1.6.0: @@ -26093,7 +25766,7 @@ snapshots: citty@0.1.6: dependencies: - consola: 3.4.0 + consola: 3.4.2 cjs-module-lexer@1.4.0: {} @@ -26301,8 +25974,6 @@ snapshots: consola@3.2.3: {} - consola@3.4.0: {} - consola@3.4.2: {} console-control-strings@1.1.0: {} @@ -26674,10 +26345,6 @@ snapshots: dependencies: ms: 2.1.3 - debug@4.3.5: - dependencies: - ms: 2.1.2 - debug@4.3.7: dependencies: ms: 2.1.3 @@ -27407,7 +27074,7 @@ snapshots: eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint@8.30.0))(eslint@8.30.0): dependencies: - debug: 4.4.1 + debug: 4.4.3 enhanced-resolve: 5.17.0 eslint: 8.30.0 eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint@8.30.0))(eslint@8.30.0))(eslint@8.30.0) @@ -27425,7 +27092,7 @@ snapshots: eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.30.0): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.0 + debug: 4.4.3 enhanced-resolve: 5.17.1 eslint: 8.30.0 eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.3.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.30.0) @@ -27444,7 +27111,7 @@ snapshots: eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint@8.30.0))(eslint@8.30.0): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.0 + debug: 4.4.3 enhanced-resolve: 5.17.1 eslint: 8.30.0 eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@8.30.0)(typescript@5.8.3))(eslint@8.30.0))(eslint@8.30.0))(eslint@8.30.0) @@ -27908,7 +27575,7 @@ snapshots: estree-walker@3.0.3: dependencies: - '@types/estree': 1.0.6 + '@types/estree': 1.0.8 esutils@2.0.3: {} @@ -28000,7 +27667,7 @@ snapshots: content-type: 1.0.5 cookie: 0.7.1 cookie-signature: 1.2.2 - debug: 4.4.1 + debug: 4.4.3 encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 @@ -28106,14 +27773,6 @@ snapshots: optionalDependencies: picomatch: 4.0.2 - fdir@6.4.3(picomatch@4.0.2): - optionalDependencies: - picomatch: 4.0.2 - - fdir@6.4.6(picomatch@4.0.2): - optionalDependencies: - picomatch: 4.0.2 - fdir@6.5.0(picomatch@4.0.3): optionalDependencies: picomatch: 4.0.3 @@ -28148,7 +27807,7 @@ snapshots: finalhandler@2.1.0: dependencies: - debug: 4.4.1 + debug: 4.4.3 encodeurl: 2.0.0 escape-html: 1.0.3 on-finished: 2.4.1 @@ -28581,7 +28240,7 @@ snapshots: giget@2.0.0: dependencies: citty: 0.1.6 - consola: 3.4.0 + consola: 3.4.2 defu: 6.1.4 node-fetch-native: 1.6.7 nypm: 0.6.2 @@ -28942,7 +28601,7 @@ snapshots: http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.1 - debug: 4.4.0 + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -28963,7 +28622,7 @@ snapshots: https-proxy-agent@7.0.5: dependencies: agent-base: 7.1.1 - debug: 4.4.1 + debug: 4.4.3 transitivePeerDependencies: - supports-color @@ -29548,7 +29207,7 @@ snapshots: content-disposition: 0.5.4 content-type: 1.0.5 cookies: 0.9.1 - debug: 4.4.0 + debug: 4.4.3 delegates: 1.0.0 depd: 2.0.0 destroy: 1.2.0 @@ -29790,7 +29449,7 @@ snapshots: magic-string@0.30.8: dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 make-dir@3.1.0: dependencies: @@ -30409,8 +30068,6 @@ snapshots: ms@2.0.0: {} - ms@2.1.2: {} - ms@2.1.3: {} mute-stream@1.0.0: {} @@ -30684,7 +30341,7 @@ snapshots: - '@babel/core' - babel-plugin-macros - next@16.1.5(@babel/core@7.28.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): + next@16.1.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): dependencies: '@next/env': 16.1.5 '@swc/helpers': 0.5.15 @@ -30693,7 +30350,7 @@ snapshots: postcss: 8.4.31 react: 19.2.3 react-dom: 19.2.3(react@19.2.3) - styled-jsx: 5.1.6(@babel/core@7.28.0)(react@19.2.3) + styled-jsx: 5.1.6(@babel/core@7.26.0)(react@19.2.3) optionalDependencies: '@next/swc-darwin-arm64': 16.1.5 '@next/swc-darwin-x64': 16.1.5 @@ -31258,23 +30915,16 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.5.6 - postcss-load-config@4.0.2(postcss@8.4.47): - dependencies: - lilconfig: 3.1.2 - yaml: 2.6.0 - optionalDependencies: - postcss: 8.4.47 - postcss-load-config@4.0.2(postcss@8.5.3): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 yaml: 2.6.0 optionalDependencies: postcss: 8.5.3 postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0)(yaml@2.8.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 1.21.7 postcss: 8.5.6 @@ -31283,7 +30933,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(yaml@2.8.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.4.47 @@ -31292,7 +30942,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.2)(tsx@4.21.0)(yaml@2.8.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.5.2 @@ -31301,7 +30951,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(yaml@2.4.5): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.5.6 @@ -31310,7 +30960,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.19.3)(yaml@2.6.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.5.6 @@ -31319,7 +30969,7 @@ snapshots: postcss-load-config@6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.21.0)(yaml@2.8.0): dependencies: - lilconfig: 3.1.2 + lilconfig: 3.1.3 optionalDependencies: jiti: 2.4.2 postcss: 8.5.6 @@ -32316,28 +31966,6 @@ snapshots: robust-predicates@3.0.2: {} - rollup@4.18.0: - dependencies: - '@types/estree': 1.0.5 - optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.18.0 - '@rollup/rollup-android-arm64': 4.18.0 - '@rollup/rollup-darwin-arm64': 4.18.0 - '@rollup/rollup-darwin-x64': 4.18.0 - '@rollup/rollup-linux-arm-gnueabihf': 4.18.0 - '@rollup/rollup-linux-arm-musleabihf': 4.18.0 - '@rollup/rollup-linux-arm64-gnu': 4.18.0 - '@rollup/rollup-linux-arm64-musl': 4.18.0 - '@rollup/rollup-linux-powerpc64le-gnu': 4.18.0 - '@rollup/rollup-linux-riscv64-gnu': 4.18.0 - '@rollup/rollup-linux-s390x-gnu': 4.18.0 - '@rollup/rollup-linux-x64-gnu': 4.18.0 - '@rollup/rollup-linux-x64-musl': 4.18.0 - '@rollup/rollup-win32-arm64-msvc': 4.18.0 - '@rollup/rollup-win32-ia32-msvc': 4.18.0 - '@rollup/rollup-win32-x64-msvc': 4.18.0 - fsevents: 2.3.3 - rollup@4.24.4: dependencies: '@types/estree': 1.0.6 @@ -32423,7 +32051,7 @@ snapshots: router@2.2.0: dependencies: - debug: 4.4.1 + debug: 4.4.3 depd: 2.0.0 is-promise: 4.0.0 parseurl: 1.3.3 @@ -32544,7 +32172,7 @@ snapshots: send@1.2.0: dependencies: - debug: 4.4.1 + debug: 4.4.3 encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 @@ -32994,12 +32622,12 @@ snapshots: optionalDependencies: '@babel/core': 7.26.0 - styled-jsx@5.1.6(@babel/core@7.28.0)(react@19.2.3): + styled-jsx@5.1.6(@babel/core@7.26.0)(react@19.2.3): dependencies: client-only: 0.0.1 react: 19.2.3 optionalDependencies: - '@babel/core': 7.28.0 + '@babel/core': 7.26.0 styled-jsx@5.1.6(@babel/core@7.28.5)(react@19.2.3): dependencies: @@ -33014,7 +32642,7 @@ snapshots: sucrase@3.35.0: dependencies: - '@jridgewell/gen-mapping': 0.3.5 + '@jridgewell/gen-mapping': 0.3.13 commander: 4.1.1 glob: 10.4.5 lines-and-columns: 1.2.4 @@ -33242,6 +32870,18 @@ snapshots: '@swc/core': 1.3.101(@swc/helpers@0.5.15) esbuild: 0.24.2 + terser-webpack-plugin@5.3.14(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)): + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + jest-worker: 27.5.1 + schema-utils: 4.3.3 + serialize-javascript: 6.0.2 + terser: 5.44.0 + webpack: 5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11) + optionalDependencies: + '@swc/core': 1.3.101(@swc/helpers@0.5.15) + esbuild: 0.25.11 + terser@5.44.0: dependencies: '@jridgewell/source-map': 0.3.11 @@ -33337,11 +32977,6 @@ snapshots: fdir: 6.4.2(picomatch@4.0.2) picomatch: 4.0.2 - tinyglobby@0.2.12: - dependencies: - fdir: 6.4.3(picomatch@4.0.2) - picomatch: 4.0.2 - tinyglobby@0.2.15: dependencies: fdir: 6.5.0(picomatch@4.0.3) @@ -33428,30 +33063,6 @@ snapshots: tsscmp@1.0.6: {} - tsup@8.1.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(postcss@8.4.47)(typescript@5.8.3): - dependencies: - bundle-require: 4.2.1(esbuild@0.21.5) - cac: 6.7.14 - chokidar: 3.6.0 - debug: 4.3.5 - esbuild: 0.21.5 - execa: 5.1.1 - globby: 11.1.0 - joycon: 3.1.1 - postcss-load-config: 4.0.2(postcss@8.4.47) - resolve-from: 5.0.0 - rollup: 4.18.0 - source-map: 0.8.0-beta.0 - sucrase: 3.35.0 - tree-kill: 1.2.2 - optionalDependencies: - '@swc/core': 1.3.101(@swc/helpers@0.5.15) - postcss: 8.4.47 - typescript: 5.8.3 - transitivePeerDependencies: - - supports-color - - ts-node - tsup@8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0): dependencies: bundle-require: 5.0.0(esbuild@0.24.2) @@ -33480,34 +33091,6 @@ snapshots: - tsx - yaml - tsup@8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(typescript@5.8.3)(yaml@2.4.5): - dependencies: - bundle-require: 5.0.0(esbuild@0.24.2) - cac: 6.7.14 - chokidar: 4.0.1 - consola: 3.2.3 - debug: 4.3.7 - esbuild: 0.24.2 - joycon: 3.1.1 - picocolors: 1.1.1 - postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(yaml@2.4.5) - resolve-from: 5.0.0 - rollup: 4.24.4 - source-map: 0.8.0-beta.0 - sucrase: 3.35.0 - tinyexec: 0.3.1 - tinyglobby: 0.2.10 - tree-kill: 1.2.2 - optionalDependencies: - '@swc/core': 1.3.101(@swc/helpers@0.5.15) - postcss: 8.5.6 - typescript: 5.8.3 - transitivePeerDependencies: - - jiti - - supports-color - - tsx - - yaml - tsup@8.3.5(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.19.3)(typescript@5.3.3)(yaml@2.6.0): dependencies: bundle-require: 5.0.0(esbuild@0.24.2) @@ -33536,23 +33119,51 @@ snapshots: - tsx - yaml - tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.2)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0): + tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0): dependencies: - bundle-require: 5.1.0(esbuild@0.25.3) + bundle-require: 5.1.0(esbuild@0.25.11) cac: 6.7.14 chokidar: 4.0.3 - consola: 3.4.0 - debug: 4.4.0 - esbuild: 0.25.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.25.11 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.4.47)(tsx@4.21.0)(yaml@2.8.0) + resolve-from: 5.0.0 + rollup: 4.50.1 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tree-kill: 1.2.2 + optionalDependencies: + '@swc/core': 1.3.101(@swc/helpers@0.5.15) + postcss: 8.4.47 + typescript: 5.8.3 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + + tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.2)(tsx@4.21.0)(typescript@5.8.3)(yaml@2.8.0): + dependencies: + bundle-require: 5.1.0(esbuild@0.25.11) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.25.11 joycon: 3.1.1 picocolors: 1.1.1 postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.2)(tsx@4.21.0)(yaml@2.8.0) resolve-from: 5.0.0 - rollup: 4.34.8 + rollup: 4.50.1 source-map: 0.8.0-beta.0 sucrase: 3.35.0 tinyexec: 0.3.2 - tinyglobby: 0.2.12 + tinyglobby: 0.2.15 tree-kill: 1.2.2 optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.15) @@ -33564,23 +33175,51 @@ snapshots: - tsx - yaml - tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.3.3)(yaml@2.8.0): + tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(typescript@5.8.3)(yaml@2.4.5): dependencies: - bundle-require: 5.1.0(esbuild@0.25.3) + bundle-require: 5.1.0(esbuild@0.25.11) cac: 6.7.14 chokidar: 4.0.3 - consola: 3.4.0 - debug: 4.4.0 - esbuild: 0.25.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.25.11 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.15.5)(yaml@2.4.5) + resolve-from: 5.0.0 + rollup: 4.50.1 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tree-kill: 1.2.2 + optionalDependencies: + '@swc/core': 1.3.101(@swc/helpers@0.5.15) + postcss: 8.5.6 + typescript: 5.8.3 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + + tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.3.3)(yaml@2.8.0): + dependencies: + bundle-require: 5.1.0(esbuild@0.25.11) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.25.11 joycon: 3.1.1 picocolors: 1.1.1 postcss-load-config: 6.0.1(jiti@2.4.2)(postcss@8.5.6)(tsx@4.21.0)(yaml@2.8.0) resolve-from: 5.0.0 - rollup: 4.34.8 + rollup: 4.50.1 source-map: 0.8.0-beta.0 sucrase: 3.35.0 tinyexec: 0.3.2 - tinyglobby: 0.2.12 + tinyglobby: 0.2.15 tree-kill: 1.2.2 optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.15) @@ -33800,7 +33439,7 @@ snapshots: dependencies: acorn: 8.15.0 chokidar: 3.6.0 - webpack-sources: 3.2.3 + webpack-sources: 3.3.3 webpack-virtual-modules: 0.5.0 update-browserslist-db@1.0.16(browserslist@4.23.1): @@ -34026,7 +33665,7 @@ snapshots: vite-node@1.6.0(@types/node@20.17.6)(lightningcss@1.30.1)(terser@5.44.0): dependencies: cac: 6.7.14 - debug: 4.4.0 + debug: 4.4.3 pathe: 1.1.2 picocolors: 1.1.1 vite: 5.4.21(@types/node@20.17.6)(lightningcss@1.30.1)(terser@5.44.0) @@ -34056,7 +33695,7 @@ snapshots: dependencies: esbuild: 0.21.5 postcss: 8.5.6 - rollup: 4.34.8 + rollup: 4.50.1 optionalDependencies: '@types/node': 20.17.6 fsevents: 2.3.3 @@ -34221,8 +33860,6 @@ snapshots: webidl-conversions@7.0.0: {} - webpack-sources@3.2.3: {} - webpack-sources@3.3.3: {} webpack-virtual-modules@0.5.0: {} @@ -34258,6 +33895,37 @@ snapshots: - esbuild - uglify-js + webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11): + dependencies: + '@types/eslint-scope': 3.7.7 + '@types/estree': 1.0.8 + '@webassemblyjs/ast': 1.14.1 + '@webassemblyjs/wasm-edit': 1.14.1 + '@webassemblyjs/wasm-parser': 1.14.1 + acorn: 8.15.0 + acorn-import-attributes: 1.9.5(acorn@8.15.0) + browserslist: 4.27.0 + chrome-trace-event: 1.0.4 + enhanced-resolve: 5.18.3 + es-module-lexer: 1.7.0 + eslint-scope: 5.1.1 + events: 3.3.0 + glob-to-regexp: 0.4.1 + graceful-fs: 4.2.11 + json-parse-even-better-errors: 2.3.1 + loader-runner: 4.3.1 + mime-types: 2.1.35 + neo-async: 2.6.2 + schema-utils: 3.3.0 + tapable: 2.3.0 + terser-webpack-plugin: 5.3.14(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)(webpack@5.92.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.25.11)) + watchpack: 2.4.4 + webpack-sources: 3.3.3 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + whatwg-encoding@3.1.1: dependencies: iconv-lite: 0.6.3