From f2f44086d8f4b5b2bfb691dde0501182ce7e77fc Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Mon, 9 Feb 2026 11:20:15 -0800 Subject: [PATCH] Merge existing DB sync migrations --- AGENTS.md | 3 +- .../migration.sql | 126 ++++++++++++------ .../migration.sql | 24 ---- apps/backend/prisma/schema.prisma | 6 +- 4 files changed, 87 insertions(+), 72 deletions(-) delete mode 100644 apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql diff --git a/AGENTS.md b/AGENTS.md index 8fe66efc9..d05907b13 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,7 +96,8 @@ To see all development ports, refer to the index.html of `apps/dev-launchpad/pub - When building frontend or React code for the dashboard, refer to DESIGN-GUIDE.md. - NEVER implement a hacky solution without EXPLICIT approval from the user. Always go the extra mile to make sure the solution is clean, maintainable, and robust. - Fail early, fail loud. Fail fast with an error instead of silently continuing. -- Do NOT use `as`/`any`/type casts or anything else like that to bypass the type system unless you specifically asked the user about it. Most of the time a place where you would use type casts is not one where you actually need them. Avoid wherever possible. +- Do NOT use `as`/`any`/type casts or anything else like that to bypass the type system unless you specifically asked the user about it. Most of the time a place where you would use type casts is not one where you actually need them. Avoid wherever possible. +- When writing database migration files, assume that we have >1,000,000 rows in every table (unless otherwise specified). This means you may have to use CONDITIONALLY_REPEAT_MIGRATION_SENTINEL to avoid running the migration and things like concurrent index builds; see the existing migrations for examples. ### Code-related - Use ES6 maps instead of records wherever you can. diff --git a/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql b/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql index e90bcf245..2c66a9e82 100644 --- a/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql +++ b/apps/backend/prisma/migrations/20251125030551_external_db_sync/migration.sql @@ -1,6 +1,6 @@ -- Creates a global sequence starting at 1 with increment of 11 for tracking row changes. -- This sequence is used to order data changes across all tables in the database. -CREATE SEQUENCE global_seq_id +CREATE SEQUENCE global_seq_id AS BIGINT START 1 INCREMENT BY 11 @@ -8,33 +8,24 @@ CREATE SEQUENCE global_seq_id NO MAXVALUE; -- SPLIT_STATEMENT_SENTINEL --- Adds sequenceId column to ContactChannel and ProjectUser tables. --- This column stores the sequence number from global_seq_id to track when each row was last modified. -ALTER TABLE "ContactChannel" ADD COLUMN "sequenceId" BIGINT; +-- Adds sequenceId and shouldUpdateSequenceId columns to ContactChannel and ProjectUser tables. +-- sequenceId stores the sequence number from global_seq_id to track when each row was last modified. +-- shouldUpdateSequenceId is a flag to track which rows need their sequenceId updated. +ALTER TABLE "ContactChannel" ADD COLUMN "sequenceId" BIGINT; -- SPLIT_STATEMENT_SENTINEL -ALTER TABLE "ProjectUser" ADD COLUMN "sequenceId" BIGINT; +ALTER TABLE "ContactChannel" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; -- SPLIT_STATEMENT_SENTINEL --- Creates unique indexes on sequenceId columns to ensure no duplicate sequence IDs exist. --- This guarantees each row has a unique position in the change sequence. -CREATE UNIQUE INDEX "ContactChannel_sequenceId_key" ON "ContactChannel"("sequenceId"); +ALTER TABLE "ProjectUser" ADD COLUMN "sequenceId" BIGINT; -- SPLIT_STATEMENT_SENTINEL -CREATE UNIQUE INDEX "ProjectUser_sequenceId_key" ON "ProjectUser"("sequenceId"); - --- SPLIT_STATEMENT_SENTINEL --- Creates composite indexes on (tenancyId, sequenceId) for efficient sync-engine queries. --- These allow fast lookups of rows by tenant ordered by sequence number. -CREATE INDEX "ProjectUser_tenancyId_sequenceId_idx" ON "ProjectUser"("tenancyId", "sequenceId"); - --- SPLIT_STATEMENT_SENTINEL -CREATE INDEX "ContactChannel_tenancyId_sequenceId_idx" ON "ContactChannel"("tenancyId", "sequenceId"); +ALTER TABLE "ProjectUser" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; -- SPLIT_STATEMENT_SENTINEL -- Creates OutgoingRequest table to queue sync requests to external databases. -- Each request stores the QStash options for making HTTP requests and tracks when fulfillment started. -CREATE TABLE "OutgoingRequest" ( +CREATE TABLE "OutgoingRequest" ( "id" UUID NOT NULL, "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "deduplicationKey" TEXT, @@ -45,14 +36,6 @@ CREATE TABLE "OutgoingRequest" ( CONSTRAINT "OutgoingRequest_deduplicationKey_key" UNIQUE ("deduplicationKey") ); --- SPLIT_STATEMENT_SENTINEL -CREATE INDEX "OutgoingRequest_startedFulfillingAt_deduplicationKey_idx" ON "OutgoingRequest"("startedFulfillingAt", "deduplicationKey"); - --- SPLIT_STATEMENT_SENTINEL --- Creates composite index on startedFulfillingAt and createdAt for efficient querying of pending requests in order. --- This allows fast lookups of pending requests (WHERE startedFulfillingAt IS NULL) ordered by createdAt. -CREATE INDEX "OutgoingRequest_startedFulfillingAt_createdAt_idx" ON "OutgoingRequest"("startedFulfillingAt", "createdAt"); - -- SPLIT_STATEMENT_SENTINEL -- Creates DeletedRow table to log information about deleted rows from other tables. -- Stores the primary key and full data of deleted rows so external databases can be notified of deletions. @@ -65,41 +48,96 @@ CREATE TABLE "DeletedRow" ( "data" JSONB, "deletedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "startedFulfillingAt" TIMESTAMP(3), + "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE, CONSTRAINT "DeletedRow_pkey" PRIMARY KEY ("id") ); -- SPLIT_STATEMENT_SENTINEL --- Creates indexes on DeletedRow table for efficient querying by sequence, table name, and tenant. -CREATE UNIQUE INDEX "DeletedRow_sequenceId_key" ON "DeletedRow"("sequenceId"); +-- Creates ExternalDbSyncMetadata table to store external database sync configuration. +-- Uses a singleton constraint to ensure only one row exists. +CREATE TABLE "ExternalDbSyncMetadata" ( + "id" TEXT NOT NULL DEFAULT gen_random_uuid(), + "singleton" "BooleanTrue" NOT NULL DEFAULT 'TRUE'::"BooleanTrue", + "sequencerEnabled" BOOLEAN NOT NULL DEFAULT true, + "pollerEnabled" BOOLEAN NOT NULL DEFAULT true, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "ExternalDbSyncMetadata_pkey" PRIMARY KEY ("id") +); -- SPLIT_STATEMENT_SENTINEL -CREATE INDEX "DeletedRow_tableName_idx" ON "DeletedRow"("tableName"); +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +-- Creates unique indexes on sequenceId columns to ensure no duplicate sequence IDs exist. +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ContactChannel_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ContactChannel"("sequenceId"); -- SPLIT_STATEMENT_SENTINEL -CREATE INDEX "DeletedRow_tenancyId_idx" ON "DeletedRow"("tenancyId"); +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUser_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUser"("sequenceId"); -- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ExternalDbSyncMetadata_singleton_key" ON /* SCHEMA_NAME_SENTINEL */."ExternalDbSyncMetadata"("singleton"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +-- Creates composite indexes on (tenancyId, sequenceId) for efficient sync-engine queries. +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUser_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUser"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ContactChannel_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ContactChannel"("tenancyId", "sequenceId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "OutgoingRequest_startedFulfillingAt_deduplicationKey_idx" ON /* SCHEMA_NAME_SENTINEL */."OutgoingRequest"("startedFulfillingAt", "deduplicationKey"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "OutgoingRequest_startedFulfillingAt_createdAt_idx" ON /* SCHEMA_NAME_SENTINEL */."OutgoingRequest"("startedFulfillingAt", "createdAt"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_tableName_idx" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("tableName"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_tenancyId_idx" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("tenancyId"); + +-- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL -- Creates composite index for efficient querying of deleted rows by tenant and table, ordered by sequence. -CREATE INDEX "DeletedRow_tenancyId_tableName_sequenceId_idx" ON "DeletedRow"("tenancyId", "tableName", "sequenceId"); - --- SPLIT_STATEMENT_SENTINEL --- Adds shouldUpdateSequenceId flag to track which rows need their sequenceId updated. -ALTER TABLE "ProjectUser" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; - --- SPLIT_STATEMENT_SENTINEL -ALTER TABLE "ContactChannel" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; - --- SPLIT_STATEMENT_SENTINEL -ALTER TABLE "DeletedRow" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE; +CREATE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_tenancyId_tableName_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("tenancyId", "tableName", "sequenceId"); -- SPLIT_STATEMENT_SENTINEL +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL -- Creates indexes on (shouldUpdateSequenceId, tenancyId) to quickly find rows that need updates -- and support ORDER BY tenancyId for less fragmented updates. -CREATE INDEX "ProjectUser_shouldUpdateSequenceId_idx" ON "ProjectUser"("shouldUpdateSequenceId", "tenancyId"); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUser_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUser"("shouldUpdateSequenceId", "tenancyId"); -- SPLIT_STATEMENT_SENTINEL -CREATE INDEX "ContactChannel_shouldUpdateSequenceId_idx" ON "ContactChannel"("shouldUpdateSequenceId", "tenancyId"); +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "ContactChannel_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ContactChannel"("shouldUpdateSequenceId", "tenancyId"); -- SPLIT_STATEMENT_SENTINEL -CREATE INDEX "DeletedRow_shouldUpdateSequenceId_idx" ON "DeletedRow"("shouldUpdateSequenceId", "tenancyId"); +-- SINGLE_STATEMENT_SENTINEL +-- RUN_OUTSIDE_TRANSACTION_SENTINEL +CREATE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("shouldUpdateSequenceId", "tenancyId"); diff --git a/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql b/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql deleted file mode 100644 index aa0a767c0..000000000 --- a/apps/backend/prisma/migrations/20260204014127_external_db_metadata/migration.sql +++ /dev/null @@ -1,24 +0,0 @@ --- DropIndex -DROP INDEX "ContactChannel_shouldUpdateSequenceId_idx"; - --- DropIndex -DROP INDEX "DeletedRow_shouldUpdateSequenceId_idx"; - --- DropIndex -DROP INDEX "ProjectUser_shouldUpdateSequenceId_idx"; - --- CreateTable -CREATE TABLE "ExternalDbSyncMetadata" ( - "id" TEXT NOT NULL DEFAULT gen_random_uuid(), - "singleton" "BooleanTrue" NOT NULL DEFAULT 'TRUE', - "sequencerEnabled" BOOLEAN NOT NULL DEFAULT true, - "pollerEnabled" BOOLEAN NOT NULL DEFAULT true, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "ExternalDbSyncMetadata_pkey" PRIMARY KEY ("id") -); - --- CreateIndex -CREATE UNIQUE INDEX "ExternalDbSyncMetadata_singleton_key" ON "ExternalDbSyncMetadata"("singleton"); - diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index e0cf9744d..46e39119e 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -243,7 +243,7 @@ model ProjectUser { @@index([tenancyId, createdAt(sort: Asc)], name: "ProjectUser_createdAt_asc") @@index([tenancyId, createdAt(sort: Desc)], name: "ProjectUser_createdAt_desc") @@index([tenancyId, sequenceId], name: "ProjectUser_tenancyId_sequenceId_idx") - // Partial index for external db sync backfill lives in migration SQL. + @@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUser_shouldUpdateSequenceId_idx") } // This should be renamed to "OAuthAccount" as it is not always bound to a user @@ -309,7 +309,7 @@ model ContactChannel { // only one contact channel per project with the same value and type can be used for auth @@unique([tenancyId, type, value, usedForAuth]) @@index([tenancyId, sequenceId], name: "ContactChannel_tenancyId_sequenceId_idx") - // Partial index for external db sync backfill lives in migration SQL (WHERE shouldUpdateSequenceId = TRUE). + @@index([shouldUpdateSequenceId, tenancyId], name: "ContactChannel_shouldUpdateSequenceId_idx") } model AuthMethod { @@ -1113,5 +1113,5 @@ model DeletedRow { @@index([tenancyId]) // composite index for efficient querying of deleted rows by tenant and table, ordered by sequence @@index([tenancyId, tableName, sequenceId]) - // Partial index for external db sync backfill lives in migration SQL (WHERE shouldUpdateSequenceId = TRUE). + @@index([shouldUpdateSequenceId, tenancyId], name: "DeletedRow_shouldUpdateSequenceId_idx") }