mirror of
https://github.com/stack-auth/stack.git
synced 2026-06-04 21:04:37 +08:00
Merge existing DB sync migrations
This commit is contained in:
Konstantin Wohlwend
parent
2072dd4b3d
commit
f2f44086d8
@ -96,7 +96,8 @@ To see all development ports, refer to the index.html of `apps/dev-launchpad/pub
|
||||
- When building frontend or React code for the dashboard, refer to DESIGN-GUIDE.md.
|
||||
- NEVER implement a hacky solution without EXPLICIT approval from the user. Always go the extra mile to make sure the solution is clean, maintainable, and robust.
|
||||
- Fail early, fail loud. Fail fast with an error instead of silently continuing.
|
||||
- Do NOT use `as`/`any`/type casts or anything else like that to bypass the type system unless you specifically asked the user about it. Most of the time a place where you would use type casts is not one where you actually need them. Avoid wherever possible.
|
||||
- Do NOT use `as`/`any`/type casts or anything else like that to bypass the type system unless you specifically asked the user about it. Most of the time a place where you would use type casts is not one where you actually need them. Avoid wherever possible.
|
||||
- When writing database migration files, assume that we have >1,000,000 rows in every table (unless otherwise specified). This means you may have to use CONDITIONALLY_REPEAT_MIGRATION_SENTINEL to avoid running the migration and things like concurrent index builds; see the existing migrations for examples.
|
||||
|
||||
### Code-related
|
||||
- Use ES6 maps instead of records wherever you can.
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
-- Creates a global sequence starting at 1 with increment of 11 for tracking row changes.
|
||||
-- This sequence is used to order data changes across all tables in the database.
|
||||
CREATE SEQUENCE global_seq_id
|
||||
CREATE SEQUENCE global_seq_id
|
||||
AS BIGINT
|
||||
START 1
|
||||
INCREMENT BY 11
|
||||
@ -8,33 +8,24 @@ CREATE SEQUENCE global_seq_id
|
||||
NO MAXVALUE;
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- Adds sequenceId column to ContactChannel and ProjectUser tables.
|
||||
-- This column stores the sequence number from global_seq_id to track when each row was last modified.
|
||||
ALTER TABLE "ContactChannel" ADD COLUMN "sequenceId" BIGINT;
|
||||
-- Adds sequenceId and shouldUpdateSequenceId columns to ContactChannel and ProjectUser tables.
|
||||
-- sequenceId stores the sequence number from global_seq_id to track when each row was last modified.
|
||||
-- shouldUpdateSequenceId is a flag to track which rows need their sequenceId updated.
|
||||
ALTER TABLE "ContactChannel" ADD COLUMN "sequenceId" BIGINT;
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
ALTER TABLE "ProjectUser" ADD COLUMN "sequenceId" BIGINT;
|
||||
ALTER TABLE "ContactChannel" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- Creates unique indexes on sequenceId columns to ensure no duplicate sequence IDs exist.
|
||||
-- This guarantees each row has a unique position in the change sequence.
|
||||
CREATE UNIQUE INDEX "ContactChannel_sequenceId_key" ON "ContactChannel"("sequenceId");
|
||||
ALTER TABLE "ProjectUser" ADD COLUMN "sequenceId" BIGINT;
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
CREATE UNIQUE INDEX "ProjectUser_sequenceId_key" ON "ProjectUser"("sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- Creates composite indexes on (tenancyId, sequenceId) for efficient sync-engine queries.
|
||||
-- These allow fast lookups of rows by tenant ordered by sequence number.
|
||||
CREATE INDEX "ProjectUser_tenancyId_sequenceId_idx" ON "ProjectUser"("tenancyId", "sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
CREATE INDEX "ContactChannel_tenancyId_sequenceId_idx" ON "ContactChannel"("tenancyId", "sequenceId");
|
||||
ALTER TABLE "ProjectUser" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- Creates OutgoingRequest table to queue sync requests to external databases.
|
||||
-- Each request stores the QStash options for making HTTP requests and tracks when fulfillment started.
|
||||
CREATE TABLE "OutgoingRequest" (
|
||||
CREATE TABLE "OutgoingRequest" (
|
||||
"id" UUID NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"deduplicationKey" TEXT,
|
||||
@ -45,14 +36,6 @@ CREATE TABLE "OutgoingRequest" (
|
||||
CONSTRAINT "OutgoingRequest_deduplicationKey_key" UNIQUE ("deduplicationKey")
|
||||
);
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
CREATE INDEX "OutgoingRequest_startedFulfillingAt_deduplicationKey_idx" ON "OutgoingRequest"("startedFulfillingAt", "deduplicationKey");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- Creates composite index on startedFulfillingAt and createdAt for efficient querying of pending requests in order.
|
||||
-- This allows fast lookups of pending requests (WHERE startedFulfillingAt IS NULL) ordered by createdAt.
|
||||
CREATE INDEX "OutgoingRequest_startedFulfillingAt_createdAt_idx" ON "OutgoingRequest"("startedFulfillingAt", "createdAt");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- Creates DeletedRow table to log information about deleted rows from other tables.
|
||||
-- Stores the primary key and full data of deleted rows so external databases can be notified of deletions.
|
||||
@ -65,41 +48,96 @@ CREATE TABLE "DeletedRow" (
|
||||
"data" JSONB,
|
||||
"deletedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"startedFulfillingAt" TIMESTAMP(3),
|
||||
"shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
|
||||
CONSTRAINT "DeletedRow_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- Creates indexes on DeletedRow table for efficient querying by sequence, table name, and tenant.
|
||||
CREATE UNIQUE INDEX "DeletedRow_sequenceId_key" ON "DeletedRow"("sequenceId");
|
||||
-- Creates ExternalDbSyncMetadata table to store external database sync configuration.
|
||||
-- Uses a singleton constraint to ensure only one row exists.
|
||||
CREATE TABLE "ExternalDbSyncMetadata" (
|
||||
"id" TEXT NOT NULL DEFAULT gen_random_uuid(),
|
||||
"singleton" "BooleanTrue" NOT NULL DEFAULT 'TRUE'::"BooleanTrue",
|
||||
"sequencerEnabled" BOOLEAN NOT NULL DEFAULT true,
|
||||
"pollerEnabled" BOOLEAN NOT NULL DEFAULT true,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "ExternalDbSyncMetadata_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
CREATE INDEX "DeletedRow_tableName_idx" ON "DeletedRow"("tableName");
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
-- Creates unique indexes on sequenceId columns to ensure no duplicate sequence IDs exist.
|
||||
CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ContactChannel_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ContactChannel"("sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
CREATE INDEX "DeletedRow_tenancyId_idx" ON "DeletedRow"("tenancyId");
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUser_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."ProjectUser"("sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_sequenceId_key" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "ExternalDbSyncMetadata_singleton_key" ON /* SCHEMA_NAME_SENTINEL */."ExternalDbSyncMetadata"("singleton");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
-- Creates composite indexes on (tenancyId, sequenceId) for efficient sync-engine queries.
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUser_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUser"("tenancyId", "sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "ContactChannel_tenancyId_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ContactChannel"("tenancyId", "sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "OutgoingRequest_startedFulfillingAt_deduplicationKey_idx" ON /* SCHEMA_NAME_SENTINEL */."OutgoingRequest"("startedFulfillingAt", "deduplicationKey");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "OutgoingRequest_startedFulfillingAt_createdAt_idx" ON /* SCHEMA_NAME_SENTINEL */."OutgoingRequest"("startedFulfillingAt", "createdAt");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_tableName_idx" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("tableName");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_tenancyId_idx" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("tenancyId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
-- Creates composite index for efficient querying of deleted rows by tenant and table, ordered by sequence.
|
||||
CREATE INDEX "DeletedRow_tenancyId_tableName_sequenceId_idx" ON "DeletedRow"("tenancyId", "tableName", "sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- Adds shouldUpdateSequenceId flag to track which rows need their sequenceId updated.
|
||||
ALTER TABLE "ProjectUser" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
ALTER TABLE "ContactChannel" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
ALTER TABLE "DeletedRow" ADD COLUMN "shouldUpdateSequenceId" BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_tenancyId_tableName_sequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("tenancyId", "tableName", "sequenceId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
-- Creates indexes on (shouldUpdateSequenceId, tenancyId) to quickly find rows that need updates
|
||||
-- and support ORDER BY tenancyId for less fragmented updates.
|
||||
CREATE INDEX "ProjectUser_shouldUpdateSequenceId_idx" ON "ProjectUser"("shouldUpdateSequenceId", "tenancyId");
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "ProjectUser_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ProjectUser"("shouldUpdateSequenceId", "tenancyId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
CREATE INDEX "ContactChannel_shouldUpdateSequenceId_idx" ON "ContactChannel"("shouldUpdateSequenceId", "tenancyId");
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "ContactChannel_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."ContactChannel"("shouldUpdateSequenceId", "tenancyId");
|
||||
|
||||
-- SPLIT_STATEMENT_SENTINEL
|
||||
CREATE INDEX "DeletedRow_shouldUpdateSequenceId_idx" ON "DeletedRow"("shouldUpdateSequenceId", "tenancyId");
|
||||
-- SINGLE_STATEMENT_SENTINEL
|
||||
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "DeletedRow_shouldUpdateSequenceId_idx" ON /* SCHEMA_NAME_SENTINEL */."DeletedRow"("shouldUpdateSequenceId", "tenancyId");
|
||||
|
||||
@ -1,24 +0,0 @@
|
||||
-- DropIndex
|
||||
DROP INDEX "ContactChannel_shouldUpdateSequenceId_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "DeletedRow_shouldUpdateSequenceId_idx";
|
||||
|
||||
-- DropIndex
|
||||
DROP INDEX "ProjectUser_shouldUpdateSequenceId_idx";
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "ExternalDbSyncMetadata" (
|
||||
"id" TEXT NOT NULL DEFAULT gen_random_uuid(),
|
||||
"singleton" "BooleanTrue" NOT NULL DEFAULT 'TRUE',
|
||||
"sequencerEnabled" BOOLEAN NOT NULL DEFAULT true,
|
||||
"pollerEnabled" BOOLEAN NOT NULL DEFAULT true,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "ExternalDbSyncMetadata_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "ExternalDbSyncMetadata_singleton_key" ON "ExternalDbSyncMetadata"("singleton");
|
||||
|
||||
@ -243,7 +243,7 @@ model ProjectUser {
|
||||
@@index([tenancyId, createdAt(sort: Asc)], name: "ProjectUser_createdAt_asc")
|
||||
@@index([tenancyId, createdAt(sort: Desc)], name: "ProjectUser_createdAt_desc")
|
||||
@@index([tenancyId, sequenceId], name: "ProjectUser_tenancyId_sequenceId_idx")
|
||||
// Partial index for external db sync backfill lives in migration SQL.
|
||||
@@index([shouldUpdateSequenceId, tenancyId], name: "ProjectUser_shouldUpdateSequenceId_idx")
|
||||
}
|
||||
|
||||
// This should be renamed to "OAuthAccount" as it is not always bound to a user
|
||||
@ -309,7 +309,7 @@ model ContactChannel {
|
||||
// only one contact channel per project with the same value and type can be used for auth
|
||||
@@unique([tenancyId, type, value, usedForAuth])
|
||||
@@index([tenancyId, sequenceId], name: "ContactChannel_tenancyId_sequenceId_idx")
|
||||
// Partial index for external db sync backfill lives in migration SQL (WHERE shouldUpdateSequenceId = TRUE).
|
||||
@@index([shouldUpdateSequenceId, tenancyId], name: "ContactChannel_shouldUpdateSequenceId_idx")
|
||||
}
|
||||
|
||||
model AuthMethod {
|
||||
@ -1113,5 +1113,5 @@ model DeletedRow {
|
||||
@@index([tenancyId])
|
||||
// composite index for efficient querying of deleted rows by tenant and table, ordered by sequence
|
||||
@@index([tenancyId, tableName, sequenceId])
|
||||
// Partial index for external db sync backfill lives in migration SQL (WHERE shouldUpdateSequenceId = TRUE).
|
||||
@@index([shouldUpdateSequenceId, tenancyId], name: "DeletedRow_shouldUpdateSequenceId_idx")
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user