From b86864ca60102bab60d495b0de2f34402ccb04b6 Mon Sep 17 00:00:00 2001 From: Baptiste Arnaud Date: Wed, 24 Jun 2026 18:38:42 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Improve=20expired=20data=20clean?= =?UTF-8?q?up=20reliability=20(#2536)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reduces expired chat session delete chunks to keep cleanup transactions smaller. - Forces old chat session lookups to use the primary database before deletion. - Adds structured progress, timing, and error logging to expired data cleanup steps. --- .../scripts/src/helpers/cleanExpiredData.ts | 209 ++++++++++++++---- 1 file changed, 164 insertions(+), 45 deletions(-) diff --git a/packages/scripts/src/helpers/cleanExpiredData.ts b/packages/scripts/src/helpers/cleanExpiredData.ts index 3e096b410..f3e3bc88c 100644 --- a/packages/scripts/src/helpers/cleanExpiredData.ts +++ b/packages/scripts/src/helpers/cleanExpiredData.ts @@ -1,48 +1,99 @@ import prisma from "@typebot.io/prisma/withReadReplica"; export const cleanExpiredData = async () => { + console.log("[cleanExpiredData] Starting expired data cleanup..."); + const startedAt = Date.now(); const { totalDeletedChatSessions } = await deleteOldChatSessions(); await deleteExpiredAppSessions(); await deleteExpiredVerificationTokens(); + console.log( + `[cleanExpiredData] Finished expired data cleanup in ${formatElapsedTime(startedAt)}.`, + ); return { totalDeletedChatSessions }; }; const CHAT_SESSIONS_BATCH_SIZE = 80000; +const CHAT_SESSIONS_DELETE_CHUNK_SIZE = 400; const deleteOldChatSessions = async () => { const twoDaysAgo = new Date(); twoDaysAgo.setDate(twoDaysAgo.getDate() - 2); twoDaysAgo.setHours(0, 0, 0, 0); let totalDeletedChatSessions = 0; let deletingChatSessions: number; + let batchNumber = 0; + console.log( + `[cleanExpiredData] Looking for chat sessions updated before ${twoDaysAgo.toISOString()}.`, + ); do { - console.log(`🔨 Deleting ${CHAT_SESSIONS_BATCH_SIZE} old chat sessions...`); - const chatSessions = await prisma.chatSession.findMany({ - where: { - updatedAt: { - lte: twoDaysAgo, + batchNumber += 1; + console.log( + `[cleanExpiredData] Chat sessions batch ${batchNumber}: fetching up to ${CHAT_SESSIONS_BATCH_SIZE} records from primary...`, + ); + const fetchStartedAt = Date.now(); + const chatSessions = await prisma + .$primary() + .chatSession.findMany({ + where: { + updatedAt: { + lte: twoDaysAgo, + }, }, - }, - select: { - id: true, - }, - take: CHAT_SESSIONS_BATCH_SIZE, - }); + select: { + id: true, + }, + take: CHAT_SESSIONS_BATCH_SIZE, + }) + .catch((error) => { + logCleanupError( + `[cleanExpiredData] Chat sessions batch ${batchNumber}: fetch failed after ${formatElapsedTime(fetchStartedAt)}.`, + error, + ); + throw error; + }); deletingChatSessions = chatSessions.length; totalDeletedChatSessions += deletingChatSessions; + console.log( + `[cleanExpiredData] Chat sessions batch ${batchNumber}: fetched ${deletingChatSessions} records in ${formatElapsedTime(fetchStartedAt)}.`, + ); - const chunkSize = 500; - for (let i = 0; i < chatSessions.length; i += chunkSize) { - const chunk = chatSessions.slice(i, i + chunkSize); - await prisma.chatSession.deleteMany({ - where: { - id: { - in: chunk.map((chatSession) => chatSession.id), + const totalChunks = Math.ceil( + chatSessions.length / CHAT_SESSIONS_DELETE_CHUNK_SIZE, + ); + for ( + let i = 0; + i < chatSessions.length; + i += CHAT_SESSIONS_DELETE_CHUNK_SIZE + ) { + const chunk = chatSessions.slice(i, i + CHAT_SESSIONS_DELETE_CHUNK_SIZE); + const chunkNumber = Math.floor(i / CHAT_SESSIONS_DELETE_CHUNK_SIZE) + 1; + const deleteStartedAt = Date.now(); + console.log( + `[cleanExpiredData] Chat sessions batch ${batchNumber}: deleting chunk ${chunkNumber}/${totalChunks} (${chunk.length} records, offset ${i})...`, + ); + const { count } = await prisma.chatSession + .deleteMany({ + where: { + id: { + in: chunk.map((chatSession) => chatSession.id), + }, }, - }, - }); + }) + .catch((error) => { + logCleanupError( + `[cleanExpiredData] Chat sessions batch ${batchNumber}: delete chunk ${chunkNumber}/${totalChunks} failed after ${formatElapsedTime(deleteStartedAt)} (${chunk.length} records, offset ${i}).`, + error, + ); + throw error; + }); + console.log( + `[cleanExpiredData] Chat sessions batch ${batchNumber}: deleted ${count}/${chunk.length} records from chunk ${chunkNumber}/${totalChunks} in ${formatElapsedTime(deleteStartedAt)}.`, + ); } } while (deletingChatSessions === CHAT_SESSIONS_BATCH_SIZE); + console.log( + `[cleanExpiredData] Deleted ${totalDeletedChatSessions} old chat sessions.`, + ); return { totalDeletedChatSessions }; }; @@ -50,14 +101,28 @@ const deleteExpiredAppSessions = async () => { const threeDaysAgo = new Date(); threeDaysAgo.setDate(threeDaysAgo.getDate() - 3); threeDaysAgo.setHours(0, 0, 0, 0); - const { count } = await prisma.session.deleteMany({ - where: { - expires: { - lte: threeDaysAgo, + console.log( + `[cleanExpiredData] Deleting app sessions expiring before ${threeDaysAgo.toISOString()}...`, + ); + const startedAt = Date.now(); + const { count } = await prisma.session + .deleteMany({ + where: { + expires: { + lte: threeDaysAgo, + }, }, - }, - }); - console.log(`Deleted ${count} expired user sessions.`); + }) + .catch((error) => { + logCleanupError( + `[cleanExpiredData] Expired app sessions delete failed after ${formatElapsedTime(startedAt)}.`, + error, + ); + throw error; + }); + console.log( + `[cleanExpiredData] Deleted ${count} expired user sessions in ${formatElapsedTime(startedAt)}.`, + ); }; const deleteExpiredVerificationTokens = async () => { @@ -65,33 +130,87 @@ const deleteExpiredVerificationTokens = async () => { threeDaysAgo.setDate(threeDaysAgo.getDate() - 3); threeDaysAgo.setHours(0, 0, 0, 0); let totalVerificationTokens: number; + let batchNumber = 0; + console.log( + `[cleanExpiredData] Looking for verification tokens expiring before ${threeDaysAgo.toISOString()}.`, + ); do { - const verificationTokens = await prisma.verificationToken.findMany({ - where: { - expires: { - lte: threeDaysAgo, + batchNumber += 1; + const fetchStartedAt = Date.now(); + console.log( + `[cleanExpiredData] Verification tokens batch ${batchNumber}: fetching up to 80000 records...`, + ); + const verificationTokens = await prisma.verificationToken + .findMany({ + where: { + expires: { + lte: threeDaysAgo, + }, }, - }, - select: { - token: true, - }, - take: 80000, - }); + select: { + token: true, + }, + take: 80000, + }) + .catch((error) => { + logCleanupError( + `[cleanExpiredData] Verification tokens batch ${batchNumber}: fetch failed after ${formatElapsedTime(fetchStartedAt)}.`, + error, + ); + throw error; + }); totalVerificationTokens = verificationTokens.length; - console.log(`Deleting ${verificationTokens.length} expired tokens...`); + console.log( + `[cleanExpiredData] Verification tokens batch ${batchNumber}: fetched ${verificationTokens.length} records in ${formatElapsedTime(fetchStartedAt)}.`, + ); const chunkSize = 1000; + const totalChunks = Math.ceil(verificationTokens.length / chunkSize); for (let i = 0; i < verificationTokens.length; i += chunkSize) { const chunk = verificationTokens.slice(i, i + chunkSize); - await prisma.verificationToken.deleteMany({ - where: { - token: { - in: chunk.map((verificationToken) => verificationToken.token), + const chunkNumber = Math.floor(i / chunkSize) + 1; + const deleteStartedAt = Date.now(); + console.log( + `[cleanExpiredData] Verification tokens batch ${batchNumber}: deleting chunk ${chunkNumber}/${totalChunks} (${chunk.length} records, offset ${i})...`, + ); + const { count } = await prisma.verificationToken + .deleteMany({ + where: { + token: { + in: chunk.map((verificationToken) => verificationToken.token), + }, }, - }, - }); + }) + .catch((error) => { + logCleanupError( + `[cleanExpiredData] Verification tokens batch ${batchNumber}: delete chunk ${chunkNumber}/${totalChunks} failed after ${formatElapsedTime(deleteStartedAt)} (${chunk.length} records, offset ${i}).`, + error, + ); + throw error; + }); + console.log( + `[cleanExpiredData] Verification tokens batch ${batchNumber}: deleted ${count}/${chunk.length} records from chunk ${chunkNumber}/${totalChunks} in ${formatElapsedTime(deleteStartedAt)}.`, + ); } } while (totalVerificationTokens === 80000); - console.log("Done!"); + console.log("[cleanExpiredData] Done deleting expired verification tokens."); +}; + +const formatElapsedTime = (startedAt: number) => + `${((Date.now() - startedAt) / 1000).toFixed(1)}s`; + +const logCleanupError = (message: string, error: unknown) => { + console.error(message); + if (error instanceof Error) { + console.error(`[cleanExpiredData] Error name: ${error.name}`); + console.error(`[cleanExpiredData] Error message: ${error.message}`); + console.error(`[cleanExpiredData] Error stack: ${error.stack}`); + } + if (typeof error === "object" && error !== null && "status" in error) { + console.error("[cleanExpiredData] Error status:", error.status); + } + if (typeof error === "object" && error !== null && "body" in error) { + console.error("[cleanExpiredData] Error body:", error.body); + } };