CodeCow/stack
1
0
Fork 0
mirror of https://github.com/stack-auth/stack.git synced 2026-06-04 21:04:37 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[Fix]: Investigate Memory Leak on Verify Data Integrity (#1269)

Browse Source 
### Context
We encountered an out of memory error when running verify-data-integrity
against the prod database. This was the error:
`FATAL ERROR: Ineffective mark-compacts near heap limit Allocation
failed - JavaScript heap out of memory`. This was one of the things
preventing verify-data-integrity from running successfully in prod.

### Summary of Changes
Local stress testing with constrained heap and memory telemetry revealed
that the rise in used heap memory was directly proportional to the
number of api calls. Investigation revealed that the `currentOutputData`
array was growing with each api call and was kept in memory. Since it
was still being appended to, it was actively kept in the heap. We
refactor the script to no longer use it, and for the two flags
`--save-output` and `--verify-output` that used it before, we refactor
them to not need to. `--save-output` now streams responses to disk as
JSONL and `--verify-output` now compares each response immediately and
discards it.
We also note a potential source of a future memory leak in the
`allUsers` array that is populated in memory for each project. We
refactor to paginate instead. Note that this didn't cause a memory leak
on local, this is a preventive measure.

### Out of Scope
fetching all transactions in the payments section of the script is
another potential cause for concern, but since the payments section of
the script will be refactored soon, we defer that discussion.
This commit is contained in:
Aman Ganapathy 2026-03-23 08:55:10 -07:00 committed by GitHub
parent 0886586aa7
commit 1d00ed2c64
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 201 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
apps/backend/scripts/verify-data-integrity/api.ts
View File

@ -1,3 +1,4 @@
import fs from "fs";
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
import { deepPlainEquals, filterUndefined } from "@stackframe/stack-shared/dist/utils/objects";
@ -8,7 +9,7 @@ export type EndpointOutput = {
responseJson: any,
};
export type OutputData = Record<string, EndpointOutput[]>;
export type OutputData = Map<string, EndpointOutput[]>;
export type ExpectStatusCode = <T = any>(
expectedStatusCode: number,
@ -16,40 +17,109 @@ export type ExpectStatusCode = <T = any>(
request: RequestInit,
) => Promise<T>;
/**
* Reads an output file that may be in either format:
* - Legacy: a single JSON object keyed by endpoint. This was old
* - JSONL: one JSON object per line, each `{ endpoint, output }`
*/
export function loadOutputData(filePath: string): OutputData {
const content = fs.readFileSync(filePath, "utf8").trim();
const data: OutputData = new Map();
if (!content) return data;
const lines = content.split(/\r?\n/);
const firstLine = lines[0];
try {
const parsed = JSON.parse(firstLine);
if (typeof parsed === "object" && parsed !== null && "endpoint" in parsed && "output" in parsed) {
for (const line of lines) {
if (!line.trim()) continue;
const { endpoint, output } = JSON.parse(line);
if (!data.has(endpoint)) data.set(endpoint, []);
data.get(endpoint)!.push(output);
}
return data;
}
} catch {
// Not JSONL — fall through to legacy parse
}
const legacy = JSON.parse(content) as Record<string, EndpointOutput[]>;
for (const [endpoint, outputs] of Object.entries(legacy)) {
data.set(endpoint, outputs);
}
return data;
}
export function createApiHelpers(options: {
currentOutputData: OutputData,
targetOutputData?: OutputData,
/**
* When set, each API response is streamed to this file as JSONL
* (one `{ endpoint, output }` object per line). This avoids
* accumulating all responses in memory. Writes go to a temporary
* file first; call `finalizeOutput()` to rename it to the final path.
*/
outputFilePath?: string,
}) {
const { currentOutputData, targetOutputData } = options;
const { targetOutputData, outputFilePath } = options;
const outputCountByEndpoint = new Map<string, number>();
const tmpFilePath = outputFilePath ? `${outputFilePath}.tmp` : undefined;
if (tmpFilePath) {
fs.writeFileSync(tmpFilePath, "");
}
function appendOutputData(endpoint: string, output: EndpointOutput) {
if (!(endpoint in currentOutputData)) {
currentOutputData[endpoint] = [];
}
const newLength = currentOutputData[endpoint].push(output);
const count = (outputCountByEndpoint.get(endpoint) ?? 0) + 1;
outputCountByEndpoint.set(endpoint, count);
if (targetOutputData) {
if (!(endpoint in targetOutputData)) {
const targetEndpointOutputs = targetOutputData.get(endpoint);
if (!targetEndpointOutputs) {
throw new StackAssertionError(deindent`
Output data mismatch for endpoint ${endpoint}:
Expected ${endpoint} to be in targetOutputData, but it is not.
`, { endpoint });
}
if (targetOutputData[endpoint].length < newLength) {
if (targetEndpointOutputs.length < count) {
throw new StackAssertionError(deindent`
Output data mismatch for endpoint ${endpoint}:
Expected ${targetOutputData[endpoint].length} outputs but got at least ${newLength}.
Expected ${targetEndpointOutputs.length} outputs but got at least ${count}.
`, { endpoint });
}
if (!(deepPlainEquals(targetOutputData[endpoint][newLength - 1], output))) {
if (!(deepPlainEquals(targetEndpointOutputs[count - 1], output))) {
throw new StackAssertionError(deindent`
Output data mismatch for endpoint ${endpoint}:
Expected output[${JSON.stringify(endpoint)}][${newLength - 1}] to be:
${JSON.stringify(targetOutputData[endpoint][newLength - 1], null, 2)}
Expected output[${JSON.stringify(endpoint)}][${count - 1}] to be:
${JSON.stringify(targetEndpointOutputs[count - 1], null, 2)}
but got:
${JSON.stringify(output, null, 2)}.
`, { endpoint });
}
}
if (tmpFilePath) {
fs.appendFileSync(tmpFilePath, JSON.stringify({ endpoint, output }) + "\n");
}
}
function verifyOutputCompleteness() {
if (!targetOutputData) return;
for (const [endpoint, expectedOutputs] of targetOutputData) {
const actualCount = outputCountByEndpoint.get(endpoint) ?? 0;
if (actualCount !== expectedOutputs.length) {
throw new StackAssertionError(deindent`
Output data mismatch for endpoint ${endpoint}:
Expected ${expectedOutputs.length} outputs but got ${actualCount}.
`, { endpoint, expectedCount: expectedOutputs.length, actualCount });
}
}
}
function finalizeOutput() {
if (tmpFilePath && outputFilePath) {
fs.renameSync(tmpFilePath, outputFilePath);
}
}
const expectStatusCode: ExpectStatusCode = async (expectedStatusCode, endpoint, request) => {
@ -87,6 +157,7 @@ export function createApiHelpers(options: {
return {
appendOutputData,
expectStatusCode,
verifyOutputCompleteness,
finalizeOutput,
};
}

233
apps/backend/scripts/verify-data-integrity/index.ts
View File

@ -3,12 +3,12 @@ import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client";
import type { OrganizationRenderedConfig } from "@stackframe/stack-shared/dist/config/schema";
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
import { deepPlainEquals, omit } from "@stackframe/stack-shared/dist/utils/objects";
import { omit } from "@stackframe/stack-shared/dist/utils/objects";
import { wait } from "@stackframe/stack-shared/dist/utils/promises";
import { deindent } from "@stackframe/stack-shared/dist/utils/strings";
import fs from "fs";
import { createApiHelpers, type OutputData } from "./api";
import { createApiHelpers, loadOutputData, type OutputData } from "./api";
import { createPaymentsVerifier } from "./payments-verifier";
import { createRecurse } from "./recurse";
import { verifyStripePayoutIntegrity } from "./stripe-payout-integrity";
@ -19,7 +19,6 @@ const STRIPE_SECRET_KEY = getEnvVariable("STACK_STRIPE_SECRET_KEY", "");
const USE_MOCK_STRIPE_API = STRIPE_SECRET_KEY === "sk_test_mockstripekey";
let targetOutputData: OutputData | undefined = undefined;
const currentOutputData: OutputData = {};
async function main() {
console.log();
@ -83,9 +82,12 @@ async function main() {
const maxUsersPerProject = maxUsersPerProjectFlag
? parseInt(maxUsersPerProjectFlag.split("=")[1], 10)
: Infinity;
const { recurse, collectedErrors } = createRecurse({ noBail });
if (shouldSaveOutput && shouldVerifyOutput) {
throw new Error("Cannot use --save-output and --verify-output at the same time.");
}
if (noBail) {
console.log(`Running in no-bail mode: will continue on errors and report all at the end.`);
}
@ -102,11 +104,12 @@ async function main() {
throw new Error(`Cannot verify output: ${OUTPUT_FILE_PATH} does not exist`);
}
try {
targetOutputData = JSON.parse(fs.readFileSync(OUTPUT_FILE_PATH, "utf8"));
targetOutputData = loadOutputData(OUTPUT_FILE_PATH);
// TODO next-release these are hacks for the migration, delete them
if (targetOutputData) {
targetOutputData["/api/v1/internal/projects/current"] = targetOutputData["/api/v1/internal/projects/current"].map(output => {
const projectCurrentOutputs = targetOutputData.get("/api/v1/internal/projects/current");
if (projectCurrentOutputs) {
targetOutputData.set("/api/v1/internal/projects/current", projectCurrentOutputs.map(output => {
if ("config" in output.responseJson) {
delete output.responseJson.config.id;
output.responseJson.config.oauth_providers = output.responseJson.config.oauth_providers
@ -116,7 +119,7 @@ async function main() {
.map((provider: any) => omit(provider, ["enabled"]));
}
return output;
});
}));
}
console.log(`Loaded previous output data for verification`);
@ -125,9 +128,9 @@ async function main() {
}
}
const { expectStatusCode } = createApiHelpers({
currentOutputData,
const { expectStatusCode, verifyOutputCompleteness, finalizeOutput } = createApiHelpers({
targetOutputData,
outputFilePath: shouldSaveOutput ? OUTPUT_FILE_PATH : undefined,
});
const projects = await prismaClient.project.findMany({
@ -191,30 +194,6 @@ async function main() {
]);
void currentProject;
// Fetch users with pagination
const PAGE_LIMIT = 1000;
const allUsers: any[] = [];
let cursor: string | undefined = undefined;
while (allUsers.length < maxUsersPerProject) {
const remainingToFetch = maxUsersPerProject - allUsers.length;
const limit = Math.min(PAGE_LIMIT, remainingToFetch);
const cursorParam: string = cursor ? `&cursor=${encodeURIComponent(cursor)}` : "";
const usersPage = await expectStatusCode(200, `/api/v1/users?limit=${limit}${cursorParam}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
allUsers.push(...usersPage.items);
if (!usersPage.pagination?.next_cursor) {
break;
}
cursor = usersPage.pagination.next_cursor;
}
const users = { items: allUsers.slice(0, maxUsersPerProject) };
const tenancy = await getSoleTenancyFromProjectBranch(projectId, DEFAULT_BRANCH_ID, true);
const paymentsConfig = tenancy ? (tenancy.config as OrganizationRenderedConfig).payments : undefined;
const paymentsVerifier = tenancy && paymentsConfig
@ -241,86 +220,110 @@ async function main() {
const verifiedTeams = new Set<string>();
if (!skipUsers) {
for (let j = 0; j < users.items.length; j++) {
const user = users.items[j];
await recurse(`[user ${j + 1}/${users.items.length}] ${user.display_name ?? user.primary_email}`, async (recurse) => {
// get user individually
await expectStatusCode(200, `/api/v1/users/${user.id}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
const userCount = tenancy
? await (await getPrismaClientForTenancy(tenancy)).projectUser.count({ where: { tenancyId: tenancy.id } })
: 0;
// list project permissions
const projectPermissions = await expectStatusCode(200, `/api/v1/project-permissions?user_id=${user.id}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
for (const projectPermission of projectPermissions.items) {
// `any` because these endpoint response types aren't imported here,
// and this script is intentionally tolerant of response shape changes.
if (!projectPermissionDefinitions.items.some((p: any) => p.id === projectPermission.id)) {
throw new StackAssertionError(deindent`
Project permission ${projectPermission.id} not found in project permission definitions.
`);
}
}
// Process users page-by-page to avoid holding all users in memory at once
const PAGE_LIMIT = 1000;
let userCursor: string | undefined = undefined;
let usersProcessed = 0;
let hasMore = true;
// list teams
const teams = await expectStatusCode(200, `/api/v1/teams?user_id=${user.id}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
for (const team of teams.items) {
await recurse(`[team ${team.id}] ${team.name}`, async (recurse) => {
// list team permissions
const teamPermissions = await expectStatusCode(200, `/api/v1/team-permissions?team_id=${team.id}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
for (const teamPermission of teamPermissions.items) {
// `any` because these endpoint response types aren't imported here,
// and this script is intentionally tolerant of response shape changes.
if (!teamPermissionDefinitions.items.some((p: any) => p.id === teamPermission.id)) {
throw new StackAssertionError(deindent`
Team permission ${teamPermission.id} not found in team permission definitions.
`);
}
}
});
if (paymentsVerifier && !verifiedTeams.has(team.id)) {
await paymentsVerifier.verifyCustomerPayments({
customerType: "team",
customerId: team.id,
});
verifiedTeams.add(team.id);
}
}
if (paymentsVerifier) {
await paymentsVerifier.verifyCustomerPayments({
customerType: "user",
customerId: user.id,
});
}
while (hasMore && usersProcessed < maxUsersPerProject) {
const remainingToFetch = maxUsersPerProject - usersProcessed;
const limit = Math.min(PAGE_LIMIT, remainingToFetch);
const cursorParam: string = userCursor ? `&cursor=${encodeURIComponent(userCursor)}` : "";
const usersPage = await expectStatusCode(200, `/api/v1/users?limit=${limit}${cursorParam}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
for (const user of usersPage.items) {
if (usersProcessed >= maxUsersPerProject) break;
usersProcessed++;
await recurse(`[user ${usersProcessed}/${Math.min(userCount, maxUsersPerProject)}] ${user.display_name ?? user.primary_email}`, async (recurse) => {
await expectStatusCode(200, `/api/v1/users/${user.id}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
const projectPermissions = await expectStatusCode(200, `/api/v1/project-permissions?user_id=${user.id}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
for (const projectPermission of projectPermissions.items) {
// `any` because these endpoint response types aren't imported here,
// and this script is intentionally tolerant of response shape changes.
if (!projectPermissionDefinitions.items.some((p: any) => p.id === projectPermission.id)) {
throw new StackAssertionError(deindent`
Project permission ${projectPermission.id} not found in project permission definitions.
`);
}
}
const teams = await expectStatusCode(200, `/api/v1/teams?user_id=${user.id}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
for (const team of teams.items) {
await recurse(`[team ${team.id}] ${team.name}`, async (recurse) => {
const teamPermissions = await expectStatusCode(200, `/api/v1/team-permissions?team_id=${team.id}`, {
method: "GET",
headers: {
"x-stack-project-id": projectId,
"x-stack-access-type": "admin",
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
},
});
for (const teamPermission of teamPermissions.items) {
// `any` because these endpoint response types aren't imported here,
// and this script is intentionally tolerant of response shape changes.
if (!teamPermissionDefinitions.items.some((p: any) => p.id === teamPermission.id)) {
throw new StackAssertionError(deindent`
Team permission ${teamPermission.id} not found in team permission definitions.
`);
}
}
});
if (paymentsVerifier && !verifiedTeams.has(team.id)) {
await paymentsVerifier.verifyCustomerPayments({
customerType: "team",
customerId: team.id,
});
verifiedTeams.add(team.id);
}
}
if (paymentsVerifier) {
await paymentsVerifier.verifyCustomerPayments({
customerType: "user",
customerId: user.id,
});
}
});
}
hasMore = !!usersPage.pagination?.next_cursor;
userCursor = usersPage.pagination?.next_cursor ?? undefined;
}
if (paymentsVerifier) {
@ -335,13 +338,9 @@ async function main() {
});
}
if (targetOutputData && !deepPlainEquals(currentOutputData, targetOutputData)) {
throw new StackAssertionError(deindent`
Output data mismatch between final and target output data.
`);
}
verifyOutputCompleteness();
if (shouldSaveOutput) {
fs.writeFileSync(OUTPUT_FILE_PATH, JSON.stringify(currentOutputData, null, 2));
finalizeOutput();
console.log(`Output saved to ${OUTPUT_FILE_PATH}`);
}