mirror of
https://github.com/stack-auth/stack.git
synced 2026-06-13 21:01:21 +08:00
<img width="567" height="249" alt="Screenshot 2025-10-20 at 11 23 10 AM" src="https://github.com/user-attachments/assets/340df844-f619-489f-8d41-cc26bc165018" /> <img width="595" height="255" alt="Screenshot 2025-10-20 at 11 24 00 AM" src="https://github.com/user-attachments/assets/9321bda1-e6f0-4f53-8c6b-e29d0fc16038" /> <!-- Make sure you've read the CONTRIBUTING.md guidelines: https://github.com/stack-auth/stack-auth/blob/dev/CONTRIBUTING.md --> <!-- RECURSEML_SUMMARY:START --> ## High-level PR Summary This PR optimizes the performance of user list and metrics endpoints by refactoring SQL queries to use more efficient patterns. The changes include rewriting queries to use `LATERAL` joins and CTEs with proper filtering, extracting common user mapping logic into reusable functions, and adding performance tests with SQL scripts to generate realistic test data (10,000 mock users and activity events across 100 countries). ⏱️ Estimated Review Time: 30-90 minutes <details> <summary>💡 Review Order Suggestion</summary> | Order | File Path | |-------|-----------| | 1 | `apps/e2e/tests/backend/performance/mock-users.sql` | | 2 | `apps/e2e/tests/backend/performance/mock-metric-events.sql` | | 3 | `apps/e2e/tests/backend/performance/users-list.test.ts` | | 4 | `apps/backend/src/app/api/latest/users/crud.tsx` | | 5 | `apps/backend/src/app/api/latest/internal/metrics/route.tsx` | </details> [](https://discord.gg/n3SsVDAW6U) [ <!-- RECURSEML_SUMMARY:END --> <!-- ELLIPSIS_HIDDEN --> ---- > [!IMPORTANT] > Optimize metrics and user list endpoints with SQL refactoring, caching, and performance tests, adding a `CacheEntry` model and mock data scripts. > > - **Performance Optimization**: > - Refactor SQL queries in `route.tsx` to use `LATERAL` joins and CTEs for efficient data retrieval. > - Implement caching in `route.tsx` using `getOrSetCacheValue()` to reduce database load. > - **Database Changes**: > - Add `CacheEntry` model to `schema.prisma` and create corresponding table and index in `migration.sql`. > - Remove auto-migration metadata step from `check-prisma-migrations.yaml`. > - **Testing**: > - Add performance tests in `metrics.test.ts` to benchmark metrics and user endpoints. > - Create mock data scripts `mock-users.sql` and `mock-metric-events.sql` for testing with 10,000 users and events across 100 countries. > - **Miscellaneous**: > - Update `db-migrations.ts` to include new migration file generation logic. > - Add `cache.tsx` for caching logic implementation. > > <sup>This description was created by </sup>[<img alt="Ellipsis" src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=stack-auth%2Fstack-auth&utm_source=github&utm_medium=referral)<sup> for4d9be71063. You can [customize](https://app.ellipsis.dev/stack-auth/settings/summaries) this summary. It will automatically update as commits are pushed.</sup> ---- <!-- ELLIPSIS_HIDDEN --> <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Metrics now use a cache layer with per-entry TTL and tenancy-aware loaders. * **Bug Fixes** * Improved accuracy of daily active and related metrics with tenancy-aware counting and more robust last-active computation. * **Performance** * Faster metrics responses via batched reads and cache-backed endpoints. * **Tests** * Added end-to-end performance benchmarks and SQL seed scripts for metrics/user load testing. * **Chores** * DB migration added support for cached entries; CI migration check flow adjusted; migration tooling improved. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Konsti Wohlwend <n2d4xc@gmail.com>
222 lines
6.4 KiB
TypeScript
222 lines
6.4 KiB
TypeScript
import { applyMigrations } from "@/auto-migrations";
|
|
import { MIGRATION_FILES_DIR, getMigrationFiles } from "@/auto-migrations/utils";
|
|
import { globalPrismaClient, globalPrismaSchema, sqlQuoteIdent } from "@/prisma-client";
|
|
import { Prisma } from "@prisma/client";
|
|
import { spawnSync } from "child_process";
|
|
import fs from "fs";
|
|
import path from "path";
|
|
import * as readline from "readline";
|
|
import { seed } from "../prisma/seed";
|
|
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
|
|
|
|
const dropSchema = async () => {
|
|
await globalPrismaClient.$executeRaw(Prisma.sql`DROP SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} CASCADE`);
|
|
await globalPrismaClient.$executeRaw(Prisma.sql`CREATE SCHEMA ${sqlQuoteIdent(globalPrismaSchema)}`);
|
|
await globalPrismaClient.$executeRaw(Prisma.sql`GRANT ALL ON SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} TO postgres`);
|
|
await globalPrismaClient.$executeRaw(Prisma.sql`GRANT ALL ON SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} TO public`);
|
|
};
|
|
|
|
|
|
const askQuestion = (question: string) => {
|
|
const rl = readline.createInterface({
|
|
input: process.stdin,
|
|
output: process.stdout,
|
|
});
|
|
|
|
return new Promise<string>((resolve) => {
|
|
rl.question(question, (answer) => {
|
|
rl.close();
|
|
resolve(answer);
|
|
});
|
|
});
|
|
};
|
|
|
|
const promptDropDb = async () => {
|
|
const answer = (await askQuestion(
|
|
'Are you sure you want to drop everything in the database? This action cannot be undone. (y/N): ',
|
|
)).trim();
|
|
|
|
if (answer.toLowerCase() !== 'y') {
|
|
console.log('Operation cancelled');
|
|
process.exit(0);
|
|
}
|
|
};
|
|
|
|
const formatMigrationName = (input: string) =>
|
|
input
|
|
.trim()
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9]+/g, '_')
|
|
.replace(/^_+|_+$/g, '');
|
|
|
|
const promptMigrationName = async () => {
|
|
while (true) {
|
|
const rawName = (await askQuestion('Enter a migration name: ')).trim();
|
|
const formattedName = formatMigrationName(rawName);
|
|
|
|
if (!formattedName) {
|
|
console.log('Migration name cannot be empty. Please try again.');
|
|
continue;
|
|
}
|
|
|
|
if (formattedName !== rawName) {
|
|
console.log(`Using sanitized migration name: ${formattedName}`);
|
|
}
|
|
|
|
return formattedName;
|
|
}
|
|
};
|
|
|
|
const timestampPrefix = () => new Date().toISOString().replace(/\D/g, '').slice(0, 14);
|
|
|
|
const generateMigrationFile = async () => {
|
|
const migrationName = await promptMigrationName();
|
|
const folderName = `${timestampPrefix()}_${migrationName}`;
|
|
const migrationDir = path.join(MIGRATION_FILES_DIR, folderName);
|
|
const migrationSqlPath = path.join(migrationDir, 'migration.sql');
|
|
const diffUrl = getEnvVariable('STACK_DIRECT_DATABASE_CONNECTION_STRING');
|
|
|
|
console.log(`Generating migration ${folderName}...`);
|
|
const diffResult = spawnSync(
|
|
'pnpm',
|
|
[
|
|
'-s',
|
|
'prisma',
|
|
'migrate',
|
|
'diff',
|
|
'--from-url',
|
|
diffUrl,
|
|
'--to-schema-datamodel',
|
|
'prisma/schema.prisma',
|
|
'--script',
|
|
],
|
|
{
|
|
cwd: process.cwd(),
|
|
encoding: 'utf8',
|
|
},
|
|
);
|
|
|
|
if (diffResult.error || diffResult.status !== 0) {
|
|
console.error(diffResult.stdout);
|
|
console.error(diffResult.stderr);
|
|
throw diffResult.error ?? new Error(`Failed to generate migration (exit code ${diffResult.status})`);
|
|
}
|
|
|
|
const sql = diffResult.stdout;
|
|
|
|
if (!sql.trim()) {
|
|
console.log('No schema changes detected. Migration file was not created.');
|
|
} else {
|
|
fs.mkdirSync(migrationDir, { recursive: true });
|
|
fs.writeFileSync(migrationSqlPath, sql, 'utf8');
|
|
console.log(`Migration written to ${path.relative(process.cwd(), migrationSqlPath)}`);
|
|
console.log('Applying migration...');
|
|
await migrate([{ migrationName: folderName, sql }]);
|
|
}
|
|
};
|
|
|
|
const migrate = async (selectedMigrationFiles?: { migrationName: string, sql: string }[]) => {
|
|
const startTime = performance.now();
|
|
const migrationFiles = selectedMigrationFiles ?? getMigrationFiles(MIGRATION_FILES_DIR);
|
|
const totalMigrations = migrationFiles.length;
|
|
|
|
const result = await applyMigrations({
|
|
prismaClient: globalPrismaClient,
|
|
migrationFiles,
|
|
logging: true,
|
|
schema: globalPrismaSchema,
|
|
});
|
|
|
|
const endTime = performance.now();
|
|
const duration = ((endTime - startTime) / 1000).toFixed(2);
|
|
|
|
// Print summary
|
|
console.log('\n' + '='.repeat(60));
|
|
console.log('📊 MIGRATION SUMMARY');
|
|
console.log('='.repeat(60));
|
|
console.log(`✅ Migrations completed successfully`);
|
|
console.log(`⏱️ Duration: ${duration} seconds`);
|
|
console.log(`📁 Total migrations in folder: ${totalMigrations}`);
|
|
console.log(`🆕 Newly applied migrations: ${result.newlyAppliedMigrationNames.length}`);
|
|
console.log(`✓ Already applied migrations: ${totalMigrations - result.newlyAppliedMigrationNames.length}`);
|
|
|
|
if (result.newlyAppliedMigrationNames.length > 0) {
|
|
console.log('\n📝 Newly applied migrations:');
|
|
result.newlyAppliedMigrationNames.forEach((name, index) => {
|
|
console.log(` ${index + 1}. ${name}`);
|
|
});
|
|
} else {
|
|
console.log('\n✨ Database is already up to date!');
|
|
}
|
|
|
|
console.log('='.repeat(60) + '\n');
|
|
|
|
return result;
|
|
};
|
|
|
|
const showHelp = () => {
|
|
console.log(`Database Migration Script
|
|
|
|
Usage: pnpm db-migrations <command>
|
|
|
|
Commands:
|
|
reset Drop all data and recreate the database, then apply migrations and seed
|
|
generate-migration-file Generate a new migration file using Prisma, then reset and migrate
|
|
seed [Advanced] Run database seeding only
|
|
init Apply migrations and seed the database
|
|
migrate Apply migrations
|
|
help Show this help message
|
|
`);
|
|
};
|
|
|
|
const main = async () => {
|
|
const args = process.argv.slice(2);
|
|
const command = args[0];
|
|
|
|
switch (command) {
|
|
case 'reset': {
|
|
await promptDropDb();
|
|
await dropSchema();
|
|
await migrate();
|
|
await seed();
|
|
break;
|
|
}
|
|
case 'generate-migration-file': {
|
|
await promptDropDb();
|
|
await dropSchema();
|
|
await migrate();
|
|
await generateMigrationFile();
|
|
await seed();
|
|
break;
|
|
}
|
|
case 'seed': {
|
|
await seed();
|
|
break;
|
|
}
|
|
case 'init': {
|
|
await migrate();
|
|
await seed();
|
|
break;
|
|
}
|
|
case 'migrate': {
|
|
await migrate();
|
|
break;
|
|
}
|
|
case 'help': {
|
|
showHelp();
|
|
break;
|
|
}
|
|
default: {
|
|
console.error('Unknown command.');
|
|
showHelp();
|
|
process.exit(1);
|
|
}
|
|
}
|
|
};
|
|
|
|
// eslint-disable-next-line no-restricted-syntax
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|