mirror of
https://github.com/stack-auth/stack.git
synced 2026-06-30 21:01:54 +08:00
Merge branch 'dev' into skill-issue
This commit is contained in:
Mantra
GitHub
commit
349ff27eb9
@ -57,6 +57,10 @@ const nextConfig = {
|
||||
serverMinification: false, // needs to be disabled for oidc-provider to work, which relies on the original constructor names
|
||||
},
|
||||
|
||||
outputFileTracingIncludes: {
|
||||
"/api/**": ["../../packages/private/dist/**"],
|
||||
},
|
||||
|
||||
serverExternalPackages: [
|
||||
'oidc-provider',
|
||||
],
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import fs from "fs";
|
||||
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
|
||||
import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
|
||||
import { deepPlainEquals, filterUndefined } from "@stackframe/stack-shared/dist/utils/objects";
|
||||
@ -8,7 +9,7 @@ export type EndpointOutput = {
|
||||
responseJson: any,
|
||||
};
|
||||
|
||||
export type OutputData = Record<string, EndpointOutput[]>;
|
||||
export type OutputData = Map<string, EndpointOutput[]>;
|
||||
|
||||
export type ExpectStatusCode = <T = any>(
|
||||
expectedStatusCode: number,
|
||||
@ -16,40 +17,109 @@ export type ExpectStatusCode = <T = any>(
|
||||
request: RequestInit,
|
||||
) => Promise<T>;
|
||||
|
||||
/**
|
||||
* Reads an output file that may be in either format:
|
||||
* - Legacy: a single JSON object keyed by endpoint. This was old
|
||||
* - JSONL: one JSON object per line, each `{ endpoint, output }`
|
||||
*/
|
||||
export function loadOutputData(filePath: string): OutputData {
|
||||
const content = fs.readFileSync(filePath, "utf8").trim();
|
||||
const data: OutputData = new Map();
|
||||
if (!content) return data;
|
||||
|
||||
const lines = content.split(/\r?\n/);
|
||||
const firstLine = lines[0];
|
||||
try {
|
||||
const parsed = JSON.parse(firstLine);
|
||||
if (typeof parsed === "object" && parsed !== null && "endpoint" in parsed && "output" in parsed) {
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
const { endpoint, output } = JSON.parse(line);
|
||||
if (!data.has(endpoint)) data.set(endpoint, []);
|
||||
data.get(endpoint)!.push(output);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
} catch {
|
||||
// Not JSONL — fall through to legacy parse
|
||||
}
|
||||
|
||||
const legacy = JSON.parse(content) as Record<string, EndpointOutput[]>;
|
||||
for (const [endpoint, outputs] of Object.entries(legacy)) {
|
||||
data.set(endpoint, outputs);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
export function createApiHelpers(options: {
|
||||
currentOutputData: OutputData,
|
||||
targetOutputData?: OutputData,
|
||||
/**
|
||||
* When set, each API response is streamed to this file as JSONL
|
||||
* (one `{ endpoint, output }` object per line). This avoids
|
||||
* accumulating all responses in memory. Writes go to a temporary
|
||||
* file first; call `finalizeOutput()` to rename it to the final path.
|
||||
*/
|
||||
outputFilePath?: string,
|
||||
}) {
|
||||
const { currentOutputData, targetOutputData } = options;
|
||||
const { targetOutputData, outputFilePath } = options;
|
||||
const outputCountByEndpoint = new Map<string, number>();
|
||||
const tmpFilePath = outputFilePath ? `${outputFilePath}.tmp` : undefined;
|
||||
|
||||
if (tmpFilePath) {
|
||||
fs.writeFileSync(tmpFilePath, "");
|
||||
}
|
||||
|
||||
function appendOutputData(endpoint: string, output: EndpointOutput) {
|
||||
if (!(endpoint in currentOutputData)) {
|
||||
currentOutputData[endpoint] = [];
|
||||
}
|
||||
const newLength = currentOutputData[endpoint].push(output);
|
||||
const count = (outputCountByEndpoint.get(endpoint) ?? 0) + 1;
|
||||
outputCountByEndpoint.set(endpoint, count);
|
||||
|
||||
if (targetOutputData) {
|
||||
if (!(endpoint in targetOutputData)) {
|
||||
const targetEndpointOutputs = targetOutputData.get(endpoint);
|
||||
if (!targetEndpointOutputs) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Output data mismatch for endpoint ${endpoint}:
|
||||
Expected ${endpoint} to be in targetOutputData, but it is not.
|
||||
`, { endpoint });
|
||||
}
|
||||
if (targetOutputData[endpoint].length < newLength) {
|
||||
if (targetEndpointOutputs.length < count) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Output data mismatch for endpoint ${endpoint}:
|
||||
Expected ${targetOutputData[endpoint].length} outputs but got at least ${newLength}.
|
||||
Expected ${targetEndpointOutputs.length} outputs but got at least ${count}.
|
||||
`, { endpoint });
|
||||
}
|
||||
if (!(deepPlainEquals(targetOutputData[endpoint][newLength - 1], output))) {
|
||||
if (!(deepPlainEquals(targetEndpointOutputs[count - 1], output))) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Output data mismatch for endpoint ${endpoint}:
|
||||
Expected output[${JSON.stringify(endpoint)}][${newLength - 1}] to be:
|
||||
${JSON.stringify(targetOutputData[endpoint][newLength - 1], null, 2)}
|
||||
Expected output[${JSON.stringify(endpoint)}][${count - 1}] to be:
|
||||
${JSON.stringify(targetEndpointOutputs[count - 1], null, 2)}
|
||||
but got:
|
||||
${JSON.stringify(output, null, 2)}.
|
||||
`, { endpoint });
|
||||
}
|
||||
}
|
||||
|
||||
if (tmpFilePath) {
|
||||
fs.appendFileSync(tmpFilePath, JSON.stringify({ endpoint, output }) + "\n");
|
||||
}
|
||||
}
|
||||
|
||||
function verifyOutputCompleteness() {
|
||||
if (!targetOutputData) return;
|
||||
for (const [endpoint, expectedOutputs] of targetOutputData) {
|
||||
const actualCount = outputCountByEndpoint.get(endpoint) ?? 0;
|
||||
if (actualCount !== expectedOutputs.length) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Output data mismatch for endpoint ${endpoint}:
|
||||
Expected ${expectedOutputs.length} outputs but got ${actualCount}.
|
||||
`, { endpoint, expectedCount: expectedOutputs.length, actualCount });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function finalizeOutput() {
|
||||
if (tmpFilePath && outputFilePath) {
|
||||
fs.renameSync(tmpFilePath, outputFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
const expectStatusCode: ExpectStatusCode = async (expectedStatusCode, endpoint, request) => {
|
||||
@ -87,6 +157,7 @@ export function createApiHelpers(options: {
|
||||
return {
|
||||
appendOutputData,
|
||||
expectStatusCode,
|
||||
verifyOutputCompleteness,
|
||||
finalizeOutput,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -3,12 +3,12 @@ import { getPrismaClientForTenancy, globalPrismaClient } from "@/prisma-client";
|
||||
import type { OrganizationRenderedConfig } from "@stackframe/stack-shared/dist/config/schema";
|
||||
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
|
||||
import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
|
||||
import { deepPlainEquals, omit } from "@stackframe/stack-shared/dist/utils/objects";
|
||||
import { omit } from "@stackframe/stack-shared/dist/utils/objects";
|
||||
import { wait } from "@stackframe/stack-shared/dist/utils/promises";
|
||||
import { deindent } from "@stackframe/stack-shared/dist/utils/strings";
|
||||
import fs from "fs";
|
||||
|
||||
import { createApiHelpers, type OutputData } from "./api";
|
||||
import { createApiHelpers, loadOutputData, type OutputData } from "./api";
|
||||
import { createPaymentsVerifier } from "./payments-verifier";
|
||||
import { createRecurse } from "./recurse";
|
||||
import { verifyStripePayoutIntegrity } from "./stripe-payout-integrity";
|
||||
@ -19,7 +19,6 @@ const STRIPE_SECRET_KEY = getEnvVariable("STACK_STRIPE_SECRET_KEY", "");
|
||||
const USE_MOCK_STRIPE_API = STRIPE_SECRET_KEY === "sk_test_mockstripekey";
|
||||
|
||||
let targetOutputData: OutputData | undefined = undefined;
|
||||
const currentOutputData: OutputData = {};
|
||||
|
||||
async function main() {
|
||||
console.log();
|
||||
@ -83,9 +82,12 @@ async function main() {
|
||||
const maxUsersPerProject = maxUsersPerProjectFlag
|
||||
? parseInt(maxUsersPerProjectFlag.split("=")[1], 10)
|
||||
: Infinity;
|
||||
|
||||
const { recurse, collectedErrors } = createRecurse({ noBail });
|
||||
|
||||
if (shouldSaveOutput && shouldVerifyOutput) {
|
||||
throw new Error("Cannot use --save-output and --verify-output at the same time.");
|
||||
}
|
||||
|
||||
if (noBail) {
|
||||
console.log(`Running in no-bail mode: will continue on errors and report all at the end.`);
|
||||
}
|
||||
@ -102,11 +104,12 @@ async function main() {
|
||||
throw new Error(`Cannot verify output: ${OUTPUT_FILE_PATH} does not exist`);
|
||||
}
|
||||
try {
|
||||
targetOutputData = JSON.parse(fs.readFileSync(OUTPUT_FILE_PATH, "utf8"));
|
||||
targetOutputData = loadOutputData(OUTPUT_FILE_PATH);
|
||||
|
||||
// TODO next-release these are hacks for the migration, delete them
|
||||
if (targetOutputData) {
|
||||
targetOutputData["/api/v1/internal/projects/current"] = targetOutputData["/api/v1/internal/projects/current"].map(output => {
|
||||
const projectCurrentOutputs = targetOutputData.get("/api/v1/internal/projects/current");
|
||||
if (projectCurrentOutputs) {
|
||||
targetOutputData.set("/api/v1/internal/projects/current", projectCurrentOutputs.map(output => {
|
||||
if ("config" in output.responseJson) {
|
||||
delete output.responseJson.config.id;
|
||||
output.responseJson.config.oauth_providers = output.responseJson.config.oauth_providers
|
||||
@ -116,7 +119,7 @@ async function main() {
|
||||
.map((provider: any) => omit(provider, ["enabled"]));
|
||||
}
|
||||
return output;
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
console.log(`Loaded previous output data for verification`);
|
||||
@ -125,9 +128,9 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
const { expectStatusCode } = createApiHelpers({
|
||||
currentOutputData,
|
||||
const { expectStatusCode, verifyOutputCompleteness, finalizeOutput } = createApiHelpers({
|
||||
targetOutputData,
|
||||
outputFilePath: shouldSaveOutput ? OUTPUT_FILE_PATH : undefined,
|
||||
});
|
||||
|
||||
const projects = await prismaClient.project.findMany({
|
||||
@ -191,30 +194,6 @@ async function main() {
|
||||
]);
|
||||
void currentProject;
|
||||
|
||||
// Fetch users with pagination
|
||||
const PAGE_LIMIT = 1000;
|
||||
const allUsers: any[] = [];
|
||||
let cursor: string | undefined = undefined;
|
||||
while (allUsers.length < maxUsersPerProject) {
|
||||
const remainingToFetch = maxUsersPerProject - allUsers.length;
|
||||
const limit = Math.min(PAGE_LIMIT, remainingToFetch);
|
||||
const cursorParam: string = cursor ? `&cursor=${encodeURIComponent(cursor)}` : "";
|
||||
const usersPage = await expectStatusCode(200, `/api/v1/users?limit=${limit}${cursorParam}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
allUsers.push(...usersPage.items);
|
||||
if (!usersPage.pagination?.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = usersPage.pagination.next_cursor;
|
||||
}
|
||||
const users = { items: allUsers.slice(0, maxUsersPerProject) };
|
||||
|
||||
const tenancy = await getSoleTenancyFromProjectBranch(projectId, DEFAULT_BRANCH_ID, true);
|
||||
const paymentsConfig = tenancy ? (tenancy.config as OrganizationRenderedConfig).payments : undefined;
|
||||
const paymentsVerifier = tenancy && paymentsConfig
|
||||
@ -241,86 +220,110 @@ async function main() {
|
||||
const verifiedTeams = new Set<string>();
|
||||
|
||||
if (!skipUsers) {
|
||||
for (let j = 0; j < users.items.length; j++) {
|
||||
const user = users.items[j];
|
||||
await recurse(`[user ${j + 1}/${users.items.length}] ${user.display_name ?? user.primary_email}`, async (recurse) => {
|
||||
// get user individually
|
||||
await expectStatusCode(200, `/api/v1/users/${user.id}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
const userCount = tenancy
|
||||
? await (await getPrismaClientForTenancy(tenancy)).projectUser.count({ where: { tenancyId: tenancy.id } })
|
||||
: 0;
|
||||
|
||||
// list project permissions
|
||||
const projectPermissions = await expectStatusCode(200, `/api/v1/project-permissions?user_id=${user.id}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
for (const projectPermission of projectPermissions.items) {
|
||||
// `any` because these endpoint response types aren't imported here,
|
||||
// and this script is intentionally tolerant of response shape changes.
|
||||
if (!projectPermissionDefinitions.items.some((p: any) => p.id === projectPermission.id)) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Project permission ${projectPermission.id} not found in project permission definitions.
|
||||
`);
|
||||
}
|
||||
}
|
||||
// Process users page-by-page to avoid holding all users in memory at once
|
||||
const PAGE_LIMIT = 1000;
|
||||
let userCursor: string | undefined = undefined;
|
||||
let usersProcessed = 0;
|
||||
let hasMore = true;
|
||||
|
||||
// list teams
|
||||
const teams = await expectStatusCode(200, `/api/v1/teams?user_id=${user.id}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
|
||||
for (const team of teams.items) {
|
||||
await recurse(`[team ${team.id}] ${team.name}`, async (recurse) => {
|
||||
// list team permissions
|
||||
const teamPermissions = await expectStatusCode(200, `/api/v1/team-permissions?team_id=${team.id}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
for (const teamPermission of teamPermissions.items) {
|
||||
// `any` because these endpoint response types aren't imported here,
|
||||
// and this script is intentionally tolerant of response shape changes.
|
||||
if (!teamPermissionDefinitions.items.some((p: any) => p.id === teamPermission.id)) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Team permission ${teamPermission.id} not found in team permission definitions.
|
||||
`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (paymentsVerifier && !verifiedTeams.has(team.id)) {
|
||||
await paymentsVerifier.verifyCustomerPayments({
|
||||
customerType: "team",
|
||||
customerId: team.id,
|
||||
});
|
||||
verifiedTeams.add(team.id);
|
||||
}
|
||||
}
|
||||
|
||||
if (paymentsVerifier) {
|
||||
await paymentsVerifier.verifyCustomerPayments({
|
||||
customerType: "user",
|
||||
customerId: user.id,
|
||||
});
|
||||
}
|
||||
while (hasMore && usersProcessed < maxUsersPerProject) {
|
||||
const remainingToFetch = maxUsersPerProject - usersProcessed;
|
||||
const limit = Math.min(PAGE_LIMIT, remainingToFetch);
|
||||
const cursorParam: string = userCursor ? `&cursor=${encodeURIComponent(userCursor)}` : "";
|
||||
const usersPage = await expectStatusCode(200, `/api/v1/users?limit=${limit}${cursorParam}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
|
||||
for (const user of usersPage.items) {
|
||||
if (usersProcessed >= maxUsersPerProject) break;
|
||||
usersProcessed++;
|
||||
await recurse(`[user ${usersProcessed}/${Math.min(userCount, maxUsersPerProject)}] ${user.display_name ?? user.primary_email}`, async (recurse) => {
|
||||
await expectStatusCode(200, `/api/v1/users/${user.id}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
|
||||
const projectPermissions = await expectStatusCode(200, `/api/v1/project-permissions?user_id=${user.id}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
for (const projectPermission of projectPermissions.items) {
|
||||
// `any` because these endpoint response types aren't imported here,
|
||||
// and this script is intentionally tolerant of response shape changes.
|
||||
if (!projectPermissionDefinitions.items.some((p: any) => p.id === projectPermission.id)) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Project permission ${projectPermission.id} not found in project permission definitions.
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
const teams = await expectStatusCode(200, `/api/v1/teams?user_id=${user.id}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
|
||||
for (const team of teams.items) {
|
||||
await recurse(`[team ${team.id}] ${team.name}`, async (recurse) => {
|
||||
const teamPermissions = await expectStatusCode(200, `/api/v1/team-permissions?team_id=${team.id}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-stack-project-id": projectId,
|
||||
"x-stack-access-type": "admin",
|
||||
"x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
|
||||
},
|
||||
});
|
||||
for (const teamPermission of teamPermissions.items) {
|
||||
// `any` because these endpoint response types aren't imported here,
|
||||
// and this script is intentionally tolerant of response shape changes.
|
||||
if (!teamPermissionDefinitions.items.some((p: any) => p.id === teamPermission.id)) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Team permission ${teamPermission.id} not found in team permission definitions.
|
||||
`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (paymentsVerifier && !verifiedTeams.has(team.id)) {
|
||||
await paymentsVerifier.verifyCustomerPayments({
|
||||
customerType: "team",
|
||||
customerId: team.id,
|
||||
});
|
||||
verifiedTeams.add(team.id);
|
||||
}
|
||||
}
|
||||
|
||||
if (paymentsVerifier) {
|
||||
await paymentsVerifier.verifyCustomerPayments({
|
||||
customerType: "user",
|
||||
customerId: user.id,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
hasMore = !!usersPage.pagination?.next_cursor;
|
||||
userCursor = usersPage.pagination?.next_cursor ?? undefined;
|
||||
}
|
||||
|
||||
if (paymentsVerifier) {
|
||||
@ -335,13 +338,9 @@ async function main() {
|
||||
});
|
||||
}
|
||||
|
||||
if (targetOutputData && !deepPlainEquals(currentOutputData, targetOutputData)) {
|
||||
throw new StackAssertionError(deindent`
|
||||
Output data mismatch between final and target output data.
|
||||
`);
|
||||
}
|
||||
verifyOutputCompleteness();
|
||||
if (shouldSaveOutput) {
|
||||
fs.writeFileSync(OUTPUT_FILE_PATH, JSON.stringify(currentOutputData, null, 2));
|
||||
finalizeOutput();
|
||||
console.log(`Output saved to ${OUTPUT_FILE_PATH}`);
|
||||
}
|
||||
|
||||
|
||||
@ -10,6 +10,28 @@ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0
|
||||
|
||||
const MAX_EVENTS = 500;
|
||||
|
||||
// Lone surrogates (\uD800-\uDFFF not part of a valid pair) are technically
|
||||
// representable in JS strings but rejected by ClickHouse's JSON parser.
|
||||
// The client-side event tracker can produce these when .substring() truncates
|
||||
// text in the middle of a surrogate pair (e.g. emoji characters).
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const LONE_SURROGATE_RE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
|
||||
|
||||
function stripLoneSurrogates(value: unknown): unknown {
|
||||
if (typeof value === "string") {
|
||||
return value.replace(LONE_SURROGATE_RE, "\uFFFD");
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(stripLoneSurrogates);
|
||||
}
|
||||
if (value !== null && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([k, v]) => [k, stripLoneSurrogates(v)])
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export const POST = createSmartRouteHandler({
|
||||
metadata: {
|
||||
summary: "Upload analytics event batch",
|
||||
@ -69,7 +91,7 @@ export const POST = createSmartRouteHandler({
|
||||
const rows = body.events.map((event) => ({
|
||||
event_type: event.event_type,
|
||||
event_at: new Date(event.event_at_ms),
|
||||
data: event.data,
|
||||
data: stripLoneSurrogates(event.data),
|
||||
project_id: projectId,
|
||||
branch_id: branchId,
|
||||
user_id: userId,
|
||||
|
||||
@ -2,10 +2,9 @@ import { getPrismaClientForTenancy, getPrismaSchemaForTenancy, sqlQuoteIdent } f
|
||||
import type { SignUpRiskScoresCrud } from "@stackframe/stack-shared/dist/interface/crud/users";
|
||||
import type { SignUpAuthMethod } from "@stackframe/stack-shared/dist/utils/auth-methods";
|
||||
import { captureError, StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
|
||||
import { isIpAddress } from "@stackframe/stack-shared/dist/utils/ips";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { checkEmailWithEmailable } from "./emailable";
|
||||
import { normalizeEmail } from "./emails";
|
||||
import { createNeutralSignUpHeuristicFacts, type DerivedSignUpHeuristicFacts } from "./sign-up-heuristics";
|
||||
import type { Tenancy } from "./tenancies";
|
||||
import type { SignUpTurnstileAssessment } from "./turnstile";
|
||||
@ -44,144 +43,95 @@ export type SignUpRiskRecentStats = {
|
||||
similarEmailCount: number,
|
||||
};
|
||||
|
||||
export type SignUpRiskEngineDependencies = {
|
||||
now: () => Date,
|
||||
normalizeEmail: (email: string) => string,
|
||||
isIpAddress: (ipAddress: string) => boolean,
|
||||
createAssertionError: (message: string, details: Record<string, unknown>) => Error,
|
||||
checkPrimaryEmailRisk: (email: string) => Promise<{ emailableScore: number | null }>,
|
||||
loadRecentSignUpStats: (request: SignUpRiskRecentStatsRequest) => Promise<SignUpRiskRecentStats>,
|
||||
};
|
||||
|
||||
export type SignUpRiskEngine = {
|
||||
type SignUpRiskEngine = {
|
||||
calculateRiskAssessment: (
|
||||
context: SignUpRiskScoreContext,
|
||||
dependencies: SignUpRiskEngineDependencies,
|
||||
dependencies: {
|
||||
checkPrimaryEmailRisk: (email: string) => Promise<{ emailableScore: number | null }>,
|
||||
loadRecentSignUpStats: (request: SignUpRiskRecentStatsRequest) => Promise<SignUpRiskRecentStats>,
|
||||
},
|
||||
) => Promise<SignUpRiskAssessment>,
|
||||
};
|
||||
|
||||
|
||||
// ── Fallback engine (zero scores) ──────────────────────────────────────
|
||||
// ── Private engine ─────────────────────────────────────────────────────
|
||||
|
||||
const ZERO_SCORES: SignUpRiskScores = { bot: 0, free_trial_abuse: 0 };
|
||||
|
||||
const fallbackEngine: SignUpRiskEngine = {
|
||||
async calculateRiskAssessment(_context, deps) {
|
||||
return { scores: ZERO_SCORES, heuristicFacts: createNeutralSignUpHeuristicFacts(deps.now()) };
|
||||
export const PRIVATE_ENGINE_PATH: string | null = (() => {
|
||||
const cwd = process.cwd();
|
||||
for (const relative of ["packages/private/dist/index.js", "../../packages/private/dist/index.js"]) {
|
||||
const resolved = path.resolve(cwd, relative);
|
||||
if (fs.existsSync(resolved)) return resolved;
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
function createZeroRiskAssessment(now: Date): SignUpRiskAssessment {
|
||||
return { scores: ZERO_SCORES, heuristicFacts: createNeutralSignUpHeuristicFacts(now) };
|
||||
}
|
||||
|
||||
const ZERO_SCORE_ENGINE: SignUpRiskEngine = {
|
||||
async calculateRiskAssessment() {
|
||||
return createZeroRiskAssessment(new Date());
|
||||
},
|
||||
};
|
||||
|
||||
let cachedEnginePromise: Promise<SignUpRiskEngine> | null = null;
|
||||
|
||||
// ── Private engine loader ──────────────────────────────────────────────
|
||||
|
||||
const PRIVATE_MODULE_PATH = "dist/sign-up-risk-engine.js";
|
||||
const PRIVATE_PACKAGE_IMPORT = "@stackframe/private/dist/sign-up-risk-engine.js";
|
||||
|
||||
const _testOverrides = {
|
||||
rootPath: null as string | null,
|
||||
importer: null as ((modulePath: string) => Promise<unknown>) | null,
|
||||
};
|
||||
|
||||
let cachedEngine: Promise<SignUpRiskEngine> | null = null;
|
||||
|
||||
function isEngine(value: unknown): value is SignUpRiskEngine {
|
||||
return typeof value === "object" && value !== null
|
||||
&& "calculateRiskAssessment" in value
|
||||
&& typeof (value as Record<string, unknown>).calculateRiskAssessment === "function";
|
||||
}
|
||||
|
||||
function extractEngine(mod: unknown): SignUpRiskEngine {
|
||||
const nested = (obj: unknown, key: string): unknown =>
|
||||
typeof obj === "object" && obj !== null && key in obj ? (obj as Record<string, unknown>)[key] : undefined;
|
||||
|
||||
const defaultExport = nested(mod, "default");
|
||||
for (const candidate of [mod, nested(mod, "signUpRiskEngine"), defaultExport, nested(defaultExport, "signUpRiskEngine")]) {
|
||||
if (isEngine(candidate)) return candidate;
|
||||
}
|
||||
|
||||
throw new StackAssertionError("Private sign-up risk module does not export a valid signUpRiskEngine");
|
||||
}
|
||||
|
||||
function getFallbackPaths(): string[] {
|
||||
if (_testOverrides.rootPath != null) {
|
||||
return [path.join(_testOverrides.rootPath, PRIVATE_MODULE_PATH)];
|
||||
}
|
||||
const cwd = process.cwd();
|
||||
return [
|
||||
path.join(cwd, "packages/private", PRIVATE_MODULE_PATH), // monorepo root
|
||||
path.join(cwd, "../../packages/private", PRIVATE_MODULE_PATH), // workspace dir (e.g. apps/backend)
|
||||
];
|
||||
}
|
||||
|
||||
function isModuleNotFound(e: unknown): boolean {
|
||||
if (typeof e === "object" && e !== null && "code" in e) {
|
||||
const code = (e as { code: unknown }).code;
|
||||
return code === "MODULE_NOT_FOUND" || code === "ERR_MODULE_NOT_FOUND";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Native dynamic import — the webpackIgnore comment prevents bundler transformation.
|
||||
function nativeImport(modulePath: string): Promise<unknown> {
|
||||
return import(/* webpackIgnore: true */ modulePath);
|
||||
function isSignUpRiskEngine(value: unknown): value is SignUpRiskEngine {
|
||||
return value != null && typeof value === "object" && typeof (value as Record<string, unknown>).calculateRiskAssessment === "function";
|
||||
}
|
||||
|
||||
async function loadEngine(): Promise<SignUpRiskEngine> {
|
||||
const importer = _testOverrides.importer ?? nativeImport;
|
||||
const fallbackPaths = getFallbackPaths();
|
||||
if (PRIVATE_ENGINE_PATH == null) {
|
||||
console.debug("[risk-scores] Private sign-up risk engine not found; using zero scores");
|
||||
return ZERO_SCORE_ENGINE;
|
||||
}
|
||||
|
||||
// 1. Try package-name resolution (works when @stackframe/private is a proper dependency)
|
||||
return await loadEngineFromPath(PRIVATE_ENGINE_PATH);
|
||||
}
|
||||
|
||||
async function loadEngineFromPath(privateEnginePath: string): Promise<SignUpRiskEngine> {
|
||||
let mod: Record<string, unknown>;
|
||||
try {
|
||||
const engine = extractEngine(await importer(PRIVATE_PACKAGE_IMPORT));
|
||||
console.info("[risk-scores] Loaded private sign-up risk engine via package import");
|
||||
return engine;
|
||||
} catch (e: unknown) {
|
||||
if (!isModuleNotFound(e)) {
|
||||
captureError("sign-up-risk-engine-load", new StackAssertionError(
|
||||
"Failed to load private sign-up risk engine via package import",
|
||||
{ importPath: PRIVATE_PACKAGE_IMPORT, cause: e },
|
||||
));
|
||||
}
|
||||
mod = await import(/* webpackIgnore: true */ privateEnginePath) as Record<string, unknown>;
|
||||
} catch (error) {
|
||||
captureError("sign-up-risk-engine-load", new StackAssertionError(
|
||||
"Failed to import private sign-up risk engine; using zero scores fallback",
|
||||
{
|
||||
cause: error,
|
||||
path: privateEnginePath,
|
||||
},
|
||||
));
|
||||
return ZERO_SCORE_ENGINE;
|
||||
}
|
||||
|
||||
// 2. Fall back to path-based resolution for monorepo setups
|
||||
for (const fullPath of fallbackPaths) {
|
||||
try {
|
||||
const engine = extractEngine(await importer(fullPath));
|
||||
console.info("[risk-scores] Loaded private sign-up risk engine from path:", fullPath);
|
||||
return engine;
|
||||
} catch (e: unknown) {
|
||||
if (!isModuleNotFound(e)) {
|
||||
captureError("sign-up-risk-engine-load", new StackAssertionError(
|
||||
"Failed to load private sign-up risk engine from path",
|
||||
{ fullPath, cause: e },
|
||||
));
|
||||
}
|
||||
}
|
||||
const engine = mod.signUpRiskEngine;
|
||||
if (!isSignUpRiskEngine(engine)) {
|
||||
captureError("sign-up-risk-engine-invalid", new StackAssertionError(
|
||||
"Private engine does not export a valid signUpRiskEngine; using zero scores fallback",
|
||||
{ path: privateEnginePath },
|
||||
));
|
||||
return ZERO_SCORE_ENGINE;
|
||||
}
|
||||
|
||||
// 3. No engine found — fall back to zero scores
|
||||
captureError("sign-up-risk-engine-not-found", new StackAssertionError(
|
||||
"Private sign-up risk engine not found — using fallback (zero scores)",
|
||||
{ searchedPaths: [PRIVATE_PACKAGE_IMPORT, ...fallbackPaths] },
|
||||
));
|
||||
return fallbackEngine;
|
||||
console.info("[risk-scores] Loaded private sign-up risk engine from", privateEnginePath);
|
||||
return engine;
|
||||
}
|
||||
|
||||
function getEngine(): Promise<SignUpRiskEngine> {
|
||||
if (cachedEngine == null) {
|
||||
cachedEngine = loadEngine().catch((e) => {
|
||||
cachedEngine = null; // clear so next call retries
|
||||
throw e;
|
||||
});
|
||||
}
|
||||
return cachedEngine;
|
||||
}
|
||||
async function getEngine(): Promise<SignUpRiskEngine> {
|
||||
if (cachedEnginePromise != null) return await cachedEnginePromise;
|
||||
|
||||
function resetForTests() {
|
||||
cachedEngine = null;
|
||||
_testOverrides.rootPath = null;
|
||||
_testOverrides.importer = null;
|
||||
const enginePromise = loadEngine();
|
||||
cachedEnginePromise = enginePromise;
|
||||
|
||||
try {
|
||||
return await enginePromise;
|
||||
} catch (error) {
|
||||
if (cachedEnginePromise === enginePromise) {
|
||||
cachedEnginePromise = null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -227,19 +177,43 @@ async function loadRecentSignUpStats(
|
||||
};
|
||||
}
|
||||
|
||||
function createDependencies(tenancy: Tenancy): SignUpRiskEngineDependencies {
|
||||
function createDependencies(tenancy: Tenancy) {
|
||||
return {
|
||||
now: () => new Date(),
|
||||
normalizeEmail,
|
||||
isIpAddress,
|
||||
createAssertionError: (message, details) => new StackAssertionError(message, details),
|
||||
checkPrimaryEmailRisk: async (email) => ({
|
||||
checkPrimaryEmailRisk: async (email: string) => ({
|
||||
emailableScore: (await checkEmailWithEmailable(email)).emailableScore,
|
||||
}),
|
||||
loadRecentSignUpStats: (request) => loadRecentSignUpStats(tenancy, request),
|
||||
loadRecentSignUpStats: (request: SignUpRiskRecentStatsRequest) => loadRecentSignUpStats(tenancy, request),
|
||||
};
|
||||
}
|
||||
|
||||
async function calculateRiskAssessmentWithFallback(
|
||||
engine: SignUpRiskEngine,
|
||||
context: SignUpRiskScoreContext,
|
||||
dependencies: Parameters<SignUpRiskEngine["calculateRiskAssessment"]>[1],
|
||||
): Promise<SignUpRiskAssessment> {
|
||||
try {
|
||||
return await engine.calculateRiskAssessment(context, dependencies);
|
||||
} catch (error) {
|
||||
captureError("sign-up-risk-assessment-failed", new StackAssertionError(
|
||||
"Sign-up risk assessment failed; using zero scores fallback",
|
||||
{
|
||||
cause: error,
|
||||
privateEnginePath: PRIVATE_ENGINE_PATH,
|
||||
context: {
|
||||
authMethod: context.authMethod,
|
||||
oauthProvider: context.oauthProvider,
|
||||
hasPrimaryEmail: context.primaryEmail != null,
|
||||
primaryEmailVerified: context.primaryEmailVerified,
|
||||
hasIpAddress: context.ipAddress != null,
|
||||
ipTrusted: context.ipTrusted,
|
||||
turnstileAssessment: context.turnstileAssessment,
|
||||
},
|
||||
},
|
||||
));
|
||||
return createZeroRiskAssessment(new Date());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ── Public API ─────────────────────────────────────────────────────────
|
||||
|
||||
@ -248,12 +222,7 @@ export async function calculateSignUpRiskAssessment(
|
||||
context: SignUpRiskScoreContext,
|
||||
): Promise<SignUpRiskAssessment> {
|
||||
const engine = await getEngine();
|
||||
try {
|
||||
return await engine.calculateRiskAssessment(context, createDependencies(tenancy));
|
||||
} catch (error) {
|
||||
captureError("sign-up-risk-engine-error", error);
|
||||
return { scores: ZERO_SCORES, heuristicFacts: createNeutralSignUpHeuristicFacts(new Date()) };
|
||||
}
|
||||
return await calculateRiskAssessmentWithFallback(engine, context, createDependencies(tenancy));
|
||||
}
|
||||
|
||||
export async function calculateSignUpRiskScores(
|
||||
@ -266,52 +235,75 @@ export async function calculateSignUpRiskScores(
|
||||
|
||||
// ── Tests ──────────────────────────────────────────────────────────────
|
||||
|
||||
import.meta.vitest?.test("fallback engine returns zero scores", async ({ expect }) => {
|
||||
const now = new Date("2026-03-11T00:00:00.000Z");
|
||||
const assessment = await fallbackEngine.calculateRiskAssessment({
|
||||
primaryEmail: "user@example.com",
|
||||
primaryEmailVerified: false,
|
||||
authMethod: "password",
|
||||
oauthProvider: null,
|
||||
ipAddress: "127.0.0.1",
|
||||
ipTrusted: true,
|
||||
turnstileAssessment: { status: "invalid" },
|
||||
}, {
|
||||
now: () => now,
|
||||
normalizeEmail,
|
||||
isIpAddress,
|
||||
createAssertionError: (msg, details) => new StackAssertionError(msg, details),
|
||||
checkPrimaryEmailRisk: async () => ({ emailableScore: 100 }),
|
||||
loadRecentSignUpStats: async () => ({ sameIpCount: 10, similarEmailCount: 10 }),
|
||||
});
|
||||
|
||||
expect(assessment).toEqual({
|
||||
scores: ZERO_SCORES,
|
||||
heuristicFacts: createNeutralSignUpHeuristicFacts(now),
|
||||
});
|
||||
import.meta.vitest?.test.skipIf(!PRIVATE_ENGINE_PATH)("PRIVATE_ENGINE_PATH resolves in the monorepo", ({ expect }) => {
|
||||
expect(PRIVATE_ENGINE_PATH).toMatch(/packages\/private\/dist\/index\.js$/);
|
||||
});
|
||||
|
||||
import.meta.vitest?.test("loader falls back when private submodule is absent", async ({ expect }) => {
|
||||
resetForTests();
|
||||
_testOverrides.rootPath = path.join(process.cwd(), "packages", `private-missing-${Date.now()}`);
|
||||
|
||||
import.meta.vitest?.test.skipIf(!PRIVATE_ENGINE_PATH)("getEngine loads the real engine when available", async ({ expect }) => {
|
||||
cachedEnginePromise = null;
|
||||
try {
|
||||
expect(await getEngine()).toBe(fallbackEngine);
|
||||
const engine = await getEngine();
|
||||
await engine.calculateRiskAssessment({
|
||||
primaryEmail: null,
|
||||
primaryEmailVerified: false,
|
||||
authMethod: "password",
|
||||
oauthProvider: null,
|
||||
ipAddress: null,
|
||||
ipTrusted: null,
|
||||
turnstileAssessment: { status: "ok" },
|
||||
}, {
|
||||
checkPrimaryEmailRisk: async () => ({ emailableScore: null }),
|
||||
loadRecentSignUpStats: async () => ({ sameIpCount: 0, similarEmailCount: 0 }),
|
||||
});
|
||||
expect(typeof engine.calculateRiskAssessment).toBe("function");
|
||||
expect(engine).not.toBe(ZERO_SCORE_ENGINE);
|
||||
} finally {
|
||||
resetForTests();
|
||||
cachedEnginePromise = null;
|
||||
}
|
||||
});
|
||||
|
||||
import.meta.vitest?.test("loader falls back when private engine import fails", async ({ expect }) => {
|
||||
resetForTests();
|
||||
_testOverrides.rootPath = path.join(process.cwd(), "packages", "private");
|
||||
_testOverrides.importer = async () => {
|
||||
throw new Error("private engine exploded");
|
||||
};
|
||||
import.meta.vitest?.test("loadEngine returns zero-score engine when private engine import fails", async ({ expect }) => {
|
||||
const missingPrivateEnginePath = path.join(process.cwd(), "__missing-risk-engine__.js");
|
||||
const engine = await loadEngineFromPath(missingPrivateEnginePath);
|
||||
expect(engine).toBe(ZERO_SCORE_ENGINE);
|
||||
});
|
||||
|
||||
import.meta.vitest?.test("loadEngineFromPath returns zero-score engine when private engine export is invalid", async ({ expect }) => {
|
||||
const invalidPrivateEnginePath = path.join(process.cwd(), "__invalid-risk-engine__.mjs");
|
||||
const invalidPrivateEngineSource = "export const signUpRiskEngine = {};\n";
|
||||
fs.writeFileSync(invalidPrivateEnginePath, invalidPrivateEngineSource);
|
||||
|
||||
try {
|
||||
expect(await getEngine()).toBe(fallbackEngine);
|
||||
const engine = await loadEngineFromPath(invalidPrivateEnginePath);
|
||||
expect(engine).toBe(ZERO_SCORE_ENGINE);
|
||||
} finally {
|
||||
resetForTests();
|
||||
fs.unlinkSync(invalidPrivateEnginePath);
|
||||
}
|
||||
});
|
||||
|
||||
import.meta.vitest?.test("calculateRiskAssessmentWithFallback returns zero scores on engine error", async ({ expect }) => {
|
||||
const { vi } = import.meta.vitest!;
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-03-20T00:00:00.000Z"));
|
||||
|
||||
try {
|
||||
const assessment = await calculateRiskAssessmentWithFallback({
|
||||
async calculateRiskAssessment() { throw new Error("boom"); },
|
||||
}, {
|
||||
primaryEmail: "user@example.com",
|
||||
primaryEmailVerified: false,
|
||||
authMethod: "password",
|
||||
oauthProvider: null,
|
||||
ipAddress: "127.0.0.1",
|
||||
ipTrusted: true,
|
||||
turnstileAssessment: { status: "ok" },
|
||||
}, {
|
||||
checkPrimaryEmailRisk: async () => ({ emailableScore: null }),
|
||||
loadRecentSignUpStats: async () => ({ sameIpCount: 0, similarEmailCount: 0 }),
|
||||
});
|
||||
|
||||
expect(assessment).toEqual(createZeroRiskAssessment(new Date("2026-03-20T00:00:00.000Z")));
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
@ -160,6 +160,51 @@ it("accepts valid $click events", async ({ expect }) => {
|
||||
`);
|
||||
});
|
||||
|
||||
it("handles click event data containing a truncated surrogate pair (lone high surrogate)", async ({ expect }) => {
|
||||
await Project.createAndSwitch({ config: { magic_link_enabled: true } });
|
||||
await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } });
|
||||
await Auth.Otp.signIn();
|
||||
|
||||
// Simulate what the client-side event tracker does: .substring(0, 200) can
|
||||
// cut a string in the middle of a surrogate pair when emoji characters are
|
||||
// near the boundary. For example, 🍉 is "\uD83C\uDF49" in UTF-16; cutting
|
||||
// after the high surrogate leaves a lone "\uD83C" that ClickHouse cannot parse.
|
||||
const paddedText = "a".repeat(199) + "\uD83C"; // lone high surrogate at position 199
|
||||
|
||||
const now = Date.now();
|
||||
const res = await uploadEventBatch({
|
||||
sessionReplaySegmentId: randomUUID(),
|
||||
batchId: randomUUID(),
|
||||
sentAtMs: now,
|
||||
events: [
|
||||
{
|
||||
event_type: "$click",
|
||||
event_at_ms: now - 50,
|
||||
data: {
|
||||
tag_name: "div",
|
||||
text: paddedText,
|
||||
href: null,
|
||||
selector: "div.container",
|
||||
x: 100,
|
||||
y: 200,
|
||||
page_x: 100,
|
||||
page_y: 500,
|
||||
viewport_width: 375,
|
||||
viewport_height: 647,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(res).toMatchInlineSnapshot(`
|
||||
NiceResponse {
|
||||
"status": 200,
|
||||
"body": { "inserted": 1 },
|
||||
"headers": Headers { <some fields may have been hidden> },
|
||||
}
|
||||
`);
|
||||
});
|
||||
|
||||
it("rejects empty events array", async ({ expect }) => {
|
||||
await Project.createAndSwitch({ config: { magic_link_enabled: true } });
|
||||
await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } });
|
||||
|
||||
@ -1 +1 @@
|
||||
Subproject commit 6f708e4206abc6ec0903dd93629c7bd137dbcb0b
|
||||
Subproject commit 2f2c03135725c2cf8273304725a93d12f2bc45ec
|
||||
Loading…
Reference in New Issue
Block a user