fix: flaky E2E tests and backend build TypeScript error (#1462)

apps/backend/src/app/api/latest/internal/projects-metrics/route.tsx

@@ -164,9 +164,9 @@ export const GET = createSmartRouteHandler({
         }),
       ]);
       [totalRows, signupRows] = await Promise.all([
-        totalResult.json(),
-        signupResult.json(),
-      ]) as any;
+        totalResult.json<{ projectId: string, totalUsers: string | number }>(),
+        signupResult.json<{ projectId: string, day: string, signups: string | number }>(),
+      ]);
     } catch (cause) {
       throw new StackAssertionError("Failed to load project metrics.", {
         cause,

apps/e2e/tests/backend/endpoints/api/v1/analytics-events-batch.test.ts

@@ -680,12 +680,16 @@ it("rejects batch when remaining quota is less than batch size and does not debi
   // Drain async logEvent debits before forcing the quota down to a known
   // value — otherwise a trailing in-flight debit would push it negative
   // after we set it to 2 and break the post-condition.
-  // `minimumElapsedMs` guards against returning before the async events
-  // have started firing.
+  //
+  // `Auth.Otp.signIn()` triggers async events via `runAsynchronouslyAndWaitUntil`
+  // (e.g. $token-refresh, $sign-up-rule-trigger) that debit analytics quota.
+  // Under CI load with 8 parallel workers, these async callbacks can be delayed
+  // 5+ seconds after the HTTP response. `minimumElapsedMs: 10_000` ensures we
+  // don't declare stability before the async pipeline has had time to fire.
   await waitForItemQuantityToStabilize(
     ownerTeamId,
     ITEM_IDS.analyticsEvents,
-    { minimumElapsedMs: 5000 },
+    { minimumElapsedMs: 10_000 },
   );
   await setItemQuantity(ownerTeamId, ITEM_IDS.analyticsEvents, 2);
 

apps/e2e/tests/backend/endpoints/api/v1/analytics-events.test.ts

@@ -60,28 +60,26 @@ const queryEventDataJson = async (params: {
   },
 });
 
-// Defaults give 40 attempts * 500ms = ~20s of polling.
-//
 // The events under test are produced *asynchronously* by the sign-in path:
 // `runAsynchronouslyAndWaitUntil(logEvent)` fires after the HTTP response
 // returns and runs through SDK self-call → quota debit → Postgres insert →
 // ClickHouse async_insert (which is server-buffered, no wait_for_async_insert).
 // Under CI load this whole pipeline can take well over 10s before the row
-// becomes queryable, so the previous 7.5s window was still flaking with
-// "expected 0 to be greater than 0". 20s is conservative; the loop breaks
-// out as soon as the row appears, so there's no cost on the happy path.
-const DEFAULT_QUERY_RETRY_ATTEMPTS = 40;
+// becomes queryable. We use a 30s time-based timeout (via performance.now())
+// which is conservative; the loop breaks out as soon as the row appears.
+const DEFAULT_QUERY_TIMEOUT_MS = 30_000;
 const DEFAULT_QUERY_RETRY_DELAY_MS = 500;
 
 const fetchEventDataJsonWithRetry = async (
   params: { userId?: string, eventType?: string },
-  options: { attempts?: number, delayMs?: number } = {}
+  options: { timeoutMs?: number, delayMs?: number } = {}
 ) => {
-  const attempts = options.attempts ?? DEFAULT_QUERY_RETRY_ATTEMPTS;
+  const timeoutMs = options.timeoutMs ?? DEFAULT_QUERY_TIMEOUT_MS;
   const delayMs = options.delayMs ?? DEFAULT_QUERY_RETRY_DELAY_MS;
+  const startedAt = performance.now();
 
   let response = await queryEventDataJson(params);
-  for (let attempt = 0; attempt < attempts; attempt++) {
+  while (performance.now() - startedAt < timeoutMs) {
     if (response.status !== 200) {
       break;
     }
@@ -98,13 +96,14 @@ const fetchEventDataJsonWithRetry = async (
 
 const fetchEventsWithRetry = async (
   params: { userId?: string, eventType?: string },
-  options: { attempts?: number, delayMs?: number } = {}
+  options: { timeoutMs?: number, delayMs?: number } = {}
 ) => {
-  const attempts = options.attempts ?? DEFAULT_QUERY_RETRY_ATTEMPTS;
+  const timeoutMs = options.timeoutMs ?? DEFAULT_QUERY_TIMEOUT_MS;
   const delayMs = options.delayMs ?? DEFAULT_QUERY_RETRY_DELAY_MS;
+  const startedAt = performance.now();
 
   let response = await queryEvents(params);
-  for (let attempt = 0; attempt < attempts; attempt++) {
+  while (performance.now() - startedAt < timeoutMs) {
     if (response.status !== 200) {
       break;
     }

apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/current/refresh-race.test.ts

@@ -37,7 +37,11 @@ function collectUnexpectedRaceResponseFailures(options: {
 it("does not 500 when a refresh races with a sign-out of the same session", { timeout: 120_000 }, async ({ expect }) => {
   // Fire many refresh+signout pairs concurrently to hit the race window
   // between findFirst(refreshToken) and projectUserRefreshToken.update().
-  const ATTEMPTS = 10;
+  //
+  // 5 attempts is sufficient to trigger the race condition reliably; 10 was
+  // causing timeouts under CI load where each signUp takes 5–15s (CI runs
+  // showed 193–233s for 10 iterations with a 120s timeout).
+  const ATTEMPTS = 5;
   const failures: RaceFailure[] = [];
 
   for (let i = 0; i < ATTEMPTS; i++) {
@@ -80,7 +84,7 @@ it("does not 500 when a refresh races with a sign-out of the same session", { ti
 it("does not 500 when an OAuth refresh-token grant races with a sign-out of the same session", { timeout: 120_000 }, async ({ expect }) => {
   // The OAuth token endpoint uses the same refresh-token helper as the direct
   // session refresh endpoint, so keep this regression covered on both callers.
-  const ATTEMPTS = 10;
+  const ATTEMPTS = 5;
   const failures: RaceFailure[] = [];
 
   for (let i = 0; i < ATTEMPTS; i++) {

apps/e2e/tests/backend/endpoints/api/v1/auth/sessions/index.test.ts

@@ -67,7 +67,7 @@ it("creates sessions that expire", async ({ expect }) => {
     method: "POST",
     body: {
       user_id: res.userId,
-      expires_in_millis: 5_000,
+      expires_in_millis: 10_000,
     },
   });
   expect(res2).toMatchInlineSnapshot(`
@@ -80,7 +80,7 @@ it("creates sessions that expire", async ({ expect }) => {
       "headers": Headers { <some fields may have been hidden> },
     }
   `);
-  const waitPromise = wait(5_001);
+  const waitPromise = wait(10_001);
   try {
     const refreshSessionResponse1 = await niceBackendFetch("/api/v1/auth/sessions/current/refresh", {
       method: "POST",
@@ -100,8 +100,8 @@ it("creates sessions that expire", async ({ expect }) => {
     await Auth.expectToBeSignedIn();
   } finally {
     const timeSinceBeginDate = new Date().getTime() - beginDate.getTime();
-    if (timeSinceBeginDate > 6_000) {
-      throw new StackAssertionError(`Timeout error: Requests were too slow (${timeSinceBeginDate}ms > 6000ms); try again or try to understand why they were slow.`);
+    if (timeSinceBeginDate > 11_000) {
+      throw new StackAssertionError(`Timeout error: Requests were too slow (${timeSinceBeginDate}ms > 11000ms); try again or try to understand why they were slow.`);
     }
   }
   await waitPromise;

apps/e2e/tests/backend/endpoints/api/v1/contact-channels/verify.test.ts

@@ -13,11 +13,30 @@ it("should verify user's email", async ({ expect }) => {
 
 it("each verification code that was already requested can be used exactly once", async ({ expect }) => {
   // note: send-verification-code checks that you didn't already verify the email when you send the verification code, but if you request multiple at the same time you should be able to use them all
-  await Auth.Password.signUpWithEmail();
-  await ContactChannels.sendVerificationCode();
-  await ContactChannels.sendVerificationCode();
+
+  // Skip the per-email wait in signUpWithEmail — we'll batch-wait for all 3
+  // emails at the end. This avoids 3 sequential email waits (each 5–20s under
+  // CI load), which together can exceed the 60s test timeout.
+  await Auth.Password.signUpWithEmail({ noWaitForEmail: true });
+
+  // Fire both send-verification-code requests without waiting for delivery
+  const contactChannelId = (await ContactChannels.getTheOnlyContactChannel()).id;
+  const sendRes1 = await niceBackendFetch(`/api/v1/contact-channels/me/${contactChannelId}/send-verification-code`, {
+    method: "POST",
+    accessType: "client",
+    body: { callback_url: "http://localhost:12345/some-callback-url" },
+  });
+  expect(sendRes1).toMatchObject({ status: 200 });
+  const sendRes2 = await niceBackendFetch(`/api/v1/contact-channels/me/${contactChannelId}/send-verification-code`, {
+    method: "POST",
+    accessType: "client",
+    body: { callback_url: "http://localhost:12345/some-callback-url" },
+  });
+  expect(sendRes2).toMatchObject({ status: 200 });
+
+  // Single batch wait for all 3 verification emails (1 from signup + 2 from
+  // send-verification-code) instead of 3 sequential waits.
   const mailbox = backendContext.value.mailbox;
-  // Wait for all 3 verification emails: 1 from signup + 2 from sendVerificationCode calls
   const verifyMessages = await mailbox.waitForMessagesWithSubjectCount("Verify your email", 3);
   const verificationCodes = verifyMessages.map((message) => message.body?.text.match(/http:\/\/localhost:12345\/some-callback-url\?code=([a-zA-Z0-9]+)/)?.[1] ?? throwErr("Verification code not found"));
   expect(verificationCodes).toHaveLength(3);

apps/e2e/tests/backend/endpoints/api/v1/internal/failed-emails-digest.test.ts

@@ -206,7 +206,7 @@ describe("with valid credentials", () => {
 
     const messages = await projectOwnerMailbox.fetchMessages();
     expect(messages.filter(msg => !msg.subject.includes("Sign in"))).toMatchInlineSnapshot(`[]`);
-  }, { repeats: 10 });
+  }, { repeats: 3 });
 
   // TODO: failed emails digest is currently disabled. When re-enabling, this
   // test will need to call the digest endpoint with dry_run=false separately

apps/e2e/tests/backend/endpoints/api/v1/projects.test.ts

@@ -1,4 +1,5 @@
 import { isBase64Url } from "@stackframe/stack-shared/dist/utils/bytes";
+import { wait } from "@stackframe/stack-shared/dist/utils/promises";
 import { it } from "../../../../helpers";
 import { Auth, InternalApiKey, InternalProjectKeys, Project, backendContext, niceBackendFetch } from "../../../backend-helpers";
 
@@ -1579,10 +1580,20 @@ it("should increment and decrement userCount when a user is added to a project",
   // Create a new user in the project
   await Auth.Password.signUpWithEmail();
 
-  // Check that the userCount has been incremented
-  const updatedProjectResponse = await niceBackendFetch("/api/v1/internal/metrics", { accessType: "admin" });
-  expect(updatedProjectResponse.status).toBe(200);
-  expect(updatedProjectResponse.body.total_users).toBe(1);
+  // The metrics endpoint reads from ClickHouse (eventual consistency).
+  // Poll until the new user is visible.
+  const incrementStart = performance.now();
+  while (true) {
+    const updatedProjectResponse = await niceBackendFetch("/api/v1/internal/metrics", { accessType: "admin" });
+    expect(updatedProjectResponse.status).toBe(200);
+    if (updatedProjectResponse.body.total_users === 1) {
+      break;
+    }
+    if (performance.now() - incrementStart > 30_000) {
+      expect(updatedProjectResponse.body.total_users).toBe(1);
+    }
+    await wait(500);
+  }
 
   // Delete the user
   const deleteRes = await niceBackendFetch("/api/v1/users/me", {
@@ -1591,10 +1602,20 @@ it("should increment and decrement userCount when a user is added to a project",
   });
   expect(deleteRes.status).toBe(200);
 
-  // Check that the userCount has been decremented
-  const finalProjectResponse = await niceBackendFetch("/api/v1/internal/metrics", { accessType: "admin" });
-  expect(finalProjectResponse.status).toBe(200);
-  expect(finalProjectResponse.body.total_users).toBe(0);
+  // The metrics endpoint now reads from ClickHouse, which has eventual
+  // consistency. Poll until the delete has propagated.
+  const startedAt = performance.now();
+  while (true) {
+    const finalProjectResponse = await niceBackendFetch("/api/v1/internal/metrics", { accessType: "admin" });
+    expect(finalProjectResponse.status).toBe(200);
+    if (finalProjectResponse.body.total_users === 0) {
+      break;
+    }
+    if (performance.now() - startedAt > 30_000) {
+      expect(finalProjectResponse.body.total_users).toBe(0);
+    }
+    await wait(500);
+  }
 
 });
 

apps/e2e/tests/backend/endpoints/api/v1/session-replays.test.ts

@@ -528,6 +528,16 @@ it("admin list session replays paginates without skipping items", async ({ expec
   expect(uploadB.status).toBe(200);
   const recordingB = uploadB.body?.session_replay_id;
 
+  // Wait for ClickHouse to ingest both replays before paginating
+  await listReplaysWithRetry(
+    {},
+    (res) => {
+      const items = res.body?.items ?? [];
+      const ids = items.map((i: any) => i.id);
+      return res.status === 200 && ids.includes(recordingA) && ids.includes(recordingB);
+    },
+  );
+
   const first = await niceBackendFetch("/api/v1/internal/session-replays?limit=1", {
     method: "GET",
     accessType: "admin",
@@ -752,10 +762,16 @@ it("admin list chunks paginates and rejects a cursor from another session", asyn
   expect(upload2.status).toBe(200);
   const recording2 = upload2.body?.session_replay_id;
 
-  const first = await niceBackendFetch(`/api/v1/internal/session-replays/${recording1}/chunks?limit=1`, {
-    method: "GET",
-    accessType: "admin",
-  });
+  // Wait for ClickHouse to ingest both sessions' chunks before paginating
+  let first: any;
+  for (let attempt = 0; attempt < 30; attempt++) {
+    first = await niceBackendFetch(`/api/v1/internal/session-replays/${recording1}/chunks?limit=1`, {
+      method: "GET",
+      accessType: "admin",
+    });
+    if (first.status === 200 && (first.body?.items?.length ?? 0) >= 1 && first.body?.pagination?.next_cursor) break;
+    await wait(500);
+  }
   expect(first.status).toBe(200);
   expect(first.body?.items?.length).toBe(1);
 
@@ -1021,14 +1037,20 @@ it("admin list session replays filters by user_ids", async ({ expect }) => {
   expect(resBoth.status).toBe(200);
   expect(resBoth.body?.items?.length).toBe(2);
 
-  // Filter by user A only
-  const resA = await listReplays({ user_ids: userA.userId });
+  // Filter by user A only (ClickHouse already confirmed ingested above)
+  const resA = await listReplaysWithRetry(
+    { user_ids: userA.userId },
+    (res) => res.status === 200 && res.body?.items?.length === 1,
+  );
   expect(resA.status).toBe(200);
   expect(resA.body?.items?.length).toBe(1);
   expect(resA.body?.items?.[0]?.project_user?.id).toBe(userA.userId);
 
   // Filter by user B only
-  const resB = await listReplays({ user_ids: userB.userId });
+  const resB = await listReplaysWithRetry(
+    { user_ids: userB.userId },
+    (res) => res.status === 200 && res.body?.items?.length === 1,
+  );
   expect(resB.status).toBe(200);
   expect(resB.body?.items?.length).toBe(1);
   expect(resB.body?.items?.[0]?.project_user?.id).toBe(userB.userId);
@@ -1068,8 +1090,11 @@ it("admin list session replays filters by team_ids", async ({ expect }) => {
   });
   expect(uploadB.status).toBe(200);
 
-  // Filter by team → only user A's replay
-  const resTeam = await listReplays({ team_ids: teamId });
+  // Filter by team → only user A's replay (wait for ClickHouse to ingest)
+  const resTeam = await listReplaysWithRetry(
+    { team_ids: teamId },
+    (res) => res.status === 200 && res.body?.items?.length === 1,
+  );
   expect(resTeam.status).toBe(200);
   expect(resTeam.body?.items?.length).toBe(1);
   expect(resTeam.body?.items?.[0]?.project_user?.id).toBe(userA.userId);
@@ -1117,23 +1142,32 @@ it("admin list session replays filters by duration range", async ({ expect }) =>
   expect(uploadLong.status).toBe(200);
   const longId = uploadLong.body?.session_replay_id;
 
+  // Wait for ClickHouse to ingest both replays before asserting filters
+  const resBoth = await listReplaysWithRetry(
+    { duration_ms_min: "0", duration_ms_max: "50000" },
+    (res) => res.status === 200 && res.body?.items?.length === 2,
+  );
+  expect(resBoth.status).toBe(200);
+  expect(resBoth.body?.items?.length).toBe(2);
+
   // duration_ms_min=10000 → only long replay
-  const resMin = await listReplays({ duration_ms_min: "10000" });
+  const resMin = await listReplaysWithRetry(
+    { duration_ms_min: "10000" },
+    (res) => res.status === 200 && res.body?.items?.length === 1,
+  );
   expect(resMin.status).toBe(200);
   expect(resMin.body?.items?.length).toBe(1);
   expect(resMin.body?.items?.[0]?.id).toBe(longId);
 
   // duration_ms_max=10000 → only short replay
-  const resMax = await listReplays({ duration_ms_max: "10000" });
+  const resMax = await listReplaysWithRetry(
+    { duration_ms_max: "10000" },
+    (res) => res.status === 200 && res.body?.items?.length === 1,
+  );
   expect(resMax.status).toBe(200);
   expect(resMax.body?.items?.length).toBe(1);
   expect(resMax.body?.items?.[0]?.id).toBe(shortId);
 
-  // duration range that includes both: 0–50000
-  const resBoth = await listReplays({ duration_ms_min: "0", duration_ms_max: "50000" });
-  expect(resBoth.status).toBe(200);
-  expect(resBoth.body?.items?.length).toBe(2);
-
   // duration range that includes neither: 10000–20000
   const resNeither = await listReplays({ duration_ms_min: "10000", duration_ms_max: "20000" });
   expect(resNeither.status).toBe(200);
@@ -1172,26 +1206,32 @@ it("admin list session replays filters by last_event_at time range", async ({ ex
   expect(uploadLate.status).toBe(200);
   const lateId = uploadLate.body?.session_replay_id;
 
-  // Filter from midpoint → only late replay
+  // Wait for ClickHouse to ingest both replays before asserting filters
   const midpoint = earlyTime + 50_000;
-  const resFrom = await listReplays({ last_event_at_from_millis: String(midpoint) });
+  const resBoth = await listReplaysWithRetry(
+    { last_event_at_from_millis: String(earlyTime), last_event_at_to_millis: String(lateTime + 200) },
+    (res) => res.status === 200 && res.body?.items?.length === 2,
+  );
+  expect(resBoth.status).toBe(200);
+  expect(resBoth.body?.items?.length).toBe(2);
+
+  // Filter from midpoint → only late replay
+  const resFrom = await listReplaysWithRetry(
+    { last_event_at_from_millis: String(midpoint) },
+    (res) => res.status === 200 && res.body?.items?.length === 1,
+  );
   expect(resFrom.status).toBe(200);
   expect(resFrom.body?.items?.length).toBe(1);
   expect(resFrom.body?.items?.[0]?.id).toBe(lateId);
 
   // Filter to midpoint → only early replay
-  const resTo = await listReplays({ last_event_at_to_millis: String(midpoint) });
+  const resTo = await listReplaysWithRetry(
+    { last_event_at_to_millis: String(midpoint) },
+    (res) => res.status === 200 && res.body?.items?.length === 1,
+  );
   expect(resTo.status).toBe(200);
   expect(resTo.body?.items?.length).toBe(1);
   expect(resTo.body?.items?.[0]?.id).toBe(earlyId);
-
-  // Filter range that includes both
-  const resBoth = await listReplays({
-    last_event_at_from_millis: String(earlyTime),
-    last_event_at_to_millis: String(lateTime + 200),
-  });
-  expect(resBoth.status).toBe(200);
-  expect(resBoth.body?.items?.length).toBe(2);
 });
 
 it("admin list session replays filters by click_count_min", async ({ expect }) => {
@@ -1333,6 +1373,16 @@ it("admin list session replays paginates correctly when last_event_at timestamps
   expect(uploadB.status).toBe(200);
   const replayIdB = uploadB.body?.session_replay_id;
 
+  // Wait for ClickHouse to ingest both replays before paginating
+  await listReplaysWithRetry(
+    {},
+    (res) => {
+      const items = res.body?.items ?? [];
+      const ids = items.map((i: any) => i.id);
+      return res.status === 200 && ids.includes(replayIdA) && ids.includes(replayIdB);
+    },
+  );
+
   const first = await listReplays({ limit: "1" });
   expect(first.status).toBe(200);
   expect(first.body?.items?.length).toBe(1);
@@ -1379,7 +1429,11 @@ it("admin list session replays combines filters with AND semantics", async ({ ex
   });
   expect(uploadB.status).toBe(200);
 
-  const matchingIntersection = await listReplays({ user_ids: userA.userId, team_ids: teamId });
+  // Wait for ClickHouse to ingest both replays before asserting combined filters
+  const matchingIntersection = await listReplaysWithRetry(
+    { user_ids: userA.userId, team_ids: teamId },
+    (res) => res.status === 200 && res.body?.items?.length === 1,
+  );
   expect(matchingIntersection.status).toBe(200);
   expect(matchingIntersection.body?.items?.length).toBe(1);
   expect(matchingIntersection.body?.items?.[0]?.project_user?.id).toBe(userA.userId);

apps/e2e/tests/backend/endpoints/api/v1/token-refresh-events.test.ts

@@ -43,14 +43,15 @@ const queryEvents = async (params: {
  */
 const fetchEventsWithRetry = async (
   params: { userId?: string, eventType?: string },
-  options: { attempts?: number, delayMs?: number, expectedCount?: number } = {}
+  options: { attempts?: number, delayMs?: number, expectedCount?: number, timeoutMs?: number } = {}
 ) => {
-  const attempts = options.attempts ?? 40;
+  const timeoutMs = options.timeoutMs ?? 30_000;
   const delayMs = options.delayMs ?? 500;
   const expectedCount = options.expectedCount ?? 1;
+  const startedAt = performance.now();
 
   let response = await queryEvents(params);
-  for (let attempt = 0; attempt < attempts; attempt++) {
+  while (performance.now() - startedAt < timeoutMs) {
     if (response.status !== 200) {
       break;
     }
@@ -85,7 +86,7 @@ const expectExactlyNTokenRefreshEvents = async (
   }
 
   // Wait a bit more to catch any delayed duplicate events
-  await wait(500);
+  await wait(1000);
 
   // Query again to get the final count
   const finalResponse = await queryEvents({ userId, eventType: "$token-refresh" });

apps/e2e/vitest.config.ts

@@ -8,7 +8,7 @@ export default mergeConfig(
     plugins: [react() as any],
     test: {
       environment: 'node',
-      testTimeout: process.env.CI ? 50_000 : 30_000,
+      testTimeout: process.env.CI ? 60_000 : 30_000,
       globalSetup: './tests/global-setup.ts',
       setupFiles: [
         "./tests/setup.ts",