Merge dev into update-oauth-docs

2026-06-13 21:01:21 +08:00 · 2025-09-04 04:31:34 -07:00 · 2025-09-04 04:31:34 -07:00 · a4de74a284
commit a4de74a284
parent 35f0a7bd45 663a0184cf
24 changed files with 550 additions and 144 deletions
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -76,6 +76,7 @@
    "Proxied",
    "psql",
    "qrcode",
+    "QSTASH",
    "quetzallabs",
    "rehype",
    "reqs",
--- a/apps/backend/.env
+++ b/apps/backend/.env
@ -63,7 +63,13 @@ STACK_AWS_REGION=
 STACK_AWS_KMS_ENDPOINT=
 STACK_AWS_ACCESS_KEY_ID=
 STACK_AWS_SECRET_ACCESS_KEY=
+STACK_AWS_VERCEL_OIDC_ROLE_ARN=

+# Upstash configuration
+STACK_QSTASH_URL=
+STACK_QSTASH_TOKEN=
+STACK_QSTASH_CURRENT_SIGNING_KEY=
+STACK_QSTASH_NEXT_SIGNING_KEY=


 # Misc, optional
--- a/apps/backend/.env.development
+++ b/apps/backend/.env.development
@ -62,3 +62,9 @@ STACK_AWS_REGION=us-east-1
 STACK_AWS_KMS_ENDPOINT=http://localhost:8124
 STACK_AWS_ACCESS_KEY_ID=test
 STACK_AWS_SECRET_ACCESS_KEY=test
+
+# Upstash defaults to one of the pre-build test users of the local emulator
+STACK_QSTASH_URL=http://localhost:8125
+STACK_QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc3N3b3JkIn0=
+STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r
+STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs
--- a/apps/backend/package.json
+++ b/apps/backend/package.json
@ -64,6 +64,7 @@
    "@sentry/nextjs": "^8.40.0",
    "@simplewebauthn/server": "^11.0.0",
    "@stackframe/stack-shared": "workspace:*",
+    "@upstash/qstash": "^2.8.2",
    "@vercel/functions": "^2.0.0",
    "@vercel/otel": "^1.10.4",
    "ai": "^4.3.17",
--- a/apps/backend/prisma/schema.prisma
+++ b/apps/backend/prisma/schema.prisma
@ -810,6 +810,7 @@ model WorkflowTrigger {

  // the following fields determine the state of the trigger:
  // - scheduledAt && !compiledWorkflowId && !output && !error: the trigger is scheduled to be executed
+  // - !scheduledAt && !compiledWorkflowId: the trigger was scheduled, but its workflow subsequently deleted. The trigger never ran
  // - !scheduledAt && compiledWorkflowId && !output && !error: the trigger is currently executing
  // - !scheduledAt && compiledWorkflowId && output && !error: the trigger has successfully completed execution
  // - !scheduledAt && compiledWorkflowId && !output && error: the trigger has failed execution
--- a/apps/backend/src/app/api/latest/internal/trigger/run-scheduled/route.tsx
+++ b/apps/backend/src/app/api/latest/internal/trigger/run-scheduled/route.tsx
@ -0,0 +1,41 @@
+import { getTenancy } from "@/lib/tenancies";
+import { ensureUpstashSignature } from "@/lib/upstash";
+import { triggerScheduledWorkflows } from "@/lib/workflows";
+import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { yupNumber, yupObject, yupString, yupTuple } from "@stackframe/stack-shared/dist/schema-fields";
+import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
+
+export const POST = createSmartRouteHandler({
+  metadata: {
+    hidden: true,
+  },
+  request: yupObject({
+    headers: yupObject({
+      "upstash-signature": yupTuple([yupString().defined()]).defined(),
+    }).defined(),
+    body: yupObject({
+      tenancyId: yupString().defined(),
+    }).defined(),
+    method: yupString().oneOf(["POST"]).defined(),
+  }),
+  response: yupObject({
+    statusCode: yupNumber().oneOf([200]).defined(),
+    bodyType: yupString().oneOf(["success"]).defined(),
+  }),
+  handler: async (req, fullReq) => {
+    await ensureUpstashSignature(fullReq);
+
+    const tenancy = await getTenancy(req.body.tenancyId);
+    if (!tenancy) {
+      throw new StackAssertionError(`Tenancy not found for scheduled trigger`, { tenancyId: req.body.tenancyId });
+    }
+
+    await triggerScheduledWorkflows(tenancy);
+
+    return {
+      statusCode: 200,
+      bodyType: "success",
+    } as const;
+  },
+});
+
--- a/apps/backend/src/lib/upstash.tsx
+++ b/apps/backend/src/lib/upstash.tsx
@ -0,0 +1,35 @@
+import { SmartRequest } from "@/route-handlers/smart-request";
+import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env";
+import { StatusError } from "@stackframe/stack-shared/dist/utils/errors";
+import { Client, Receiver } from "@upstash/qstash";
+
+export const upstash = new Client({
+  baseUrl: getEnvVariable("STACK_QSTASH_URL", ""),
+  token: getEnvVariable("STACK_QSTASH_TOKEN", ""),
+});
+
+export const upstashReceiver = new Receiver({
+  currentSigningKey: getEnvVariable("STACK_QSTASH_CURRENT_SIGNING_KEY", ""),
+  nextSigningKey: getEnvVariable("STACK_QSTASH_NEXT_SIGNING_KEY", ""),
+});
+
+export async function ensureUpstashSignature(fullReq: SmartRequest): Promise<void> {
+  const upstashSignature = fullReq.headers["upstash-signature"]?.[0];
+  if (!upstashSignature) {
+    throw new StatusError(400, "upstash-signature header is required");
+  }
+
+  const url = new URL(fullReq.url);
+  if ((getNodeEnvironment().includes("development") || getNodeEnvironment().includes("test")) && url.hostname === "localhost") {
+    url.hostname = "host.docker.internal";
+  }
+
+  const isValid = await upstashReceiver.verify({
+    signature: upstashSignature,
+    url: url.toString(),
+    body: new TextDecoder().decode(fullReq.bodyBuffer),
+  });
+  if (!isValid) {
+    throw new StatusError(400, "Invalid Upstash signature");
+  }
+}
--- a/apps/backend/src/lib/workflows.tsx
+++ b/apps/backend/src/lib/workflows.tsx
@ -7,16 +7,15 @@ import { encodeBase64 } from "@stackframe/stack-shared/dist/utils/bytes";
 import { generateSecureRandomString, hash } from "@stackframe/stack-shared/dist/utils/crypto";
 import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
 import { StackAssertionError, captureError, errorToNiceString, throwErr } from "@stackframe/stack-shared/dist/utils/errors";
-import { bundleJavaScript } from "@stackframe/stack-shared/dist/utils/esbuild";
+import { bundleJavaScript, initializeEsbuild } from "@stackframe/stack-shared/dist/utils/esbuild";
 import { runAsynchronously, timeout, wait } from "@stackframe/stack-shared/dist/utils/promises";
 import { Result } from "@stackframe/stack-shared/dist/utils/results";
 import { generateUuid } from "@stackframe/stack-shared/dist/utils/uuids";
 import { Freestyle } from "./freestyle";
 import { Tenancy } from "./tenancies";
+import { upstash } from "./upstash";

-const externalPackages = {
-  '@stackframe/stack': 'latest',
-};
+const externalPackages: Record<string, string> = {};

 type WorkflowRegisteredTriggerType = "sign-up";

@ -55,7 +54,9 @@ export async function compileWorkflowSource(source: string): Promise<Result<stri
  const bundleResult = await bundleJavaScript({
    "/source.tsx": source,
    "/entry.js": `
-      import { StackServerApp } from '@stackframe/stack';
+      import { StackServerApp } from 'https://esm.sh/@stackframe/js@2.8.36?target=es2021&standalone';
+
+      globalThis.navigator.onLine = true;

      export default async () => {
        globalThis.stackApp = new StackServerApp({
@ -82,7 +83,7 @@ export async function compileWorkflowSource(source: string): Promise<Result<stri
          if (!callbackFunc) {
            throw new Error(\`Callback \${callbackId} not found. Was it maybe deleted from the workflow?\`);
          }
-          return callbackFunc(JSON.parse(data.dataJson));
+          return callbackFunc(data);
        });
        let scheduledCallback = undefined;
        globalThis.scheduleCallback = ({ callbackId, data, scheduleAt }) => {
@ -124,6 +125,7 @@ export async function compileWorkflowSource(source: string): Promise<Result<stri
  }, {
    format: 'esm',
    keepAsImports: Object.keys(externalPackages),
+    allowHttpImports: true,
  });
  if (bundleResult.status === "error") {
    return Result.error(bundleResult.error);
@ -138,17 +140,23 @@ async function compileWorkflow(tenancy: Tenancy, workflowId: string): Promise<Re
    }
    const workflow = tenancy.config.workflows.availableWorkflows[workflowId];
    const res = await timeout(async () => {
+      console.log(`Compiling workflow ${workflowId}...`);
      const compiledCodeResult = await compileWorkflowSource(workflow.tsSource);
      if (compiledCodeResult.status === "error") {
        return Result.error({ compileError: `Failed to compile workflow: ${compiledCodeResult.error}` });
      }

+      console.log(`Compiled workflow source for ${workflowId}, running compilation trigger...`, { compiledCodeLength: compiledCodeResult.data.length });
+
      const compileTriggerResult = await triggerWorkflowRaw(tenancy, compiledCodeResult.data, {
        type: "compile",
      });
      if (compileTriggerResult.status === "error") {
        return Result.error({ compileError: `Failed to initialize workflow: ${compileTriggerResult.error}` });
      }
+
+      console.log(`Compilation trigger completed!`);
+
      const compileTriggerOutputResult = compileTriggerResult.data;
      if (typeof compileTriggerOutputResult !== "object" || !compileTriggerOutputResult || !("triggerOutput" in compileTriggerOutputResult)) {
        captureError("workflows-compile-trigger-output", new StackAssertionError(`Failed to parse compile trigger output`, { compileTriggerOutputResult }));
@ -160,13 +168,16 @@ async function compileWorkflow(tenancy: Tenancy, workflowId: string): Promise<Re
        return Result.error({ compileError: `Failed to parse compile trigger output, should be array of strings` });
      }

+      console.log(`Workflow ${workflowId} compiled successfully, returning result...`, { registeredTriggers });
+
      return Result.ok({
        compiledCode: compiledCodeResult.data,
        registeredTriggers: registeredTriggers,
      });
-    }, 10_000);
+    }, 30_000);

    if (res.status === "error") {
+      console.warn(`Timed out compiling workflow ${workflowId} after ${res.error.ms}ms`, { res });
      return Result.error({ compileError: `Timed out compiling workflow ${workflowId} after ${res.error.ms}ms` });
    }
    return res.data;
@ -229,6 +240,9 @@ import.meta.vitest?.test("compileWorkflow", async ({ expect }) => {
 });

 async function compileAndGetEnabledWorkflows(tenancy: Tenancy): Promise<Map<string, CompiledWorkflow>> {
+  // initialize ESBuild early so it doesn't count towards the 10s compilation timeout later
+  await initializeEsbuild();
+
  const compilationVersion = 1;
  const enabledWorkflows = new Map(await Promise.all(Object.entries(tenancy.config.workflows.availableWorkflows)
    .filter(([_, workflow]) => workflow.enabled)
@ -333,7 +347,6 @@ async function compileAndGetEnabledWorkflows(tenancy: Tenancy): Promise<Map<stri
            },
          },
        });
-        console.log(`Compiled workflow ${workflowId}`);
      } finally {
        await prisma.currentlyCompilingWorkflow.delete({
          where: {
@ -351,11 +364,11 @@ async function compileAndGetEnabledWorkflows(tenancy: Tenancy): Promise<Map<stri
    const { count } = await prisma.currentlyCompilingWorkflow.deleteMany({
      where: {
        tenancyId: tenancy.id,
-        startedCompilingAt: { lt: new Date(Date.now() - 20_000) },
+        startedCompilingAt: { lt: new Date(Date.now() - 40_000) },
      },
    });
    if (count > 0) {
-      captureError("workflows-compile-timeout", new StackAssertionError(`Deleted ${count} currently compiling workflows that were compiling for more than 20 seconds; this probably indicates a bug in the workflow compilation code`));
+      captureError("workflows-compile-timeout", new StackAssertionError(`Deleted ${count} currently compiling workflows that were compiling for more than 40 seconds; this probably indicates a bug in the workflow compilation code (as they should time out after 30 seconds)`));
    }

    await wait(1000);
@ -365,46 +378,56 @@ async function compileAndGetEnabledWorkflows(tenancy: Tenancy): Promise<Map<stri
 }

 async function triggerWorkflowRaw(tenancy: Tenancy, compiledWorkflowCode: string, trigger: WorkflowTrigger): Promise<Result<unknown, string>> {
-  const workflowToken = generateSecureRandomString();
-  const workflowTriggerToken = await globalPrismaClient.workflowTriggerToken.create({
-    data: {
-      expiresAt: new Date(Date.now() + 1000 * 35),
-      tenancyId: tenancy.id,
-      tokenHash: await hashWorkflowTriggerToken(workflowToken),
-    },
-  });
-
-  const tokenRefreshInterval = setInterval(() => {
-    runAsynchronously(async () => {
-      await globalPrismaClient.workflowTriggerToken.update({
-        where: {
-          tenancyId_id: {
-            tenancyId: tenancy.id,
-            id: workflowTriggerToken.id,
-          },
-        },
-        data: { expiresAt: new Date(Date.now() + 1000 * 35) },
-      });
-    });
-  }, 10_000);
-
-  try {
-    const freestyle = new Freestyle();
-    const freestyleRes = await freestyle.executeScript(compiledWorkflowCode, {
-      envVars: {
-        STACK_WORKFLOW_TRIGGER_DATA: JSON.stringify(trigger),
-        NEXT_PUBLIC_STACK_PROJECT_ID: tenancy.project.id,
-        NEXT_PUBLIC_STACK_API_URL: getEnvVariable("NEXT_PUBLIC_STACK_API_URL").replace("http://localhost", "http://host.docker.internal"),  // the replace is a hardcoded hack for the Freestyle mock server
-        NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY: "<placeholder publishable client key; the actual auth happens with the workflow token>",
-        STACK_SECRET_SERVER_KEY: "<placeholder secret server key; the actual auth happens with the workflow token>",
-        STACK_WORKFLOW_TOKEN_SECRET: workflowToken,
+  return await traceSpan({ description: `triggerWorkflowRaw ${trigger.type}` }, async () => {
+    const workflowToken = generateSecureRandomString();
+    const workflowTriggerToken = await globalPrismaClient.workflowTriggerToken.create({
+      data: {
+        expiresAt: new Date(Date.now() + 1000 * 35),
+        tenancyId: tenancy.id,
+        tokenHash: await hashWorkflowTriggerToken(workflowToken),
      },
-      nodeModules: Object.fromEntries(Object.entries(externalPackages).map(([packageName, version]) => [packageName, version])),
    });
-    return Result.map(freestyleRes, (data) => data.result);
-  } finally {
-    clearInterval(tokenRefreshInterval);
-  }
+
+    const tokenRefreshInterval = setInterval(() => {
+      runAsynchronously(async () => {
+        await globalPrismaClient.workflowTriggerToken.update({
+          where: {
+            tenancyId_id: {
+              tenancyId: tenancy.id,
+              id: workflowTriggerToken.id,
+            },
+          },
+          data: { expiresAt: new Date(Date.now() + 1000 * 35) },
+        });
+      });
+    }, 10_000);
+
+    try {
+      const freestyle = new Freestyle();
+      const apiUrl = new URL("/", getEnvVariable("NEXT_PUBLIC_STACK_API_URL").replace("http://localhost", "http://host.docker.internal"));
+      const freestyleRes = await freestyle.executeScript(compiledWorkflowCode, {
+        envVars: {
+          STACK_WORKFLOW_TRIGGER_DATA: JSON.stringify(trigger),
+          NEXT_PUBLIC_STACK_PROJECT_ID: tenancy.project.id,
+          NEXT_PUBLIC_STACK_API_URL: apiUrl.toString(),
+          NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY: "<placeholder publishable client key; the actual auth happens with the workflow token>",
+          STACK_SECRET_SERVER_KEY: "<placeholder secret server key; the actual auth happens with the workflow token>",
+          STACK_WORKFLOW_TOKEN_SECRET: workflowToken,
+        },
+        nodeModules: Object.fromEntries(Object.entries(externalPackages).map(([packageName, version]) => [packageName, version])),
+        networkPermissions: [
+          {
+            action: "allow",
+            behavior: "exact",
+            query: apiUrl.host,
+          },
+        ],
+      });
+      return Result.map(freestyleRes, (data) => data.result);
+    } finally {
+      clearInterval(tokenRefreshInterval);
+    }
+  });
 }

 async function createScheduledTrigger(tenancy: Tenancy, workflowId: string, trigger: WorkflowTrigger, scheduledAt: Date) {
@ -431,74 +454,119 @@ async function createScheduledTrigger(tenancy: Tenancy, workflowId: string, trig
      },
    },
  });
+
+  await upstash.publishJSON({
+    url: new URL(`/api/v1/internal/trigger/run-scheduled`, getEnvVariable("NEXT_PUBLIC_STACK_API_URL").replace("http://localhost", "http://host.docker.internal")).toString(),
+    body: {
+      tenancyId: tenancy.id,
+    },
+    notBefore: Math.floor(scheduledAt.getTime() / 1000),
+  });
+
  return dbTrigger;
 }

-async function triggerWorkflow(tenancy: Tenancy, compiledWorkflow: CompiledWorkflow, triggerId: string): Promise<Result<void, string>> {
-  if (compiledWorkflow.compiledCode === null) {
-    return Result.error(`Workflow ${compiledWorkflow.id} failed to compile: ${compiledWorkflow.compileError}`);
-  }
-
+export async function triggerScheduledWorkflows(tenancy: Tenancy) {
  const prisma = await getPrismaClientForTenancy(tenancy);
-  const trigger = await prisma.workflowTrigger.update({
-    where: {
-      tenancyId_id: {
-        tenancyId: tenancy.id,
-        id: triggerId,
-      },
-    },
-    data: {
-      compiledWorkflowId: compiledWorkflow.id,
-      scheduledAt: null,
-      output: Prisma.DbNull,
-      error: Prisma.DbNull,
-    },
-  });
+  const compiledWorkflows = await compileAndGetEnabledWorkflows(tenancy);

-  const res = await triggerWorkflowRaw(tenancy, compiledWorkflow.compiledCode, trigger.triggerData as WorkflowTrigger);
-  if (res.status === "error") {
-    console.log(`Compiled workflow failed to process trigger: ${res.error}`, { trigger, compiledWorkflowId: compiledWorkflow.id, res });
-  } else {
-    if (res.data && typeof res.data === "object" && "scheduledCallback" in res.data && res.data.scheduledCallback && typeof res.data.scheduledCallback === "object") {
-      const scheduledCallback: any = res.data.scheduledCallback;
-      const callbackId = `${scheduledCallback.callbackId}`;
-      const scheduleAt = new Date(scheduledCallback.scheduleAtMillis);
-      const callbackData = scheduledCallback.data;
-      await createScheduledTrigger(
-        tenancy,
-        compiledWorkflow.id,
-        {
-          type: "callback",
-          callbackId,
-          data: callbackData,
-          scheduledAtMillis: scheduleAt.getTime(),
-          callerTriggerId: triggerId,
-          executionId: trigger.executionId,
+  const toTrigger = await retryTransaction(prisma, async (tx) => {
+    const triggers = await tx.workflowTrigger.findMany({
+      where: {
+        tenancyId: tenancy.id,
+        scheduledAt: { lt: new Date(Date.now() + 5_000) },
+      },
+      include: {
+        execution: true,
+      },
+      orderBy: {
+        scheduledAt: "asc",
+      },
+      // let's take multiple triggers so we can catch up on the backlog, in case some triggers never went through (eg. if the queue was down)
+      // however, to prevent deadlocks as we are doing multiple writes in this transaction, we randomize it (so there's
+      // a chance that we only take one trigger, which would never deadlock)
+      take: Math.floor(1 + Math.random() * 3),
+    });
+    const toTrigger = [];
+    for (const trigger of triggers) {
+      const compiledWorkflow = compiledWorkflows.get(trigger.execution.workflowId);
+      const updatedTrigger = await tx.workflowTrigger.update({
+        where: {
+          tenancyId_id: {
+            tenancyId: tenancy.id,
+            id: trigger.id,
+          },
        },
-        scheduleAt
-      );
+        data: {
+          scheduledAt: null,
+          compiledWorkflowId: compiledWorkflow?.id ?? null,
+          output: Prisma.DbNull,
+          error: Prisma.DbNull,
+        },
+        include: {
+          execution: true,
+        },
+      });
+
+      if (compiledWorkflow) {
+        toTrigger.push(updatedTrigger);
+      } else {
+        // the workflow was deleted; we don't run the trigger, but we still mark it in the DB
+      }
    }
-  }
-  await prisma.workflowTrigger.update({
-    where: {
-      tenancyId_id: {
-        tenancyId: tenancy.id,
-        id: triggerId,
-      },
-    },
-    data: {
-      ...res.status === "ok" ? {
-        output: res.data as any,
-      } : {
-        error: res.error,
-      },
-    },
-  });
-  return Result.ok(undefined);
-}
+    return toTrigger;
+  }, { level: "serializable" });

-export async function triggerScheduledCallbacks(tenancy: Tenancy) {
+  await allPromisesAndWaitUntilEach(toTrigger.map(async (trigger) => {
+    const compiledWorkflow = compiledWorkflows.get(trigger.execution.workflowId) ?? throwErr(`Compiled workflow ${trigger.execution.workflowId} not found in trigger execution; this should not happen because we should've already checked for this in the transaction!`);
+    if (compiledWorkflow.compiledCode === null) {
+      return Result.error(`Workflow ${compiledWorkflow.id} failed to compile: ${compiledWorkflow.compileError}`);
+    }

+    const res = await triggerWorkflowRaw(tenancy, compiledWorkflow.compiledCode, trigger.triggerData as WorkflowTrigger);
+    if (res.status === "error") {
+      // This is probably fine and just a user error, but let's log it regardless
+      console.log(`Compiled workflow failed to process trigger: ${res.error}`, { trigger, compiledWorkflowId: compiledWorkflow.id, res });
+    } else {
+      if (res.data && typeof res.data === "object" && "scheduledCallback" in res.data && res.data.scheduledCallback && typeof res.data.scheduledCallback === "object") {
+        const scheduledCallback: any = res.data.scheduledCallback;
+        const callbackId = `${scheduledCallback.callbackId}`;
+        const scheduleAt = new Date(scheduledCallback.scheduleAtMillis);
+        const callbackData = scheduledCallback.data;
+        await createScheduledTrigger(
+          tenancy,
+          compiledWorkflow.id,
+          {
+            type: "callback",
+            callbackId,
+            data: callbackData,
+            scheduledAtMillis: scheduleAt.getTime(),
+            callerTriggerId: trigger.id,
+            executionId: trigger.executionId,
+          },
+          scheduleAt
+        );
+      }
+    }
+
+    const prisma = await getPrismaClientForTenancy(tenancy);
+    await prisma.workflowTrigger.update({
+      where: {
+        tenancyId_id: {
+          tenancyId: tenancy.id,
+          id: trigger.id,
+        },
+      },
+      data: {
+        ...res.status === "ok" ? {
+          output: res.data as any,
+        } : {
+          error: res.error,
+        },
+      },
+    });
+    return Result.ok(undefined);
+  }));
 }

 export async function triggerWorkflows(tenancy: Tenancy, trigger: WorkflowTrigger & { type: WorkflowRegisteredTriggerType }) {
@ -507,9 +575,8 @@ export async function triggerWorkflows(tenancy: Tenancy, trigger: WorkflowTrigge
    const promises = [...compiledWorkflows]
      .filter(([_, compiledWorkflow]) => compiledWorkflow.registeredTriggers.includes(trigger.type))
      .map(async ([workflowId, compiledWorkflow]) => {
-        const dbTrigger = await createScheduledTrigger(tenancy, workflowId, trigger, new Date());
-        await triggerWorkflow(tenancy, compiledWorkflow, dbTrigger.id);
+        await createScheduledTrigger(tenancy, workflowId, trigger, new Date());
      });
-    await Promise.all(promises);
+    await allPromisesAndWaitUntilEach(promises);
  });
 }
--- a/apps/backend/src/prisma-client.tsx
+++ b/apps/backend/src/prisma-client.tsx
@ -125,6 +125,16 @@ export async function retryTransaction<T>(client: PrismaClient, fn: (tx: PrismaC
  // serializable transactions are currently off by default, later we may turn them on
  const enableSerializable = options.level === "serializable";

+  const isRetryablePrismaError = (e: unknown) => {
+    if (e instanceof Prisma.PrismaClientKnownRequestError) {
+      return [
+        "P2028", // Serializable/repeatable read conflict
+        "P2034", // Transaction already closed (eg. timeout)
+      ];
+    }
+    return false;
+  };
+
  return await traceSpan('Prisma transaction', async (span) => {
    const res = await Result.retry(async (attemptIndex) => {
      return await traceSpan(`transaction attempt #${attemptIndex}`, async (attemptSpan) => {
@ -140,7 +150,7 @@ export async function retryTransaction<T>(client: PrismaClient, fn: (tx: PrismaC
                // to other (nested) transactions failing
                // however, we make an exception for "Transaction already closed", as those are (annoyingly) thrown on
                // the actual query, not the $transaction function itself
-                if (e instanceof Prisma.PrismaClientKnownRequestError && e.code === "P2028") { // Transaction already closed
+                if (isRetryablePrismaError(e)) {
                  throw new TransactionErrorThatShouldBeRetried(e);
                }
                throw new TransactionErrorThatShouldNotBeRetried(e);
@ -165,11 +175,7 @@ export async function retryTransaction<T>(client: PrismaClient, fn: (tx: PrismaC
            if (e instanceof TransactionErrorThatShouldNotBeRetried) {
              throw e.cause;
            }
-            if ([
-              "Transaction failed due to a write conflict or a deadlock. Please retry your transaction",
-              "Transaction already closed: A commit cannot be executed on an expired transaction. The timeout for this transaction",
-            ].some(s => e instanceof Prisma.PrismaClientKnownRequestError && e.message.includes(s))) {
-              // transaction timeout, retry
+            if (isRetryablePrismaError(e)) {
              return Result.error(e);
            }
            throw e;
--- a/apps/dashboard/src/app/(main)/(protected)/(outside-dashboard)/new-project/page-client.tsx
+++ b/apps/dashboard/src/app/(main)/(protected)/(outside-dashboard)/new-project/page-client.tsx
@ -5,7 +5,7 @@ import { yupResolver } from "@hookform/resolvers/yup";
 import { AuthPage, TeamSwitcher, useUser } from "@stackframe/stack";
 import { allProviders } from "@stackframe/stack-shared/dist/utils/oauth";
 import { runAsynchronouslyWithAlert, wait } from "@stackframe/stack-shared/dist/utils/promises";
-import { BrowserFrame, Button, Form, FormControl, FormField, FormItem, FormMessage, Label, Separator, Typography } from "@stackframe/stack-ui";
+import { BrowserFrame, Button, Form, FormControl, FormField, FormItem, FormMessage, Separator, Typography } from "@stackframe/stack-ui";
 import { useSearchParams } from "next/navigation";
 import { useState } from "react";
 import { useForm } from "react-hook-form";
@ -47,7 +47,7 @@ export default function PageClient() {
      credentialEnabled: form.watch("signInMethods").includes("credential"),
      magicLinkEnabled: form.watch("signInMethods").includes("magicLink"),
      passkeyEnabled: form.watch("signInMethods").includes("passkey"),
-      oauthProviders: form.watch('signInMethods').filter((method) => ["google", "github", "microsoft", "spotify"].includes(method)).map(provider => ({ id: provider, type: 'shared' })),
+      oauthProviders: form.watch('signInMethods').filter((method) => ["google", "github", "microsoft"].includes(method)).map(provider => ({ id: provider, type: 'shared' })),
    }
  };

@ -133,7 +133,6 @@ export default function PageClient() {
                  { value: "google", label: "Google" },
                  { value: "github", label: "GitHub" },
                  { value: "microsoft", label: "Microsoft" },
-                  { value: "spotify", label: "Spotify" },
                ]}
                info="More sign-in methods are available on the dashboard later."
              />
--- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/data-vault/page.tsx
+++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/data-vault/page.tsx
@ -1,5 +1,5 @@
 import { redirect } from "next/navigation";

 export default function Page() {
-  redirect("stores");
+  redirect("./data-vault/stores");
 }
--- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/workflows/[workflowId]/page.tsx
+++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/workflows/[workflowId]/page.tsx
@ -22,6 +22,7 @@ export default function WorkflowDetailPage() {
  const workflow = workflowId in availableWorkflows ? availableWorkflows[workflowId] : undefined;
  const [workflowContent, setWorkflowContent] = useState("");
  const [isLoading, setIsLoading] = useState(false);
+  const [isToggling, setIsToggling] = useState(false);

  useEffect(() => {
    if (workflow && workflow.tsSource) {
@ -47,6 +48,21 @@ export default function WorkflowDetailPage() {
    router.push(`/projects/${projectId}/workflows`);
  };

+  const handleToggleEnabled = async () => {
+    if (!workflow) return;
+    setIsToggling(true);
+    try {
+      await project.updateConfig({
+        [`workflows.availableWorkflows.${workflowId}.enabled`]: !workflow.enabled,
+      });
+      toast({ title: workflow.enabled ? "Workflow disabled" : "Workflow enabled" });
+    } catch (error) {
+      toast({ title: "Failed to toggle workflow", variant: "destructive" });
+    } finally {
+      setIsToggling(false);
+    }
+  };
+
  if (workflow === undefined) {
    return (
      <PageLayout title="Workflow Not Found">
@ -70,6 +86,9 @@ export default function WorkflowDetailPage() {
            <ArrowLeft className="h-4 w-4 mr-2" />
            Back
          </Button>
+          <Button onClick={handleToggleEnabled} size="sm" variant={workflow.enabled ? "outline" : "default"} disabled={isToggling}>
+            {workflow.enabled ? "Disable" : "Enable"}
+          </Button>
          <Button onClick={handleSave} size="sm" disabled={isLoading}>
            <Save className="h-4 w-4 mr-2" />
            {isLoading ? "Saving..." : "Save"}
--- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/workflows/page.tsx
+++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/workflows/page.tsx
@ -69,7 +69,7 @@ function CreateWorkflowDialog({
 }: {
  open: boolean,
  onOpenChange: (open: boolean) => void,
-  onSave: (id: string, displayName: string, tsSource: string) => Promise<void>,
+  onSave: (id: string, displayName: string, tsSource: string, enabled: boolean) => Promise<void>,
 }) {
  const [displayName, setDisplayName] = useState("");
  const [isSubmitting, setIsSubmitting] = useState(false);
@ -95,7 +95,7 @@ function CreateWorkflowDialog({
        registerCallback("in-7-days", async (data) => {
          await stackApp.sendEmail({ userIds: [data.userId], subject: "Welcome to the app!", html: "<p>Example email</p>" });
        });
-      `);
+      `, false);
      onOpenChange(false);
      setDisplayName("");
    } finally {
@ -151,12 +151,12 @@ export default function WorkflowsPage() {
    setShowCreateDialog(true);
  };

-  const handleSaveWorkflow = async (id: string, displayName: string, tsSource: string) => {
+  const handleSaveWorkflow = async (id: string, displayName: string, tsSource: string, enabled: boolean) => {
    await project.updateConfig({
      [`workflows.availableWorkflows.${id}`]: {
        displayName,
        tsSource,
-        enabled: true
+        enabled,
      }
    });
    toast({ title: "Workflow created successfully" });
--- a/apps/dev-launchpad/public/index.html
+++ b/apps/dev-launchpad/public/index.html
@ -119,6 +119,9 @@
      <li>
        8124: LocalStack Gateway (AWS mock)
      </li>
+      <li>
+        8125: QStash mock
+      </li>
      <li>
        8150-8199: Reserved for LocalStack (external services)
      </li>
--- a/apps/e2e/tests/backend/backend-helpers.ts
+++ b/apps/e2e/tests/backend/backend-helpers.ts
@ -108,12 +108,17 @@ export async function niceBackendFetch(url: string | URL, options?: Omit<NiceReq
  accessType?: null | "client" | "server" | "admin",
  body?: unknown,
  headers?: Record<string, string | undefined>,
+  userAuth?: {
+    accessToken?: string,
+    refreshToken?: string,
+  },
 }): Promise<NiceResponse> {
-  const { body, headers, accessType, ...otherOptions } = options ?? {};
+  const { body, headers, accessType, userAuth: userAuthOverride, ...otherOptions } = options ?? {};
  if (typeof body === "object") {
    expectSnakeCase(body, "req.body");
  }
-  const { projectKeys, userAuth } = backendContext.value;
+  const projectKeys = backendContext.value.projectKeys;
+  const userAuth = userAuthOverride ?? backendContext.value.userAuth;
  const fullUrl = new URL(url, STACK_BACKEND_BASE_URL);
  if (fullUrl.origin !== new URL(STACK_BACKEND_BASE_URL).origin) throw new StackAssertionError(`Invalid niceBackendFetch origin: ${fullUrl.origin}`);
  if (fullUrl.protocol !== new URL(STACK_BACKEND_BASE_URL).protocol) throw new StackAssertionError(`Invalid niceBackendFetch protocol: ${fullUrl.protocol}`);
@ -203,6 +208,27 @@ export namespace Auth {
    }
  }

+  export async function refreshAccessToken() {
+    const response = await niceBackendFetch("/api/v1/auth/sessions/current/refresh", {
+      method: "POST",
+      accessType: "client",
+      userAuth: {
+        refreshToken: backendContext.value.userAuth?.refreshToken,
+      },
+    });
+    expect(response).toMatchInlineSnapshot(`
+      NiceResponse {
+        "status": 200,
+        "body": { "access_token": <stripped field 'access_token'> },
+        "headers": Headers { <some fields may have been hidden> },
+      }
+    `);
+    backendContext.set({ userAuth: { accessToken: response.body.access_token, refreshToken: response.body.refresh_token } });
+    return {
+      refreshAccessTokenResponse: response,
+    };
+  }
+
  /**
   * Valid session & valid access token: OK
   * Valid session & invalid access token: OK
--- a/apps/e2e/tests/backend/workflows.test.ts
+++ b/apps/e2e/tests/backend/workflows.test.ts
@ -27,26 +27,40 @@ async function configureEmailAndWorkflow(workflowId: string, tsSource: string, e
  });
 }

+const waitRetries = 25;
+
 async function waitForMailboxSubject(mailbox: Mailbox, subject: string) {
-  for (let i = 0; i < 10; i++) {
+  for (let i = 0; i < waitRetries; i++) {
    const messages = await mailbox.fetchMessages();
    const message = messages.find((m) => m.subject === subject);
    if (message) return;
    await wait(1_000);
  }
-  throw new Error(`Message with subject ${subject} not found after 10 tries`);
+  throw new Error(`Message with subject ${subject} not found after ${waitRetries} tries`);
+}
+
+async function waitForServerMetadataNotNull(userId: string, key: string) {
+  for (let i = 0; i < waitRetries; i++) {
+    const user = await niceBackendFetch(`/api/v1/users/${userId}`, { accessType: "server" });
+    if (user.body.server_metadata?.[key]) return;
+    await wait(1_000);
+  }
+  throw new Error(`Server metadata for user ${userId} with key ${key} not found after ${waitRetries} tries`);
 }

 test("onSignUp workflow sends email for client sign-up", async ({ expect }) => {
  await Project.createAndSwitch();
+  await InternalApiKey.createAndSetProjectKeys();
  const mailbox = await bumpEmailAddress({ unindexed: true });
  const subject = `WF client signup ${crypto.randomUUID()}`;

  await configureEmailAndWorkflow("wf-email", `
    onSignUp(async (user) => {
      await stackApp.sendEmail({ userIds: [user.id], subject: ${JSON.stringify(subject)}, html: "<p>hi</p>" });
+
+      // schedule a callback as an example (we don't actually test whether it executed successfully)
      return scheduleCallback({
-        scheduleAt: new Date(Date.now() + 120_000),
+        scheduleAt: new Date(Date.now() + 7_000),
        data: { "example": "data" },
        callbackId: "my-callback",
      });
@ -87,6 +101,64 @@ test("onSignUp workflow sends email for client sign-up", async ({ expect }) => {
      },
    ]
  `);
+}, {
+  timeout: 60_000,
+});
+
+test("onSignUp workflow can schedule callbacks", async ({ expect }) => {
+  await Project.createAndSwitch();
+  await InternalApiKey.createAndSetProjectKeys();
+  const mailbox = await bumpEmailAddress({ unindexed: true });
+  const subject = `WF client signup ${crypto.randomUUID()}`;
+
+  await configureEmailAndWorkflow("wf-email", `
+    onSignUp(async (user) => {
+      return scheduleCallback({
+        scheduleAt: new Date(Date.now() + 7_000),
+        data: { "userId": user.id },
+        callbackId: "my-callback",
+      });
+    });
+
+    registerCallback("my-callback", async (data) => {
+      await stackApp.sendEmail({ userIds: [data.userId], subject: ${JSON.stringify(subject)}, html: "<p>hi</p>" });
+    });
+  `);
+
+  await Auth.Password.signUpWithEmail({ password: "password" });
+
+  // since we wait for the callback, add some extra time
+  await wait(10_000);
+  await waitForMailboxSubject(mailbox, subject);
+
+  expect(await mailbox.fetchMessages()).toMatchInlineSnapshot(`
+    [
+      MailboxMessage {
+        "attachments": [],
+        "body": {
+          "html": "http://localhost:12345/some-callback-url?code=%3Cstripped+query+param%3E",
+          "text": "http://localhost:12345/some-callback-url?code=%3Cstripped+query+param%3E",
+        },
+        "from": "Test Project <test@example.com>",
+        "subject": "Verify your email at New Project",
+        "to": ["<unindexed-mailbox--<stripped UUID>@stack-generated.example.com>"],
+        <some fields may have been hidden>,
+      },
+      MailboxMessage {
+        "attachments": [],
+        "body": {
+          "html": "<!DOCTYPE html PUBLIC \\"-//W3C//DTD XHTML 1.0 Transitional//EN\\" \\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\\"><html dir=\\"ltr\\" lang=\\"en\\"><head><meta content=\\"text/html; charset=UTF-8\\" http-equiv=\\"Content-Type\\"/><meta name=\\"x-apple-disable-message-reformatting\\"/></head><body style=\\"background-color:rgb(250,251,251);font-family:ui-sans-serif, system-ui, sans-serif, &quot;Apple Color Emoji&quot;, &quot;Segoe UI Emoji&quot;, &quot;Segoe UI Symbol&quot;, &quot;Noto Color Emoji&quot;;font-size:1rem;line-height:1.5rem\\"><!--$--><table align=\\"center\\" width=\\"100%\\" border=\\"0\\" cellPadding=\\"0\\" cellSpacing=\\"0\\" role=\\"presentation\\" style=\\"background-color:rgb(255,255,255);padding:45px;border-radius:0.5rem;max-width:37.5em\\"><tbody><tr style=\\"width:100%\\"><td><div><p>hi</p></div></td></tr></tbody></table><!--7--><!--/$--></body></html>",
+          "text": "hi",
+        },
+        "from": "Test Project <test@example.com>",
+        "subject": "WF client signup <stripped UUID>",
+        "to": ["<unindexed-mailbox--<stripped UUID>@stack-generated.example.com>"],
+        <some fields may have been hidden>,
+      },
+    ]
+  `);
+}, {
+  timeout: 60_000,
 });

 test("onSignUp workflow sends email for server-created user", async ({ expect }) => {
@ -128,10 +200,13 @@ test("onSignUp workflow sends email for server-created user", async ({ expect })
      },
    ]
  `);
+}, {
+  timeout: 60_000,
 });

 test("disabled workflows do not trigger", async ({ expect }) => {
  await Project.createAndSwitch();
+  await InternalApiKey.createAndSetProjectKeys();
  const mailbox = await bumpEmailAddress({ unindexed: true });
  const subject = `WF disabled ${crypto.randomUUID()}`;

@ -143,7 +218,8 @@ test("disabled workflows do not trigger", async ({ expect }) => {

  await Auth.Password.signUpWithEmail({ password: "password" });

-  await wait(12_000);
+  await wait(waitRetries * 1_000 * 1.3);
+  await Auth.refreshAccessToken();

  expect(await mailbox.fetchMessages()).toMatchInlineSnapshot(`
    [
@ -160,10 +236,13 @@ test("disabled workflows do not trigger", async ({ expect }) => {
      },
    ]
  `);
+}, {
+  timeout: 90_000,
 });

 test("compile/runtime errors in one workflow don't block others", async ({ expect }) => {
  await Project.createAndSwitch();
+  await InternalApiKey.createAndSetProjectKeys();
  const mailbox = await bumpEmailAddress({ unindexed: true });
  const subject = `WF ok ${crypto.randomUUID()}`;

@ -207,10 +286,13 @@ test("compile/runtime errors in one workflow don't block others", async ({ expec
      },
    ]
  `);
+}, {
+  timeout: 60_000,
 });

 test("anonymous sign-up does not trigger; upgrade triggers workflow", async ({ expect }) => {
  await Project.createAndSwitch();
+  await InternalApiKey.createAndSetProjectKeys();
  const markerKey = `wfMarker-${crypto.randomUUID()}`;

  await Project.updateConfig({
@ -229,7 +311,8 @@ test("anonymous sign-up does not trigger; upgrade triggers workflow", async ({ e
  const { userId: anonUserId } = await Auth.Anonymous.signUp();

  // ensure marker not present yet
-  await wait(12_000);
+  await wait(waitRetries * 1_000 * 1.3);
+  await Auth.refreshAccessToken();
  const me1 = await niceBackendFetch("/api/v1/users/me", { accessType: "client" });
  expect(me1.body.server_metadata?.[markerKey]).toBeUndefined();

@ -237,16 +320,17 @@ test("anonymous sign-up does not trigger; upgrade triggers workflow", async ({ e
  const { userId } = await Auth.Password.signUpWithEmail({ password: "password" });
  expect(userId).toEqual(anonUserId);

-  await wait(12_000);
+  await waitForServerMetadataNotNull(anonUserId, markerKey);
  const me2 = await niceBackendFetch("/api/v1/users/me", { accessType: "server" });
  expect(me2.body.is_anonymous).toBe(false);
  expect(me2.body.server_metadata?.[markerKey]).toBe(me2.body.primary_email);
 }, {
-  timeout: 40_000,
+  timeout: 90_000,
 });

 test("workflow source changes take effect for subsequent sign-ups", async ({ expect }) => {
  await Project.createAndSwitch();
+  await InternalApiKey.createAndSetProjectKeys();
  const markerKey = `versionMarker-${crypto.randomUUID()}`;

  // v1
@ -263,7 +347,7 @@ test("workflow source changes take effect for subsequent sign-ups", async ({ exp
  });
  await bumpEmailAddress({ unindexed: true });
  await Auth.Password.signUpWithEmail({ password: "password" });
-  await wait(12_000);
+  await waitForServerMetadataNotNull("me", markerKey);
  const me1 = await niceBackendFetch("/api/v1/users/me", { accessType: "server" });
  expect(me1.body.server_metadata?.[markerKey]).toBe("v1");

@ -281,9 +365,9 @@ test("workflow source changes take effect for subsequent sign-ups", async ({ exp
  });
  await bumpEmailAddress({ unindexed: true });
  await Auth.Password.signUpWithEmail({ password: "password" });
-  await wait(12_000);
+  await waitForServerMetadataNotNull("me", markerKey);
  const me2 = await niceBackendFetch("/api/v1/users/me", { accessType: "server" });
  expect(me2.body.server_metadata?.[markerKey]).toBe("v2");
 }, {
-  timeout: 40_000,
+  timeout: 90_000,
 });
--- a/docker/dependencies/docker.compose.yaml
+++ b/docker/dependencies/docker.compose.yaml
@ -177,7 +177,7 @@ services:
    ports:
      - "8122:8080"       # POST http://localhost:8119/execute/v1/script
    extra_hosts:
-      - "host.docker.internal:host-gateway"  # noop on Docker Desktop/Orbstack enables host.docker.internal on Linux
+      - "host.docker.internal:host-gateway"  # noop on Docker Desktop/Orbstack, enables host.docker.internal on Linux
    environment:
      DENO_DIR: /deno-cache
      HOST_ON_HOST: host.docker.internal
@ -193,6 +193,18 @@ services:
    environment:
      - STRIPE_API_KEY=sk_test_1234567890

+  # ================= QStash =================
+
+  qstash:
+    image: public.ecr.aws/upstash/qstash:latest
+    ports:
+      - 8125:8080
+    command: qstash dev
+    extra_hosts:
+      - "host.docker.internal:host-gateway"  # noop on Docker Desktop/Orbstack, enables host.docker.internal on Linux
+    environment:
+      HOST_ON_HOST: host.docker.internal
+

 # ================= volumes =================

--- a/packages/stack-shared/package.json
+++ b/packages/stack-shared/package.json
@ -56,6 +56,7 @@
    "@aws-sdk/client-kms": "^3.876.0",
    "@opentelemetry/api": "^1.9.0",
    "@simplewebauthn/browser": "^11.0.0",
+    "@vercel/functions": "^2.0.0",
    "async-mutex": "^0.5.0",
    "bcryptjs": "^3.0.2",
    "crc": "^4.3.2",
--- a/packages/stack-shared/src/helpers/vault/server-side.ts
+++ b/packages/stack-shared/src/helpers/vault/server-side.ts
@ -6,19 +6,24 @@ import {
  GenerateDataKeyCommand,
  KMSClient
 } from "@aws-sdk/client-kms";
+import { awsCredentialsProvider } from '@vercel/functions/oidc';
 import { decodeBase64, encodeBase64 } from "../../utils/bytes";
 import { decrypt, encrypt } from "../../utils/crypto";
 import { getEnvVariable } from "../../utils/env";
 import { Result } from "../../utils/results";

+
 function getKmsClient() {
+  const roleArn = getEnvVariable("STACK_AWS_VERCEL_OIDC_ROLE_ARN", "");
  return new KMSClient({
    region: getEnvVariable("STACK_AWS_REGION"),
    endpoint: getEnvVariable("STACK_AWS_KMS_ENDPOINT"),
-    credentials: {
+    credentials: roleArn ? awsCredentialsProvider({
+      roleArn,
+    }) : {
      accessKeyId: getEnvVariable("STACK_AWS_ACCESS_KEY_ID"),
-      secretAccessKey: getEnvVariable("STACK_AWS_SECRET_ACCESS_KEY")
-    }
+      secretAccessKey: getEnvVariable("STACK_AWS_SECRET_ACCESS_KEY"),
+    },
  });
 }

--- a/packages/stack-shared/src/interface/client-interface.ts
+++ b/packages/stack-shared/src/interface/client-interface.ts
@ -125,8 +125,8 @@ export class StackClientInterface {

    // try to diagnose the error for the user
    if (retriedResult.status === "error") {
-      if (globalVar.navigator && !globalVar.navigator.onLine) {
-        throw new Error("You are offline. Please check your internet connection and try again. (window.navigator.onLine is falsy)", { cause: retriedResult.error });
+      if (globalVar.navigator && globalVar.navigator.onLine === false) {
+        throw new Error("You are offline. Please check your internet connection and try again. (window.navigator.onLine is false)", { cause: retriedResult.error });
      }
      throw await this._createNetworkError(retriedResult.error, session, requestType);
    }
--- a/packages/stack-shared/src/schema-fields.ts
+++ b/packages/stack-shared/src/schema-fields.ts
@ -1,7 +1,7 @@
 import * as yup from "yup";
 import { KnownErrors } from ".";
 import { isBase64 } from "./utils/bytes";
-import { type Currency, type MoneyAmount, SUPPORTED_CURRENCIES } from "./utils/currency-constants";
+import { SUPPORTED_CURRENCIES, type Currency, type MoneyAmount } from "./utils/currency-constants";
 import { DayInterval, Interval } from "./utils/dates";
 import { StackAssertionError, throwErr } from "./utils/errors";
 import { decodeBasicAuthorizationHeader } from "./utils/http";
@ -423,7 +423,7 @@ export const dayIntervalOrNeverSchema = yupUnion(dayIntervalSchema.defined(), yu
 * This schema is useful for fields where the user can specify the ID, such as price IDs. It is particularly common
 * for IDs in the config schema.
 */
-export const userSpecifiedIdSchema = (idName: `${string}Id`) => yupString().matches(/^[a-zA-Z_][a-zA-Z0-9_-]*$/, `${idName} must start with a letter or underscore and contain only letters, numbers, underscores, and hyphens`);
+export const userSpecifiedIdSchema = (idName: `${string}Id`) => yupString().max(63).matches(/^[a-zA-Z_][a-zA-Z0-9_-]*$/, `${idName} must start with a letter or underscore and contain only letters, numbers, underscores, and hyphens`);
 export const moneyAmountSchema = (currency: Currency) => yupString<MoneyAmount>().test('money-amount', 'Invalid money amount', (value, context) => {
  if (value == null) return true;
  const regex = /^([0-9]+)(\.([0-9]+))?$/;
--- a/packages/stack-shared/src/utils/esbuild.tsx
+++ b/packages/stack-shared/src/utils/esbuild.tsx
@ -35,6 +35,7 @@ export async function bundleJavaScript(sourceFiles: Record<string, string> & { '
  externalPackages?: Record<string, string>,
  keepAsImports?: string[],
  sourcemap?: false | 'inline',
+  allowHttpImports?: boolean,
 } = {}): Promise<Result<string, string>> {
  await initializeEsbuild();

@ -42,6 +43,8 @@ export async function bundleJavaScript(sourceFiles: Record<string, string> & { '
  const externalPackagesMap = new Map(Object.entries(options.externalPackages ?? {}));
  const keepAsImports = options.keepAsImports ?? [];

+  const httpImportCache = new Map<string, { contents: string; loader: esbuild.Loader; resolveDir: string }>();
+
  const extToLoader: Map<string, esbuild.Loader> = new Map([
    ['tsx', 'tsx'],
    ['ts', 'ts'],
@ -63,6 +66,68 @@ export async function bundleJavaScript(sourceFiles: Record<string, string> & { '
      sourcemap: options.sourcemap ?? 'inline',
      external: keepAsImports,
      plugins: [
+        ...options.allowHttpImports ? [{
+          name: "esm-sh-only",
+          setup(build: esbuild.PluginBuild) {
+            // Handle absolute URLs and relative imports from esm.sh modules.
+            build.onResolve({ filter: /.*/ }, (args) => {
+              // Only touch absolute http(s) specifiers or children of our own namespace
+              const isHttp = args.path.startsWith("http://") || args.path.startsWith("https://");
+              const fromEsmNs = args.namespace === "esm-sh";
+      
+              if (!isHttp && !fromEsmNs) return null; // Let other plugins handle bare/relative/local
+      
+              // Resolve relative URLs inside esm.sh-fetched modules
+              const url = new URL(args.path, fromEsmNs ? args.importer : undefined);
+      
+              if (url.protocol !== "https:" || url.host !== "esm.sh") {
+                throw new Error(`Blocked non-esm.sh URL import: ${url.href}`);
+              }
+      
+              return { path: url.href, namespace: "esm-sh" };
+            });
+      
+            build.onLoad({ filter: /.*/, namespace: "esm-sh" }, async (args) => {
+              if (httpImportCache.has(args.path)) return httpImportCache.get(args.path)!;
+      
+              const res = await fetch(args.path, { redirect: "follow" });
+              if (!res.ok) throw new Error(`Fetch ${res.status} ${res.statusText} for ${args.path}`);
+              const finalUrl = new URL(res.url);
+              // Defensive: follow shouldn’t leave esm.sh, but re-check.
+              if (finalUrl.host !== "esm.sh") {
+                throw new Error(`Redirect escaped esm.sh: ${finalUrl.href}`);
+              }
+      
+              const ct = (res.headers.get("content-type") || "").toLowerCase();
+              let loader: esbuild.Loader =
+                ct.includes("css") ? "css" :
+                ct.includes("json") ? "json" :
+                ct.includes("typescript") ? "ts" :
+                ct.includes("jsx") ? "jsx" :
+                ct.includes("tsx") ? "tsx" :
+                "js";
+      
+              // Fallback by extension (esm.sh sometimes omits CT)
+              const p = finalUrl.pathname;
+              if (p.endsWith(".css")) loader = "css";
+              else if (p.endsWith(".json")) loader = "json";
+              else if (p.endsWith(".ts")) loader = "ts";
+              else if (p.endsWith(".tsx")) loader = "tsx";
+              else if (p.endsWith(".jsx")) loader = "jsx";
+      
+              const contents = await res.text();
+              const result = {
+                contents,
+                loader,
+                // Ensures relative imports inside that module resolve against the file’s URL
+                resolveDir: new URL(".", finalUrl.href).toString(),
+                watchFiles: [finalUrl.href],
+              };
+              httpImportCache.set(args.path, result);
+              return result;
+            });
+          },
+        } as esbuild.Plugin] : [],
        {
          name: 'replace-packages-with-globals',
          setup(build) {
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -177,6 +177,9 @@ importers:
      '@stackframe/stack-shared':
        specifier: workspace:*
        version: link:../../packages/stack-shared
+      '@upstash/qstash':
+        specifier: ^2.8.2
+        version: 2.8.2
      '@vercel/functions':
        specifier: ^2.0.0
        version: 2.0.0(@aws-sdk/credential-provider-web-identity@3.876.0)
@ -1461,6 +1464,9 @@ importers:
      '@types/react-dom':
        specifier: ^18.2.0
        version: 18.3.1
+      '@vercel/functions':
+        specifier: ^2.0.0
+        version: 2.0.0(@aws-sdk/credential-provider-web-identity@3.876.0)
      async-mutex:
        specifier: ^0.5.0
        version: 0.5.0
@ -7857,6 +7863,9 @@ packages:
  '@ungap/structured-clone@1.3.0':
    resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==}

+  '@upstash/qstash@2.8.2':
+    resolution: {integrity: sha512-gQMCs2YXmRJWGh28t3fsWuPTzGgVFVRBd4o5QeWM9l3HW8TMNwt1qzv5wtzzSlG7hv1ylEy/PQCznkxGcAwTfw==}
+
  '@urql/core@5.2.0':
    resolution: {integrity: sha512-/n0ieD0mvvDnVAXEQgX/7qJiVcvYvNkOHeBvkwtylfjydar123caCXcl58PXFY11oU1oquJocVXHxLAbtv4x1A==}

@ -9109,6 +9118,9 @@ packages:
    resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
    engines: {node: '>= 8'}

+  crypto-js@4.2.0:
+    resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==}
+
  crypto-random-string@2.0.0:
    resolution: {integrity: sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==}
    engines: {node: '>=8'}
@ -12359,6 +12371,10 @@ packages:
  nested-error-stacks@2.0.1:
    resolution: {integrity: sha512-SrQrok4CATudVzBS7coSz26QRSmlK9TzzoFbeKfcPBUFPjcQM9Rqvr/DlJkOrwI/0KcgvMub1n1g5Jt9EgRn4A==}

+  neverthrow@7.2.0:
+    resolution: {integrity: sha512-iGBUfFB7yPczHHtA8dksKTJ9E8TESNTAx1UQWW6TzMF280vo9jdPYpLUXrMN1BCkPdHFdNG3fxOt2CUad8KhAw==}
+    engines: {node: '>=18'}
+
  next-intl@3.19.1:
    resolution: {integrity: sha512-KlJSomzbB5dNkWBIiSIRaoy5zqwLgHNV5Zw0ULhkHjNnPN7aLFFv2G+VOQKle630sNH2JiKc9nsmi6PM1GdkhA==}
    peerDependencies:
@ -23539,6 +23555,12 @@ snapshots:

  '@ungap/structured-clone@1.3.0': {}

+  '@upstash/qstash@2.8.2':
+    dependencies:
+      crypto-js: 4.2.0
+      jose: 5.6.3
+      neverthrow: 7.2.0
+
  '@urql/core@5.2.0':
    dependencies:
      '@0no-co/graphql.web': 1.1.2
@ -25033,6 +25055,8 @@ snapshots:
      shebang-command: 2.0.0
      which: 2.0.2

+  crypto-js@4.2.0: {}
+
  crypto-random-string@2.0.0: {}

  css-select@5.1.0:
@ -29468,6 +29492,8 @@ snapshots:

  nested-error-stacks@2.0.1: {}

+  neverthrow@7.2.0: {}
+
  next-intl@3.19.1(next@14.2.5(@babel/core@7.26.0)(@opentelemetry/api@1.9.0)(react-dom@18.3.1(react@18.2.0))(react@18.2.0))(react@18.3.1):
    dependencies:
      '@formatjs/intl-localematcher': 0.5.4
--- a/vitest.shared.ts
+++ b/vitest.shared.ts
@ -4,6 +4,8 @@ import { defineConfig } from 'vitest/config';
 export default defineConfig({
  plugins: [tsconfigPaths() as any],
  test: {
+    pool: 'threads',
+    maxWorkers: 8,
    include: ['**/*.test.{js,ts,jsx,tsx}'],
    includeSource: ['**/*.{js,ts,jsx,tsx}'],
  },