Fix STACK-BACKEND-5K

.github/workflows/e2e-api-tests.yaml

@@ -125,6 +125,9 @@ jobs:
       - name: Run tests again, to make sure they are stable (attempt 3)
         run: pnpm test
 
+      - name: Verify data integrity
+        run: pnpm run verify-data-integrity
+
       - name: Print Docker Compose logs
         if: always()
         run: docker compose -f dependencies.compose.yaml logs

apps/backend/package.json

@@ -25,7 +25,8 @@
     "watch-docs": "pnpm run with-env tsx watch --clear-screen=false scripts/generate-docs.ts",
     "generate-docs": "pnpm run with-env tsx scripts/generate-docs.ts",
     "generate-keys": "pnpm run with-env tsx scripts/generate-keys.ts",
-    "db-seed-script": "pnpm run with-env tsx prisma/seed.ts"
+    "db-seed-script": "pnpm run with-env tsx prisma/seed.ts",
+    "verify-data-integrity": "pnpm run with-env tsx scripts/verify-data-integrity.ts"
   },
   "prisma": {
     "seed": "pnpm run db-seed-script"

apps/backend/prisma/migrations/20241223225110_fill_empty_project_config_values/migration.sql

@@ -0,0 +1,5 @@
+-- Some older versions allowed the empty string for OAuth provider clientId and clientSecret values.
+-- We fix that.
+
+UPDATE "StandardOAuthProviderConfig" SET "clientId" = 'invalid' WHERE "clientId" = '';
+UPDATE "StandardOAuthProviderConfig" SET "clientSecret" = 'invalid' WHERE "clientSecret" = '';

apps/backend/prisma/migrations/20241223231022_remove_empty_team_profile_images/migration.sql

@@ -0,0 +1,4 @@
+-- Some older versions allowed the empty string as a team profile image.
+-- We fix that.
+
+UPDATE "Team" SET "profileImageUrl" = NULL WHERE "profileImageUrl" = '';

apps/backend/prisma/migrations/20241223231023_onlyhttps_domains/migration.sql

@@ -0,0 +1,4 @@
+-- Some older versions allowed http:// URLs as trusted domains, instead of just https://.
+-- We fix that.
+
+UPDATE "ProjectDomain" SET "domain" = 'https://example.com' WHERE "domain" LIKE 'http://%';

apps/backend/scripts/verify-data-integrity.ts

@@ -0,0 +1,163 @@
+import { PrismaClient } from "@prisma/client";
+import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
+import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
+import { filterUndefined } from "@stackframe/stack-shared/dist/utils/objects";
+import { wait } from "@stackframe/stack-shared/dist/utils/promises";
+import { deindent } from "@stackframe/stack-shared/dist/utils/strings";
+
+const prismaClient = new PrismaClient();
+
+async function main() {
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log("===================================================");
+  console.log("Welcome to verify-data-integrity.ts.");
+  console.log();
+  console.log("This script will ensure that the data in the");
+  console.log("database is not corrupted.");
+  console.log();
+  console.log("It will call the most important endpoints for");
+  console.log("each project and every user, and ensure that");
+  console.log("the status codes are what they should be.");
+  console.log();
+  console.log("It's a good idea to run this script on REPLICAS");
+  console.log("of the production database regularly (not the actual");
+  console.log("prod db!); it should never fail at any point in time.");
+  console.log();
+  console.log("");
+  console.log("\x1b[41mIMPORTANT\x1b[0m: This script may modify");
+  console.log("the database during its execution in all sorts of");
+  console.log("ways, so don't run it on production!");
+  console.log();
+  console.log("===================================================");
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log("Starting in 3 seconds...");
+  await wait(1000);
+  console.log("2...");
+  await wait(1000);
+  console.log("1...");
+  await wait(1000);
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+
+
+  const projects = await prismaClient.project.findMany({
+    select: {
+      id: true,
+      displayName: true,
+    },
+    orderBy: {
+      id: "asc",
+    },
+  });
+  console.log(`Found ${projects.length} projects, iterating over them.`);
+
+  for (let i = 0; i < projects.length; i++) {
+    const projectId = projects[i].id;
+    await recurse(`[project ${i + 1}/${projects.length}] ${projectId} ${projects[i].displayName}`, async (recurse) => {
+      await Promise.all([
+        expectStatusCode(200, `/api/v1/projects/current`, {
+          method: "GET",
+          headers: {
+            "x-stack-project-id": projectId,
+            "x-stack-access-type": "admin",
+            "x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
+          },
+        }),
+        expectStatusCode(200, `/api/v1/users`, {
+          method: "GET",
+          headers: {
+            "x-stack-project-id": projectId,
+            "x-stack-access-type": "admin",
+            "x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
+          },
+        }),
+      ]);
+    });
+  }
+
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log("===================================================");
+  console.log("All good!");
+  console.log();
+  console.log("Goodbye.");
+  console.log("===================================================");
+  console.log();
+  console.log();
+}
+main().catch((...args) => {
+  console.error();
+  console.error();
+  console.error(`\x1b[41mERROR\x1b[0m! Could not verify data integrity. See the error message for more details.`);
+  console.error(...args);
+  process.exit(1);
+});
+
+async function expectStatusCode(expectedStatusCode: number, endpoint: string, request: RequestInit) {
+  const apiUrl = new URL(getEnvVariable("NEXT_PUBLIC_STACK_API_URL"));
+  const response = await fetch(new URL(endpoint, apiUrl), {
+    ...request,
+    headers: {
+      "x-stack-disable-artificial-development-delay": "yes",
+      "x-stack-development-disable-extended-logging": "yes",
+      ...filterUndefined(request.headers ?? {}),
+    },
+  });
+  if (response.status !== expectedStatusCode) {
+    throw new StackAssertionError(deindent`
+      Expected status code ${expectedStatusCode} but got ${response.status} for ${endpoint}:
+
+          ${await response.text()}
+    `, { request, response });
+  }
+  const json = await response.json();
+  return json;
+}
+
+let lastProgress = performance.now() - 9999999999;
+
+type RecurseFunction = (progressPrefix: string, inner: (recurse: RecurseFunction) => Promise<void>) => Promise<void>;
+
+const _recurse = async (progressPrefix: string | ((...args: any[]) => void), inner: Parameters<RecurseFunction>[1]): Promise<void> => {
+  const progressFunc = typeof progressPrefix === "function" ? progressPrefix : (...args: any[]) => {
+    console.log(`${progressPrefix}`, ...args);
+  };
+  if (performance.now() - lastProgress > 1000) {
+    progressFunc();
+    lastProgress = performance.now();
+  }
+  try {
+    return await inner(
+      (progressPrefix, inner) => _recurse(
+        (...args) => progressFunc(progressPrefix, ...args),
+        inner,
+      ),
+    );
+  } catch (error) {
+    progressFunc(`\x1b[41mERROR\x1b[0m!`);
+    throw error;
+  }
+};
+const recurse: RecurseFunction = _recurse;

apps/backend/src/app/api/v1/integrations/neon/oauth-providers/crud.tsx

@@ -14,8 +14,14 @@ import * as yup from "yup";
 const oauthProviderReadSchema = yupObject({
   id: schemaFields.oauthIdSchema.defined(),
   type: schemaFields.oauthTypeSchema.defined(),
-  client_id: schemaFields.yupDefinedWhen(schemaFields.oauthClientIdSchema, 'type', 'standard'),
-  client_secret: schemaFields.yupDefinedWhen(schemaFields.oauthClientSecretSchema, 'type', 'standard'),
+  client_id: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientIdSchema, {
+    when: 'type',
+    is: 'standard',
+  }),
+  client_secret: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientSecretSchema, {
+    when: 'type',
+    is: 'standard',
+  }),
 
   // extra params
   facebook_config_id: schemaFields.oauthFacebookConfigIdSchema.optional(),
@@ -24,8 +30,14 @@ const oauthProviderReadSchema = yupObject({
 
 const oauthProviderUpdateSchema = yupObject({
   type: schemaFields.oauthTypeSchema.optional(),
-  client_id: schemaFields.yupDefinedWhen(schemaFields.oauthClientIdSchema, 'type', 'standard').optional(),
-  client_secret: schemaFields.yupDefinedWhen(schemaFields.oauthClientSecretSchema, 'type', 'standard').optional(),
+  client_id: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientIdSchema, {
+    when: 'type',
+    is: 'standard',
+  }).optional(),
+  client_secret: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientSecretSchema, {
+    when: 'type',
+    is: 'standard',
+  }).optional(),
 
   // extra params
   facebook_config_id: schemaFields.oauthFacebookConfigIdSchema.optional(),

apps/backend/src/route-handlers/crud-handler.tsx

@@ -292,7 +292,7 @@ async function validate<T>(obj: unknown, schema: yup.ISchema<T>, currentUser: Us
           Errors:
             ${error.errors.join("\n")}
         `,
-        { obj: JSON.stringify(obj), schema, cause: error },
+        { obj: obj, schema, cause: error },
       );
     }
     throw error;

apps/backend/src/route-handlers/smart-request.tsx

@@ -9,6 +9,7 @@ import { ProjectsCrud } from "@stackframe/stack-shared/dist/interface/crud/proje
 import { UsersCrud } from "@stackframe/stack-shared/dist/interface/crud/users";
 import { StackAdaptSentinel, yupValidate } from "@stackframe/stack-shared/dist/schema-fields";
 import { groupBy, typedIncludes } from "@stackframe/stack-shared/dist/utils/arrays";
+import { getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env";
 import { StackAssertionError, StatusError, throwErr } from "@stackframe/stack-shared/dist/utils/errors";
 import { ignoreUnhandledRejection } from "@stackframe/stack-shared/dist/utils/promises";
 import { deindent } from "@stackframe/stack-shared/dist/utils/strings";
@@ -145,6 +146,7 @@ async function parseAuth(req: NextRequest): Promise<SmartRequestAuth | null> {
   const adminAccessToken = req.headers.get("x-stack-admin-access-token");
   const accessToken = req.headers.get("x-stack-access-token");
   const refreshToken = req.headers.get("x-stack-refresh-token");
+  const developmentKeyOverride = req.headers.get("x-stack-development-override-key");  // in development, the internal project's API key can optionally be used to access any project
 
   const extractUserFromAccessToken = async (options: { token: string, projectId: string }) => {
     const result = await decodeAccessToken(options.token);
@@ -212,7 +214,13 @@ async function parseAuth(req: NextRequest): Promise<SmartRequestAuth | null> {
   if (!typedIncludes(["client", "server", "admin"] as const, requestType)) throw new KnownErrors.InvalidAccessType(requestType);
   if (!projectId) throw new KnownErrors.AccessTypeWithoutProjectId(requestType);
 
-  if (adminAccessToken) {
+  if (developmentKeyOverride) {
+    if (getNodeEnvironment() !== "development") {
+      throw new StatusError(401, "Development key override is only allowed in development mode");
+    }
+    const result = await checkApiKeySet("internal", { superSecretAdminKey: developmentKeyOverride });
+    if (!result) throw new StatusError(401, "Invalid development key override");
+  } else if (adminAccessToken) {
     if (await queries.internalUser) {
       if (!await queries.project) {
         // this happens if the project is still in the user's managedProjectIds, but has since been deleted

apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/auth-methods/providers.tsx

@@ -3,6 +3,7 @@ import { FormDialog } from "@/components/form-dialog";
 import { InputField, SwitchField } from "@/components/form-fields";
 import { SettingIconButton, SettingSwitch } from "@/components/settings";
 import { AdminProject } from "@stackframe/stack";
+import { yupBoolean, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
 import { sharedProviders } from "@stackframe/stack-shared/dist/utils/oauth";
 import { runAsynchronously } from "@stackframe/stack-shared/dist/utils/promises";
 import { ActionDialog, Badge, InlineCode, Label, SimpleTooltip, Typography } from "@stackframe/stack-ui";
@@ -31,22 +32,22 @@ function toTitle(id: string) {
   }[id];
 }
 
-export const providerFormSchema = yup.object({
-  shared: yup.boolean().defined(),
-  clientId: yup.string()
+export const providerFormSchema = yupObject({
+  shared: yupBoolean().defined(),
+  clientId: yupString()
     .when('shared', {
       is: false,
-      then: (schema) => schema.defined(),
+      then: (schema) => schema.defined().nonEmpty(),
       otherwise: (schema) => schema.optional()
     }),
-  clientSecret: yup.string()
+  clientSecret: yupString()
     .when('shared', {
       is: false,
-      then: (schema) => schema.defined(),
+      then: (schema) => schema.defined().nonEmpty(),
       otherwise: (schema) => schema.optional()
     }),
-  facebookConfigId: yup.string().optional(),
-  microsoftTenantId: yup.string().optional(),
+  facebookConfigId: yupString().optional(),
+  microsoftTenantId: yupString().optional(),
 });
 
 export type ProviderFormValues = yup.InferType<typeof providerFormSchema>
@@ -104,7 +105,8 @@ export function ProviderSettingDialog(props: Props & { open: boolean, onClose: (
               Shared keys are created by the Stack team for development. It helps you get started, but will show a Stack logo and name on the OAuth screen. This should never be enabled in production.
             </Typography> :
             <div className="flex flex-col gap-2">
-              <Label>Redirect URL for the OAuth provider settings
+              <Label>
+                Redirect URL for the OAuth provider settings
               </Label>
               <Typography type="footnote">
                 <InlineCode>{`${process.env.NEXT_PUBLIC_STACK_API_URL}/api/v1/auth/oauth/callback/${props.id}`}</InlineCode>

apps/e2e/tests/backend/endpoints/api/v1/projects.test.ts

@@ -520,6 +520,37 @@ it("updates the project email configuration", async ({ expect }) => {
   `);
 });
 
+it("does not update project email config to empty host", async ({ expect }) => {
+  const { adminAccessToken } = await Project.createAndGetAdminToken();
+  const { updateProjectResponse: response } = await Project.updateCurrent(adminAccessToken, {
+    config: {
+      email_config: {
+        type: "standard",
+        host: "",
+        port: 587,
+        username: "test username",
+        password: "test password",
+        sender_name: "Test Sender",
+        sender_email: "test@email.com",
+      },
+    },
+  });
+  expect(response).toMatchInlineSnapshot(`
+    NiceResponse {
+      "status": 400,
+      "body": {
+        "code": "SCHEMA_ERROR",
+        "details": { "message": "Request validation failed on PATCH /api/v1/projects/current:\\n  - body.config.email_config.host must not be empty" },
+        "error": "Request validation failed on PATCH /api/v1/projects/current:\\n  - body.config.email_config.host must not be empty",
+      },
+      "headers": Headers {
+        "x-stack-known-error": "SCHEMA_ERROR",
+        <some fields may have been hidden>,
+      },
+    }
+  `);
+});
+
 it("updates the project email configuration with invalid parameters", async ({ expect }) => {
   await Auth.Otp.signIn();
   const { adminAccessToken } = await Project.createAndGetAdminToken();
@@ -869,6 +900,39 @@ it("updates the project oauth configuration", async ({ expect }) => {
   `);
 });
 
+it("fails when trying to update OAuth provider with empty client_secret", async ({ expect }) => {
+  await Project.createAndSwitch();
+  const response = await niceBackendFetch(`/api/v1/projects/current`, {
+    accessType: "admin",
+    method: "PATCH",
+    body: {
+      config: {
+        oauth_providers: [{
+          id: "google",
+          type: "standard",
+          enabled: true,
+          client_id: "client_id",
+          client_secret: ""
+        }]
+      }
+    }
+  });
+  expect(response).toMatchInlineSnapshot(`
+    NiceResponse {
+      "status": 400,
+      "body": {
+        "code": "SCHEMA_ERROR",
+        "details": { "message": "Request validation failed on PATCH /api/v1/projects/current:\\n  - body.config.oauth_providers[0].client_secret must not be empty" },
+        "error": "Request validation failed on PATCH /api/v1/projects/current:\\n  - body.config.oauth_providers[0].client_secret must not be empty",
+      },
+      "headers": Headers {
+        "x-stack-known-error": "SCHEMA_ERROR",
+        <some fields may have been hidden>,
+      },
+    }
+  `);
+});
+
 it("deletes a project with admin access", async ({ expect }) => {
   await Auth.Otp.signIn();
   const { adminAccessToken } = await Project.createAndGetAdminToken();

apps/e2e/tests/general/verify-data-integrity.test.ts

@@ -0,0 +1,14 @@
+import { exec } from "child_process";
+import { describe } from "vitest";
+import { it } from "../helpers";
+
+describe("Data integrity verification", () => {
+  it("completes successfully", async ({ expect }) => {
+    const [error, stdout, stderr] = await new Promise<[Error | null, string, string]>((resolve) => {
+      exec("pnpm run verify-data-integrity", (error, stdout, stderr) => {
+        resolve([error, stdout, stderr]);
+      });
+    });
+    expect(error, `Expected no error to be thrown!\n\n\n\nstdout: ${stdout}\n\n\n\nstderr: ${stderr}`).toBeNull();
+  }, 120_000);
+});

package.json

@@ -35,7 +35,8 @@
     "changeset": "only-allow pnpm && changeset",
     "test": "vitest",
     "generate-docs": "only-allow pnpm && turbo run generate-docs",
-    "generate-keys": "only-allow pnpm && turbo run generate-keys"
+    "generate-keys": "only-allow pnpm && turbo run generate-keys",
+    "verify-data-integrity": "only-allow pnpm && pnpm -C apps/backend run verify-data-integrity"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.9",

packages/stack-shared/src/interface/crud/projects.ts

@@ -1,6 +1,6 @@
 import { CrudTypeOf, createCrud } from "../../crud";
 import * as schemaFields from "../../schema-fields";
-import { yupArray, yupDefinedWhen, yupObject, yupString } from "../../schema-fields";
+import { yupArray, yupObject, yupString } from "../../schema-fields";
 
 const teamPermissionSchema = yupObject({
   id: yupString().defined(),
@@ -10,8 +10,20 @@ const oauthProviderSchema = yupObject({
   id: schemaFields.oauthIdSchema.defined(),
   enabled: schemaFields.oauthEnabledSchema.defined(),
   type: schemaFields.oauthTypeSchema.defined(),
-  client_id: yupDefinedWhen(schemaFields.oauthClientIdSchema, 'type', 'standard'),
-  client_secret: yupDefinedWhen(schemaFields.oauthClientSecretSchema, 'type', 'standard'),
+  client_id: schemaFields.yupDefinedAndNonEmptyWhen(
+    schemaFields.oauthClientIdSchema,
+    {
+      type: 'standard',
+      enabled: true,
+    },
+  ),
+  client_secret: schemaFields.yupDefinedAndNonEmptyWhen(
+    schemaFields.oauthClientSecretSchema,
+    {
+      type: 'standard',
+      enabled: true,
+    },
+  ),
 
   // extra params
   facebook_config_id: schemaFields.oauthFacebookConfigIdSchema.optional(),
@@ -24,12 +36,24 @@ const enabledOAuthProviderSchema = yupObject({
 
 export const emailConfigSchema = yupObject({
   type: schemaFields.emailTypeSchema.defined(),
-  host: yupDefinedWhen(schemaFields.emailHostSchema, 'type', 'standard'),
-  port: yupDefinedWhen(schemaFields.emailPortSchema, 'type', 'standard'),
-  username: yupDefinedWhen(schemaFields.emailUsernameSchema, 'type', 'standard'),
-  password: yupDefinedWhen(schemaFields.emailPasswordSchema, 'type', 'standard'),
-  sender_name: yupDefinedWhen(schemaFields.emailSenderNameSchema, 'type', 'standard'),
-  sender_email: yupDefinedWhen(schemaFields.emailSenderEmailSchema, 'type', 'standard'),
+  host: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.emailHostSchema, {
+    type: 'standard',
+  }),
+  port: schemaFields.yupDefinedWhen(schemaFields.emailPortSchema, {
+    type: 'standard',
+  }),
+  username: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.emailUsernameSchema, {
+    type: 'standard',
+  }),
+  password: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.emailPasswordSchema, {
+    type: 'standard',
+  }),
+  sender_name: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.emailSenderNameSchema, {
+    type: 'standard',
+  }),
+  sender_email: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.emailSenderEmailSchema, {
+    type: 'standard',
+  }),
 });
 
 const domainSchema = yupObject({

packages/stack-shared/src/schema-fields.ts

@@ -12,13 +12,19 @@ declare module "yup" {
   // eslint-disable-next-line @typescript-eslint/consistent-type-definitions
   interface StringSchema<TType, TContext, TDefault, TFlags> {
     nonEmpty(message?: string): StringSchema<TType, TContext, TDefault, TFlags>,
+    empty(): StringSchema<TType, TContext, TDefault, TFlags>,
   }
 }
+
 // eslint-disable-next-line no-restricted-syntax
 yup.addMethod(yup.string, "nonEmpty", function (message?: string) {
-  return this.test("non-empty", message ?? "String must not be empty", (value) => {
-    return value !== "";
-  });
+  return this.test(
+    "non-empty",
+    message ?? (({ path }) => `${path} must not be empty`),
+    (value) => {
+      return value !== "";
+    }
+  );
 });
 
 
@@ -394,12 +400,24 @@ export const neonAuthorizationHeaderSchema = basicAuthorizationHeaderSchema.test
 // Utils
 export function yupDefinedWhen<S extends yup.AnyObject>(
   schema: S,
-  triggerName: string,
-  isValue: any
+  triggers: Record<string, any>,
 ): S {
-  return schema.when(triggerName, {
-    is: isValue,
+  const entries = Object.entries(triggers);
+  return schema.when(entries.map(([key]) => key), {
+    is: (...values: any[]) => entries.every(([key, value], index) => value === values[index]),
     then: (schema: S) => schema.defined(),
     otherwise: (schema: S) => schema.optional()
   });
 }
+
+export function yupDefinedAndNonEmptyWhen<S extends yup.StringSchema>(
+  schema: S,
+  triggers: Record<string, any>,
+): S {
+  const entries = Object.entries(triggers);
+  return schema.when(entries.map(([key]) => key), {
+    is: (...values: any[]) => entries.every(([key, value], index) => value === values[index]),
+    then: (schema: S) => schema.defined().nonEmpty(),
+    otherwise: (schema: S) => schema.optional()
+  });
+}