encrypted connection strings, updates, tests

This commit is contained in:
Bilal Godil 2025-09-05 15:41:17 -07:00
parent 11b6b93fe2
commit 8b0563dd76
14 changed files with 330 additions and 27 deletions

View File

@ -2,6 +2,9 @@
NEXT_PUBLIC_STACK_API_URL=# the base URL of Stack's backend/API. For local development, this is `http://localhost:8102`; for the managed service, this is `https://api.stack-auth.com`.
NEXT_PUBLIC_STACK_DASHBOARD_URL=# the URL of Stack's dashboard. For local development, this is `http://localhost:8101`; for the managed service, this is `https://app.stack-auth.com`.
STACK_SECRET_SERVER_KEY=# a random, unguessable secret key generated by `pnpm generate-keys`
STACK_INTERNAL_PROJECT_CLIENT_KEY=# enter your Stack publishable client key here. For local development, just enter a random string, then run `pnpm db:reset`
STACK_INTERNAL_PROJECT_SERVER_KEY=# enter your Stack secret server key here. For local development, do the same as above
# seed script settings
STACK_SEED_INTERNAL_PROJECT_SIGN_UP_ENABLED=# true to enable user sign up to the dashboard when seeding
@ -76,3 +79,4 @@ STACK_OPENAI_API_KEY=# enter your openai api key
STACK_FEATUREBASE_API_KEY=# enter your featurebase api key
STACK_STRIPE_SECRET_KEY=# enter your stripe api key
STACK_STRIPE_WEBHOOK_SECRET=# enter your stripe webhook secret

View File

@ -1,5 +1,7 @@
NEXT_PUBLIC_STACK_API_URL=http://localhost:8102
NEXT_PUBLIC_STACK_DASHBOARD_URL=http://localhost:8101
STACK_INTERNAL_PROJECT_CLIENT_KEY=this-publishable-client-key-is-for-local-development-only
STACK_INTERNAL_PROJECT_SERVER_KEY=this-secret-server-key-is-for-local-development-only
STACK_SERVER_SECRET=23-wuNpik0gIW4mruTz25rbIvhuuvZFrLOLtL7J4tyo
STACK_SEED_INTERNAL_PROJECT_SIGN_UP_ENABLED=true

View File

@ -63,6 +63,7 @@
"@prisma/instrumentation": "^6.12.0",
"@sentry/nextjs": "^8.40.0",
"@simplewebauthn/server": "^11.0.0",
"@stackframe/stack": "workspace:*",
"@stackframe/stack-shared": "workspace:*",
"@vercel/functions": "^2.0.0",
"@vercel/otel": "^1.10.4",

View File

@ -90,6 +90,13 @@ async function seed() {
projectId: 'internal',
branchId: DEFAULT_BRANCH_ID,
environmentConfigOverrideOverride: {
dataVault: {
stores: {
'neon-connection-strings': {
displayName: 'Neon Connection Strings',
}
}
},
payments: {
groups: {
plans: {

View File

@ -18,7 +18,7 @@ export const sessionsCrudHandlers = createLazyProxy(() => createCrudHandlers(ses
}).defined(),
onList: async ({ auth, query }) => {
const prisma = await getPrismaClientForTenancy(auth.tenancy);
const schema = getPrismaSchemaForTenancy(auth.tenancy);
const schema = await getPrismaSchemaForTenancy(auth.tenancy);
const listImpersonations = auth.type === 'admin';
if (auth.type === 'client') {

View File

@ -39,7 +39,6 @@ export const POST = createSmartRouteHandler({
// note that encryptedValue is encrypted by client-side encryption, while encrypted is encrypted by both client-side
// and server-side encryption.
const encrypted = await encryptWithKms(encryptedValue);
// Store or update the entry
await prisma.dataVaultEntry.upsert({
where: {
@ -59,7 +58,6 @@ export const POST = createSmartRouteHandler({
encrypted,
},
});
return {
statusCode: 200,
bodyType: "success",

View File

@ -0,0 +1,82 @@
import { overrideProjectConfigOverride } from "@/lib/config";
import { getPrismaClientForSourceOfTruth, globalPrismaClient } from "@/prisma-client";
import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
import { stackServerApp } from "@/stack";
import { KnownErrors } from "@stackframe/stack-shared";
import { neonAuthorizationHeaderSchema, yupArray, yupNumber, yupObject, yupString, yupTuple } from "@stackframe/stack-shared/dist/schema-fields";
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
import { decodeBasicAuthorizationHeader } from "@stackframe/stack-shared/dist/utils/http";
import { generateUuid } from "@stackframe/stack-shared/dist/utils/uuids";
export const POST = createSmartRouteHandler({
metadata: {
hidden: true,
},
request: yupObject({
query: yupObject({
project_id: yupString().defined(),
}).defined(),
body: yupObject({
connection_strings: yupArray(yupObject({
branch_id: yupString().defined(),
connection_string: yupString().defined(),
}).defined()).defined(),
}).defined(),
headers: yupObject({
authorization: yupTuple([neonAuthorizationHeaderSchema.defined()]).defined(),
}).defined(),
}),
response: yupObject({
statusCode: yupNumber().oneOf([200]).defined(),
bodyType: yupString().oneOf(["json"]).defined(),
body: yupObject({
project_id: yupString().defined(),
}).defined(),
}),
handler: async (req) => {
const [clientId] = decodeBasicAuthorizationHeader(req.headers.authorization[0])!;
const provisionedProject = await globalPrismaClient.provisionedProject.findUnique({
where: {
projectId: req.query.project_id,
clientId: clientId,
},
});
if (!provisionedProject) {
throw new KnownErrors.ProjectNotFound(req.query.project_id);
}
const uuidConnectionStrings: Record<string, string> = {};
const store = await stackServerApp.getDataVaultStore('neon-connection-strings');
const secret = getEnvVariable('STACK_SERVER_SECRET');
for (const c of req.body.connection_strings) {
const uuid = generateUuid();
await store.setValue(uuid, c.connection_string, { secret });
uuidConnectionStrings[c.branch_id] = uuid;
}
const sourceOfTruthPersisted = {
type: 'neon' as const,
connectionStrings: uuidConnectionStrings,
};
await overrideProjectConfigOverride({
projectId: provisionedProject.projectId,
projectConfigOverrideOverride: {
sourceOfTruth: sourceOfTruthPersisted,
},
});
await Promise.all(req.body.connection_strings.map(({ branch_id, connection_string }) => getPrismaClientForSourceOfTruth({
type: 'neon',
connectionString: undefined,
connectionStrings: { [branch_id]: connection_string },
} as const, branch_id)));
return {
statusCode: 200,
bodyType: "json",
body: {
project_id: provisionedProject.projectId,
},
};
},
});

View File

@ -2,8 +2,11 @@ import { createApiKeySet } from "@/lib/internal-api-keys";
import { createOrUpdateProjectWithLegacyConfig } from "@/lib/projects";
import { getPrismaClientForSourceOfTruth, globalPrismaClient } from "@/prisma-client";
import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
import { stackServerApp } from "@/stack";
import { neonAuthorizationHeaderSchema, projectDisplayNameSchema, yupArray, yupNumber, yupObject, yupString, yupTuple } from "@stackframe/stack-shared/dist/schema-fields";
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
import { decodeBasicAuthorizationHeader } from "@stackframe/stack-shared/dist/utils/http";
import { generateUuid } from "@stackframe/stack-shared/dist/utils/uuids";
export const POST = createSmartRouteHandler({
metadata: {
@ -32,14 +35,30 @@ export const POST = createSmartRouteHandler({
handler: async (req) => {
const [clientId] = decodeBasicAuthorizationHeader(req.headers.authorization[0])!;
const sourceOfTruth = req.body.connection_strings ? {
type: 'neon',
const hasNeonConnections = req.body.connection_strings && req.body.connection_strings.length > 0;
const realConnectionStrings: Record<string, string> = {};
const uuidConnectionStrings: Record<string, string> = {};
if (hasNeonConnections) {
const store = await stackServerApp.getDataVaultStore('neon-connection-strings');
const secret = getEnvVariable('STACK_SERVER_SECRET');
for (const c of req.body.connection_strings!) {
const uuid = generateUuid();
await store.setValue(uuid, c.connection_string, { secret });
realConnectionStrings[c.branch_id] = c.connection_string;
uuidConnectionStrings[c.branch_id] = uuid;
}
}
const sourceOfTruthPersisted = hasNeonConnections ? {
type: 'neon' as const,
connectionString: undefined,
connectionStrings: Object.fromEntries(req.body.connection_strings.map((c) => [c.branch_id, c.connection_string])),
} as const : { type: 'hosted', connectionString: undefined, connectionStrings: undefined } as const;
connectionStrings: uuidConnectionStrings,
} : { type: 'hosted' as const, connectionString: undefined, connectionStrings: undefined };
const createdProject = await createOrUpdateProjectWithLegacyConfig({
sourceOfTruth,
sourceOfTruth: sourceOfTruthPersisted,
type: 'create',
data: {
display_name: req.body.display_name,
@ -63,10 +82,14 @@ export const POST = createSmartRouteHandler({
});
if (sourceOfTruth.type === 'neon') {
// Get the Prisma client for all branches in parallel, as doing so will run migrations
const branchIds = Object.keys(sourceOfTruth.connectionStrings);
await Promise.all(branchIds.map((branchId) => getPrismaClientForSourceOfTruth(sourceOfTruth, branchId)));
if (hasNeonConnections) {
// Run migrations using the real connection strings (do not persist them)
const branchIds = Object.keys(realConnectionStrings);
await Promise.all(branchIds.map((branchId) => getPrismaClientForSourceOfTruth({
type: 'neon',
connectionString: undefined,
connectionStrings: realConnectionStrings,
} as const, branchId)));
}
@ -86,6 +109,7 @@ export const POST = createSmartRouteHandler({
has_super_secret_admin_key: true,
});
return {
statusCode: 200,
bodyType: "json",

View File

@ -46,7 +46,7 @@ async function loadUsersByCountry(tenancy: Tenancy, includeAnonymous: boolean =
}
async function loadTotalUsers(tenancy: Tenancy, now: Date, includeAnonymous: boolean = false): Promise<DataPoints> {
const schema = getPrismaSchemaForTenancy(tenancy);
const schema = await getPrismaSchemaForTenancy(tenancy);
const prisma = await getPrismaClientForTenancy(tenancy);
return (await prisma.$queryRaw<{date: Date, dailyUsers: bigint, cumUsers: bigint}[]>`
WITH date_series AS (
@ -109,7 +109,7 @@ async function loadDailyActiveUsers(tenancy: Tenancy, now: Date, includeAnonymou
}
async function loadLoginMethods(tenancy: Tenancy): Promise<{method: string, count: number }[]> {
const schema = getPrismaSchemaForTenancy(tenancy);
const schema = await getPrismaSchemaForTenancy(tenancy);
const prisma = await getPrismaClientForTenancy(tenancy);
return await prisma.$queryRaw<{ method: string, count: number }[]>`
WITH tab AS (

View File

@ -206,7 +206,7 @@ export const getUsersLastActiveAtMillis = async (projectId: string, branchId: st
const tenancy = await getSoleTenancyFromProjectBranch(projectId, branchId);
const prisma = await getPrismaClientForTenancy(tenancy);
const schema = getPrismaSchemaForTenancy(tenancy);
const schema = await getPrismaSchemaForTenancy(tenancy);
const events = await prisma.$queryRaw<Array<{ userId: string, lastActiveAt: Date }>>`
SELECT data->>'userId' as "userId", MAX("eventStartedAt") as "lastActiveAt"
FROM ${sqlQuoteIdent(schema)}."Event"
@ -401,7 +401,7 @@ export async function getUser(options: { userId: string } & ({ projectId: string
const environmentConfig = await rawQuery(globalPrismaClient, getRenderedEnvironmentConfigQuery({ projectId, branchId }));
const prisma = await getPrismaClientForSourceOfTruth(environmentConfig.sourceOfTruth, branchId);
const schema = getPrismaSchemaForSourceOfTruth(environmentConfig.sourceOfTruth, branchId);
const schema = await getPrismaSchemaForSourceOfTruth(environmentConfig.sourceOfTruth, branchId);
const result = await rawQuery(prisma, getUserQuery(projectId, branchId, options.userId, schema));
return result;
}

View File

@ -1,3 +1,4 @@
import { stackServerApp } from "@/stack";
import { PrismaNeon } from "@prisma/adapter-neon";
import { PrismaPg } from '@prisma/adapter-pg';
import { Prisma, PrismaClient } from "@prisma/client";
@ -9,6 +10,7 @@ import { deepPlainEquals, filterUndefined, typedFromEntries, typedKeys } from "@
import { concatStacktracesIfRejected, ignoreUnhandledRejection } from "@stackframe/stack-shared/dist/utils/promises";
import { Result } from "@stackframe/stack-shared/dist/utils/results";
import { traceSpan } from "@stackframe/stack-shared/dist/utils/telemetry";
import { isUuid } from "@stackframe/stack-shared/dist/utils/uuids";
import { isPromise } from "util/types";
import { runMigrationNeeded } from "./auto-migrations";
import { Tenancy } from "./lib/tenancies";
@ -32,6 +34,9 @@ export const globalPrismaClient = prismaClientsStore.global;
const dbString = getEnvVariable("STACK_DIRECT_DATABASE_CONNECTION_STRING", "");
export const globalPrismaSchema = dbString === "" ? "public" : getSchemaFromConnectionString(dbString);
const vaultUuidToConnectionString = new Map<string, string>();
const vaultUuidToSchema = new Map<string, string>();
function getNeonPrismaClient(connectionString: string) {
let neonPrismaClient = prismaClientsStore.neon.get(connectionString);
if (!neonPrismaClient) {
@ -40,7 +45,6 @@ function getNeonPrismaClient(connectionString: string) {
neonPrismaClient = new PrismaClient({ adapter });
prismaClientsStore.neon.set(connectionString, neonPrismaClient);
}
return neonPrismaClient;
}
@ -48,14 +52,27 @@ function getSchemaFromConnectionString(connectionString: string) {
return (new URL(connectionString)).searchParams.get('schema') ?? "public";
}
async function resolveNeonConnectionString(entry: string): Promise<string> {
if (!isUuid(entry)) {
return entry;
}
const store = await stackServerApp.getDataVaultStore('neon-connection-strings');
const secret = getEnvVariable('STACK_SERVER_SECRET');
const value = await store.getValue(entry, { secret });
if (!value) throw new Error('No Neon connection string found for UUID');
vaultUuidToSchema.set(entry, getSchemaFromConnectionString(value));
return value;
}
export async function getPrismaClientForTenancy(tenancy: Tenancy) {
return await getPrismaClientForSourceOfTruth(tenancy.config.sourceOfTruth, tenancy.branchId);
}
export function getPrismaSchemaForTenancy(tenancy: Tenancy) {
return getPrismaSchemaForSourceOfTruth(tenancy.config.sourceOfTruth, tenancy.branchId);
export async function getPrismaSchemaForTenancy(tenancy: Tenancy) {
return await getPrismaSchemaForSourceOfTruth(tenancy.config.sourceOfTruth, tenancy.branchId);
}
function getPostgresPrismaClient(connectionString: string) {
let postgresPrismaClient = prismaClientsStore.postgres.get(connectionString);
if (!postgresPrismaClient) {
@ -76,7 +93,8 @@ export async function getPrismaClientForSourceOfTruth(sourceOfTruth: CompleteCon
if (!(branchId in sourceOfTruth.connectionStrings)) {
throw new Error(`No connection string provided for Neon source of truth for branch ${branchId}`);
}
const connectionString = sourceOfTruth.connectionStrings[branchId];
const entry = sourceOfTruth.connectionStrings[branchId];
const connectionString = await resolveNeonConnectionString(entry);
const neonPrismaClient = getNeonPrismaClient(connectionString);
await runMigrationNeeded({ prismaClient: neonPrismaClient, schema: getSchemaFromConnectionString(connectionString) });
return neonPrismaClient;
@ -92,13 +110,21 @@ export async function getPrismaClientForSourceOfTruth(sourceOfTruth: CompleteCon
}
}
export function getPrismaSchemaForSourceOfTruth(sourceOfTruth: CompleteConfig["sourceOfTruth"], branchId: string) {
export async function getPrismaSchemaForSourceOfTruth(sourceOfTruth: CompleteConfig["sourceOfTruth"], branchId: string) {
switch (sourceOfTruth.type) {
case 'postgres': {
return getSchemaFromConnectionString(sourceOfTruth.connectionString);
}
case 'neon': {
return getSchemaFromConnectionString(sourceOfTruth.connectionStrings[branchId]);
if (!(branchId in sourceOfTruth.connectionStrings)) {
throw new Error(`No connection string provided for Neon source of truth for branch ${branchId}`);
}
const entry = sourceOfTruth.connectionStrings[branchId];
if (isUuid(entry)) {
const connectionString = await resolveNeonConnectionString(entry);
return getSchemaFromConnectionString(connectionString);
}
return getSchemaFromConnectionString(entry);
}
case 'hosted': {
return globalPrismaSchema;
@ -223,19 +249,19 @@ export const RawQuery = {
supportedPrismaClients,
sql: Prisma.sql`
WITH ${Prisma.join(queries.map((q, index) => {
return Prisma.sql`${Prisma.raw("q" + index)} AS (
return Prisma.sql`${Prisma.raw("q" + index)} AS (
${q.sql}
)`;
}), ",\n")}
}), ",\n")}
${Prisma.join(queries.map((q, index) => {
return Prisma.sql`
return Prisma.sql`
SELECT
${"q" + index} AS type,
row_to_json(c) AS json
FROM (SELECT * FROM ${Prisma.raw("q" + index)}) c
`;
}), "\nUNION ALL\n")}
}), "\nUNION ALL\n")}
`,
postProcess: (rows) => {
const unprocessed = new Array(queries.length).fill(null).map(() => [] as any[]);

View File

@ -0,0 +1,10 @@
import { StackServerApp } from '@stackframe/stack';
import { getEnvVariable } from '@stackframe/stack-shared/dist/utils/env';
export const stackServerApp = new StackServerApp({
projectId: 'internal',
tokenStore: 'memory',
baseUrl: getEnvVariable('NEXT_PUBLIC_STACK_API_URL'),
publishableClientKey: getEnvVariable('STACK_INTERNAL_PROJECT_CLIENT_KEY'),
secretServerKey: getEnvVariable('STACK_INTERNAL_PROJECT_SERVER_KEY'),
});

View File

@ -1,5 +1,6 @@
import { decryptValue, hashKey } from "@stackframe/stack-shared/dist/helpers/vault/client-side";
import { it } from "../../../../../../../helpers";
import { Auth, InternalApiKey, backendContext, niceBackendFetch } from "../../../../../../backend-helpers";
import { Auth, InternalApiKey, InternalProjectKeys, Project, backendContext, niceBackendFetch } from "../../../../../../backend-helpers";
export async function provisionProject() {
return await niceBackendFetch("/api/v1/integrations/neon/projects/provision", {
@ -193,3 +194,148 @@ it("should fail if the neon client details are missing", async ({ expect }) => {
}
`);
});
it("should accept empty connection_strings without attempting migrations", async ({ expect }) => {
const response = await niceBackendFetch("/api/v1/integrations/neon/projects/provision", {
method: "POST",
body: {
display_name: "Test project",
connection_strings: [],
},
headers: {
"Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==",
},
});
expect(response.status).toBe(200);
expect(response.body).toMatchObject({
project_id: expect.any(String),
super_secret_admin_key: expect.any(String),
});
});
it("should validate connection_strings item shape", async ({ expect }) => {
const response = await niceBackendFetch("/api/v1/integrations/neon/projects/provision", {
method: "POST",
body: {
display_name: "Test project",
// missing connection_string in the item
connection_strings: [
{ branch_id: "main" } as any,
],
},
headers: {
"Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==",
},
});
expect(response.status).toBe(400);
expect(response.headers.get("x-stack-known-error")).toBe("SCHEMA_ERROR");
});
it("can provision with a Neon connection string when provided via env (optional)", async ({ expect }) => {
// this test only runs with a neon connection string set
const neonConnectionString = process.env.STACK_TEST_NEON_CONNECTION_STRING;
if (!neonConnectionString) {
return;
}
const response = await niceBackendFetch("/api/v1/integrations/neon/projects/provision", {
method: "POST",
body: {
display_name: "Test project (neon)",
connection_strings: [
{
branch_id: "main",
connection_string: neonConnectionString,
},
],
},
headers: {
"Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==",
},
});
expect(response.status).toBe(200);
backendContext.set({
projectKeys: {
projectId: response.body.project_id,
superSecretAdminKey: response.body.super_secret_admin_key,
},
});
const r = await niceBackendFetch(`/api/latest/internal/config`, {
accessType: "admin",
});
expect(response.status).toBe(200);
expect(r.body.config_string).toBeDefined();
const sourceOfTruth = JSON.parse(r.body.config_string).sourceOfTruth;
expect(sourceOfTruth).toMatchInlineSnapshot(`
{
"connectionStrings": { "main": "<stripped UUID>" },
"type": "neon",
}
`);
backendContext.set({
projectKeys: InternalProjectKeys,
});
const getConnectionResponse = await niceBackendFetch(`/api/latest/data-vault/stores/neon-connection-strings/get`, {
method: "POST",
accessType: "server",
body: {
hashed_key: await hashKey("23-wuNpik0gIW4mruTz25rbIvhuuvZFrLOLtL7J4tyo", sourceOfTruth.connectionStrings.main),
},
});
expect(getConnectionResponse.status).toBe(200);
const connectionString = await decryptValue(
"23-wuNpik0gIW4mruTz25rbIvhuuvZFrLOLtL7J4tyo",
sourceOfTruth.connectionStrings.main,
getConnectionResponse.body.encrypted_value
);
expect(connectionString).toBe(neonConnectionString);
});
it("can update the connection_strings for an existing provisioned project", async ({ expect }) => {
// this test only runs with a neon connection string set
const neonConnectionString = process.env.STACK_TEST_NEON_CONNECTION_STRING;
if (!neonConnectionString) {
return;
}
const provisionResponse = await provisionProject();
const response = await niceBackendFetch(`/api/v1/integrations/neon/projects/connection?project_id=${provisionResponse.body.project_id}`, {
method: "POST",
body: {
connection_strings: [
{ branch_id: "branch1", connection_string: neonConnectionString },
],
},
headers: {
"Authorization": "Basic bmVvbi1sb2NhbDpuZW9uLWxvY2FsLXNlY3JldA==",
},
});
expect(response).toMatchInlineSnapshot(`
NiceResponse {
"status": 200,
"body": { "project_id": "<stripped UUID>" },
"headers": Headers { <some fields may have been hidden> },
}
`);
backendContext.set({
projectKeys: {
projectId: provisionResponse.body.project_id,
superSecretAdminKey: provisionResponse.body.super_secret_admin_key,
},
});
const configResponse = await niceBackendFetch(`/api/latest/internal/config`, {
accessType: "admin",
});
expect(configResponse.status).toBe(200);
expect(configResponse.body.config_string).toBeDefined();
const sourceOfTruth = JSON.parse(configResponse.body.config_string).sourceOfTruth;
expect(sourceOfTruth).toMatchInlineSnapshot(`
{
"connectionStrings": { "branch1": "<stripped UUID>" },
"type": "neon",
}
`);
});

View File

@ -174,6 +174,9 @@ importers:
'@simplewebauthn/server':
specifier: ^11.0.0
version: 11.0.0(encoding@0.1.13)
'@stackframe/stack':
specifier: workspace:*
version: link:../../packages/stack
'@stackframe/stack-shared':
specifier: workspace:*
version: link:../../packages/stack-shared