diff --git a/apps/backend/prisma/migrations/20260622000000_add_stripe_webhook_event/migration.sql b/apps/backend/prisma/migrations/20260622000000_add_stripe_webhook_event/migration.sql new file mode 100644 index 000000000..151ff73c0 --- /dev/null +++ b/apps/backend/prisma/migrations/20260622000000_add_stripe_webhook_event/migration.sql @@ -0,0 +1,21 @@ +-- CreateEnum +CREATE TYPE "StripeWebhookEventStatus" AS ENUM ('PENDING', 'PROCESSED', 'FAILED'); + +-- CreateTable +CREATE TABLE "StripeWebhookEvent" ( + "id" UUID NOT NULL, + "stripeEventId" TEXT NOT NULL, + "eventType" TEXT NOT NULL, + "stripeAccountId" TEXT, + "payload" JSONB NOT NULL, + "status" "StripeWebhookEventStatus" NOT NULL DEFAULT 'PENDING', + "lastError" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "processedAt" TIMESTAMP(3), + + CONSTRAINT "StripeWebhookEvent_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "StripeWebhookEvent_stripeEventId_key" ON "StripeWebhookEvent"("stripeEventId"); diff --git a/apps/backend/prisma/migrations/20260622000000_add_stripe_webhook_event/tests/unique-stripe-event-id.ts b/apps/backend/prisma/migrations/20260622000000_add_stripe_webhook_event/tests/unique-stripe-event-id.ts new file mode 100644 index 000000000..8bca6123e --- /dev/null +++ b/apps/backend/prisma/migrations/20260622000000_add_stripe_webhook_event/tests/unique-stripe-event-id.ts @@ -0,0 +1,60 @@ +import { randomUUID } from "crypto"; +import type { Sql } from "postgres"; +import { expect } from "vitest"; + +export const preMigration = async (sql: Sql) => { + // Table does not exist before the migration, so nothing to seed. + return {}; +}; + +export const postMigration = async (sql: Sql) => { + const tables = await sql<{ table_name: string }[]>` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'StripeWebhookEvent' + `; + expect(Array.from(tables)).toMatchInlineSnapshot(` + [ + { + "table_name": "StripeWebhookEvent", + }, + ] + `); + + const eventId = `evt_${randomUUID()}`; + + await sql` + INSERT INTO "StripeWebhookEvent" ("id", "stripeEventId", "eventType", "payload", "updatedAt") + VALUES (${randomUUID()}::uuid, ${eventId}, 'invoice.payment_succeeded', '{"id":"evt"}'::jsonb, NOW()) + `; + + // Status defaults to PENDING. + const inserted = await sql` + SELECT "status" FROM "StripeWebhookEvent" WHERE "stripeEventId" = ${eventId} + `; + expect(Array.from(inserted)).toMatchInlineSnapshot(` + [ + { + "status": "PENDING", + }, + ] + `); + + // The same Stripe event id cannot be inserted twice (idempotency guarantee). + await expect(sql` + INSERT INTO "StripeWebhookEvent" ("id", "stripeEventId", "eventType", "payload", "updatedAt") + VALUES (${randomUUID()}::uuid, ${eventId}, 'invoice.payment_succeeded', '{"id":"evt2"}'::jsonb, NOW()) + `).rejects.toThrow(/StripeWebhookEvent_stripeEventId_key/); + + // A different event id is fine, and the status enum rejects invalid values. + await sql` + INSERT INTO "StripeWebhookEvent" ("id", "stripeEventId", "eventType", "payload", "status", "updatedAt") + VALUES (${randomUUID()}::uuid, ${`evt_${randomUUID()}`}, 'invoice.paid', '{}'::jsonb, 'PROCESSED', NOW()) + `; + + await expect(sql` + INSERT INTO "StripeWebhookEvent" ("id", "stripeEventId", "eventType", "payload", "status", "updatedAt") + VALUES (${randomUUID()}::uuid, ${`evt_${randomUUID()}`}, 'invoice.paid', '{}'::jsonb, 'NOT_A_STATUS', NOW()) + `).rejects.toThrow(); +}; diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index 1b4d8544f..d152621c6 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -1424,6 +1424,32 @@ model OutgoingRequest { @@index([startedFulfillingAt, deduplicationKey]) } +enum StripeWebhookEventStatus { + PENDING + PROCESSED + FAILED +} + +// Idempotency + recovery log for incoming Stripe webhook events. Each event is +// persisted synchronously (keyed on the Stripe `event.id`) before we ack 200 to +// Stripe, so redeliveries are deduped and the full `payload` of any +// PENDING/FAILED row can be replayed manually. Not tenancy-scoped: the Stripe +// account -> tenancy resolution happens during processing, not here. +model StripeWebhookEvent { + id String @id @default(uuid()) @db.Uuid + + stripeEventId String @unique + eventType String + stripeAccountId String? + payload Json + status StripeWebhookEventStatus @default(PENDING) + lastError String? + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + processedAt DateTime? +} + // BulldozerStorageEngine is managed externally (see prisma.config.ts // `tables.external`). It's created by migrations and interacted with // via raw SQL — not through the Prisma client. Keeping it out of the diff --git a/apps/backend/src/app/api/latest/integrations/stripe/webhooks/route.tsx b/apps/backend/src/app/api/latest/integrations/stripe/webhooks/route.tsx index 0388a0e80..7656826eb 100644 --- a/apps/backend/src/app/api/latest/integrations/stripe/webhooks/route.tsx +++ b/apps/backend/src/app/api/latest/integrations/stripe/webhooks/route.tsx @@ -1,5 +1,7 @@ import { sendEmailToMany, type EmailOutboxRecipient } from "@/lib/emails"; import { bulldozerWriteOneTimePurchase } from "@/lib/payments/bulldozer-dual-write"; +import { claimStripeEvent, markStripeEventFailed, markStripeEventProcessed } from "@/lib/stripe-webhook-events"; +import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks"; import { listPermissions } from "@/lib/permissions"; import { getHexclaveStripe, getStripeForAccount, resolveProductFromStripeMetadata, syncStripeSubscriptions, upsertStripeInvoice } from "@/lib/stripe"; import type { StripeOverridesMap } from "@/lib/stripe-proxy"; @@ -466,13 +468,29 @@ export const POST = createSmartRouteHandler({ throw new StatusError(400, "Invalid stripe-signature header"); } - try { - await processStripeWebhookEvent(event); - } catch (error) { - captureError("stripe-webhook-receiver", error); - throw error; + // Persist the event for idempotency + recovery BEFORE acking. Stripe + // delivers at-least-once + const { shouldProcess } = await claimStripeEvent(event); + if (!shouldProcess) { + return { + statusCode: 200, + bodyType: "json", + body: { received: true, deduplicated: true }, + }; } + // Ack Stripe immediately and process in the background. + // Stripe recommends ACKing ASAP to avoid timeouts and redeliveries + runAsynchronouslyAndWaitUntil(async () => { + try { + await processStripeWebhookEvent(event); + await markStripeEventProcessed(event.id); + } catch (error) { + captureError("stripe-webhook-receiver", error); + await markStripeEventFailed(event.id, error); + } + }); + return { statusCode: 200, bodyType: "json", diff --git a/apps/backend/src/lib/stripe-webhook-events.test.ts b/apps/backend/src/lib/stripe-webhook-events.test.ts new file mode 100644 index 000000000..65aae6ce3 --- /dev/null +++ b/apps/backend/src/lib/stripe-webhook-events.test.ts @@ -0,0 +1,87 @@ +import { randomUUID } from "node:crypto"; +import type Stripe from "stripe"; +import { describe, expect, it } from "vitest"; +import { StripeWebhookEventStatus } from "@/generated/prisma/client"; +import { globalPrismaClient } from "@/prisma-client"; +import { claimStripeEvent, markStripeEventFailed, markStripeEventProcessed } from "./stripe-webhook-events"; + +// Test fixtures only need the fields the helper reads (id/type/account) plus a +// JSON-serializable body. Building a full Stripe.Event is impractical, so we +// cast a minimal object — any drift in the fields we actually use is still +// caught because the helper reads them directly. +function makeEvent(): Stripe.Event { + return { + id: `evt_${randomUUID()}`, + type: "invoice.payment_succeeded", + account: "acct_test_123", + data: { object: { id: "in_test", note: "fixture" } }, + } as unknown as Stripe.Event; +} + +describe("stripe webhook event idempotency (real DB)", () => { + it("claims a brand new event and persists it as PENDING", async ({ expect }) => { + const event = makeEvent(); + + const { shouldProcess } = await claimStripeEvent(event); + expect(shouldProcess).toBe(true); + + const row = await globalPrismaClient.stripeWebhookEvent.findUnique({ + where: { stripeEventId: event.id }, + }); + expect(row).not.toBeNull(); + expect(row?.status).toBe(StripeWebhookEventStatus.PENDING); + expect(row?.eventType).toBe(event.type); + expect(row?.stripeAccountId).toBe(event.account); + expect(row?.processedAt).toBeNull(); + expect(row?.lastError).toBeNull(); + // The full event payload is stored so dropped/failed events can be replayed. + expect(row?.payload).toMatchObject({ id: event.id, type: event.type }); + }); + + it("allows reprocessing while the prior delivery is still PENDING", async ({ expect }) => { + const event = makeEvent(); + + const first = await claimStripeEvent(event); + expect(first.shouldProcess).toBe(true); + + // Redelivery before the background work finished: we must NOT skip, otherwise + // a crash between claim and processing would silently drop the event forever. + const second = await claimStripeEvent(event); + expect(second.shouldProcess).toBe(true); + }); + + it("deduplicates once the event has been fully PROCESSED", async ({ expect }) => { + const event = makeEvent(); + + await claimStripeEvent(event); + await markStripeEventProcessed(event.id); + + const processedRow = await globalPrismaClient.stripeWebhookEvent.findUnique({ + where: { stripeEventId: event.id }, + }); + expect(processedRow?.status).toBe(StripeWebhookEventStatus.PROCESSED); + expect(processedRow?.processedAt).not.toBeNull(); + expect(processedRow?.lastError).toBeNull(); + + // A Stripe redelivery of an already-processed event must be a no-op. + const redelivery = await claimStripeEvent(event); + expect(redelivery.shouldProcess).toBe(false); + }); + + it("records the error on failure and allows recovery via redelivery", async ({ expect }) => { + const event = makeEvent(); + + await claimStripeEvent(event); + await markStripeEventFailed(event.id, new Error("boom while processing")); + + const failedRow = await globalPrismaClient.stripeWebhookEvent.findUnique({ + where: { stripeEventId: event.id }, + }); + expect(failedRow?.status).toBe(StripeWebhookEventStatus.FAILED); + expect(failedRow?.lastError).toContain("boom while processing"); + + // FAILED rows must reprocess so a manual Stripe "Resend" can recover them. + const recovery = await claimStripeEvent(event); + expect(recovery.shouldProcess).toBe(true); + }); +}); diff --git a/apps/backend/src/lib/stripe-webhook-events.ts b/apps/backend/src/lib/stripe-webhook-events.ts new file mode 100644 index 000000000..0d1e0124d --- /dev/null +++ b/apps/backend/src/lib/stripe-webhook-events.ts @@ -0,0 +1,69 @@ +import { Prisma, StripeWebhookEventStatus } from "@/generated/prisma/client"; +import { globalPrismaClient } from "@/prisma-client"; +import { errorToNiceString, HexclaveAssertionError } from "@hexclave/shared/dist/utils/errors"; +import type Stripe from "stripe"; + +/** + * Idempotency + recovery layer for incoming Stripe webhook events. + * + * Each event is persisted (keyed on the Stripe `event.id`) synchronously before + * we ack 200 to Stripe. Processing then runs in the background. Because Stripe + * delivers at-least-once, this is what guarantees the receipt fan-out happens at + * most once per event. The full `payload` is stored so PENDING/FAILED rows can + * be replayed manually if the background work is dropped (e.g. instance recycle). + */ + +/** + * Records the event (or detects a prior one) and decides whether processing + * should run. Returns `shouldProcess: false` only when the event has already + * been fully PROCESSED — PENDING/FAILED rows are allowed to reprocess so that a + * manual Stripe "Resend" can recover a dropped/failed event. + */ +export async function claimStripeEvent(event: Stripe.Event): Promise<{ shouldProcess: boolean }> { + try { + await globalPrismaClient.stripeWebhookEvent.create({ + data: { + stripeEventId: event.id, + eventType: event.type, + stripeAccountId: event.account ?? null, + payload: JSON.parse(JSON.stringify(event)) as Prisma.InputJsonValue, + status: StripeWebhookEventStatus.PENDING, + }, + }); + return { shouldProcess: true }; + } catch (error) { + // Unique violation on stripeEventId => we've seen this event before. + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { + const existing = await globalPrismaClient.stripeWebhookEvent.findUnique({ + where: { stripeEventId: event.id }, + }); + return { shouldProcess: existing?.status !== StripeWebhookEventStatus.PROCESSED }; + } + + throw new HexclaveAssertionError( + `Failed to claim Stripe webhook event for idempotency: ${event.id}`, + { cause: error, stripeEventId: event.id, eventType: event.type }, + ); + } +} + +export async function markStripeEventProcessed(stripeEventId: string): Promise { + await globalPrismaClient.stripeWebhookEvent.update({ + where: { stripeEventId }, + data: { + status: StripeWebhookEventStatus.PROCESSED, + processedAt: new Date(), + lastError: null, + }, + }); +} + +export async function markStripeEventFailed(stripeEventId: string, error: unknown): Promise { + await globalPrismaClient.stripeWebhookEvent.update({ + where: { stripeEventId }, + data: { + status: StripeWebhookEventStatus.FAILED, + lastError: errorToNiceString(error), + }, + }); +} diff --git a/apps/e2e/tests/backend/endpoints/api/v1/stripe-webhooks.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/stripe-webhooks.test.ts index 82de0dbdd..aa29a8f19 100644 --- a/apps/e2e/tests/backend/endpoints/api/v1/stripe-webhooks.test.ts +++ b/apps/e2e/tests/backend/endpoints/api/v1/stripe-webhooks.test.ts @@ -1,9 +1,42 @@ +import { randomUUID } from "node:crypto"; import { throwErr } from "@hexclave/shared/dist/utils/errors"; import { wait } from "@hexclave/shared/dist/utils/promises"; import { it } from "../../../../helpers"; import { Auth, bumpEmailAddress, niceBackendFetch, Payments, Project, Team } from "../../../backend-helpers"; import { getOutboxEmails } from "./emails/email-helpers"; +// Stripe webhook events are now deduplicated globally by their `event.id` (see +// the StripeWebhookEvent table). The dev DB is NOT reset between test runs, so +// every claimed event needs a per-run unique id, otherwise a second run would +// hit the dedupe path and skip processing. +function uniqueEventId(prefix: string) { + return `evt_${prefix}_${randomUUID()}`; +} + +// Webhook processing now happens in the background after a fast 200 ack, so DB +// state is eventually-consistent from the test's perspective. Poll instead of +// reading immediately after the webhook returns. +async function waitForItemQuantity( + args: { customerType: "user" | "team", customerId: string, itemId: string, expected: number }, +) { + let last: number | undefined; + for (let i = 0; i < 30; i++) { + const res = await niceBackendFetch( + `/api/latest/payments/items/${args.customerType}/${args.customerId}/${args.itemId}`, + { accessType: "client" }, + ); + if (res.status !== 200) { + throw new Error(`Unexpected ${res.status} reading item ${args.itemId}`); + } + last = res.body.quantity; + if (last === args.expected) { + return; + } + await wait(500); + } + throw new Error(`Item ${args.itemId} quantity never reached ${args.expected} (last seen: ${last})`); +} + async function waitForOutboxEmail(subject: string) { for (let i = 0; i < 30; i++) { const emails = await getOutboxEmails({ subject }); @@ -26,14 +59,20 @@ async function waitForNoOutboxEmail(subject: string) { } -it("rejects signed mock_event.succeeded webhook", async ({ expect }) => { +it("acks unknown signed webhook types (errors handled in background)", async ({ expect }) => { + // We now persist + ack the event synchronously and process it in the + // background, so an unknown type no longer surfaces a 500 to Stripe. The + // "Unknown stripe webhook type" error is captured async and the event row is + // marked FAILED (covered deterministically in stripe-webhook-events.test.ts). const payload = { - id: "evt_test_1", + id: uniqueEventId("mock_event_succeeded"), type: "mock_event.succeeded", account: "acct_test123", data: { object: { customer: "cus_test123", metadata: {} } }, }; - await expect(Payments.sendStripeWebhook(payload)).rejects.toThrow(/Unknown stripe webhook type received/); + const res = await Payments.sendStripeWebhook(payload); + expect(res.status).toBe(200); + expect(res.body).toEqual({ received: true }); }); it("returns 400 on invalid signature", async ({ expect }) => { @@ -53,15 +92,17 @@ it("returns 400 on invalid signature", async ({ expect }) => { `); }); -it("returns 500 on unknown webhook type", async ({ expect }) => { +it("acks unknown webhook types with 200 (errors handled in background)", async ({ expect }) => { const payload = { - id: "evt_test_unknown", + id: uniqueEventId("unknown_event"), type: "unknown.event", account: "acct_test123", data: { object: {} }, }; - await expect(Payments.sendStripeWebhook(payload)).rejects.toThrow(/Unknown stripe webhook type received/); + const res = await Payments.sendStripeWebhook(payload); + expect(res.status).toBe(200); + expect(res.body).toEqual({ received: true }); }); it("returns 400 when signature header is missing (schema validation)", async ({ expect }) => { @@ -86,7 +127,7 @@ it("accepts chargeback webhooks", async ({ expect }) => { const accountId: string = accountInfo.body.account_id; const payload = { - id: "evt_chargeback_test", + id: uniqueEventId("chargeback"), type: "charge.dispute.created", account: accountId, data: { @@ -172,7 +213,7 @@ it("deduplicates one-time purchase on payment_intent.succeeded retry", async ({ const fullCode = purchaseUrl.split("/purchase/")[1]; const stackTestTenancyId = fullCode.split("_")[0]; const payloadObj = { - id: "evt_retry_test", + id: uniqueEventId("retry"), type: "payment_intent.succeeded", account: accountId, data: { @@ -198,18 +239,18 @@ it("deduplicates one-time purchase on payment_intent.succeeded retry", async ({ }; const res = await Payments.sendStripeWebhook(payloadObj); expect(res.status).toBe(200); - expect(res.body).toEqual({ received: true }); + expect(res.body.received).toBe(true); + + // First grant must land before we redeliver, so the duplicate deterministically + // hits the event-dedupe path (PROCESSED) rather than racing in-flight work. + await waitForItemQuantity({ customerType: "user", customerId: userId, itemId, expected: 1 }); const res2 = await Payments.sendStripeWebhook(payloadObj); expect(res2.status).toBe(200); - expect(res2.body).toEqual({ received: true }); + expect(res2.body).toEqual({ received: true, deduplicated: true }); - // After duplicate deliveries, quantity should reflect a single OneTimePurchase grant - const getAfter = await niceBackendFetch(`/api/latest/payments/items/user/${userId}/${itemId}`, { - accessType: "client", - }); - expect(getAfter.status).toBe(200); - expect(getAfter.body.quantity).toBe(1); + // After the deduplicated redelivery, quantity stays at a single grant. + await waitForItemQuantity({ customerType: "user", customerId: userId, itemId, expected: 1 }); }); it("sends a payment receipt email for one-time purchases", async ({ expect }) => { @@ -268,7 +309,7 @@ it("sends a payment receipt email for one-time purchases", async ({ expect }) => const receiptLink = "https://example.com/receipt/pi_test_receipt_1"; const paymentIntentId = "pi_test_receipt_1"; const payloadObj = { - id: "evt_receipt_test_1", + id: uniqueEventId("receipt"), type: "payment_intent.succeeded", account: accountId, data: { @@ -313,6 +354,117 @@ it("sends a payment receipt email for one-time purchases", async ({ expect }) => `); }); +it("sends exactly one receipt when Stripe redelivers the same event", async ({ expect }) => { + // Regression test for the duplicate-receipt bug: Stripe delivers at-least-once, + // and slow synchronous processing used to time out and trigger redeliveries, + // each re-sending the receipt fan-out. The StripeWebhookEvent dedupe must keep + // the fan-out to exactly once per event id. + const projectDisplayName = `Receipt Idempotency ${randomUUID()}`; + await Project.createAndSwitch({ display_name: projectDisplayName }); + await Payments.setup(); + + const itemId = "idem-receipt-credits"; + const productId = "idem-receipt-ot"; + const product = { + displayName: "Idem Receipt Pack", + customerType: "user", + serverOnly: false, + stackable: true, + prices: { one: { USD: "500" } }, + includedItems: { [itemId]: { quantity: 1 } }, + }; + + await Project.updateConfig({ + payments: { + items: { + [itemId]: { displayName: "Credits", customerType: "user" }, + }, + products: { + [productId]: product, + }, + }, + }); + + const mailbox = await bumpEmailAddress(); + const { userId } = await Auth.fastSignUp({ + primary_email: mailbox.emailAddress, + primary_email_verified: true, + }); + + const accountInfo = await niceBackendFetch("/api/latest/internal/payments/stripe/account-info", { + accessType: "admin", + }); + expect(accountInfo.status).toBe(200); + const accountId: string = accountInfo.body.account_id; + + const createUrlResponse = await niceBackendFetch("/api/latest/payments/purchases/create-purchase-url", { + method: "POST", + accessType: "client", + body: { + customer_type: "user", + customer_id: userId, + product_id: productId, + }, + }); + expect(createUrlResponse.status).toBe(200); + const purchaseUrl = (createUrlResponse.body as { url: string }).url; + const fullCode = purchaseUrl.split("/purchase/")[1]; + const stackTestTenancyId = fullCode.split("_")[0]; + + const receiptLink = "https://example.com/receipt/pi_idem_receipt"; + const eventId = uniqueEventId("idem_receipt"); + const payloadObj = { + id: eventId, + type: "payment_intent.succeeded", + account: accountId, + data: { + object: { + id: "pi_idem_receipt", + customer: userId, + amount_received: 500, + currency: "usd", + charges: { data: [{ receipt_url: receiptLink }] }, + stack_stripe_mock_data: { + "accounts.retrieve": { metadata: { tenancyId: stackTestTenancyId } }, + "customers.retrieve": { metadata: { customerId: userId, customerType: "USER" } }, + "subscriptions.list": { data: [] }, + }, + metadata: { + productId, + product: JSON.stringify(product), + customerId: userId, + customerType: "user", + purchaseQuantity: "1", + purchaseKind: "ONE_TIME", + priceId: "one", + }, + }, + }, + }; + + const first = await Payments.sendStripeWebhook(payloadObj); + expect(first.status).toBe(200); + expect(first.body).toEqual({ received: true }); + + // Wait for the receipt to land, which proves the first event finished + // processing (and is now PROCESSED), so the redeliveries deterministically + // take the dedupe path. + const subject = `Your receipt from ${projectDisplayName}`; + await waitForOutboxEmail(subject); + + for (let i = 0; i < 2; i++) { + const redelivery = await Payments.sendStripeWebhook(payloadObj); + expect(redelivery.status).toBe(200); + expect(redelivery.body).toEqual({ received: true, deduplicated: true }); + } + + // Give any (incorrectly) re-triggered fan-out a chance to show up, then assert + // there is still exactly one receipt email for this project. + await wait(1500); + const receipts = await getOutboxEmails({ subject }); + expect(receipts.length).toBe(1); +}); + it("sends a payment failed email for invoice.payment_failed", async ({ expect }) => { const projectDisplayName = "Payments Failed Email Test"; await Project.createAndSwitch({ display_name: projectDisplayName }); @@ -365,7 +517,7 @@ it("sends a payment failed email for invoice.payment_failed", async ({ expect }) const invoiceId = "in_test_failed_1"; const invoiceUrl = "https://example.com/billing/update"; const payloadObj = { - id: "evt_invoice_failed_1", + id: uniqueEventId("invoice_failed"), type: "invoice.payment_failed", account: accountId, data: { @@ -458,7 +610,7 @@ it("skips payment failed email when invoice is not uncollectible", async ({ expe const invoiceId = "in_test_failed_open_1"; const invoiceUrl = "https://example.com/billing/open"; const payloadObj = { - id: "evt_invoice_failed_open_1", + id: uniqueEventId("invoice_failed_open"), type: "invoice.payment_failed", account: accountId, data: { @@ -569,7 +721,7 @@ it("syncs subscriptions from webhook and is idempotent", async ({ expect }) => { }; const payloadObj = { - id: "evt_sub_sync_1", + id: uniqueEventId("sub_sync"), type: "invoice.paid", account: accountId, data: { @@ -586,23 +738,16 @@ it("syncs subscriptions from webhook and is idempotent", async ({ expect }) => { const res = await Payments.sendStripeWebhook(payloadObj); expect(res.status).toBe(200); - expect(res.body).toEqual({ received: true }); + expect(res.body.received).toBe(true); - const getAfter1 = await niceBackendFetch(`/api/latest/payments/items/user/${userId}/${itemId}`, { - accessType: "client", - }); - expect(getAfter1.status).toBe(200); - expect(getAfter1.body.quantity).toBe(1); + await waitForItemQuantity({ customerType: "user", customerId: userId, itemId, expected: 1 }); + // Redelivery of the same event id is deduplicated and leaves state untouched. const res2 = await Payments.sendStripeWebhook(payloadObj); expect(res2.status).toBe(200); - expect(res2.body).toEqual({ received: true }); + expect(res2.body).toEqual({ received: true, deduplicated: true }); - const getAfter2 = await niceBackendFetch(`/api/latest/payments/items/user/${userId}/${itemId}`, { - accessType: "client", - }); - expect(getAfter2.status).toBe(200); - expect(getAfter2.body.quantity).toBe(1); + await waitForItemQuantity({ customerType: "user", customerId: userId, itemId, expected: 1 }); }); @@ -682,7 +827,7 @@ it("updates a user's subscriptions via webhook (add then remove)", async ({ expe }; const payloadAdd = { - id: "evt_sub_add", + id: uniqueEventId("sub_add"), type: "invoice.paid", account: accountId, data: { @@ -701,11 +846,7 @@ it("updates a user's subscriptions via webhook (add then remove)", async ({ expe expect(resAdd.status).toBe(200); expect(resAdd.body).toEqual({ received: true }); - const afterAdd = await niceBackendFetch(`/api/latest/payments/items/user/${userId}/${itemId}`, { - accessType: "client", - }); - expect(afterAdd.status).toBe(200); - expect(afterAdd.body.quantity).toBe(1); + await waitForItemQuantity({ customerType: "user", customerId: userId, itemId, expected: 1 }); const canceledSubscription = { ...activeSubscription, @@ -722,7 +863,7 @@ it("updates a user's subscriptions via webhook (add then remove)", async ({ expe }; const payloadRemove = { - id: "evt_sub_remove", + id: uniqueEventId("sub_remove"), type: "customer.subscription.updated", account: accountId, data: { @@ -741,11 +882,7 @@ it("updates a user's subscriptions via webhook (add then remove)", async ({ expe expect(resRemove.status).toBe(200); expect(resRemove.body).toEqual({ received: true }); - const afterRemove = await niceBackendFetch(`/api/latest/payments/items/user/${userId}/${itemId}`, { - accessType: "client", - }); - expect(afterRemove.status).toBe(200); - expect(afterRemove.body.quantity).toBe(0); + await waitForItemQuantity({ customerType: "user", customerId: userId, itemId, expected: 0 }); }); @@ -796,7 +933,7 @@ it("does NOT auto-grant `free` when a non-internal tenancy's sub is canceled via const nowSec = Math.floor(Date.now() / 1000); const webhookResponse = await Payments.sendStripeWebhook({ - id: "evt_customer_cancel", + id: uniqueEventId("customer_cancel"), type: "customer.subscription.deleted", account: accountId, data: {