Keep the dashboard loading when bulldozer is down (internal billing degrades gracefully) (#1740)

apps/backend/src/lib/plan-usage.test.ts

@@ -1,6 +1,6 @@
 import { ITEM_IDS, UNLIMITED } from "@hexclave/shared/dist/plans";
 import type { SubscriptionRow } from "./payments/schema/types";
-import { buildUsageRow, getNextPlanId, getPlanUsagePeriod } from "./plan-usage";
+import { buildUsageRow, getNextPlanId, getPlanUsagePeriod, readBillingSubscriptionMapOrSkip } from "./plan-usage";
 import { describe, expect, it } from "vitest";
 
 function createSubscriptionPeriod(startMillis: number, endMillis: number): SubscriptionRow {
@@ -134,6 +134,32 @@ describe("plan upgrade targets", () => {
   });
 });
 
+describe("readBillingSubscriptionMapOrSkip", () => {
+  it("returns the subscription map when the bulldozer read succeeds", async () => {
+    const start = Date.UTC(2026, 4, 15);
+    const end = Date.UTC(2026, 5, 15);
+    const subMap = { sub_1: createSubscriptionPeriod(start, end) };
+    const result = await readBillingSubscriptionMapOrSkip(() => Promise.resolve(subMap));
+    expect(result).toBe(subMap);
+  });
+
+  it("falls back to an empty map (free plan) instead of throwing when bulldozer is down", async () => {
+    // Regression: plan usage is read on the dashboard's own pages (overview
+    // limit banners, usage page). A bulldozer outage here used to bubble a 500
+    // and take the whole dashboard down. It must now degrade to "no
+    // subscription" (free plan) rather than failing the page.
+    const result = await readBillingSubscriptionMapOrSkip(() => {
+      throw new Error("bulldozer unreachable");
+    });
+    expect(result).toEqual({});
+  });
+
+  it("also swallows async rejections from the bulldozer read", async () => {
+    const result = await readBillingSubscriptionMapOrSkip(() => Promise.reject(new Error("bulldozer timed out")));
+    expect(result).toEqual({});
+  });
+});
+
 describe("billing period selection", () => {
   it("uses the subscription period when available", () => {
     const start = Date.UTC(2026, 4, 15);

apps/backend/src/lib/plan-usage.ts

@@ -12,7 +12,7 @@ import { DEFAULT_BRANCH_ID, getSoleTenancyFromProjectBranch, getTenancy, type Te
 import { getPrismaClientForTenancy, getPrismaSchemaForTenancy, globalPrismaClient, sqlQuoteIdent } from "@/prisma-client";
 import { BASE_PLAN_IDS_BY_TIER, ITEM_IDS, PLAN_LIMITS, UNLIMITED, type ItemId, type PlanId } from "@hexclave/shared/dist/plans";
 import type { PlanUsageResponse } from "@hexclave/shared/dist/interface/admin-interface";
-import { HexclaveAssertionError, throwErr } from "@hexclave/shared/dist/utils/errors";
+import { captureError, HexclaveAssertionError, throwErr } from "@hexclave/shared/dist/utils/errors";
 import { mapWithConcurrency } from "@hexclave/shared/dist/utils/promises";
 import type { SubscriptionRow } from "./payments/schema/types";
 
@@ -136,6 +136,27 @@ function getUsageItemLabel(itemId: ItemId): string {
   return USAGE_ITEM_LABELS.get(itemId) ?? throwErr(`Missing usage item label for ${itemId}`);
 }
 
+/**
+ * Reads the billing team's subscription map, but never lets a bulldozer outage
+ * break the caller. Plan usage is surfaced only on the dashboard's own
+ * (internal project) surfaces — the overview/analytics limit banners and the
+ * usage page — so a hard dependency on bulldozer here would take the whole
+ * dashboard down whenever bulldozer is unreachable. On failure we report the
+ * error and fall back to an empty subscription map, which resolves to the free
+ * plan: we skip whatever we would normally bill/enforce rather than failing the
+ * page. The real plan is re-resolved on the next request once bulldozer is back.
+ */
+export async function readBillingSubscriptionMapOrSkip(
+  read: () => Promise<Record<string, SubscriptionRow>>,
+): Promise<Record<string, SubscriptionRow>> {
+  try {
+    return await read();
+  } catch (error) {
+    captureError("plan-usage:subscription-map-unavailable", error);
+    return {};
+  }
+}
+
 function resolveActivePlanSubscription(subscriptions: Record<string, SubscriptionRow>): SubscriptionRow | null {
   const activeSubscriptions = Object.values(subscriptions).filter(isActiveSubscription);
   for (const planId of BASE_PLAN_IDS_BY_TIER) {
@@ -394,12 +415,12 @@ export async function getPlanUsageForProject(project: UsageSourceProject, now: D
 
   const internalTenancy = await getInternalBillingTenancy();
   const internalPrisma = await getPrismaClientForTenancy(internalTenancy);
-  const subscriptions = await getSubscriptionMapForCustomer({
+  const subscriptions = await readBillingSubscriptionMapOrSkip(() => getSubscriptionMapForCustomer({
     prisma: internalPrisma,
     tenancyId: internalTenancy.id,
     customerType: "team",
     customerId: ownerTeamId,
-  });
+  }));
   const activePlanSubscription = resolveActivePlanSubscription(subscriptions);
   const planId = resolveActivePlanId(activePlanSubscription);
   const period = getPlanUsagePeriod(activePlanSubscription, now);

apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/shared.tsx

@@ -18,9 +18,11 @@ import { Alert, AlertDescription, Button } from "@/components/ui";
 import { Link } from "@/components/link";
 import { useDashboardInternalUser } from "@/lib/dashboard-user";
 import { PLAN_LIMITS, resolvePlanId } from "@hexclave/shared/dist/plans";
+import { captureError } from "@hexclave/shared/dist/utils/errors";
 import { runAsynchronouslyWithAlert } from "@hexclave/shared/dist/utils/promises";
 import { useVirtualizer } from "@tanstack/react-virtual";
-import { useMemo, useRef } from "react";
+import { ErrorBoundary } from "next/dist/client/components/error-boundary";
+import { Suspense, useEffect, useMemo, useRef, type ReactNode } from "react";
 import { useAdminApp } from "../use-admin-app";
 
 // ============================================================================
@@ -322,11 +324,45 @@ export function ErrorDisplay({ error, onRetry }: { error: unknown, onRetry: () =
   );
 }
 
+/**
+ * The plan-limit banners live on core dashboard pages (overview, analytics,
+ * session replays) but read the internal project's billing state from
+ * bulldozer (plan usage, owned products, item quantities). Billing is not
+ * essential to those pages, so a bulldozer outage must not take them down:
+ * on any read failure we report it and render nothing rather than letting the
+ * error escape to the page. `Suspense` keeps the banner from blocking the page
+ * while its data loads.
+ */
+function BillingBannerErrorFallback() {
+  useEffect(() => {
+    captureError("analytics-limit-banner:billing-read-failed", new Error("Failed to load plan-limit banner data; hiding the banner"));
+  }, []);
+  return null;
+}
+
+function ResilientBillingBanner(props: { children: ReactNode }) {
+  return (
+    <ErrorBoundary errorComponent={BillingBannerErrorFallback}>
+      <Suspense fallback={null}>
+        {props.children}
+      </Suspense>
+    </ErrorBoundary>
+  );
+}
+
 /**
  * Shows a warning banner when analytics event usage is at 80%+ or 100%.
  * Fetches the billing team's analytics_events item and computes usage against the plan's total allocation.
  */
 export function AnalyticsEventLimitBanner() {
+  return (
+    <ResilientBillingBanner>
+      <AnalyticsEventLimitBannerContent />
+    </ResilientBillingBanner>
+  );
+}
+
+function AnalyticsEventLimitBannerContent() {
   const adminApp = useAdminApp();
   const project = adminApp.useProject();
   const planUsage = adminApp.usePlanUsage();
@@ -350,6 +386,14 @@ export function AnalyticsEventLimitBanner() {
  * Since the limit is the same across all plans, no upgrade button is shown.
  */
 export function SessionReplayLimitBanner() {
+  return (
+    <ResilientBillingBanner>
+      <SessionReplayLimitBannerContent />
+    </ResilientBillingBanner>
+  );
+}
+
+function SessionReplayLimitBannerContent() {
   const adminApp = useAdminApp();
   const project = adminApp.useProject();
   const planUsage = adminApp.usePlanUsage();

apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/auth-methods/page-client.tsx

@@ -37,12 +37,13 @@ import type { CompleteConfig } from "@hexclave/shared/dist/config/schema";
 import type { RestrictedReason } from "@hexclave/shared/dist/schema-fields";
 import { urlSchema, yupObject, yupString } from "@hexclave/shared/dist/schema-fields";
 import { runAsynchronouslyWithAlert } from "@hexclave/shared/dist/utils/promises";
-import { HexclaveAssertionError, throwErr } from "@hexclave/shared/dist/utils/errors";
+import { captureError, HexclaveAssertionError, throwErr } from "@hexclave/shared/dist/utils/errors";
 import { allProviders } from "@hexclave/shared/dist/utils/oauth";
 import { typedFromEntries, typedEntries } from "@hexclave/shared/dist/utils/objects";
 import { resolvePlanId } from "@hexclave/shared/dist/plans";
 import { generateUuid } from "@hexclave/shared/dist/utils/uuids";
-import { useId, useMemo, useState } from "react";
+import { ErrorBoundary } from "next/dist/client/components/error-boundary";
+import { Suspense, useEffect, useId, useMemo, useState } from "react";
 import { AppEnabledGuard } from "../app-enabled-guard";
 import { PageLayout } from "../page-layout";
 import { useAdminApp } from "../use-admin-app";
@@ -193,7 +194,24 @@ function AddCustomOidcButton({ onClick }: { onClick: () => void }) {
     return <AddCustomOidcButtonDisabled onClick={onClick} isTeamPlanOrAbove={false} />;
   }
 
-  return <AddCustomOidcButtonInner team={ownerTeam} onClick={onClick} />;
+  // The plan check reads the internal project's owned products from bulldozer.
+  // That gate must not break the auth-methods page if bulldozer is down: on a
+  // read failure (or while it loads) we report it and fall back to the locked
+  // (non-Team) state, which is the same safe default as having no owner team.
+  return (
+    <ErrorBoundary errorComponent={() => <AddCustomOidcButtonPlanReadFailed onClick={onClick} />}>
+      <Suspense fallback={<AddCustomOidcButtonDisabled onClick={onClick} isTeamPlanOrAbove={false} />}>
+        <AddCustomOidcButtonInner team={ownerTeam} onClick={onClick} />
+      </Suspense>
+    </ErrorBoundary>
+  );
+}
+
+function AddCustomOidcButtonPlanReadFailed({ onClick }: { onClick: () => void }) {
+  useEffect(() => {
+    captureError("auth-methods:custom-oidc-plan-gate", new Error("Failed to load owner team plan for the custom OIDC gate; defaulting to locked"));
+  }, []);
+  return <AddCustomOidcButtonDisabled onClick={onClick} isTeamPlanOrAbove={false} />;
 }
 
 function AddCustomOidcButtonInner({