Refactor AI query and MCP call logging to improve error handling and serialization

- Updated error handling in `ai-query-logger.ts` and `mcp-call-logger.ts` to return structured error information instead of a simple object, ensuring compatibility with analytics views.
- Enhanced the `getVerifiedQaContext` function to serve stale content on fetch failures while maintaining cache integrity.
- Introduced new tests for `getVerifiedQaContext` to validate caching behavior and error handling.
- Updated environment variable prefixes from `STACK_` to `HEXCLAVE_` across various files for consistency in internal tool configuration.

apps/backend/src/lib/ai/loggers/ai-query-logger.ts

@@ -42,7 +42,14 @@ function serializeSteps(steps: ReadonlyArray<StepResult<ToolSet>>): string {
     })));
   } catch (e) {
     captureError("ai-query-steps-serialize", e);
-    return JSON.stringify({ _serializationFailed: true, stepCount: steps.length });
+    // Must stay array-shaped: UsageDetail JSON.parses this as StepEntry[] and
+    // calls .map on it unguarded, so an object here crashes the analytics view.
+    return JSON.stringify([{
+      step: 0,
+      text: `[serialization of ${steps.length} steps failed]`,
+      toolCalls: [],
+      toolResults: [],
+    }]);
   }
 }
 
@@ -71,6 +78,7 @@ export function logAiQuery(args: LogAiQueryArgs): void {
         errorMessage: undefined,
       };
     } else {
+      captureError("ai-query-upstream", args.err);
       entry = {
         ...args.common,
         stepsJson: args.partialSteps && args.partialSteps.length > 0 ? serializeSteps(args.partialSteps) : "[]",

apps/backend/src/lib/ai/loggers/mcp-call-logger.test.ts

@@ -75,7 +75,7 @@ describe("logIfMcpToolCall", () => {
         "correlationId": "correlation-1",
         "durationMs": "<duration>",
         "errorMessage": undefined,
-        "innerToolCallsJson": "{"_serializationFailed":true,"stepCount":1}",
+        "innerToolCallsJson": "[{"type":"tool-call","toolName":"_serializationFailed","toolCallId":"_serializationFailed","args":{"stepCount":1},"argsText":"{\\"stepCount\\":1}","result":null}]",
         "modelId": "model-1",
         "question": "{"_serializationFailed":true}",
         "reason": "review",

apps/backend/src/lib/ai/loggers/mcp-call-logger.ts

@@ -26,8 +26,16 @@ function buildInnerToolCallsJson(steps: ReadonlyArray<StepResult<ToolSet>>): str
       }
     }
     return JSON.stringify(items);
-  } catch {
-    return JSON.stringify({ _serializationFailed: true, stepCount: steps.length });
+  } catch (e) {
+    captureError("mcp-call-serialize", e);
+    return JSON.stringify([{
+      type: "tool-call",
+      toolName: "_serializationFailed",
+      toolCallId: "_serializationFailed",
+      args: { stepCount: steps.length },
+      argsText: JSON.stringify({ stepCount: steps.length }),
+      result: null,
+    }]);
   }
 }
 

apps/backend/src/lib/ai/qa/verified-qa.test.ts

@@ -0,0 +1,78 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../internal-tool-client", () => ({
+  callInternalTool: vi.fn(),
+}));
+
+vi.mock("@hexclave/shared/dist/utils/errors", async (importOriginal) => ({
+  ...await importOriginal<Record<string, unknown>>(),
+  captureError: vi.fn(),
+}));
+
+import { captureError } from "@hexclave/shared/dist/utils/errors";
+import { callInternalTool } from "../internal-tool-client";
+
+// The module holds its cache in module-level state, so each test imports a
+// fresh copy to start from an empty cache.
+async function freshGetVerifiedQaContext() {
+  vi.resetModules();
+  const mod = await import("./verified-qa");
+  return mod.getVerifiedQaContext;
+}
+
+describe("getVerifiedQaContext", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+    vi.mocked(callInternalTool).mockReset();
+    vi.mocked(captureError).mockClear();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("caches a successful fetch for the TTL", async () => {
+    const getVerifiedQaContext = await freshGetVerifiedQaContext();
+    vi.mocked(callInternalTool).mockResolvedValue({ context: "qa-block" });
+
+    expect(await getVerifiedQaContext()).toBe("qa-block");
+    expect(await getVerifiedQaContext()).toBe("qa-block");
+    expect(callInternalTool).toHaveBeenCalledTimes(1);
+
+    vi.advanceTimersByTime(61_000);
+    expect(await getVerifiedQaContext()).toBe("qa-block");
+    expect(callInternalTool).toHaveBeenCalledTimes(2);
+  });
+
+  it("serves stale content on failure and re-arms the TTL instead of retrying every call", async () => {
+    const getVerifiedQaContext = await freshGetVerifiedQaContext();
+    vi.mocked(callInternalTool).mockResolvedValue({ context: "qa-block" });
+    expect(await getVerifiedQaContext()).toBe("qa-block");
+
+    vi.advanceTimersByTime(61_000);
+    vi.mocked(callInternalTool).mockRejectedValue(new Error("internal tool down"));
+
+    // First call past expiry hits the failing fetch, serves stale...
+    expect(await getVerifiedQaContext()).toBe("qa-block");
+    expect(callInternalTool).toHaveBeenCalledTimes(2);
+    expect(captureError).toHaveBeenCalledWith("verified-qa", expect.any(Error));
+
+    // ...and negative-caches it: calls within the re-armed TTL don't retry.
+    expect(await getVerifiedQaContext()).toBe("qa-block");
+    expect(callInternalTool).toHaveBeenCalledTimes(2);
+
+    // After the re-armed TTL expires, it retries again.
+    vi.advanceTimersByTime(61_000);
+    expect(await getVerifiedQaContext()).toBe("qa-block");
+    expect(callInternalTool).toHaveBeenCalledTimes(3);
+  });
+
+  it("negative-caches an empty value when there is no stale content yet", async () => {
+    const getVerifiedQaContext = await freshGetVerifiedQaContext();
+    vi.mocked(callInternalTool).mockRejectedValue(new Error("internal tool down"));
+
+    expect(await getVerifiedQaContext()).toBe("");
+    expect(await getVerifiedQaContext()).toBe("");
+    expect(callInternalTool).toHaveBeenCalledTimes(1);
+  });
+});

apps/backend/src/lib/ai/qa/verified-qa.ts

@@ -18,8 +18,8 @@ export async function getVerifiedQaContext(): Promise<string> {
   const result = await Result.fromPromise(getVerifiedQaContextInner());
   if (result.status === "error") {
     captureError("verified-qa", result.error);
-    // Serve stale content over nothing if we have it.
-    return cached?.value ?? "";
+    cached = { value: cached?.value ?? "", expiresAtMillis: Date.now() + CACHE_TTL_MILLIS };
+    return cached.value;
   }
   cached = { value: result.data, expiresAtMillis: Date.now() + CACHE_TTL_MILLIS };
   return result.data;

apps/internal-tool/.env

@@ -9,11 +9,11 @@ NEXT_PUBLIC_SPACETIMEDB_HOST=REPLACE_ME
 NEXT_PUBLIC_SPACETIMEDB_DB_NAME=REPLACE_ME
 # Server-side SpacetimeDB access (this app owns ALL reducer/SQL calls,
 # including telemetry ingested from the backend via /api/backend/*)
-STACK_SPACETIMEDB_URL=REPLACE_ME# SpacetimeDB HTTP base URL
-STACK_SPACETIMEDB_DB_NAME=REPLACE_ME
-STACK_OPENROUTER_API_KEY=REPLACE_ME# used by the automated + retried LLM QA reviews
+HEXCLAVE_SPACETIMEDB_URL=REPLACE_ME# SpacetimeDB HTTP base URL
+HEXCLAVE_SPACETIMEDB_DB_NAME=REPLACE_ME
+HEXCLAVE_OPENROUTER_API_KEY=REPLACE_ME# used by the automated + retried LLM QA reviews
 # This app is the OIDC issuer for SpacetimeDB tokens: it serves
 # /.well-known/openid-configuration + /api/oidc/jwks.json and mints tokens for
 # signed-in users. The issuer must be this deployment's own public base URL.
-STACK_SPACETIMEDB_TOKEN_ISSUER=REPLACE_ME
-STACK_SPACETIMEDB_SIGNING_KEY_JWK=REPLACE_ME# private ES256 JWK (JSON, one line); this app is its ONLY holder — see DEPLOYMENT.md step 2 to generate
+HEXCLAVE_SPACETIMEDB_TOKEN_ISSUER=REPLACE_ME
+HEXCLAVE_SPACETIMEDB_SIGNING_KEY_JWK=REPLACE_ME# private ES256 JWK (JSON, one line); this app is its ONLY holder — see DEPLOYMENT.md step 2 to generate

apps/internal-tool/.env.development

@@ -5,15 +5,15 @@ NEXT_PUBLIC_HEXCLAVE_DASHBOARD_URL=http://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_
 NEXT_PUBLIC_SPACETIMEDB_HOST=ws://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}39
 NEXT_PUBLIC_SPACETIMEDB_DB_NAME=hexclave-ai-analytics
 # Server-side SpacetimeDB access (this app owns all reducer/SQL calls)
-STACK_SPACETIMEDB_URL=http://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}39
-STACK_SPACETIMEDB_DB_NAME=hexclave-ai-analytics
+HEXCLAVE_SPACETIMEDB_URL=http://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}39
+HEXCLAVE_SPACETIMEDB_DB_NAME=hexclave-ai-analytics
 # Optional in dev: set to run the automated/retried LLM QA reviews locally
-STACK_OPENROUTER_API_KEY=
+HEXCLAVE_OPENROUTER_API_KEY=
 # Dev-only server key for the seeded internal project (matches the e2e/dev default)
 HEXCLAVE_SECRET_SERVER_KEY=this-secret-server-key-is-for-local-development-only
 # ES256 private key that signs SpacetimeDB tokens. Left as REPLACE_ME here so no
 # private key material lives in git: `scripts/pre-dev.mjs` mints a unique key
 # into the gitignored .env.local on first `pnpm dev`. In prod, set a real key in
 # the hosting platform env.
-STACK_SPACETIMEDB_TOKEN_ISSUER=http://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}41
-STACK_SPACETIMEDB_SIGNING_KEY_JWK=REPLACE_ME
+HEXCLAVE_SPACETIMEDB_TOKEN_ISSUER=http://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}41
+HEXCLAVE_SPACETIMEDB_SIGNING_KEY_JWK=REPLACE_ME

apps/internal-tool/scripts/pre-dev.mjs

@@ -24,7 +24,7 @@ import { exportJWK, generateKeyPair } from "jose";
 // placeholder in .env.development, and Next loads it before the dev server
 // mints its first token.
 const ENV_LOCAL = resolve(".env.local");
-const SIGNING_KEY_VAR = "STACK_SPACETIMEDB_SIGNING_KEY_JWK";
+const SIGNING_KEY_VAR = "HEXCLAVE_SPACETIMEDB_SIGNING_KEY_JWK";
 
 async function ensureSigningKey() {
   const existing = existsSync(ENV_LOCAL) ? readFileSync(ENV_LOCAL, "utf8") : "";

apps/internal-tool/scripts/spacetime-auth-config.mjs

@@ -8,9 +8,9 @@
 // document + JWKS and mints SpacetimeDB tokens after verifying the caller's
 // Stack Auth session (see src/lib/server/spacetimedb-token.ts).
 //
-// - STACK_SPACETIMEDB_ALLOWED_ISSUERS: comma-separated issuer URLs. Defaults
-//   to STACK_SPACETIMEDB_TOKEN_ISSUER, then the local dev internal tool.
-// - STACK_SPACETIMEDB_EXPECTED_AUDIENCE: JWT audience. Defaults to
+// - HEXCLAVE_SPACETIMEDB_ALLOWED_ISSUERS: comma-separated issuer URLs. Defaults
+//   to HEXCLAVE_SPACETIMEDB_TOKEN_ISSUER, then the local dev internal tool.
+// - HEXCLAVE_SPACETIMEDB_EXPECTED_AUDIENCE: JWT audience. Defaults to
 //   "spacetimedb".
 
 import { readFileSync, writeFileSync, existsSync, renameSync, unlinkSync } from "node:fs";
@@ -27,13 +27,13 @@ function defaultDevIssuer() {
 }
 
 function allowedIssuers() {
-  const raw = process.env.STACK_SPACETIMEDB_ALLOWED_ISSUERS || process.env.STACK_SPACETIMEDB_TOKEN_ISSUER;
+  const raw = process.env.HEXCLAVE_SPACETIMEDB_ALLOWED_ISSUERS || process.env.HEXCLAVE_SPACETIMEDB_TOKEN_ISSUER;
   if (!raw) return [defaultDevIssuer()];
   return raw.split(",").map((s) => s.trim().replace(/\/+$/, "")).filter((s) => s !== "");
 }
 
 function expectedAudience() {
-  return process.env.STACK_SPACETIMEDB_EXPECTED_AUDIENCE || "spacetimedb";
+  return process.env.HEXCLAVE_SPACETIMEDB_EXPECTED_AUDIENCE || "spacetimedb";
 }
 
 const action = process.argv[2];

apps/internal-tool/scripts/spacetime-publish.mjs

@@ -5,22 +5,12 @@
 
 import { spawnSync } from "node:child_process";
 
-// Dual-read shim for the HEXCLAVE_/STACK_ env var prefix migration (same as examples/demo/cli-sim.mjs).
-function resolveHexclaveStackEnvVar(hexclaveName, stackName) {
-  const hexclaveValue = process.env[hexclaveName];
-  const stackValue = process.env[stackName];
-  if (hexclaveValue && stackValue && hexclaveValue !== stackValue) {
-    throw new Error(`Environment variables ${hexclaveName} and ${stackName} are both set to different values. Remove one of them or set them to the same value.`);
-  }
-  return hexclaveValue || stackValue || undefined;
-}
-
 const target = process.argv[2]; // "local" or "prod"
-const dbName = process.env.STACK_SPACETIMEDB_DB_NAME ?? "hexclave-ai-analytics";
+const dbName = process.env.HEXCLAVE_SPACETIMEDB_DB_NAME ?? "hexclave-ai-analytics";
 
 /** HTTP API for 'spacetime publish' (matches docker/dependencies/docker.compose.yaml host port ...39). */
 function localPublishServerUrl() {
-  const publishUrl = resolveHexclaveStackEnvVar("HEXCLAVE_SPACETIME_PUBLISH_URL", "STACK_SPACETIME_PUBLISH_URL");
+  const publishUrl = process.env.HEXCLAVE_SPACETIME_PUBLISH_URL;
   if (publishUrl) {
     return publishUrl;
   }
@@ -49,8 +39,8 @@ if (!args) {
   process.exit(1);
 }
 
-if (target === "prod" && !process.env.STACK_SPACETIMEDB_ALLOWED_ISSUERS && !process.env.STACK_SPACETIMEDB_TOKEN_ISSUER) {
-  console.error("Error: STACK_SPACETIMEDB_TOKEN_ISSUER (or STACK_SPACETIMEDB_ALLOWED_ISSUERS) must be set for prod publish — the deployed internal tool's public URL, which serves the OIDC discovery document SpacetimeDB validates tokens against.");
+if (target === "prod" && !process.env.HEXCLAVE_SPACETIMEDB_ALLOWED_ISSUERS && !process.env.HEXCLAVE_SPACETIMEDB_TOKEN_ISSUER) {
+  console.error("Error: HEXCLAVE_SPACETIMEDB_TOKEN_ISSUER (or HEXCLAVE_SPACETIMEDB_ALLOWED_ISSUERS) must be set for prod publish — the deployed internal tool's public URL, which serves the OIDC discovery document SpacetimeDB validates tokens against.");
   process.exit(1);
 }
 

apps/internal-tool/src/lib/server/qa-reviewer.ts

@@ -50,7 +50,7 @@ const qaReviewSchema = z.object({
 
 function createOpenRouterProvider() {
   return createOpenRouter({
-    apiKey: getEnvVariable("STACK_OPENROUTER_API_KEY"),
+    apiKey: getEnvVariable("HEXCLAVE_OPENROUTER_API_KEY"),
   });
 }
 

apps/internal-tool/src/lib/server/spacetimedb-client.ts

@@ -14,7 +14,7 @@ function requiredEnv(name: string): string {
 }
 
 function httpBase(): string {
-  return requiredEnv("STACK_SPACETIMEDB_URL");
+  return requiredEnv("HEXCLAVE_SPACETIMEDB_URL");
 }
 
 // All calls authenticate with a token minted by the internal tool (a signed-in
@@ -24,7 +24,7 @@ function httpBase(): string {
 // full read/write.
 export async function callReducerStrict(accessToken: string, reducer: string, args: unknown[]): Promise<void> {
   const base = httpBase();
-  const dbName = requiredEnv("STACK_SPACETIMEDB_DB_NAME");
+  const dbName = requiredEnv("HEXCLAVE_SPACETIMEDB_DB_NAME");
   const res = await fetch(`${base}/v1/database/${encodeURIComponent(dbName)}/call/${encodeURIComponent(reducer)}`, {
     method: "POST",
     headers: {
@@ -58,7 +58,7 @@ export function opt<T>(value: T | null | undefined): { some: T } | { none: [] }
 
 export async function callSql(accessToken: string, sql: string): Promise<Record<string, unknown>[]> {
   const base = httpBase();
-  const dbName = requiredEnv("STACK_SPACETIMEDB_DB_NAME");
+  const dbName = requiredEnv("HEXCLAVE_SPACETIMEDB_DB_NAME");
   const res = await fetch(`${base}/v1/database/${encodeURIComponent(dbName)}/sql`, {
     method: "POST",
     headers: { "Authorization": `Bearer ${accessToken}` },

apps/internal-tool/src/lib/server/spacetimedb-token.ts

@@ -16,21 +16,21 @@ import * as jose from "jose";
 const USER_TOKEN_TTL = "10m";
 
 export function spacetimeTokenIssuer(): string {
-  const issuer = getEnvVariable("STACK_SPACETIMEDB_TOKEN_ISSUER", "").trim().replace(/\/+$/, "");
+  const issuer = getEnvVariable("HEXCLAVE_SPACETIMEDB_TOKEN_ISSUER", "").trim().replace(/\/+$/, "");
   if (issuer === "") {
-    throw new HexclaveAssertionError("STACK_SPACETIMEDB_TOKEN_ISSUER is not configured for the internal tool.");
+    throw new HexclaveAssertionError("HEXCLAVE_SPACETIMEDB_TOKEN_ISSUER is not configured for the internal tool.");
   }
   return issuer;
 }
 
 export function spacetimeTokenAudience(): string {
-  return getEnvVariable("STACK_SPACETIMEDB_EXPECTED_AUDIENCE", "spacetimedb");
+  return getEnvVariable("HEXCLAVE_SPACETIMEDB_EXPECTED_AUDIENCE", "spacetimedb");
 }
 
 function privateJwk(): jose.JWK & { kid?: string, alg?: string } {
-  const raw = getEnvVariable("STACK_SPACETIMEDB_SIGNING_KEY_JWK", "");
+  const raw = getEnvVariable("HEXCLAVE_SPACETIMEDB_SIGNING_KEY_JWK", "");
   if (raw.trim() === "") {
-    throw new HexclaveAssertionError("STACK_SPACETIMEDB_SIGNING_KEY_JWK is not configured for the internal tool.");
+    throw new HexclaveAssertionError("HEXCLAVE_SPACETIMEDB_SIGNING_KEY_JWK is not configured for the internal tool.");
   }
   return JSON.parse(raw) as jose.JWK;
 }