diff --git a/apps/backend/.env b/apps/backend/.env index 058545379..9ba054d9d 100644 --- a/apps/backend/.env +++ b/apps/backend/.env @@ -125,5 +125,5 @@ HEXCLAVE_BULLDOZER_SERVER_SECRET=# shared secret the backend sends as a Bearer t HEXCLAVE_MINTLIFY_MCP_URL=# override the Mintlify MCP server used by the backend's AI docs tool bundle. Defaults to https://stackauth-e0affa27.mintlify.app/mcp # Internal tool (AI telemetry / MCP review) -STACK_INTERNAL_TOOL_URL=# public base URL of the deployed internal tool; default empty (telemetry disabled) -STACK_INTERNAL_TOOL_PROJECT_ID=# Stack Auth project the internal tool runs on (assertion audience) +HEXCLAVE_INTERNAL_TOOL_URL=# public base URL of the deployed internal tool; default empty (telemetry disabled) +HEXCLAVE_INTERNAL_TOOL_PROJECT_ID=# Hexclave project the internal tool runs on (assertion audience) diff --git a/apps/backend/.env.development b/apps/backend/.env.development index d021f12a7..d4069c8b4 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -124,8 +124,8 @@ HEXCLAVE_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs # Internal tool (AI telemetry / MCP review) # The internal tool (port ...41) owns all SpacetimeDB access; the backend ships # telemetry to its /api/backend/* routes over HTTP. -STACK_INTERNAL_TOOL_URL=http://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}41 -STACK_INTERNAL_TOOL_PROJECT_ID=internal +HEXCLAVE_INTERNAL_TOOL_URL=http://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}41 +HEXCLAVE_INTERNAL_TOOL_PROJECT_ID=internal # Clickhouse HEXCLAVE_CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}36 diff --git a/apps/backend/src/lib/ai/internal-tool-client.ts b/apps/backend/src/lib/ai/internal-tool-client.ts index 4ef4c2623..09c27f44c 100644 --- a/apps/backend/src/lib/ai/internal-tool-client.ts +++ b/apps/backend/src/lib/ai/internal-tool-client.ts @@ -21,15 +21,15 @@ const ASSERTION_TTL_MILLIS = 5 * 60 * 1000; const ASSERTION_REFRESH_MARGIN_MILLIS = 60 * 1000; function toolBase(): string | null { - const base = getEnvVariable("STACK_INTERNAL_TOOL_URL", "").trim().replace(/\/+$/, ""); + const base = getEnvVariable("HEXCLAVE_INTERNAL_TOOL_URL", "").trim().replace(/\/+$/, ""); return base === "" ? null : base; } -// The Stack Auth project the internal tool runs on — its id is the assertion +// The Hexclave project the internal tool runs on — its id is the assertion // audience (the public JWKS endpoint serves keys per project id, so the tool // can verify without any shared secret). function internalToolProjectId(): string { - return getEnvVariable("STACK_INTERNAL_TOOL_PROJECT_ID"); + return getEnvVariable("HEXCLAVE_INTERNAL_TOOL_PROJECT_ID"); } let cachedAssertion: { token: string, expiresAtMillis: number } | null = null; @@ -55,12 +55,13 @@ async function getAssertion(): Promise { /** * Calls an internal-tool ingest route. Returns `null` (without doing - * anything) when `STACK_INTERNAL_TOOL_URL` is unset — telemetry is disabled, + * anything) when `HEXCLAVE_INTERNAL_TOOL_URL` is unset — telemetry is disabled, * mirroring the old "no SpacetimeDB configured" behavior. Throws on non-2xx. */ export async function callInternalTool(path: string, options?: { method?: "GET" | "POST", body?: unknown, + timeoutMs?: number, }): Promise { const base = toolBase(); if (!base) return null; @@ -73,7 +74,7 @@ export async function callInternalTool(path: string, options?: { ...options?.body !== undefined ? { "Content-Type": "application/json" } : {}, }, ...options?.body !== undefined ? { body: JSON.stringify(options.body) } : {}, - signal: AbortSignal.timeout(INTERNAL_TOOL_FETCH_TIMEOUT_MS), + signal: AbortSignal.timeout(options?.timeoutMs ?? INTERNAL_TOOL_FETCH_TIMEOUT_MS), }); if (!res.ok) { const preview = (await res.text()).slice(0, 200); @@ -91,11 +92,12 @@ export async function callInternalTool(path: string, options?: { export async function callInternalToolStrict(path: string, options?: { method?: "GET" | "POST", body?: unknown, + timeoutMs?: number, }): Promise { const result = await callInternalTool(path, options); if (result === null) { throw new HexclaveAssertionError( - `Internal tool is not configured; ${path} cannot run. Check STACK_INTERNAL_TOOL_URL.`, + `Internal tool is not configured; ${path} cannot run. Check HEXCLAVE_INTERNAL_TOOL_URL.`, ); } return result; diff --git a/apps/backend/src/lib/ai/openrouter-usage.test.ts b/apps/backend/src/lib/ai/openrouter-usage.test.ts index bac47c7de..1d5ac58a3 100644 --- a/apps/backend/src/lib/ai/openrouter-usage.test.ts +++ b/apps/backend/src/lib/ai/openrouter-usage.test.ts @@ -82,6 +82,64 @@ describe("OpenRouter generation usage refinement", () => { }); }); + it("prefers native token counts when both native and normalized are present", async () => { + fetchMock.mockResolvedValue(jsonResponse({ + data: { + tokens_prompt: 95, + tokens_completion: 20, + native_tokens_prompt: 130, + native_tokens_completion: 42, + native_tokens_cached: 50, + total_cost: 0.02, + cache_discount: null, + }, + })); + + const promise = refineGenerationUsage({ generationId: "gen-both", correlationId: "corr-both" }); + await vi.advanceTimersByTimeAsync(1500); + await promise; + + expect(callInternalTool).toHaveBeenCalledWith("/api/backend/update-ai-query-usage", { + body: { + correlationId: "corr-both", + inputTokens: 130, + outputTokens: 42, + cachedInputTokens: 50, + costUsd: 0.02, + cacheDiscountUsd: undefined, + }, + }); + }); + + it("falls back to normalized token counts when native counts are null", async () => { + fetchMock.mockResolvedValue(jsonResponse({ + data: { + tokens_prompt: 130, + tokens_completion: 42, + native_tokens_prompt: null, + native_tokens_completion: null, + native_tokens_cached: null, + total_cost: 0.0456, + cache_discount: null, + }, + })); + + const promise = refineGenerationUsage({ generationId: "gen-normalized", correlationId: "corr-normalized" }); + await vi.advanceTimersByTimeAsync(1500); + await promise; + + expect(callInternalTool).toHaveBeenCalledWith("/api/backend/update-ai-query-usage", { + body: { + correlationId: "corr-normalized", + inputTokens: 130, + outputTokens: 42, + cachedInputTokens: undefined, + costUsd: 0.0456, + cacheDiscountUsd: undefined, + }, + }); + }); + it("retries when generation metadata is not ready yet", async () => { fetchMock .mockResolvedValueOnce(new Response("not ready", { status: 404 })) diff --git a/apps/backend/src/lib/ai/openrouter-usage.ts b/apps/backend/src/lib/ai/openrouter-usage.ts index 7d51b31e0..74788a0cd 100644 --- a/apps/backend/src/lib/ai/openrouter-usage.ts +++ b/apps/backend/src/lib/ai/openrouter-usage.ts @@ -43,8 +43,8 @@ async function fetchGenerationOnce( if (!isOpenRouterGenerationResponse(json)) return null; const data = json.data; return { - inputTokens: data.tokens_prompt ?? undefined, - outputTokens: data.tokens_completion ?? undefined, + inputTokens: data.native_tokens_prompt ?? data.tokens_prompt ?? undefined, + outputTokens: data.native_tokens_completion ?? data.tokens_completion ?? undefined, cachedInputTokens: data.native_tokens_cached ?? undefined, costUsd: data.total_cost, cacheDiscountUsd: data.cache_discount ?? undefined, diff --git a/apps/backend/src/lib/ai/qa/verified-qa.ts b/apps/backend/src/lib/ai/qa/verified-qa.ts index f9c6e30e1..18e515a1d 100644 --- a/apps/backend/src/lib/ai/qa/verified-qa.ts +++ b/apps/backend/src/lib/ai/qa/verified-qa.ts @@ -24,9 +24,10 @@ export async function getVerifiedQaContext(): Promise { cached = { value: result.data, expiresAtMillis: Date.now() + CACHE_TTL_MILLIS }; return result.data; } +const VERIFIED_QA_FETCH_TIMEOUT_MS = 2_000; async function getVerifiedQaContextInner(): Promise { - const response = await callInternalTool<{ context: string }>("/api/backend/verified-qa", { method: "GET" }); + const response = await callInternalTool<{ context: string }>("/api/backend/verified-qa", { method: "GET", timeoutMs: VERIFIED_QA_FETCH_TIMEOUT_MS }); // Telemetry/internal tool not configured — prompts simply omit the block. if (response == null) return ""; if (typeof response.context !== "string") { diff --git a/apps/e2e/tests/spacetimedb/qa-entries-invariants.test.ts b/apps/e2e/tests/spacetimedb/qa-entries-invariants.test.ts index 04741bbd9..7cf3be7cb 100644 --- a/apps/e2e/tests/spacetimedb/qa-entries-invariants.test.ts +++ b/apps/e2e/tests/spacetimedb/qa-entries-invariants.test.ts @@ -4,8 +4,9 @@ import { callReducer, createCleanupScope, decodeOptional, + findCorrelationIdByQuestion, findManualQaEntryIdByQuestion, - isSpacetimedbReachable, signMemberToken, + isSpacetimedbReachable, opt, signMemberToken, sqlQuery, touchSession, type CleanupScope, @@ -96,15 +97,21 @@ describe.skipIf(!canRun)("qa_entries CRUD invariants", () => { const qaId = await findManualQaEntryIdByQuestion(reviewerToken, marker); expect(qaId).toBeDefined(); - const beforeQa = (await sqlQuery(reviewerToken, "SELECT * FROM my_visible_qa_entries")).rows.length; - const beforeMcp = (await sqlQuery(reviewerToken, "SELECT * FROM my_visible_mcp_call_log")).rows.length; + // Seed an mcp_call_log row of our own and assert it survives the qa-entry + // delete. Scoped by marker (NOT global row counts): the spacetimedb test + // files run concurrently against the shared DB, so other tests inserting + // or deleting their own rows would race a count-based assertion. + const mcpMarker = uniqueMarker("delete-scope-mcp"); + scope.trackMcpQuestion(mcpMarker); + const seedMcp = await callReducer(reviewerToken, "log_mcp_call", [ + mcpMarker, opt(null), "delete-scope-tool", "reason", "prompt", mcpMarker, "response", 0, "[]", 0n, "model", opt(null), + ]); + expect(seedMcp.ok, seedMcp.body).toBe(true); const del = await callReducer(reviewerToken, "delete_qa_entry", [qaId!]); expect(del.ok, del.body).toBe(true); - const afterQa = (await sqlQuery(reviewerToken, "SELECT * FROM my_visible_qa_entries")).rows.length; - const afterMcp = (await sqlQuery(reviewerToken, "SELECT * FROM my_visible_mcp_call_log")).rows.length; - expect(afterQa).toBe(beforeQa - 1); - expect(afterMcp).toBe(beforeMcp); + expect(await findManualQaEntryIdByQuestion(reviewerToken, marker)).toBeUndefined(); + expect(await findCorrelationIdByQuestion(reviewerToken, mcpMarker)).toBe(mcpMarker); }); }); diff --git a/apps/e2e/tests/spacetimedb/reducer-auth.test.ts b/apps/e2e/tests/spacetimedb/reducer-auth.test.ts index dc5973f15..775056d76 100644 --- a/apps/e2e/tests/spacetimedb/reducer-auth.test.ts +++ b/apps/e2e/tests/spacetimedb/reducer-auth.test.ts @@ -43,12 +43,26 @@ describe.skipIf(!canRun)("SpacetimeDB reducer auth", () => { }); it("an identity without Stack Auth claims sees zero rows in my_visible_mcp_call_log", async ({ expect }) => { - // Seed a row so the underlying qa/mcp tables are definitely non-empty — - // otherwise a 0-row result could be a false positive from an empty table. + // Seed mcp_call_log itself (the table the view projects) so it is + // definitely non-empty — otherwise a 0-row result could be a false + // positive from an empty table. const memberToken = await signMemberToken(); const seedMarker = uniqueMarker("reducer-auth-seed"); scope.trackMcpQuestion(seedMarker); - const seed = await callReducer(memberToken, "add_manual_qa", [seedMarker, "a", false, seedMarker]); + const seed = await callReducer(memberToken, "log_mcp_call", [ + seedMarker, // correlationId + opt(null), // conversationId + "reducer-auth-seed-tool", + "reason", + "prompt", + seedMarker, // question — cleanup looks rows up by this marker + "response", + 0, // stepCount + "[]", // innerToolCallsJson + 0n, // durationMs + "model", + opt(null), // errorMessage + ]); expect(seed.ok, seed.body).toBe(true); const stranger = await mintIdentity(); diff --git a/apps/internal-tool/src/module_bindings/index.ts b/apps/internal-tool/src/module_bindings/index.ts index 47e6f327d..f06107c98 100644 --- a/apps/internal-tool/src/module_bindings/index.ts +++ b/apps/internal-tool/src/module_bindings/index.ts @@ -162,3 +162,4 @@ export class DbConnection extends __DbConnectionImpl { return new SubscriptionBuilder(this); }; } + diff --git a/apps/internal-tool/src/module_bindings/types.ts b/apps/internal-tool/src/module_bindings/types.ts index 08b462e92..a6982ae75 100644 --- a/apps/internal-tool/src/module_bindings/types.ts +++ b/apps/internal-tool/src/module_bindings/types.ts @@ -120,3 +120,4 @@ export const Sessions = __t.object("Sessions", { expiresAt: __t.timestamp(), }); export type Sessions = __Infer; + diff --git a/apps/internal-tool/src/module_bindings/types/reducers.ts b/apps/internal-tool/src/module_bindings/types/reducers.ts index 4456c96ea..cd97b3e9e 100644 --- a/apps/internal-tool/src/module_bindings/types/reducers.ts +++ b/apps/internal-tool/src/module_bindings/types/reducers.ts @@ -35,3 +35,4 @@ export type UpdateMcpQaReviewParams = __Infer; export type UpdateQaEntryWithPublishParams = __Infer; export type UpsertQaFromCallParams = __Infer; export type UpsertQaFromCallAndMarkReviewedParams = __Infer; + diff --git a/docker/dependencies/docker.compose.yaml b/docker/dependencies/docker.compose.yaml index 91918967f..aeafd653f 100644 --- a/docker/dependencies/docker.compose.yaml +++ b/docker/dependencies/docker.compose.yaml @@ -298,6 +298,8 @@ services: pull_policy: always ports: - "${NEXT_PUBLIC_HEXCLAVE_PORT_PREFIX:-81}39:3000" + extra_hosts: + - "host.docker.internal:host-gateway" command: start spacetimedb-issuer-proxy: