From b0a329f396e6795764e8d0d456e221fc8d79c290 Mon Sep 17 00:00:00 2001 From: Aadesh Kheria Date: Tue, 14 Apr 2026 23:46:57 -0700 Subject: [PATCH] initial commit --- .../app/api/latest/ai/query/[mode]/route.ts | 158 +++- apps/backend/src/lib/ai/ai-query-logger.ts | 12 + apps/backend/src/lib/ai/mcp-logger.ts | 2 +- .../ai_query_log_table.ts | 38 + .../src/lib/ai/spacetimedb-bindings/index.ts | 14 + .../log_ai_query_reducer.ts | 37 + .../src/lib/ai/spacetimedb-bindings/types.ts | 28 + .../ai/spacetimedb-bindings/types/reducers.ts | 2 + .../src/lib/ai-dashboard/shared-prompt.ts | 154 ++-- apps/internal-tool/spacetimedb/src/index.ts | 91 ++- apps/internal-tool/src/app/app-client.tsx | 43 +- .../src/components/CallLogList.tsx | 55 +- .../src/components/ConversationReplay.tsx | 6 +- apps/internal-tool/src/components/Usage.tsx | 720 ++++++++++++++++++ .../src/components/UsageDetail.tsx | 214 ++++++ .../internal-tool/src/hooks/useSpacetimeDB.ts | 71 +- .../src/module_bindings/ai_query_log_table.ts | 38 + .../src/module_bindings/index.ts | 14 + .../module_bindings/log_ai_query_reducer.ts | 37 + .../src/module_bindings/types.ts | 28 + .../src/module_bindings/types/reducers.ts | 2 + apps/internal-tool/src/types.ts | 2 +- 22 files changed, 1617 insertions(+), 149 deletions(-) create mode 100644 apps/backend/src/lib/ai/ai-query-logger.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/ai_query_log_table.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/log_ai_query_reducer.ts create mode 100644 apps/internal-tool/src/components/Usage.tsx create mode 100644 apps/internal-tool/src/components/UsageDetail.tsx create mode 100644 apps/internal-tool/src/module_bindings/ai_query_log_table.ts create mode 100644 apps/internal-tool/src/module_bindings/log_ai_query_reducer.ts diff --git a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts index bed3aed25..af7a45a94 100644 --- a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts +++ b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts @@ -1,3 +1,4 @@ +import { logAiQuery } from "@/lib/ai/ai-query-logger"; import { logMcpCall } from "@/lib/ai/mcp-logger"; import { selectModel } from "@/lib/ai/models"; import { getFullSystemPrompt } from "@/lib/ai/prompts"; @@ -10,9 +11,37 @@ import { SmartResponse } from "@/route-handlers/smart-response"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks"; import { yupMixed, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; -import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { captureError, StatusError } from "@stackframe/stack-shared/dist/utils/errors"; import { Json } from "@stackframe/stack-shared/dist/utils/json"; -import { generateText, ModelMessage, stepCountIs, streamText } from "ai"; +import type { OpenRouterUsageAccounting } from "@openrouter/ai-sdk-provider"; +import { generateText, ModelMessage, stepCountIs, streamText, type StepResult, type ToolSet } from "ai"; + +type ProviderMetadata = { openrouter?: { usage?: OpenRouterUsageAccounting } }; + +function extractOpenRouterCost(meta: unknown): number | undefined { + return (meta as ProviderMetadata | null | undefined)?.openrouter?.usage?.cost; +} + +function extractCachedTokens(meta: unknown): number | undefined { + return (meta as ProviderMetadata | null | undefined)?.openrouter?.usage?.promptTokensDetails?.cachedTokens; +} + +function buildStepsJson(steps: ReadonlyArray>): string { + return JSON.stringify(steps.map((step, i) => ({ + step: i, + text: step.text || undefined, + toolCalls: step.toolCalls.map(tc => ({ + toolName: tc.toolName, + toolCallId: tc.toolCallId, + args: tc.input, + })), + toolResults: step.toolResults.map(tr => ({ + toolName: tr.toolName, + toolCallId: tr.toolCallId, + result: tr.output, + })), + }))); +} export const POST = createSmartRouteHandler({ metadata: { @@ -35,7 +64,6 @@ export const POST = createSmartRouteHandler({ const isAuthenticated = fullReq.auth != null; const { quality, speed, systemPrompt: systemPromptId, tools: toolNames, messages, projectId } = body; - // Verify user has access to the target project if (projectId != null) { if (fullReq.auth?.project.id !== "internal") { throw new StatusError(StatusError.Forbidden, "You do not have access to this project"); @@ -60,31 +88,113 @@ export const POST = createSmartRouteHandler({ const toolsArg = Object.keys(tools).length > 0 ? tools : undefined; const stepLimit = toolsArg == null ? 1 : isDocsOrSearch ? 50 : 5; + const correlationId = crypto.randomUUID(); + const conversationIdForLog = body.mcpCallMetadata + ? body.mcpCallMetadata.conversationId ?? crypto.randomUUID() + : undefined; + const commonLogFields = { + correlationId, + mode, + systemPromptId, + quality, + speed, + modelId: String(model.modelId), + isAuthenticated, + projectId: projectId ?? undefined, + userId: fullReq.auth?.user?.id, + requestedToolsJson: JSON.stringify(toolNames), + messagesJson: JSON.stringify(messages), + mcpCorrelationId: body.mcpCallMetadata ? correlationId : undefined, + conversationId: conversationIdForLog, + }; + + const startedAt = Date.now(); + + const USER_FACING_ERROR_MESSAGE = "The AI service is temporarily unavailable. Please try again later."; + + function logError(err: unknown) { + captureError("ai-query-upstream", err); + runAsynchronouslyAndWaitUntil(logAiQuery({ + ...commonLogFields, + stepsJson: "[]", + finalText: "", + inputTokens: undefined, + outputTokens: undefined, + cachedInputTokens: undefined, + costUsd: undefined, + stepCount: 0, + durationMs: BigInt(Date.now() - startedAt), + errorMessage: err instanceof Error ? err.message : String(err), + })); + } + + const isAnthropic = model.modelId.startsWith("anthropic/"); + const systemMessage: ModelMessage = { + role: "system", + content: systemPrompt, + ...(isAnthropic && { + providerOptions: { openrouter: { cacheControl: { type: "ephemeral" } } }, + }), + }; + const cachedMessages: ModelMessage[] = [systemMessage, ...(messages as ModelMessage[])]; + const openrouterProviderOptions = { + usage: { include: true }, + extraBody: { + stream_options: { include_usage: true }, + }, + } as const; + if (mode === "stream") { const result = streamText({ model, - system: systemPrompt, - messages: messages as ModelMessage[], + messages: cachedMessages, tools: toolsArg, stopWhen: stepCountIs(stepLimit), + providerOptions: { + openrouter: openrouterProviderOptions, + }, + onFinish: ({ text, steps, usage, providerMetadata }) => { + runAsynchronouslyAndWaitUntil(logAiQuery({ + ...commonLogFields, + stepsJson: buildStepsJson(steps), + finalText: text, + inputTokens: usage.inputTokens ?? undefined, + outputTokens: usage.outputTokens ?? undefined, + cachedInputTokens: extractCachedTokens(providerMetadata), + costUsd: extractOpenRouterCost(providerMetadata), + stepCount: steps.length, + durationMs: BigInt(Date.now() - startedAt), + errorMessage: undefined, + })); + }, + onError: ({ error }) => logError(error), }); return { statusCode: 200, bodyType: "response" as const, - body: result.toUIMessageStreamResponse(), + body: result.toUIMessageStreamResponse({ + onError: () => USER_FACING_ERROR_MESSAGE, + }), }; } else { - const startedAt = Date.now(); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 120_000); - const result = await generateText({ - model, - system: systemPrompt, - messages: messages as ModelMessage[], - tools: toolsArg, - abortSignal: controller.signal, - stopWhen: stepCountIs(stepLimit), - }).finally(() => clearTimeout(timeoutId)); + let result: Awaited>; + try { + result = await generateText({ + model, + messages: cachedMessages, + tools: toolsArg, + abortSignal: controller.signal, + stopWhen: stepCountIs(stepLimit), + providerOptions: { + openrouter: openrouterProviderOptions, + }, + }).finally(() => clearTimeout(timeoutId)); + } catch (err) { + logError(err); + throw new StatusError(StatusError.BadGateway, USER_FACING_ERROR_MESSAGE); + } const contentBlocks: Array< | { type: "text", text: string } @@ -123,10 +233,22 @@ export const POST = createSmartRouteHandler({ }); }); + runAsynchronouslyAndWaitUntil(logAiQuery({ + ...commonLogFields, + stepsJson: buildStepsJson(result.steps), + finalText: result.text, + inputTokens: result.usage.inputTokens ?? undefined, + outputTokens: result.usage.outputTokens ?? undefined, + cachedInputTokens: extractCachedTokens(result.providerMetadata), + costUsd: extractOpenRouterCost(result.providerMetadata), + stepCount: result.steps.length, + durationMs: BigInt(Date.now() - startedAt), + errorMessage: undefined, + })); + let responseConversationId: string | undefined; - if (body.mcpCallMetadata != null) { - const correlationId = crypto.randomUUID(); - const conversationId = body.mcpCallMetadata.conversationId ?? crypto.randomUUID(); + if (body.mcpCallMetadata != null && conversationIdForLog != null) { + const conversationId = conversationIdForLog; responseConversationId = conversationId; const firstUserMessage = messages.find(m => m.role === "user"); const question = typeof firstUserMessage?.content === "string" diff --git a/apps/backend/src/lib/ai/ai-query-logger.ts b/apps/backend/src/lib/ai/ai-query-logger.ts new file mode 100644 index 000000000..fea0809a2 --- /dev/null +++ b/apps/backend/src/lib/ai/ai-query-logger.ts @@ -0,0 +1,12 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { getConnection } from "./mcp-logger"; +import type { LogAiQueryParams } from "./spacetimedb-bindings/types/reducers"; + +export type AiQueryLogEntry = Omit; + +export async function logAiQuery(entry: AiQueryLogEntry): Promise { + const conn = await getConnection(); + if (!conn) return; + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.logAiQuery({ token, ...entry }); +} diff --git a/apps/backend/src/lib/ai/mcp-logger.ts b/apps/backend/src/lib/ai/mcp-logger.ts index f8a9a3d04..b6bcc9533 100644 --- a/apps/backend/src/lib/ai/mcp-logger.ts +++ b/apps/backend/src/lib/ai/mcp-logger.ts @@ -23,7 +23,7 @@ export async function getConnection(): Promise { .onApplied(() => { resolve(connInstance); }) - .subscribe("SELECT * FROM mcp_call_log"); + .subscribe(["SELECT * FROM mcp_call_log", "SELECT * FROM ai_query_log"]); }) .onConnectError((_: unknown, err: Error) => { captureError("mcp-logger", err); diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/ai_query_log_table.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/ai_query_log_table.ts new file mode 100644 index 000000000..ef4208eb4 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/ai_query_log_table.ts @@ -0,0 +1,38 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64().primaryKey(), + correlationId: __t.string().name("correlation_id"), + createdAt: __t.timestamp().name("created_at"), + mode: __t.string(), + systemPromptId: __t.string().name("system_prompt_id"), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string().name("model_id"), + isAuthenticated: __t.bool().name("is_authenticated"), + projectId: __t.option(__t.string()).name("project_id"), + userId: __t.option(__t.string()).name("user_id"), + requestedToolsJson: __t.string().name("requested_tools_json"), + messagesJson: __t.string().name("messages_json"), + stepsJson: __t.string().name("steps_json"), + finalText: __t.string().name("final_text"), + inputTokens: __t.option(__t.u32()).name("input_tokens"), + outputTokens: __t.option(__t.u32()).name("output_tokens"), + cachedInputTokens: __t.option(__t.u32()).name("cached_input_tokens"), + costUsd: __t.option(__t.f64()).name("cost_usd"), + stepCount: __t.u32().name("step_count"), + durationMs: __t.u64().name("duration_ms"), + errorMessage: __t.option(__t.string()).name("error_message"), + mcpCorrelationId: __t.option(__t.string()).name("mcp_correlation_id"), + conversationId: __t.option(__t.string()).name("conversation_id"), +}); diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts index bf16c63e0..3c22e527d 100644 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts @@ -36,6 +36,7 @@ import { // Import all reducer arg schemas import AddManualQaReducer from "./add_manual_qa_reducer"; import DeleteQaEntryReducer from "./delete_qa_entry_reducer"; +import LogAiQueryReducer from "./log_ai_query_reducer"; import LogMcpCallReducer from "./log_mcp_call_reducer"; import MarkHumanReviewedReducer from "./mark_human_reviewed_reducer"; import UpdateHumanCorrectionReducer from "./update_human_correction_reducer"; @@ -44,12 +45,24 @@ import UpdateMcpQaReviewReducer from "./update_mcp_qa_review_reducer"; // Import all procedure arg schemas // Import all table schema definitions +import AiQueryLogRow from "./ai_query_log_table"; import McpCallLogRow from "./mcp_call_log_table"; /** Type-only namespace exports for generated type groups. */ /** The schema information for all tables in this module. This is defined the same was as the tables would have been defined in the server. */ const tablesSchema = __schema({ + aiQueryLog: __table({ + name: 'ai_query_log', + indexes: [ + { accessor: 'id', name: 'ai_query_log_id_idx_btree', algorithm: 'btree', columns: [ + 'id', + ] }, + ], + constraints: [ + { name: 'ai_query_log_id_key', constraint: 'unique', columns: ['id'] }, + ], + }, AiQueryLogRow), mcpCallLog: __table({ name: 'mcp_call_log', indexes: [ @@ -67,6 +80,7 @@ const tablesSchema = __schema({ const reducersSchema = __reducers( __reducerSchema("add_manual_qa", AddManualQaReducer), __reducerSchema("delete_qa_entry", DeleteQaEntryReducer), + __reducerSchema("log_ai_query", LogAiQueryReducer), __reducerSchema("log_mcp_call", LogMcpCallReducer), __reducerSchema("mark_human_reviewed", MarkHumanReviewedReducer), __reducerSchema("update_human_correction", UpdateHumanCorrectionReducer), diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/log_ai_query_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/log_ai_query_reducer.ts new file mode 100644 index 000000000..62f5a5fab --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/log_ai_query_reducer.ts @@ -0,0 +1,37 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + mode: __t.string(), + systemPromptId: __t.string(), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string(), + isAuthenticated: __t.bool(), + projectId: __t.option(__t.string()), + userId: __t.option(__t.string()), + requestedToolsJson: __t.string(), + messagesJson: __t.string(), + stepsJson: __t.string(), + finalText: __t.string(), + inputTokens: __t.option(__t.u32()), + outputTokens: __t.option(__t.u32()), + cachedInputTokens: __t.option(__t.u32()), + costUsd: __t.option(__t.f64()), + stepCount: __t.u32(), + durationMs: __t.u64(), + errorMessage: __t.option(__t.string()), + mcpCorrelationId: __t.option(__t.string()), + conversationId: __t.option(__t.string()), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts index 4af9e7b73..44a418c43 100644 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts @@ -10,6 +10,34 @@ import { type Infer as __Infer, } from "spacetimedb"; +export const AiQueryLog = __t.object("AiQueryLog", { + id: __t.u64(), + correlationId: __t.string(), + createdAt: __t.timestamp(), + mode: __t.string(), + systemPromptId: __t.string(), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string(), + isAuthenticated: __t.bool(), + projectId: __t.option(__t.string()), + userId: __t.option(__t.string()), + requestedToolsJson: __t.string(), + messagesJson: __t.string(), + stepsJson: __t.string(), + finalText: __t.string(), + inputTokens: __t.option(__t.u32()), + outputTokens: __t.option(__t.u32()), + cachedInputTokens: __t.option(__t.u32()), + costUsd: __t.option(__t.f64()), + stepCount: __t.u32(), + durationMs: __t.u64(), + errorMessage: __t.option(__t.string()), + mcpCorrelationId: __t.option(__t.string()), + conversationId: __t.option(__t.string()), +}); +export type AiQueryLog = __Infer; + export const McpCallLog = __t.object("McpCallLog", { id: __t.u64(), correlationId: __t.string(), diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts index 87a6606ae..51e5ee749 100644 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts @@ -8,6 +8,7 @@ import { type Infer as __Infer } from "spacetimedb"; // Import all reducer arg schemas import AddManualQaReducer from "../add_manual_qa_reducer"; import DeleteQaEntryReducer from "../delete_qa_entry_reducer"; +import LogAiQueryReducer from "../log_ai_query_reducer"; import LogMcpCallReducer from "../log_mcp_call_reducer"; import MarkHumanReviewedReducer from "../mark_human_reviewed_reducer"; import UpdateHumanCorrectionReducer from "../update_human_correction_reducer"; @@ -15,6 +16,7 @@ import UpdateMcpQaReviewReducer from "../update_mcp_qa_review_reducer"; export type AddManualQaParams = __Infer; export type DeleteQaEntryParams = __Infer; +export type LogAiQueryParams = __Infer; export type LogMcpCallParams = __Infer; export type MarkHumanReviewedParams = __Infer; export type UpdateHumanCorrectionParams = __Infer; diff --git a/apps/dashboard/src/lib/ai-dashboard/shared-prompt.ts b/apps/dashboard/src/lib/ai-dashboard/shared-prompt.ts index 67ca812d1..7973c9d15 100644 --- a/apps/dashboard/src/lib/ai-dashboard/shared-prompt.ts +++ b/apps/dashboard/src/lib/ai-dashboard/shared-prompt.ts @@ -1,6 +1,16 @@ import { BUNDLED_DASHBOARD_UI_TYPES, BUNDLED_TYPE_DEFINITIONS } from "@/generated/bundled-type-definitions"; import { ALL_APPS_FRONTEND, type AppId, getItemPath, hasNavigationItems } from "@/lib/apps-frontend"; -import { buildStackAuthHeaders, type CurrentUser } from "@/lib/api-headers"; +import type { CurrentUser } from "@/lib/api-headers"; + +export type DashboardMessagePart = { + type: "text", + text: string, + providerOptions?: Record, +}; +export type DashboardMessage = { + role: string, + content: string | DashboardMessagePart[], +}; /** * Builds a formatted list of available dashboard routes based on enabled apps. @@ -17,7 +27,7 @@ export function buildAvailableRoutes(enabledAppIds: AppId[]): string { routes.push({ path: "/project-settings", label: "Project Settings" }); // Dynamic routes from enabled apps - for (const appId of enabledAppIds) { + for (const appId of [...enabledAppIds].sort()) { const appFrontend = ALL_APPS_FRONTEND[appId as keyof typeof ALL_APPS_FRONTEND]; if (!hasNavigationItems(appFrontend)) { continue; @@ -35,74 +45,8 @@ export function buildAvailableRoutes(enabledAppIds: AppId[]): string { return `\nAVAILABLE DASHBOARD ROUTES (use ONLY these with window.dashboardNavigate):\n${routeList}\nDo NOT use any paths not listed above.`; } -export async function selectRelevantFiles( - prompt: string, - backendBaseUrl: string, - currentUser?: CurrentUser, -): Promise { - const availableFiles = BUNDLED_TYPE_DEFINITIONS.map((f: { path: string }) => f.path); - - const systemPromptText = `You are a code assistant helping to generate dashboard code for Stack Auth. - -Your task is to select which Stack SDK type definition files you'll need to generate the requested dashboard. - -IMPORTANT GUIDELINES: -- DO NOT be conservative in file selection - when in doubt, INCLUDE the file -- If a file might be relevant to the dashboard, SELECT IT -- For user/team dashboards: select users and/or teams files -- For project info: select projects files -- Always select server-app.ts as it contains the main SDK interface -- It's better to include extra files than to miss necessary types - -Available files: -${availableFiles.map(f => `- ${f}`).join('\n')} - -Respond with ONLY a JSON object: { "selectedFiles": ["file1.ts", "file2.ts"] } -No markdown, no explanation — just the JSON.`; - - try { - const authHeaders = await buildStackAuthHeaders(currentUser); - const response = await fetch(`${backendBaseUrl}/api/latest/ai/query/generate`, { - method: "POST", - headers: { "content-type": "application/json", ...authHeaders }, - body: JSON.stringify({ - quality: "dumb", - speed: "fast", - systemPrompt: "command-center-ask-ai", - tools: [], - messages: [{ - role: "user", - content: `${systemPromptText}\n\nDashboard request: "${prompt}"\n\nWhich type definition files do you need? When uncertain, err on the side of INCLUDING more files rather than fewer.`, - }], - }), - }); - - const result = await response.json() as { content?: Array<{ type: string, text?: string }> }; - const content = Array.isArray(result.content) ? result.content : []; - const textBlock = content.find((b) => b.type === "text"); - const responseText = textBlock?.text; - - if (!responseText) { - return availableFiles; - } - - const jsonMatch = responseText.match(/\{[\s\S]*"selectedFiles"[\s\S]*\}/); - if (!jsonMatch) { - return availableFiles; - } - - const parsed = JSON.parse(jsonMatch[0]) as { selectedFiles?: string[] }; - if (!Array.isArray(parsed.selectedFiles) || parsed.selectedFiles.length === 0) { - return availableFiles; - } - - const selected = parsed.selectedFiles.filter((f) => availableFiles.includes(f)); - - return selected; - } catch (e) { - console.log("[selectRelevantFiles] failed, returning all files:", e); - return availableFiles; - } +function getAllTypeDefinitionFiles(): string[] { + return BUNDLED_TYPE_DEFINITIONS.map((f: { path: string }) => f.path); } function stripComments(source: string): string { @@ -134,54 +78,56 @@ ${fileContents.join('\n')} `.trim(); } -function extractUserPromptText(messages: Array<{ role: string, content: unknown }>): string { - const lastUserMessage = [...messages].reverse().find(m => m.role === "user"); - if (typeof lastUserMessage?.content === "string") { - return lastUserMessage.content; - } - if (Array.isArray(lastUserMessage?.content)) { - const textPart = (lastUserMessage.content as Array<{ type: string, text?: string }>).find(c => c.type === "text"); - return textPart?.text ?? "dashboard"; - } - return "dashboard"; -} - -export async function buildDashboardMessages( - backendBaseUrl: string, - currentUser: CurrentUser | undefined, - messages: Array<{ role: string, content: unknown }>, +export function buildDashboardMessages( + _backendBaseUrl: string, + _currentUser: CurrentUser | undefined, + _messages: Array<{ role: string, content: unknown }>, currentSource?: string, enabledAppIds?: AppId[], -): Promise> { - const promptForFileSelection = extractUserPromptText(messages); - const selectedFiles = await selectRelevantFiles(promptForFileSelection, backendBaseUrl, currentUser); - const typeDefinitions = loadSelectedTypeDefinitions(selectedFiles); - +): Promise { + const typeDefinitions = loadSelectedTypeDefinitions(getAllTypeDefinitionFiles()); const availableRoutes = enabledAppIds ? buildAvailableRoutes(enabledAppIds) : ""; - const contextMessages: Array<{ role: string, content: string }> = []; + const cachedText = `Here are the type definitions for the Stack SDK:\n${typeDefinitions}\n\nHere are the dashboard UI component types:\n${BUNDLED_DASHBOARD_UI_TYPES}`; + const contextMessages: DashboardMessage[] = []; + contextMessages.push({ + role: "user", + content: [ + { + type: "text", + text: cachedText, + providerOptions: { + openrouter: { cacheControl: { type: "ephemeral" } }, + anthropic: { cacheControl: { type: "ephemeral" } }, + }, + }, + ], + }); + contextMessages.push({ + role: "assistant", + content: "I have the SDK reference material and UI component types. What dashboard would you like me to create or edit?", + }); + + const tailParts: string[] = []; + if (availableRoutes) { + tailParts.push(availableRoutes.trimStart()); + } if (currentSource != null && currentSource.length > 0) { + tailParts.push(`Here is the current dashboard source code:\n\`\`\`tsx\n${currentSource}\n\`\`\``); + } + if (tailParts.length > 0) { contextMessages.push({ role: "user", - content: `Here is the current dashboard source code:\n\`\`\`tsx\n${currentSource}\n\`\`\`\n\nHere are the type definitions:\n${typeDefinitions}\n\nHere are the dashboard UI component types:\n${BUNDLED_DASHBOARD_UI_TYPES}${availableRoutes}`, + content: tailParts.join("\n\n"), }); contextMessages.push({ role: "assistant", - content: "I understand the current dashboard code, type definitions, available UI components, and available routes. What changes would you like to make?", - }); - } else { - contextMessages.push({ - role: "user", - content: `Here are the type definitions for the Stack SDK:\n${typeDefinitions}\n\nHere are the dashboard UI component types:\n${BUNDLED_DASHBOARD_UI_TYPES}${availableRoutes}`, - }); - contextMessages.push({ - role: "assistant", - content: "I have the type definitions, available UI components, and available routes. What dashboard would you like me to create?", + content: "Got it. What changes would you like me to make?", }); } - return contextMessages; + return Promise.resolve(contextMessages); } export { BUNDLED_DASHBOARD_UI_TYPES }; diff --git a/apps/internal-tool/spacetimedb/src/index.ts b/apps/internal-tool/spacetimedb/src/index.ts index da052f4bf..99d480def 100644 --- a/apps/internal-tool/spacetimedb/src/index.ts +++ b/apps/internal-tool/spacetimedb/src/index.ts @@ -43,7 +43,37 @@ const mcpCallLog = table( } ); -const spacetimedb = schema({ mcpCallLog }); +const aiQueryLog = table( + { name: 'ai_query_log', public: true }, + { + id: t.u64().primaryKey().autoInc(), + correlationId: t.string(), + createdAt: t.timestamp(), + mode: t.string(), + systemPromptId: t.string(), + quality: t.string(), + speed: t.string(), + modelId: t.string(), + isAuthenticated: t.bool(), + projectId: t.string().optional(), + userId: t.string().optional(), + requestedToolsJson: t.string(), + messagesJson: t.string(), + stepsJson: t.string(), + finalText: t.string(), + inputTokens: t.u32().optional(), + outputTokens: t.u32().optional(), + cachedInputTokens: t.u32().optional(), + costUsd: t.f64().optional(), + stepCount: t.u32(), + durationMs: t.u64(), + errorMessage: t.string().optional(), + mcpCorrelationId: t.string().optional(), + conversationId: t.string().optional(), + } +); + +const spacetimedb = schema({ mcpCallLog, aiQueryLog }); export default spacetimedb; export const log_mcp_call = spacetimedb.reducer( @@ -237,4 +267,63 @@ export const delete_qa_entry = spacetimedb.reducer( } ); +export const log_ai_query = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + mode: t.string(), + systemPromptId: t.string(), + quality: t.string(), + speed: t.string(), + modelId: t.string(), + isAuthenticated: t.bool(), + projectId: t.string().optional(), + userId: t.string().optional(), + requestedToolsJson: t.string(), + messagesJson: t.string(), + stepsJson: t.string(), + finalText: t.string(), + inputTokens: t.u32().optional(), + outputTokens: t.u32().optional(), + cachedInputTokens: t.u32().optional(), + costUsd: t.f64().optional(), + stepCount: t.u32(), + durationMs: t.u64(), + errorMessage: t.string().optional(), + mcpCorrelationId: t.string().optional(), + conversationId: t.string().optional(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + ctx.db.aiQueryLog.insert({ + id: 0n, + correlationId: args.correlationId, + createdAt: ctx.timestamp, + mode: args.mode, + systemPromptId: args.systemPromptId, + quality: args.quality, + speed: args.speed, + modelId: args.modelId, + isAuthenticated: args.isAuthenticated, + projectId: args.projectId, + userId: args.userId, + requestedToolsJson: args.requestedToolsJson, + messagesJson: args.messagesJson, + stepsJson: args.stepsJson, + finalText: args.finalText, + inputTokens: args.inputTokens, + outputTokens: args.outputTokens, + cachedInputTokens: args.cachedInputTokens, + costUsd: args.costUsd, + stepCount: args.stepCount, + durationMs: args.durationMs, + errorMessage: args.errorMessage, + mcpCorrelationId: args.mcpCorrelationId, + conversationId: args.conversationId, + } as Parameters[0]); + } +); + export const init = spacetimedb.init(_ctx => {}); diff --git a/apps/internal-tool/src/app/app-client.tsx b/apps/internal-tool/src/app/app-client.tsx index d360c260a..1e841b509 100644 --- a/apps/internal-tool/src/app/app-client.tsx +++ b/apps/internal-tool/src/app/app-client.tsx @@ -6,18 +6,22 @@ import { Analytics } from "../components/Analytics"; import { CallLogDetail } from "../components/CallLogDetail"; import { CallLogList } from "../components/CallLogList"; import { KnowledgeBase } from "../components/KnowledgeBase"; -import { useMcpCallLogs } from "../hooks/useSpacetimeDB"; +import { Usage } from "../components/Usage"; +import { UsageDetail } from "../components/UsageDetail"; +import { useAiQueryLogs, useMcpCallLogs } from "../hooks/useSpacetimeDB"; import { makeMcpReviewApi } from "../lib/mcp-review-api"; -import type { McpCallLogRow } from "../types"; +import type { AiQueryLogRow, McpCallLogRow } from "../types"; -type Tab = "calls" | "knowledge" | "analytics"; +type Tab = "calls" | "knowledge" | "analytics" | "usage"; export default function App() { const user = useUser({ or: process.env.NODE_ENV === "development" ? "redirect" : "return-null" }); const [selectedRow, setSelectedRow] = useState(null); + const [selectedUsageRow, setSelectedUsageRow] = useState(null); const [showAddQa, setShowAddQa] = useState(false); const [tab, setTab] = useState("calls"); const { rows, connectionState } = useMcpCallLogs(); + const { rows: usageRows, connectionState: usageConnectionState } = useAiQueryLogs(); if (!user) { return ( @@ -115,6 +119,18 @@ export default function App() { > Analytics +
@@ -193,6 +209,27 @@ export default function App() { )} + + {tab === "usage" && ( +
+
+ +
+ {selectedUsageRow && ( + + )} +
+ )}
); } diff --git a/apps/internal-tool/src/components/CallLogList.tsx b/apps/internal-tool/src/components/CallLogList.tsx index 553b228ab..b4611c5f1 100644 --- a/apps/internal-tool/src/components/CallLogList.tsx +++ b/apps/internal-tool/src/components/CallLogList.tsx @@ -1,4 +1,4 @@ -import { useState, useMemo } from "react"; +import { useState, useMemo, useEffect } from "react"; import { formatDistanceToNow, format } from "date-fns"; import type { McpCallLogRow } from "../types"; import { toDate } from "../utils"; @@ -12,6 +12,8 @@ type SortField = "time" | "tool" | "steps" | "duration" | "qa" | "status"; type SortDir = "asc" | "desc"; type StatusFilter = "all" | "ok" | "error"; type QaFilter = "all" | "pending" | "pass" | "warn" | "fail" | "error" | "needs-review" | "human-reviewed" | "not-reviewed"; +const PAGE_SIZES = [25, 50, 100, 500] as const; +type PageSize = typeof PAGE_SIZES[number]; function getSortValue(row: McpCallLogRow, field: SortField): number | string { switch (field) { @@ -56,6 +58,8 @@ export function CallLogList({ }, [rows]); const [toolFilter, setToolFilter] = useState("all"); + const [page, setPage] = useState(0); + const [pageSize, setPageSize] = useState(50); const filteredAndSorted = useMemo(() => { let result = rows; @@ -115,6 +119,14 @@ export function CallLogList({ return result; }, [rows, textFilter, toolFilter, statusFilter, qaFilter, sortField, sortDir]); + const pageCount = Math.max(1, Math.ceil(filteredAndSorted.length / pageSize)); + const currentPage = Math.min(page, pageCount - 1); + const pageRows = filteredAndSorted.slice(currentPage * pageSize, (currentPage + 1) * pageSize); + + useEffect(() => { + setPage(0); + }, [textFilter, toolFilter, statusFilter, qaFilter, sortField, sortDir, pageSize]); + if (connectionState === "connecting") { return
Connecting to SpacetimeDB...
; } @@ -241,7 +253,7 @@ export function CallLogList({ - {filteredAndSorted.map((row) => ( + {pageRows.map((row) => ( onSelect(row)} @@ -307,6 +319,45 @@ export function CallLogList({ ))} +
+
+ Page size + {PAGE_SIZES.map(s => ( + + ))} +
+
+ + {filteredAndSorted.length === 0 + ? "No results" + : `${currentPage * pageSize + 1}–${Math.min((currentPage + 1) * pageSize, filteredAndSorted.length)} of ${filteredAndSorted.length}`} + + + {currentPage + 1} / {pageCount} + +
+
)} diff --git a/apps/internal-tool/src/components/ConversationReplay.tsx b/apps/internal-tool/src/components/ConversationReplay.tsx index 02516dbe3..6e2258207 100644 --- a/apps/internal-tool/src/components/ConversationReplay.tsx +++ b/apps/internal-tool/src/components/ConversationReplay.tsx @@ -40,7 +40,7 @@ function CopyButton({ text }: { text: string }) { ); } -function ToolCallCard({ call, accent = "purple" }: { call: { toolName: string; args: unknown; result: unknown }; accent?: "purple" | "indigo" }) { +export function ToolCallCard({ call, accent = "purple" }: { call: { toolName: string; args: unknown; result: unknown }; accent?: "purple" | "indigo" }) { const [expanded, setExpanded] = useState(false); const colors = accent === "indigo" ? { dot: "text-indigo-500", name: "text-indigo-700", bg: "bg-indigo-50", ring: "ring-indigo-200", hover: "hover:bg-indigo-100" } @@ -82,7 +82,7 @@ function ToolCallCard({ call, accent = "purple" }: { call: { toolName: string; a ); } -function UserBubble({ text }: { text: string }) { +export function UserBubble({ text }: { text: string }) { return (
@@ -95,7 +95,7 @@ function UserBubble({ text }: { text: string }) { ); } -function AssistantBubble({ content, toolCalls }: { content: string; toolCalls: ToolCall[] }) { +export function AssistantBubble({ content, toolCalls }: { content: string; toolCalls: ToolCall[] }) { return (
diff --git a/apps/internal-tool/src/components/Usage.tsx b/apps/internal-tool/src/components/Usage.tsx new file mode 100644 index 000000000..88f452f26 --- /dev/null +++ b/apps/internal-tool/src/components/Usage.tsx @@ -0,0 +1,720 @@ +import { useEffect, useMemo, useState } from "react"; +import { clsx } from "clsx"; +import type { AiQueryLogRow } from "../types"; +import { toDate } from "../utils"; + +type TimeRange = "24h" | "7d" | "30d" | "all"; +type AuthFilter = "all" | "authed" | "anon"; +type ModeFilter = "all" | "stream" | "generate"; +type StatusFilter = "all" | "ok" | "error"; +type SortKey = "createdAt" | "systemPromptId" | "modelId" | "mode" | "inputTokens" | "outputTokens" | "cachedInputTokens" | "costUsd" | "durationMs" | "status"; +type SortDir = "asc" | "desc"; +const PAGE_SIZES = [25, 50, 100, 500] as const; +type PageSize = typeof PAGE_SIZES[number]; + +type Props = { + rows: AiQueryLogRow[], + connectionState: "connecting" | "connected" | "disconnected" | "error", + onSelect: (row: AiQueryLogRow) => void, + selectedId?: bigint, +}; + +const ALL_SYSTEM_PROMPTS = [ + "command-center-ask-ai", + "docs-ask-ai", + "wysiwyg-edit", + "email-wysiwyg-editor", + "email-assistant-template", + "email-assistant-theme", + "email-assistant-draft", + "create-dashboard", + "run-query", + "rewrite-template-source", +]; + +export function Usage({ rows, connectionState, onSelect, selectedId }: Props) { + const [timeRange, setTimeRange] = useState("7d"); + const [systemPromptFilter, setSystemPromptFilter] = useState>(new Set()); + const [modelFilter, setModelFilter] = useState>(new Set()); + const [modeFilter, setModeFilter] = useState("all"); + const [authFilter, setAuthFilter] = useState("all"); + const [statusFilter, setStatusFilter] = useState("all"); + const [search, setSearch] = useState(""); + const [sortKey, setSortKey] = useState("createdAt"); + const [sortDir, setSortDir] = useState("desc"); + const [page, setPage] = useState(0); + const [pageSize, setPageSize] = useState(50); + + const now = Date.now(); + const rangeStart = useMemo(() => { + switch (timeRange) { + case "24h": { + return now - 24 * 60 * 60 * 1000; + } + case "7d": { + return now - 7 * 24 * 60 * 60 * 1000; + } + case "30d": { + return now - 30 * 24 * 60 * 60 * 1000; + } + case "all": { + return 0; + } + } + }, [timeRange, now]); + + const filtered = useMemo(() => { + const q = search.trim().toLowerCase(); + return rows.filter(r => { + const ts = toDate(r.createdAt).getTime(); + if (ts < rangeStart) return false; + if (systemPromptFilter.size > 0 && !systemPromptFilter.has(r.systemPromptId)) return false; + if (modelFilter.size > 0 && !modelFilter.has(r.modelId)) return false; + if (modeFilter !== "all" && r.mode !== modeFilter) return false; + if (authFilter === "authed" && !r.isAuthenticated) return false; + if (authFilter === "anon" && r.isAuthenticated) return false; + const isError = r.errorMessage != null && r.errorMessage !== ""; + if (statusFilter === "ok" && isError) return false; + if (statusFilter === "error" && !isError) return false; + if (q) { + const hay = `${r.finalText} ${r.messagesJson}`.toLowerCase(); + if (!hay.includes(q)) return false; + } + return true; + }); + }, [rows, rangeStart, systemPromptFilter, modelFilter, modeFilter, authFilter, statusFilter, search]); + + const sorted = useMemo(() => { + const copy = [...filtered]; + const mult = sortDir === "asc" ? 1 : -1; + copy.sort((a, b) => { + let av: number | string = 0; + let bv: number | string = 0; + switch (sortKey) { + case "createdAt": { + av = toDate(a.createdAt).getTime(); + bv = toDate(b.createdAt).getTime(); + break; + } + case "systemPromptId": { + av = a.systemPromptId; + bv = b.systemPromptId; + break; + } + case "modelId": { + av = a.modelId; + bv = b.modelId; + break; + } + case "mode": { + av = a.mode; + bv = b.mode; + break; + } + case "inputTokens": { + av = a.inputTokens ?? -1; + bv = b.inputTokens ?? -1; + break; + } + case "outputTokens": { + av = a.outputTokens ?? -1; + bv = b.outputTokens ?? -1; + break; + } + case "cachedInputTokens": { + av = a.cachedInputTokens ?? -1; + bv = b.cachedInputTokens ?? -1; + break; + } + case "costUsd": { + av = a.costUsd ?? -1; + bv = b.costUsd ?? -1; + break; + } + case "durationMs": { + av = Number(a.durationMs); + bv = Number(b.durationMs); + break; + } + case "status": { + av = (a.errorMessage != null && a.errorMessage !== "") ? 1 : 0; + bv = (b.errorMessage != null && b.errorMessage !== "") ? 1 : 0; + break; + } + } + if (av < bv) return -1 * mult; + if (av > bv) return 1 * mult; + return 0; + }); + return copy; + }, [filtered, sortKey, sortDir]); + + const pageCount = Math.max(1, Math.ceil(sorted.length / pageSize)); + const currentPage = Math.min(page, pageCount - 1); + const pageRows = sorted.slice(currentPage * pageSize, (currentPage + 1) * pageSize); + + function toggleSort(key: SortKey) { + if (sortKey === key) { + setSortDir(sortDir === "asc" ? "desc" : "asc"); + } else { + setSortKey(key); + setSortDir(key === "createdAt" ? "desc" : "asc"); + } + setPage(0); + } + + useEffect(() => { + setPage(0); + }, [timeRange, systemPromptFilter, modelFilter, modeFilter, authFilter, statusFilter, search, pageSize]); + + const stats = useMemo(() => { + const totalCalls = filtered.length; + const errorCalls = filtered.filter(r => r.errorMessage != null && r.errorMessage !== "").length; + const inputTokens = filtered.reduce((a, r) => a + (r.inputTokens ?? 0), 0); + const outputTokens = filtered.reduce((a, r) => a + (r.outputTokens ?? 0), 0); + const cachedInputTokens = filtered.reduce((a, r) => a + (r.cachedInputTokens ?? 0), 0); + const totalCost = filtered.reduce((a, r) => a + (r.costUsd ?? 0), 0); + const durations = filtered.map(r => Number(r.durationMs)).filter(d => d > 0).sort((a, b) => a - b); + const avgDuration = durations.length > 0 ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length) : 0; + const p95Duration = durations.length > 0 ? durations[Math.min(Math.floor(durations.length * 0.95), durations.length - 1)] : 0; + + // Time-bucketed series + const spanMs = now - rangeStart; + const bucketMs = spanMs <= 24 * 60 * 60 * 1000 ? 60 * 60 * 1000 : 24 * 60 * 60 * 1000; + const bucketLabelFmt: Intl.DateTimeFormatOptions = bucketMs === 60 * 60 * 1000 + ? { hour: "numeric" } + : { month: "short", day: "numeric" }; + const bucketCount = Math.min(48, Math.max(1, Math.ceil(spanMs / bucketMs))); + const bucketStart = now - bucketCount * bucketMs; + const timeBuckets: Array<{ label: string, start: number, calls: number, inputTokens: number, outputTokens: number, cachedInputTokens: number }> = []; + for (let i = 0; i < bucketCount; i++) { + const start = bucketStart + i * bucketMs; + timeBuckets.push({ + label: new Date(start).toLocaleString("en-US", bucketLabelFmt), + start, + calls: 0, + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + }); + } + for (const r of filtered) { + const ts = toDate(r.createdAt).getTime(); + const idx = Math.floor((ts - bucketStart) / bucketMs); + if (idx >= 0 && idx < bucketCount) { + timeBuckets[idx].calls++; + timeBuckets[idx].inputTokens += r.inputTokens ?? 0; + timeBuckets[idx].outputTokens += r.outputTokens ?? 0; + timeBuckets[idx].cachedInputTokens += r.cachedInputTokens ?? 0; + } + } + const maxCalls = Math.max(...timeBuckets.map(b => b.calls), 1); + const maxTokenTotal = Math.max(...timeBuckets.map(b => b.inputTokens + b.outputTokens), 1); + const maxInputTokens = Math.max(...timeBuckets.map(b => b.inputTokens), 1); + + // Distributions + const sysPromptCounts = new Map(); + const modelCounts = new Map(); + const toolCounts = new Map(); + for (const r of filtered) { + sysPromptCounts.set(r.systemPromptId, (sysPromptCounts.get(r.systemPromptId) ?? 0) + 1); + modelCounts.set(r.modelId, (modelCounts.get(r.modelId) ?? 0) + 1); + try { + const tools = JSON.parse(r.requestedToolsJson) as string[]; + for (const t of tools) toolCounts.set(t, (toolCounts.get(t) ?? 0) + 1); + } catch { /* skip */ } + } + const sysPromptDist = Array.from(sysPromptCounts.entries()).sort((a, b) => b[1] - a[1]); + const modelDist = Array.from(modelCounts.entries()).sort((a, b) => b[1] - a[1]); + const toolDist = Array.from(toolCounts.entries()).sort((a, b) => b[1] - a[1]); + + // Cache Hit % per systemPromptId + const cacheBySystemPrompt = new Map(); + for (const r of filtered) { + const existing = cacheBySystemPrompt.get(r.systemPromptId) ?? { input: 0, cached: 0, calls: 0 }; + existing.input += r.inputTokens ?? 0; + existing.cached += r.cachedInputTokens ?? 0; + existing.calls += 1; + cacheBySystemPrompt.set(r.systemPromptId, existing); + } + const cacheHitBySystemPrompt = Array.from(cacheBySystemPrompt.entries()) + .map(([id, v]) => ({ + id, + calls: v.calls, + hitPct: v.input > 0 ? Math.round((v.cached / v.input) * 100) : 0, + cached: v.cached, + input: v.input, + })) + .sort((a, b) => b.input - a.input); + + // Latency histogram + const latencyBuckets = [ + { label: "<500ms", max: 500, count: 0 }, + { label: "500ms–2s", max: 2000, count: 0 }, + { label: "2–10s", max: 10000, count: 0 }, + { label: "10–30s", max: 30000, count: 0 }, + { label: ">30s", max: Infinity, count: 0 }, + ]; + for (const d of durations) { + const b = latencyBuckets.find(b => d < b.max); + if (b) b.count++; + } + const maxLatencyBucket = Math.max(...latencyBuckets.map(b => b.count), 1); + + return { + totalCalls, errorCalls, inputTokens, outputTokens, cachedInputTokens, totalCost, + avgDuration, p95Duration, + timeBuckets, maxCalls, maxTokenTotal, maxInputTokens, + sysPromptDist, modelDist, toolDist, + cacheHitBySystemPrompt, + latencyBuckets, maxLatencyBucket, + }; + }, [filtered, rangeStart, now]); + + const allSystemPrompts = useMemo(() => { + const seen = new Set(ALL_SYSTEM_PROMPTS); + for (const r of rows) seen.add(r.systemPromptId); + return Array.from(seen).sort(); + }, [rows]); + + const allModels = useMemo(() => { + const seen = new Set(); + for (const r of rows) seen.add(r.modelId); + return Array.from(seen).sort(); + }, [rows]); + + function toggle(set: Set, val: string, setter: (s: Set) => void) { + const next = new Set(set); + if (next.has(val)) next.delete(val); + else next.add(val); + setter(next); + } + + return ( +
+ {/* Filter bar */} +
+
+ Range + {(["24h", "7d", "30d", "all"] as TimeRange[]).map(r => ( + + ))} + + Mode + {(["all", "stream", "generate"] as ModeFilter[]).map(m => ( + + ))} + + Auth + {(["all", "authed", "anon"] as AuthFilter[]).map(a => ( + + ))} + + Status + {(["all", "ok", "error"] as StatusFilter[]).map(s => ( + + ))} + + setSearch(e.target.value)} + placeholder="Search messages / response" + className="px-2 py-1 text-xs border border-gray-200 rounded w-64" + /> + + {connectionState === "connected" ? `${filtered.length} / ${rows.length} calls` : connectionState} + +
+
+ System prompt + {allSystemPrompts.map(sp => ( + + ))} +
+ {allModels.length > 0 && ( +
+ Model + {allModels.map(m => ( + + ))} +
+ )} +
+ + {/* Metric cards */} +
+ + 0 ? "text-red-600" : undefined} /> + + + 0 ? `${Math.round((stats.cachedInputTokens / stats.inputTokens) * 100)}%` : "—"} + valueClass={stats.inputTokens > 0 && stats.cachedInputTokens / stats.inputTokens > 0.5 ? "text-green-600" : undefined} + /> + + + +
+ + {/* Time-series charts */} +
+ +
+ {stats.timeBuckets.map((b, i) => ( +
+
+
+
+
+ ))} +
+
+ {stats.timeBuckets[0]?.label} + {stats.timeBuckets[stats.timeBuckets.length - 1]?.label} +
+ + + +
+ {stats.timeBuckets.map((b, i) => { + const total = b.inputTokens + b.outputTokens; + const outPct = total > 0 ? (b.outputTokens / total) * 100 : 0; + return ( +
+
+
+
+ ); + })} +
+
+ input + output +
+ + + +
+ {stats.timeBuckets.map((b, i) => { + const cachedPct = b.inputTokens > 0 ? (b.cachedInputTokens / b.inputTokens) * 100 : 0; + return ( +
+
+
+
+ ); + })} +
+
+ fresh + cached +
+ + + + {stats.cacheHitBySystemPrompt.length === 0 ? ( +

No data

+ ) : ( +
+ {stats.cacheHitBySystemPrompt.map(entry => ( +
+ {entry.id} +
+
= 50 ? "bg-green-500" : entry.hitPct >= 20 ? "bg-yellow-400" : "bg-red-400" + )} + style={{ width: `${entry.hitPct}%` }} + /> +
+ {entry.hitPct}% + {entry.calls} calls +
+ ))} +
+ )} + + + + + + + + + + + + + + + +
+ {stats.latencyBuckets.map(b => ( +
+ {b.label} +
+
+
+ {b.count} +
+ ))} +
+ +
+ + {/* Call list */} + +
+ + + + toggleSort("createdAt")}>Time + toggleSort("systemPromptId")}>System Prompt + toggleSort("modelId")}>Model + toggleSort("mode")}>Mode + toggleSort("inputTokens")}>In tok + toggleSort("outputTokens")}>Out tok + toggleSort("cachedInputTokens")}>Cached + toggleSort("costUsd")}>Cost + toggleSort("durationMs")}>Duration + toggleSort("status")}>Status + + + + {pageRows.map(row => { + const isError = row.errorMessage != null && row.errorMessage !== ""; + return ( + onSelect(row)} + className={clsx( + "border-b border-gray-100 cursor-pointer hover:bg-blue-50", + selectedId === row.id && "bg-blue-50" + )} + > + + + + + + + + + + + + ); + })} + +
+ {toDate(row.createdAt).toLocaleString("en-US", { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" })} + + + {row.systemPromptId} + + {row.mcpCorrelationId != null && ( + MCP + )} + {!row.isAuthenticated && ( + anon + )} + {row.modelId}{row.mode}{row.inputTokens?.toLocaleString() ?? "—"}{row.outputTokens?.toLocaleString() ?? "—"} + {row.cachedInputTokens != null && row.cachedInputTokens > 0 ? ( + {row.cachedInputTokens.toLocaleString()} + ) : ( + — + )} + {row.costUsd != null ? formatUsd(row.costUsd) : "—"}{Number(row.durationMs).toLocaleString()}ms + {isError ? ( + error + ) : ( + ok + )} +
+
+
+ Page size + {PAGE_SIZES.map(s => ( + + ))} +
+
+ + {sorted.length === 0 + ? "No results" + : `${currentPage * pageSize + 1}–${Math.min((currentPage + 1) * pageSize, sorted.length)} of ${sorted.length}`} + + + {currentPage + 1} / {pageCount} + +
+
+
+ +
+ ); +} + +function SortHeader({ + children, + align, + active, + dir, + onClick, +}: { + children: React.ReactNode, + align: "left" | "right", + active: boolean, + dir: SortDir, + onClick: () => void, +}) { + return ( + + + + ); +} + +function formatUsd(value: number): string { + if (value === 0) return "$0"; + if (value < 0.01) return `$${value.toFixed(4)}`; + if (value < 1) return `$${value.toFixed(3)}`; + return `$${value.toFixed(2)}`; +} + +function MetricCard({ label, value, valueClass }: { label: string, value: string, valueClass?: string }) { + return ( +
+

{label}

+

{value}

+
+ ); +} + +function Card({ title, children }: { title: string, children: React.ReactNode }) { + return ( +
+

{title}

+ {children} +
+ ); +} + +function DistributionBars({ items, color }: { items: Array<[string, number]>, color: string }) { + if (items.length === 0) { + return

No data

; + } + const max = Math.max(...items.map(i => i[1]), 1); + return ( +
+ {items.map(([label, count]) => ( +
+ {label} +
+
+
+ {count} +
+ ))} +
+ ); +} diff --git a/apps/internal-tool/src/components/UsageDetail.tsx b/apps/internal-tool/src/components/UsageDetail.tsx new file mode 100644 index 000000000..83c03c8b2 --- /dev/null +++ b/apps/internal-tool/src/components/UsageDetail.tsx @@ -0,0 +1,214 @@ +import { useMemo } from "react"; +import Markdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import type { AiQueryLogRow } from "../types"; +import { toDate } from "../utils"; +import { AssistantBubble, ToolCallCard, UserBubble } from "./ConversationReplay"; +import { markdownComponents } from "./markdown-components"; + +type MessageIn = { + role: "user" | "assistant" | "tool", + content: unknown, +}; + +type StepEntry = { + step: number, + text?: string, + toolCalls?: Array<{ toolName: string, toolCallId: string, args: unknown }>, + toolResults?: Array<{ toolName: string, toolCallId: string, result: unknown }>, +}; + +function messageContentToText(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map(part => { + if (typeof part === "string") return part; + if (part && typeof part === "object" && "text" in part) return String((part as { text: unknown }).text ?? ""); + return JSON.stringify(part); + }) + .join(""); + } + if (content == null) return ""; + return JSON.stringify(content); +} + +export function UsageDetail({ row, onClose }: { row: AiQueryLogRow, onClose: () => void }) { + const messages: MessageIn[] = useMemo(() => { + try { + return JSON.parse(row.messagesJson) as MessageIn[]; + } catch { + return []; + } + }, [row.messagesJson]); + + const steps: StepEntry[] = useMemo(() => { + try { + return JSON.parse(row.stepsJson) as StepEntry[]; + } catch { + return []; + } + }, [row.stepsJson]); + + const requestedTools: string[] = useMemo(() => { + try { + return JSON.parse(row.requestedToolsJson) as string[]; + } catch { + return []; + } + }, [row.requestedToolsJson]); + + const assistantBubbles = steps.map((s, i) => { + const toolCalls = (s.toolCalls ?? []).map((tc, idx) => { + const matched = s.toolResults?.find(r => r.toolCallId === tc.toolCallId) ?? s.toolResults?.[idx]; + return { + type: "tool-call", + toolName: tc.toolName, + toolCallId: tc.toolCallId, + args: tc.args, + result: matched?.result ?? null, + }; + }); + return { key: i, text: s.text ?? "", toolCalls }; + }); + + const isError = row.errorMessage != null && row.errorMessage !== ""; + + return ( +
+
+
+
+ + {row.systemPromptId} + + + {row.modelId} + + + {row.mode} + + {isError && ( + error + )} + {row.mcpCorrelationId != null && ( + MCP + )} +
+

+ {toDate(row.createdAt).toLocaleString()} + {" · "}{Number(row.durationMs).toLocaleString()}ms + {" · "}in {row.inputTokens?.toLocaleString() ?? "?"} tok + {row.cachedInputTokens != null && row.cachedInputTokens > 0 && ( + <> (cached {row.cachedInputTokens.toLocaleString()}) + )} + {" · "}out {row.outputTokens?.toLocaleString() ?? "?"} tok + {row.costUsd != null && <>{" · "}{row.costUsd < 0.01 ? `$${row.costUsd.toFixed(4)}` : `$${row.costUsd.toFixed(3)}`}} +

+
+ +
+ +
+ {isError && ( +
+

Error

+ 
{row.errorMessage}
+
+ )} + + {/* Metadata panel */} +
+ + + {row.projectId && } + {row.userId && } + {row.conversationId && } + {row.mcpCorrelationId && } + + 0 ? requestedTools.join(", ") : "—"} /> +
+ + {/* Conversation replay */} +
+

Input Messages

+ {messages.length === 0 && ( +

No input messages.

+ )} + {messages.map((m, i) => { + const text = messageContentToText(m.content); + if (m.role === "user") { + return ; + } + if (m.role === "assistant") { + return ; + } + return ( +
+
+ T +
+
+ 
{text}
+
+
+ ); + })} + +

Assistant Steps

+ {assistantBubbles.length === 0 && ( +

No assistant output recorded.

+ )} + {assistantBubbles.map(bubble => ( +
+ {bubble.toolCalls.length > 0 && ( +
+ {bubble.toolCalls.map((call, i) => ( + + ))} +
+ )} + {bubble.text && ( +
+
+ AI +
+
+ + {bubble.text} + +
+
+ )} +
+ ))} + + {row.finalText && assistantBubbles.length === 0 && ( + <> +

Final Response

+
+ + {row.finalText} + +
+ + )} +
+
+
+ ); +} + +function MetaRow({ label, value }: { label: string, value: string }) { + return ( +
+ {label} + {value} +
+ ); +} diff --git a/apps/internal-tool/src/hooks/useSpacetimeDB.ts b/apps/internal-tool/src/hooks/useSpacetimeDB.ts index 65efa1548..d2ab78635 100644 --- a/apps/internal-tool/src/hooks/useSpacetimeDB.ts +++ b/apps/internal-tool/src/hooks/useSpacetimeDB.ts @@ -1,6 +1,6 @@ import { useEffect, useState, useRef } from "react"; import { DbConnection, type EventContext, type SubscriptionEventContext } from "../module_bindings"; -import type { McpCallLogRow } from "../types"; +import type { AiQueryLogRow, McpCallLogRow } from "../types"; const IS_DEV = process.env.NODE_ENV === "development"; const PLACEHOLDER = "REPLACE_ME"; @@ -20,8 +20,17 @@ const RETRY_DELAY_MS = 2000; type ConnectionState = "connecting" | "connected" | "disconnected" | "error"; -export function useMcpCallLogs() { - const [rows, setRows] = useState([]); +type TableBinding = { + tableName: string, + iter: (ctx: SubscriptionEventContext) => Iterable, + onInsert: (conn: DbConnection, cb: (row: Row) => void) => void, + onDelete: (conn: DbConnection, cb: (row: Row) => void) => void, +}; + +function useTableSubscription( + binding: TableBinding, +) { + const [rows, setRows] = useState([]); const [connectionState, setConnectionState] = useState("connecting"); const connRef = useRef(null); @@ -29,14 +38,15 @@ export function useMcpCallLogs() { let cancelled = false; let retryCount = 0; let retryTimer: ReturnType | null = null; + const query = `SELECT * FROM ${binding.tableName}`; - console.log("[SpacetimeDB] Connecting to", HOST, "db:", DB_NAME); + console.log("[SpacetimeDB]", query, "connecting to", HOST, "db:", DB_NAME); function retry() { if (cancelled) return; retryCount++; if (retryCount > MAX_RETRIES) { - console.error("[SpacetimeDB] Max retries reached"); + console.error("[SpacetimeDB] Max retries reached for", query); setConnectionState("error"); return; } @@ -56,7 +66,6 @@ export function useMcpCallLogs() { .withToken(localStorage.getItem(TOKEN_KEY) || undefined) .onConnect((connInstance: DbConnection, _identity: unknown, token: string) => { if (cancelled) return; - console.log("[SpacetimeDB] Connected successfully"); retryCount = 0; localStorage.setItem(TOKEN_KEY, token); connRef.current = connInstance; @@ -64,18 +73,18 @@ export function useMcpCallLogs() { connInstance.subscriptionBuilder() .onApplied((ctx: SubscriptionEventContext) => { if (cancelled) return; - const initialRows: McpCallLogRow[] = []; - for (const row of ctx.db.mcpCallLog.iter()) { - initialRows.push(row); + const initial: Row[] = []; + for (const row of binding.iter(ctx)) { + initial.push(row); } - initialRows.sort((a, b) => Number(b.id - a.id)); - console.log("[SpacetimeDB] Loaded", initialRows.length, "rows"); - setRows(initialRows); + initial.sort((a, b) => Number(b.id - a.id)); + console.log(`[SpacetimeDB] ${query} loaded ${initial.length} rows`); + setRows(initial); setConnectionState("connected"); }) - .subscribe(`SELECT * FROM mcp_call_log`); + .subscribe(query); - connInstance.db.mcpCallLog.onInsert((_ctx: EventContext, row: McpCallLogRow) => { + binding.onInsert(connInstance, (row) => { if (cancelled) return; setRows(prev => { const existing = prev.findIndex(r => r.id === row.id); @@ -88,7 +97,7 @@ export function useMcpCallLogs() { }); }); - connInstance.db.mcpCallLog.onDelete((_ctx: EventContext, row: McpCallLogRow) => { + binding.onDelete(connInstance, (row) => { if (cancelled) return; setRows(prev => prev.filter(r => r.id !== row.id)); }); @@ -120,7 +129,37 @@ export function useMcpCallLogs() { connRef.current = null; } }; - }, []); + }, [binding]); return { rows, connectionState }; } + +const mcpBinding: TableBinding = { + tableName: "mcp_call_log", + iter: (ctx) => ctx.db.mcpCallLog.iter(), + onInsert: (conn, cb) => { + conn.db.mcpCallLog.onInsert((_ctx: EventContext, row: McpCallLogRow) => cb(row)); + }, + onDelete: (conn, cb) => { + conn.db.mcpCallLog.onDelete((_ctx: EventContext, row: McpCallLogRow) => cb(row)); + }, +}; + +const aiQueryBinding: TableBinding = { + tableName: "ai_query_log", + iter: (ctx) => ctx.db.aiQueryLog.iter(), + onInsert: (conn, cb) => { + conn.db.aiQueryLog.onInsert((_ctx: EventContext, row: AiQueryLogRow) => cb(row)); + }, + onDelete: (conn, cb) => { + conn.db.aiQueryLog.onDelete((_ctx: EventContext, row: AiQueryLogRow) => cb(row)); + }, +}; + +export function useMcpCallLogs() { + return useTableSubscription(mcpBinding); +} + +export function useAiQueryLogs() { + return useTableSubscription(aiQueryBinding); +} diff --git a/apps/internal-tool/src/module_bindings/ai_query_log_table.ts b/apps/internal-tool/src/module_bindings/ai_query_log_table.ts new file mode 100644 index 000000000..ef4208eb4 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/ai_query_log_table.ts @@ -0,0 +1,38 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64().primaryKey(), + correlationId: __t.string().name("correlation_id"), + createdAt: __t.timestamp().name("created_at"), + mode: __t.string(), + systemPromptId: __t.string().name("system_prompt_id"), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string().name("model_id"), + isAuthenticated: __t.bool().name("is_authenticated"), + projectId: __t.option(__t.string()).name("project_id"), + userId: __t.option(__t.string()).name("user_id"), + requestedToolsJson: __t.string().name("requested_tools_json"), + messagesJson: __t.string().name("messages_json"), + stepsJson: __t.string().name("steps_json"), + finalText: __t.string().name("final_text"), + inputTokens: __t.option(__t.u32()).name("input_tokens"), + outputTokens: __t.option(__t.u32()).name("output_tokens"), + cachedInputTokens: __t.option(__t.u32()).name("cached_input_tokens"), + costUsd: __t.option(__t.f64()).name("cost_usd"), + stepCount: __t.u32().name("step_count"), + durationMs: __t.u64().name("duration_ms"), + errorMessage: __t.option(__t.string()).name("error_message"), + mcpCorrelationId: __t.option(__t.string()).name("mcp_correlation_id"), + conversationId: __t.option(__t.string()).name("conversation_id"), +}); diff --git a/apps/internal-tool/src/module_bindings/index.ts b/apps/internal-tool/src/module_bindings/index.ts index bf16c63e0..3c22e527d 100644 --- a/apps/internal-tool/src/module_bindings/index.ts +++ b/apps/internal-tool/src/module_bindings/index.ts @@ -36,6 +36,7 @@ import { // Import all reducer arg schemas import AddManualQaReducer from "./add_manual_qa_reducer"; import DeleteQaEntryReducer from "./delete_qa_entry_reducer"; +import LogAiQueryReducer from "./log_ai_query_reducer"; import LogMcpCallReducer from "./log_mcp_call_reducer"; import MarkHumanReviewedReducer from "./mark_human_reviewed_reducer"; import UpdateHumanCorrectionReducer from "./update_human_correction_reducer"; @@ -44,12 +45,24 @@ import UpdateMcpQaReviewReducer from "./update_mcp_qa_review_reducer"; // Import all procedure arg schemas // Import all table schema definitions +import AiQueryLogRow from "./ai_query_log_table"; import McpCallLogRow from "./mcp_call_log_table"; /** Type-only namespace exports for generated type groups. */ /** The schema information for all tables in this module. This is defined the same was as the tables would have been defined in the server. */ const tablesSchema = __schema({ + aiQueryLog: __table({ + name: 'ai_query_log', + indexes: [ + { accessor: 'id', name: 'ai_query_log_id_idx_btree', algorithm: 'btree', columns: [ + 'id', + ] }, + ], + constraints: [ + { name: 'ai_query_log_id_key', constraint: 'unique', columns: ['id'] }, + ], + }, AiQueryLogRow), mcpCallLog: __table({ name: 'mcp_call_log', indexes: [ @@ -67,6 +80,7 @@ const tablesSchema = __schema({ const reducersSchema = __reducers( __reducerSchema("add_manual_qa", AddManualQaReducer), __reducerSchema("delete_qa_entry", DeleteQaEntryReducer), + __reducerSchema("log_ai_query", LogAiQueryReducer), __reducerSchema("log_mcp_call", LogMcpCallReducer), __reducerSchema("mark_human_reviewed", MarkHumanReviewedReducer), __reducerSchema("update_human_correction", UpdateHumanCorrectionReducer), diff --git a/apps/internal-tool/src/module_bindings/log_ai_query_reducer.ts b/apps/internal-tool/src/module_bindings/log_ai_query_reducer.ts new file mode 100644 index 000000000..62f5a5fab --- /dev/null +++ b/apps/internal-tool/src/module_bindings/log_ai_query_reducer.ts @@ -0,0 +1,37 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + mode: __t.string(), + systemPromptId: __t.string(), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string(), + isAuthenticated: __t.bool(), + projectId: __t.option(__t.string()), + userId: __t.option(__t.string()), + requestedToolsJson: __t.string(), + messagesJson: __t.string(), + stepsJson: __t.string(), + finalText: __t.string(), + inputTokens: __t.option(__t.u32()), + outputTokens: __t.option(__t.u32()), + cachedInputTokens: __t.option(__t.u32()), + costUsd: __t.option(__t.f64()), + stepCount: __t.u32(), + durationMs: __t.u64(), + errorMessage: __t.option(__t.string()), + mcpCorrelationId: __t.option(__t.string()), + conversationId: __t.option(__t.string()), +}; diff --git a/apps/internal-tool/src/module_bindings/types.ts b/apps/internal-tool/src/module_bindings/types.ts index 4af9e7b73..44a418c43 100644 --- a/apps/internal-tool/src/module_bindings/types.ts +++ b/apps/internal-tool/src/module_bindings/types.ts @@ -10,6 +10,34 @@ import { type Infer as __Infer, } from "spacetimedb"; +export const AiQueryLog = __t.object("AiQueryLog", { + id: __t.u64(), + correlationId: __t.string(), + createdAt: __t.timestamp(), + mode: __t.string(), + systemPromptId: __t.string(), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string(), + isAuthenticated: __t.bool(), + projectId: __t.option(__t.string()), + userId: __t.option(__t.string()), + requestedToolsJson: __t.string(), + messagesJson: __t.string(), + stepsJson: __t.string(), + finalText: __t.string(), + inputTokens: __t.option(__t.u32()), + outputTokens: __t.option(__t.u32()), + cachedInputTokens: __t.option(__t.u32()), + costUsd: __t.option(__t.f64()), + stepCount: __t.u32(), + durationMs: __t.u64(), + errorMessage: __t.option(__t.string()), + mcpCorrelationId: __t.option(__t.string()), + conversationId: __t.option(__t.string()), +}); +export type AiQueryLog = __Infer; + export const McpCallLog = __t.object("McpCallLog", { id: __t.u64(), correlationId: __t.string(), diff --git a/apps/internal-tool/src/module_bindings/types/reducers.ts b/apps/internal-tool/src/module_bindings/types/reducers.ts index 87a6606ae..51e5ee749 100644 --- a/apps/internal-tool/src/module_bindings/types/reducers.ts +++ b/apps/internal-tool/src/module_bindings/types/reducers.ts @@ -8,6 +8,7 @@ import { type Infer as __Infer } from "spacetimedb"; // Import all reducer arg schemas import AddManualQaReducer from "../add_manual_qa_reducer"; import DeleteQaEntryReducer from "../delete_qa_entry_reducer"; +import LogAiQueryReducer from "../log_ai_query_reducer"; import LogMcpCallReducer from "../log_mcp_call_reducer"; import MarkHumanReviewedReducer from "../mark_human_reviewed_reducer"; import UpdateHumanCorrectionReducer from "../update_human_correction_reducer"; @@ -15,6 +16,7 @@ import UpdateMcpQaReviewReducer from "../update_mcp_qa_review_reducer"; export type AddManualQaParams = __Infer; export type DeleteQaEntryParams = __Infer; +export type LogAiQueryParams = __Infer; export type LogMcpCallParams = __Infer; export type MarkHumanReviewedParams = __Infer; export type UpdateHumanCorrectionParams = __Infer; diff --git a/apps/internal-tool/src/types.ts b/apps/internal-tool/src/types.ts index 50f032070..6e37c4864 100644 --- a/apps/internal-tool/src/types.ts +++ b/apps/internal-tool/src/types.ts @@ -1 +1 @@ -export type { McpCallLog as McpCallLogRow } from "./module_bindings/types"; +export type { AiQueryLog as AiQueryLogRow, McpCallLog as McpCallLogRow } from "./module_bindings/types";