mirror of
https://github.com/stack-auth/stack.git
synced 2026-07-20 21:29:36 +08:00
fix(backend): use structured output for QA reviewer instead of JSON.parse
The QA reviewer extracted JSON from the model's free-form text via a greedy
/\{[\s\S]*\}/ regex and JSON.parse. Whenever Haiku wrapped its JSON in prose
or tool-call text, the captured slice was malformed and JSON.parse threw a
SyntaxError (218 occurrences in the last 30 days, STACK-BACKEND-18F).
Switch to schema-enforced structured output via generateText's
output: Output.object({ schema }), keeping the existing tool-calling loop.
The result is read off the validated result.output, removing the fragile
text extraction, JSON.parse, and manual shape validation.
This commit is contained in:
parent
6bbc792fb3
commit
ef9b4034cc
@ -1,7 +1,8 @@
|
||||
import { createMCPClient } from "@ai-sdk/mcp";
|
||||
import { getEnvVariable } from "@hexclave/shared/dist/utils/env";
|
||||
import { captureError } from "@hexclave/shared/dist/utils/errors";
|
||||
import { generateText, stepCountIs } from "ai";
|
||||
import { generateText, Output, stepCountIs } from "ai";
|
||||
import { z } from "zod";
|
||||
import { getConnection } from "./mcp-logger";
|
||||
import { createOpenRouterProvider } from "./models";
|
||||
import { getVerifiedQaContext } from "./verified-qa";
|
||||
@ -15,15 +16,7 @@ Your tasks:
|
||||
|
||||
The repo name for all tool calls is "stack-auth/stack-auth". Only use the repository documentation tools (read_wiki_structure, read_wiki_contents, ask_question) — do not create sessions or modify any other resources.
|
||||
|
||||
You MUST respond with ONLY valid JSON matching this exact schema (no markdown, no explanation outside the JSON):
|
||||
{
|
||||
"needsHumanReview": boolean,
|
||||
"answerCorrect": boolean,
|
||||
"answerRelevant": boolean,
|
||||
"flags": [{"type": string, "severity": "low" | "medium" | "high" | "critical", "explanation": string}],
|
||||
"improvementSuggestions": string,
|
||||
"overallScore": number
|
||||
}
|
||||
Produce a structured review. For each issue you find, add an entry to "flags" with a type, severity (one of "low", "medium", "high", "critical"), and explanation.
|
||||
|
||||
Flag types: "factual_error", "incomplete_answer", "off_topic", "hallucination", "outdated_info", "missing_context", "misleading", "reason_mismatch"
|
||||
|
||||
@ -38,6 +31,19 @@ Set needsHumanReview=true if: score < 50, any critical flag, or you are uncertai
|
||||
|
||||
const REVIEW_MODEL_ID = "anthropic/claude-haiku-4.5";
|
||||
|
||||
const qaReviewSchema = z.object({
|
||||
needsHumanReview: z.boolean(),
|
||||
answerCorrect: z.boolean(),
|
||||
answerRelevant: z.boolean(),
|
||||
flags: z.array(z.object({
|
||||
type: z.string(),
|
||||
severity: z.string(),
|
||||
explanation: z.string(),
|
||||
})),
|
||||
improvementSuggestions: z.string(),
|
||||
overallScore: z.number(),
|
||||
});
|
||||
|
||||
export async function reviewMcpCall(entry: {
|
||||
logPromise: Promise<void>;
|
||||
correlationId: string;
|
||||
@ -107,6 +113,7 @@ export async function reviewMcpCall(entry: {
|
||||
system: QA_SYSTEM_PROMPT + verifiedQa,
|
||||
tools: devinTools as Parameters<typeof generateText>[0]["tools"],
|
||||
stopWhen: stepCountIs(10),
|
||||
output: Output.object({ schema: qaReviewSchema }),
|
||||
messages: [{ role: "user", content: userMessage }],
|
||||
});
|
||||
|
||||
@ -125,28 +132,9 @@ export async function reviewMcpCall(entry: {
|
||||
};
|
||||
});
|
||||
|
||||
const jsonMatch = result.text.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) {
|
||||
throw new Error("No JSON found in QA review response");
|
||||
}
|
||||
const raw = JSON.parse(jsonMatch[0]);
|
||||
if (
|
||||
typeof raw.needsHumanReview !== "boolean" ||
|
||||
typeof raw.answerCorrect !== "boolean" ||
|
||||
typeof raw.answerRelevant !== "boolean" ||
|
||||
!Array.isArray(raw.flags) ||
|
||||
typeof raw.overallScore !== "number"
|
||||
) {
|
||||
throw new Error(`Invalid QA review response shape: ${JSON.stringify(raw).slice(0, 200)}`);
|
||||
}
|
||||
const parsed = raw as {
|
||||
needsHumanReview: boolean,
|
||||
answerCorrect: boolean,
|
||||
answerRelevant: boolean,
|
||||
flags: Array<{ type: string, severity: string, explanation: string }>,
|
||||
improvementSuggestions: string,
|
||||
overallScore: number,
|
||||
};
|
||||
// The model is constrained to the schema via structured output, so the result is
|
||||
// already a validated object — no fragile text extraction or JSON.parse needed.
|
||||
const parsed = result.output;
|
||||
parsed.overallScore = Math.max(0, Math.min(100, Math.round(parsed.overallScore)));
|
||||
|
||||
update = {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user