import { db } from "../db/index"; import { courses, uploads, topics, lessons, quizQuestions } from "../db/schema"; import { eq } from "drizzle-orm"; import { askAI } from "./openrouter"; function parseJSON(raw: string): T { try { return JSON.parse(raw); } catch { const cleaned = raw.replace(/^```[a-z]*\n?/i, "").replace(/\n?```$/i, "").trim(); return JSON.parse(cleaned); } } function renderSteps(steps: any[]): string { return steps.map((s: any, i: number) => { const lines: string[] = [` Step ${i + 1} [${s.type.toUpperCase()}]`]; if (s.title) lines.push(` Title: ${s.title}`); if (s.body) lines.push(` Body: ${s.body}`); if (s.callout) lines.push(` Callout: ${s.callout}`); if (Array.isArray(s.bullets)) { lines.push(` Bullets:\n${s.bullets.map((b: string) => ` - ${b}`).join("\n")}`); } if (Array.isArray(s.options)) { lines.push(` Options:\n${s.options.map((o: string, oi: number) => ` ${["A","B","C","D"][oi]}: ${o}`).join("\n")}`); lines.push(` Answer: ${s.answer}`); if (s.explanation) lines.push(` Explanation: ${s.explanation}`); } // include branches if present if (s.branches && typeof s.branches === "object") { for (const [wrongOpt, branch] of Object.entries(s.branches as Record)) { lines.push(` Branch for wrong answer "${wrongOpt}":`); for (const bs of branch.steps ?? []) { lines.push(` [BRANCH CONCEPT] ${bs.title ?? ""}: ${bs.body ?? ""}`); } if (branch.confirmQuestion) { const cq = branch.confirmQuestion; lines.push(` [CONFIRM QUESTION] ${cq.body}`); if (Array.isArray(cq.options)) { lines.push(` Options: ${cq.options.join(" | ")}`); } lines.push(` Answer: ${cq.answer}`); } if (branch.hardStop) { lines.push(` [HARD STOP] ${branch.hardStop}`); } } } return lines.join("\n"); }).join("\n"); } export async function auditCourse(courseId: string): Promise { console.log(`[audit:${courseId.slice(0, 8)}] starting post-generation audit`); try { const course = await db.query.courses.findFirst({ where: eq(courses.id, courseId) }); if (!course) throw new Error("Course not found"); const uploadRows = await db.query.uploads.findMany({ where: eq(uploads.courseId, courseId) }); const topicRows = await db.query.topics.findMany({ where: eq(topics.courseId, courseId), orderBy: (t, { asc }) => asc(t.order), }); const lessonMap: Record = {}; for (const topic of topicRows) { const lesson = await db.query.lessons.findFirst({ where: eq(lessons.topicId, topic.id) }); if (lesson) lessonMap[topic.id] = lesson; } const quizMap: Record = {}; for (const topic of topicRows) { quizMap[topic.id] = await db.query.quizQuestions.findMany({ where: eq(quizQuestions.topicId, topic.id) }); } // full source text — no truncation const primaryParts: string[] = []; const secondaryParts: string[] = []; for (const u of uploadRows) { if (!u.extractedText) continue; if (u.type === "past_paper" || u.type === "lab_worksheet") { primaryParts.push(`--- ${u.filename} (${u.type}) ---\n${u.extractedText}`); } else { secondaryParts.push(`--- ${u.filename} (${u.type}) ---\n${u.extractedText}`); } } const primaryText = primaryParts.join("\n\n") || "(none)"; const secondaryText = secondaryParts.join("\n\n") || "(none)"; // topics block const topicsBlock = topicRows .map((t) => `${t.order + 1}. ${t.title} (difficulty ${t.difficulty}/5)\n ${t.description}`) .join("\n"); // full lesson block — every step, every branch, every confirm question, every hard stop const lessonsBlock = topicRows.map((t) => { const lesson = lessonMap[t.id]; if (!lesson) return `Topic: ${t.title}\n (no lesson generated)`; let content: any = {}; try { content = JSON.parse(lesson.content); } catch {} const keyConcepts = (content.keyConcepts ?? []).join(", "); const steps = renderSteps(content.steps ?? []); return `Topic: ${t.title}\nKey concepts: ${keyConcepts}\n${steps}`; }).join("\n\n---\n\n"); // full quiz block — question, all options, answer, explanation const quizBlock = topicRows.map((t) => { const qs = quizMap[t.id] ?? []; if (qs.length === 0) return `Topic: ${t.title}\n (no quiz questions)`; const qList = qs.map((q: any, i: number) => { const lines = [` Q${i + 1} [${q.type}]: ${q.question}`]; if (q.options) { try { const opts: string[] = JSON.parse(q.options); lines.push(` Options: ${opts.join(" | ")}`); } catch {} } lines.push(` Answer: ${q.answer}`); lines.push(` Explanation: ${q.explanation}`); return lines.join("\n"); }).join("\n"); return `Topic: ${t.title}\n${qList}`; }).join("\n\n---\n\n"); const prompt = `You are auditing an AI-generated course against a single standard: can a student who completes this course answer every question in every past paper and lab worksheet provided? This is a pass/fail standard, not a score. However, also provide a score for tracking progress toward that standard. PRIMARY SOURCES (past papers + lab worksheets): ${primaryText} SECONDARY SOURCES (lecture slides): ${secondaryText} THE GENERATED COURSE: ${course.title} TOPICS COVERED (${topicRows.length} topics): ${topicsBlock} LESSONS GENERATED: ${lessonsBlock} QUIZ QUESTIONS: ${quizBlock} YOUR AUDIT PROCESS: 1. Go through every past paper question one by one. For each question, determine: could a student who completed this course answer it fully and correctly? If not, why not — what is missing or underdeveloped? 2. Go through every lab worksheet task one by one. Same question. 3. Go through every concept in the lecture slides. Is it covered in the course to a level of full understanding? 4. Identify every gap — topics missing, algorithms not taught to implementation level, calculations not drilled, pseudocode not covered, procedures not walked through. Return only valid JSON, no markdown: { "overallScore": 0-100, "passesStandard": true/false, "examReadiness": "honest plain English verdict", "unansweredPaperQuestions": [ { "source": "2023 Q2(a)", "question": "brief description", "gap": "what the course fails to teach that would be needed" } ], "coverageAnalysis": { "totalExaminedTopics": 0, "coveredTopics": 0, "coveragePercent": 0 }, "gaps": [ { "topic": "...", "severity": "high/medium/low", "appearsInSources": "...", "courseCoverage": "..." } ], "lessonQuality": [ { "topicTitle": "...", "score": 0-100, "notes": "..." } ], "recommendations": ["..."] }`; const config = useRuntimeConfig(); const evaluatorModel = (config as any).openrouterEvaluatorModel; const result = await askAI( [{ role: "user", content: prompt }], { maxRetries: 2, maxTokens: 4000, ...(evaluatorModel ? { model: evaluatorModel } : {}) } ); const report = parseJSON<{ overallScore: number }>(result.text); await db.update(courses) .set({ auditReport: result.text, auditScore: report.overallScore ?? null }) .where(eq(courses.id, courseId)); console.log(`[audit:${courseId.slice(0, 8)}] ✓ audit complete — score: ${report.overallScore ?? "?"}/100`); } catch (err: any) { console.error(`[audit:${courseId.slice(0, 8)}] ✗ audit failed: ${err?.message ?? err}`); await db.update(courses) .set({ auditReport: null }) .where(eq(courses.id, courseId)); } }