197 lines
7.7 KiB
TypeScript
197 lines
7.7 KiB
TypeScript
import { db } from "../db/index";
|
|
import { courses, uploads, topics, lessons, quizQuestions } from "../db/schema";
|
|
import { eq } from "drizzle-orm";
|
|
import { askAI } from "./openrouter";
|
|
|
|
function parseJSON<T>(raw: string): T {
|
|
let text = raw.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch {
|
|
const cleaned = text.replace(/^```[a-z]*\n?/i, "").replace(/\n?```$/i, "").trim();
|
|
return JSON.parse(cleaned);
|
|
}
|
|
}
|
|
|
|
function renderSteps(steps: any[]): string {
|
|
return steps.map((s: any, i: number) => {
|
|
const lines: string[] = [` Step ${i + 1} [${s.type.toUpperCase()}]`];
|
|
|
|
if (s.title) lines.push(` Title: ${s.title}`);
|
|
if (s.body) lines.push(` Body: ${s.body}`);
|
|
if (s.callout) lines.push(` Callout: ${s.callout}`);
|
|
if (Array.isArray(s.bullets)) {
|
|
lines.push(` Bullets:\n${s.bullets.map((b: string) => ` - ${b}`).join("\n")}`);
|
|
}
|
|
if (Array.isArray(s.options)) {
|
|
lines.push(` Options:\n${s.options.map((o: string, oi: number) => ` ${["A","B","C","D"][oi]}: ${o}`).join("\n")}`);
|
|
lines.push(` Answer: ${s.answer}`);
|
|
if (s.explanation) lines.push(` Explanation: ${s.explanation}`);
|
|
}
|
|
|
|
// include branches if present
|
|
if (s.branches && typeof s.branches === "object") {
|
|
for (const [wrongOpt, branch] of Object.entries(s.branches as Record<string, any>)) {
|
|
lines.push(` Branch for wrong answer "${wrongOpt}":`);
|
|
for (const bs of branch.steps ?? []) {
|
|
lines.push(` [BRANCH CONCEPT] ${bs.title ?? ""}: ${bs.body ?? ""}`);
|
|
}
|
|
if (branch.confirmQuestion) {
|
|
const cq = branch.confirmQuestion;
|
|
lines.push(` [CONFIRM QUESTION] ${cq.body}`);
|
|
if (Array.isArray(cq.options)) {
|
|
lines.push(` Options: ${cq.options.join(" | ")}`);
|
|
}
|
|
lines.push(` Answer: ${cq.answer}`);
|
|
}
|
|
if (branch.hardStop) {
|
|
lines.push(` [HARD STOP] ${branch.hardStop}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
return lines.join("\n");
|
|
}).join("\n");
|
|
}
|
|
|
|
export async function auditCourse(courseId: string): Promise<void> {
|
|
console.log(`[audit:${courseId.slice(0, 8)}] starting post-generation audit`);
|
|
|
|
try {
|
|
const course = await db.query.courses.findFirst({ where: eq(courses.id, courseId) });
|
|
if (!course) throw new Error("Course not found");
|
|
|
|
const uploadRows = await db.query.uploads.findMany({ where: eq(uploads.courseId, courseId) });
|
|
const topicRows = await db.query.topics.findMany({
|
|
where: eq(topics.courseId, courseId),
|
|
orderBy: (t, { asc }) => asc(t.order),
|
|
});
|
|
|
|
const lessonMap: Record<string, any> = {};
|
|
for (const topic of topicRows) {
|
|
const lesson = await db.query.lessons.findFirst({ where: eq(lessons.topicId, topic.id) });
|
|
if (lesson) lessonMap[topic.id] = lesson;
|
|
}
|
|
|
|
const quizMap: Record<string, any[]> = {};
|
|
for (const topic of topicRows) {
|
|
quizMap[topic.id] = await db.query.quizQuestions.findMany({ where: eq(quizQuestions.topicId, topic.id) });
|
|
}
|
|
|
|
// full source text — no truncation
|
|
const primaryParts: string[] = [];
|
|
const secondaryParts: string[] = [];
|
|
for (const u of uploadRows) {
|
|
if (!u.extractedText) continue;
|
|
if (u.type === "past_paper" || u.type === "lab_worksheet") {
|
|
primaryParts.push(`--- ${u.filename} (${u.type}) ---\n${u.extractedText}`);
|
|
} else {
|
|
secondaryParts.push(`--- ${u.filename} (${u.type}) ---\n${u.extractedText}`);
|
|
}
|
|
}
|
|
|
|
const primaryText = primaryParts.join("\n\n") || "(none)";
|
|
const secondaryText = secondaryParts.join("\n\n") || "(none)";
|
|
|
|
// topics block
|
|
const topicsBlock = topicRows
|
|
.map((t) => `${t.order + 1}. ${t.title} (difficulty ${t.difficulty}/5)\n ${t.description}`)
|
|
.join("\n");
|
|
|
|
// full lesson block — every step, every branch, every confirm question, every hard stop
|
|
const lessonsBlock = topicRows.map((t) => {
|
|
const lesson = lessonMap[t.id];
|
|
if (!lesson) return `Topic: ${t.title}\n (no lesson generated)`;
|
|
let content: any = {};
|
|
try { content = JSON.parse(lesson.content); } catch {}
|
|
const keyConcepts = (content.keyConcepts ?? []).join(", ");
|
|
const steps = renderSteps(content.steps ?? []);
|
|
return `Topic: ${t.title}\nKey concepts: ${keyConcepts}\n${steps}`;
|
|
}).join("\n\n---\n\n");
|
|
|
|
// full quiz block — question, all options, answer, explanation
|
|
const quizBlock = topicRows.map((t) => {
|
|
const qs = quizMap[t.id] ?? [];
|
|
if (qs.length === 0) return `Topic: ${t.title}\n (no quiz questions)`;
|
|
const qList = qs.map((q: any, i: number) => {
|
|
const lines = [` Q${i + 1} [${q.type}]: ${q.question}`];
|
|
if (q.options) {
|
|
try {
|
|
const opts: string[] = JSON.parse(q.options);
|
|
lines.push(` Options: ${opts.join(" | ")}`);
|
|
} catch {}
|
|
}
|
|
lines.push(` Answer: ${q.answer}`);
|
|
lines.push(` Explanation: ${q.explanation}`);
|
|
return lines.join("\n");
|
|
}).join("\n");
|
|
return `Topic: ${t.title}\n${qList}`;
|
|
}).join("\n\n---\n\n");
|
|
|
|
const prompt = `You are auditing an AI-generated course against a single standard: can a student who completes this course answer every question in every past paper and lab worksheet provided?
|
|
|
|
This is a pass/fail standard, not a score. However, also provide a score for tracking progress toward that standard.
|
|
|
|
PRIMARY SOURCES (past papers + lab worksheets):
|
|
${primaryText}
|
|
|
|
SECONDARY SOURCES (lecture slides):
|
|
${secondaryText}
|
|
|
|
THE GENERATED COURSE:
|
|
${course.title}
|
|
|
|
TOPICS COVERED (${topicRows.length} topics):
|
|
${topicsBlock}
|
|
|
|
LESSONS GENERATED:
|
|
${lessonsBlock}
|
|
|
|
QUIZ QUESTIONS:
|
|
${quizBlock}
|
|
|
|
YOUR AUDIT PROCESS:
|
|
1. Go through every past paper question one by one. For each question, determine: could a student who completed this course answer it fully and correctly? If not, why not — what is missing or underdeveloped?
|
|
2. Go through every lab worksheet task one by one. Same question.
|
|
3. Go through every concept in the lecture slides. Is it covered in the course to a level of full understanding?
|
|
4. Identify every gap — topics missing, algorithms not taught to implementation level, calculations not drilled, pseudocode not covered, procedures not walked through.
|
|
|
|
Return only valid JSON, no markdown:
|
|
{
|
|
"overallScore": 0-100,
|
|
"passesStandard": true/false,
|
|
"examReadiness": "honest plain English verdict",
|
|
"unansweredPaperQuestions": [
|
|
{ "source": "2023 Q2(a)", "question": "brief description", "gap": "what the course fails to teach that would be needed" }
|
|
],
|
|
"coverageAnalysis": { "totalExaminedTopics": 0, "coveredTopics": 0, "coveragePercent": 0 },
|
|
"gaps": [
|
|
{ "topic": "...", "severity": "high/medium/low", "appearsInSources": "...", "courseCoverage": "..." }
|
|
],
|
|
"lessonQuality": [
|
|
{ "topicTitle": "...", "score": 0-100, "notes": "..." }
|
|
],
|
|
"recommendations": ["..."]
|
|
}`;
|
|
|
|
const config = useRuntimeConfig();
|
|
const evaluatorModel = (config as any).openrouterEvaluatorModel;
|
|
|
|
const result = await askAI(
|
|
[{ role: "user", content: prompt }],
|
|
{ maxRetries: 2, maxTokens: 16000, ...(evaluatorModel ? { model: evaluatorModel } : {}) }
|
|
);
|
|
const report = parseJSON<{ overallScore: number }>(result.text);
|
|
|
|
await db.update(courses)
|
|
.set({ auditReport: result.text, auditScore: report.overallScore ?? null })
|
|
.where(eq(courses.id, courseId));
|
|
|
|
console.log(`[audit:${courseId.slice(0, 8)}] ✓ audit complete — score: ${report.overallScore ?? "?"}/100`);
|
|
} catch (err: any) {
|
|
console.error(`[audit:${courseId.slice(0, 8)}] ✗ audit failed: ${err?.message ?? err}`);
|
|
await db.update(courses)
|
|
.set({ auditReport: null })
|
|
.where(eq(courses.id, courseId));
|
|
}
|
|
}
|