import { db } from "../db/index"; import { topics, lessons } from "../db/schema"; import { eq, and, lt } from "drizzle-orm"; import { generateLesson } from "../utils/generateLesson"; import { generateBranches } from "../utils/generateBranches"; import { generateStepTTS, generateQuestionTTS, generateOptionTTS, generateTTSToPath, } from "../utils/generateTTS"; const INTERVAL_MS = 5 * 60 * 1000; const STALE_MINUTES = 15; function staleCutoff() { return new Date(Date.now() - STALE_MINUTES * 60 * 1000).toISOString(); } async function repairCycle() { // ── 1. failed topic generation ────────────────────────────────────────── try { const errored = await db .update(topics) .set({ status: "pending" }) .where(eq(topics.status, "error")) .returning({ id: topics.id }); if (errored.length > 0) { console.log(`[repair] resetting ${errored.length} errored topic(s) to pending`); for (const t of errored) { generateLesson(t.id).catch((err: any) => { console.error(`[repair] generateLesson failed for ${t.id.slice(0, 8)}: ${err?.message ?? err}`); }); } } } catch (err: any) { console.error("[repair] failed to repair errored topics:", err?.message ?? err); } // ── 2. stale generating topics (server died mid-generation) ───────────── try { // topics dont have createdAt so we use a rough heuristic: if a lesson // exists for this topic, generation got far enough; otherwise something // died early. Either way reset to pending and let generateLesson figure it out. const stuckGenerating = await db.query.topics.findMany({ where: eq(topics.status, "generating"), }); const cutoff = staleCutoff(); for (const t of stuckGenerating) { const lesson = await db.query.lessons.findFirst({ where: eq(lessons.topicId, t.id) }); if (lesson) { // lesson row exists — server died after insert but before topic status update console.log(`[repair] topic ${t.id.slice(0, 8)} stuck generating but lesson exists — marking ready`); await db.update(topics).set({ status: "ready" }).where(eq(topics.id, t.id)); } else { // no lesson at all — server died before the insert, need to regenerate console.log(`[repair] topic ${t.id.slice(0, 8)} stuck generating with no lesson — resetting to pending`); await db.update(topics).set({ status: "pending" }).where(eq(topics.id, t.id)); generateLesson(t.id).catch((err: any) => { console.error(`[repair] generateLesson (stuck) failed for ${t.id.slice(0, 8)}: ${err?.message ?? err}`); }); } } } catch (err: any) { console.error("[repair] failed to repair stuck generating topics:", err?.message ?? err); } // ── 3. failed branch generation ───────────────────────────────────────── try { const errored = await db .update(lessons) .set({ branchStatus: "pending" }) .where(eq(lessons.branchStatus, "error")) .returning({ id: lessons.id, topicId: lessons.topicId }); if (errored.length > 0) { console.log(`[repair] resetting ${errored.length} errored lesson branch(es) to pending`); for (const l of errored) { generateBranches(l.topicId, l.id).catch((err: any) => { console.error(`[repair] generateBranches failed for ${l.id.slice(0, 8)}: ${err?.message ?? err}`); }); } } } catch (err: any) { console.error("[repair] failed to repair errored lesson branches:", err?.message ?? err); } // ── 4. stale generating branches (server died mid-branch) ──────────────── try { const cutoff = staleCutoff(); const stale = await db .update(lessons) .set({ branchStatus: "pending" }) .where( and( eq(lessons.branchStatus, "generating"), lt(lessons.createdAt, cutoff) ) ) .returning({ id: lessons.id, topicId: lessons.topicId }); if (stale.length > 0) { console.log(`[repair] resetting ${stale.length} stale generating branch(es) to pending`); for (const l of stale) { generateBranches(l.topicId, l.id).catch((err: any) => { console.error(`[repair] generateBranches (stale generating) failed for ${l.id.slice(0, 8)}: ${err?.message ?? err}`); }); } } } catch (err: any) { console.error("[repair] failed to repair stale generating branches:", err?.message ?? err); } // ── 5. stale pending branches (never kicked off or abandoned) ──────────── try { const cutoff = staleCutoff(); const stale = await db .update(lessons) .set({ branchStatus: "pending" }) .where( and( eq(lessons.branchStatus, "pending"), lt(lessons.createdAt, cutoff) ) ) .returning({ id: lessons.id, topicId: lessons.topicId }); if (stale.length > 0) { console.log(`[repair] restarting ${stale.length} stale pending branch(es)`); for (const l of stale) { generateBranches(l.topicId, l.id).catch((err: any) => { console.error(`[repair] generateBranches (stale pending) failed for ${l.id.slice(0, 8)}: ${err?.message ?? err}`); }); } } } catch (err: any) { console.error("[repair] failed to repair stale pending branches:", err?.message ?? err); } // ── 6. missing main lesson audio ───────────────────────────────────────── try { const readyTopics = await db.query.topics.findMany({ where: eq(topics.status, "ready") }); for (const topic of readyTopics) { const lesson = await db.query.lessons.findFirst({ where: eq(lessons.topicId, topic.id) }); if (!lesson) continue; let content: { keyConcepts: string[]; analogiesUsed: string[]; steps: any[] }; try { content = JSON.parse(lesson.content); } catch { continue; } if (!Array.isArray(content.steps)) continue; type Task = () => Promise; const tasks: Task[] = []; let dirty = false; for (let si = 0; si < content.steps.length; si++) { const step = content.steps[si]; if (step.type === "concept" || step.type === "example" || step.type === "summary") { const text = step.type === "summary" ? (Array.isArray(step.bullets) ? step.bullets.join(". ") : "") : [step.body, step.callout].filter(Boolean).join(" "); if (!text.trim() || step.audioPath) continue; const siCopy = si; tasks.push(async () => { const r = await generateStepTTS(text, lesson.id, siCopy); if (r) { content.steps[siCopy].audioPath = r.audioPath; content.steps[siCopy].audioChunks = r.audioChunks; dirty = true; } }); } else if (step.type === "question") { if (step.body?.trim() && !step.questionAudioPath) { const siCopy = si; const body = step.body; tasks.push(async () => { const r = await generateQuestionTTS(body, lesson.id, siCopy); if (r) { content.steps[siCopy].questionAudioPath = r.audioPath; content.steps[siCopy].questionAudioChunks = r.audioChunks; dirty = true; } }); } if (Array.isArray(step.options)) { if (!step.optionAudioPaths) { content.steps[si].optionAudioPaths = new Array(step.options.length).fill(null); } for (let oi = 0; oi < step.options.length; oi++) { const optText = step.options[oi]; if (!optText?.trim()) continue; if (content.steps[si].optionAudioPaths?.[oi]) continue; const siCopy = si; const oiCopy = oi; tasks.push(async () => { const r = await generateOptionTTS(optText, lesson.id, siCopy, oiCopy); if (r) { content.steps[siCopy].optionAudioPaths[oiCopy] = r.audioPath; dirty = true; } }); } } } } if (tasks.length === 0) continue; console.log(`[repair] lesson ${lesson.id.slice(0, 8)} — ${tasks.length} missing audio task(s)`); const BATCH = 3; for (let i = 0; i < tasks.length; i += BATCH) { await Promise.all(tasks.slice(i, i + BATCH).map((fn) => fn())); } if (dirty) { await db.update(lessons) .set({ content: JSON.stringify(content) }) .where(eq(lessons.id, lesson.id)); console.log(`[repair] lesson ${lesson.id.slice(0, 8)} audio repaired`); } } } catch (err: any) { console.error("[repair] failed during lesson audio repair:", err?.message ?? err); } // ── 7. missing branch audio ─────────────────────────────────────────────── try { const readyLessons = await db.query.lessons.findMany({ where: eq(lessons.branchStatus, "ready"), }); for (const lesson of readyLessons) { let content: { keyConcepts: string[]; analogiesUsed: string[]; steps: any[] }; try { content = JSON.parse(lesson.content); } catch { continue; } if (!Array.isArray(content.steps)) continue; type Task = () => Promise; const tasks: Task[] = []; let dirty = false; for (let si = 0; si < content.steps.length; si++) { const step = content.steps[si]; if (step.type !== "question" || !step.branches) continue; const wrongOptions = (step.options as string[]).filter((o: string) => o !== step.answer); for (let bi = 0; bi < wrongOptions.length; bi++) { const wrongOpt = wrongOptions[bi]; const branch = step.branches[wrongOpt]; if (!branch) continue; for (let bsi = 0; bsi < (branch.steps ?? []).length; bsi++) { const bStep = branch.steps[bsi]; const text = [bStep.body, bStep.callout].filter(Boolean).join(" "); if (!text.trim() || bStep.audioPath) continue; const siCopy = si; const biCopy = bi; const bsiCopy = bsi; const woCopy = wrongOpt; tasks.push(async () => { const r = await generateTTSToPath(text, lesson.id, `branch_${siCopy}_${biCopy}_step_${bsiCopy}.mp3`); if (r) { content.steps[siCopy].branches[woCopy].steps[bsiCopy].audioPath = r.audioPath; content.steps[siCopy].branches[woCopy].steps[bsiCopy].audioChunks = r.audioChunks; dirty = true; } }); } const cq = branch.confirmQuestion; if (cq?.body?.trim() && !cq.questionAudioPath) { const siCopy = si; const biCopy = bi; const woCopy = wrongOpt; tasks.push(async () => { const r = await generateTTSToPath(cq.body, lesson.id, `branch_${siCopy}_${biCopy}_confirm_q.mp3`); if (r) { content.steps[siCopy].branches[woCopy].confirmQuestion.questionAudioPath = r.audioPath; content.steps[siCopy].branches[woCopy].confirmQuestion.questionAudioChunks = r.audioChunks; dirty = true; } }); } if (Array.isArray(cq?.options)) { if (!cq.optionAudioPaths) { branch.confirmQuestion.optionAudioPaths = new Array(cq.options.length).fill(null); } for (let oi = 0; oi < cq.options.length; oi++) { const optText = cq.options[oi]; if (!optText?.trim()) continue; if (branch.confirmQuestion.optionAudioPaths?.[oi]) continue; const siCopy = si; const biCopy = bi; const oiCopy = oi; const woCopy = wrongOpt; tasks.push(async () => { const r = await generateTTSToPath(optText, lesson.id, `branch_${siCopy}_${biCopy}_confirm_opt_${oiCopy}.mp3`); if (r) { content.steps[siCopy].branches[woCopy].confirmQuestion.optionAudioPaths[oiCopy] = r.audioPath; dirty = true; } }); } } if (branch.hardStop?.trim() && !branch.hardStopAudioPath) { const siCopy = si; const biCopy = bi; const woCopy = wrongOpt; tasks.push(async () => { const r = await generateTTSToPath(branch.hardStop, lesson.id, `branch_${siCopy}_${biCopy}_hardstop.mp3`); if (r) { content.steps[siCopy].branches[woCopy].hardStopAudioPath = r.audioPath; dirty = true; } }); } } } if (tasks.length === 0) continue; console.log(`[repair] lesson ${lesson.id.slice(0, 8)} — ${tasks.length} missing branch audio task(s)`); const BATCH = 3; for (let i = 0; i < tasks.length; i += BATCH) { await Promise.all(tasks.slice(i, i + BATCH).map((fn) => fn())); } if (dirty) { await db.update(lessons) .set({ content: JSON.stringify(content) }) .where(eq(lessons.id, lesson.id)); console.log(`[repair] lesson ${lesson.id.slice(0, 8)} branch audio repaired`); } } } catch (err: any) { console.error("[repair] failed during branch audio repair:", err?.message ?? err); } } export default defineNitroPlugin(() => { console.log("[repair] repairBrokenLessons plugin started"); repairCycle(); setInterval(repairCycle, INTERVAL_MS); });