371 lines
No EOL
14 KiB
TypeScript
371 lines
No EOL
14 KiB
TypeScript
import { db } from "../db/index";
|
|
import { topics, lessons } from "../db/schema";
|
|
import { eq, and, lt } from "drizzle-orm";
|
|
import { generateLesson } from "../utils/generateLesson";
|
|
import { generateBranches } from "../utils/generateBranches";
|
|
import {
|
|
generateStepTTS,
|
|
generateQuestionTTS,
|
|
generateOptionTTS,
|
|
generateTTSToPath,
|
|
} from "../utils/generateTTS";
|
|
|
|
const INTERVAL_MS = 5 * 60 * 1000;
|
|
const STALE_PENDING_MINUTES = 15;
|
|
const STALE_GENERATING_MINUTES = 45; // generating can take a while for big lessons
|
|
|
|
function staleCutoff(minutes: number) {
|
|
return new Date(Date.now() - minutes * 60 * 1000).toISOString();
|
|
}
|
|
|
|
async function repairCycle() {
|
|
|
|
// ── 1. failed topic generation ──────────────────────────────────────────
|
|
try {
|
|
const errored = await db
|
|
.update(topics)
|
|
.set({ status: "pending" })
|
|
.where(eq(topics.status, "error"))
|
|
.returning({ id: topics.id });
|
|
|
|
if (errored.length > 0) {
|
|
console.log(`[repair] resetting ${errored.length} errored topic(s) to pending`);
|
|
|
|
for (const t of errored) {
|
|
generateLesson(t.id).catch((err: any) => {
|
|
console.error(`[repair] generateLesson failed for ${t.id.slice(0, 8)}: ${err?.message ?? err}`);
|
|
});
|
|
}
|
|
}
|
|
} catch (err: any) {
|
|
console.error("[repair] failed to repair errored topics:", err?.message ?? err);
|
|
}
|
|
|
|
// ── 2. stale generating topics (server died mid-generation) ─────────────
|
|
try {
|
|
// topics dont have createdAt so we use a rough heuristic: if a lesson
|
|
// exists for this topic, generation got far enough; otherwise something
|
|
// died early. Either way reset to pending and let generateLesson figure it out.
|
|
const stuckGenerating = await db.query.topics.findMany({
|
|
where: eq(topics.status, "generating"),
|
|
});
|
|
|
|
for (const t of stuckGenerating) {
|
|
const lesson = await db.query.lessons.findFirst({ where: eq(lessons.topicId, t.id) });
|
|
|
|
if (lesson) {
|
|
// lesson row exists — server died after insert but before topic status update
|
|
console.log(`[repair] topic ${t.id.slice(0, 8)} stuck generating but lesson exists — marking ready`);
|
|
await db.update(topics).set({ status: "ready" }).where(eq(topics.id, t.id));
|
|
} else {
|
|
// no lesson at all — server died before the insert, need to regenerate
|
|
console.log(`[repair] topic ${t.id.slice(0, 8)} stuck generating with no lesson — resetting to pending`);
|
|
await db.update(topics).set({ status: "pending" }).where(eq(topics.id, t.id));
|
|
generateLesson(t.id).catch((err: any) => {
|
|
console.error(`[repair] generateLesson (stuck) failed for ${t.id.slice(0, 8)}: ${err?.message ?? err}`);
|
|
});
|
|
}
|
|
}
|
|
} catch (err: any) {
|
|
console.error("[repair] failed to repair stuck generating topics:", err?.message ?? err);
|
|
}
|
|
|
|
// ── 3. failed branch generation ─────────────────────────────────────────
|
|
try {
|
|
const errored = await db
|
|
.update(lessons)
|
|
.set({ branchStatus: "pending" })
|
|
.where(eq(lessons.branchStatus, "error"))
|
|
.returning({ id: lessons.id, topicId: lessons.topicId });
|
|
|
|
if (errored.length > 0) {
|
|
console.log(`[repair] resetting ${errored.length} errored lesson branch(es) to pending`);
|
|
|
|
for (const l of errored) {
|
|
generateBranches(l.topicId, l.id).catch((err: any) => {
|
|
console.error(`[repair] generateBranches failed for ${l.id.slice(0, 8)}: ${err?.message ?? err}`);
|
|
});
|
|
}
|
|
}
|
|
} catch (err: any) {
|
|
console.error("[repair] failed to repair errored lesson branches:", err?.message ?? err);
|
|
}
|
|
|
|
// ── 4. stale generating branches (server died mid-branch) ────────────────
|
|
try {
|
|
const cutoff = staleCutoff(STALE_GENERATING_MINUTES);
|
|
|
|
const stale = await db
|
|
.update(lessons)
|
|
.set({ branchStatus: "pending" })
|
|
.where(
|
|
and(
|
|
eq(lessons.branchStatus, "generating"),
|
|
lt(lessons.createdAt, cutoff)
|
|
)
|
|
)
|
|
.returning({ id: lessons.id, topicId: lessons.topicId });
|
|
|
|
if (stale.length > 0) {
|
|
console.log(`[repair] resetting ${stale.length} stale generating branch(es) to pending`);
|
|
|
|
for (const l of stale) {
|
|
generateBranches(l.topicId, l.id).catch((err: any) => {
|
|
console.error(`[repair] generateBranches (stale generating) failed for ${l.id.slice(0, 8)}: ${err?.message ?? err}`);
|
|
});
|
|
}
|
|
}
|
|
} catch (err: any) {
|
|
console.error("[repair] failed to repair stale generating branches:", err?.message ?? err);
|
|
}
|
|
|
|
// ── 5. stale pending branches (never kicked off or abandoned) ────────────
|
|
try {
|
|
const cutoff = staleCutoff(STALE_PENDING_MINUTES);
|
|
|
|
const stale = await db
|
|
.update(lessons)
|
|
.set({ branchStatus: "pending" })
|
|
.where(
|
|
and(
|
|
eq(lessons.branchStatus, "pending"),
|
|
lt(lessons.createdAt, cutoff)
|
|
)
|
|
)
|
|
.returning({ id: lessons.id, topicId: lessons.topicId });
|
|
|
|
if (stale.length > 0) {
|
|
console.log(`[repair] restarting ${stale.length} stale pending branch(es)`);
|
|
|
|
for (const l of stale) {
|
|
generateBranches(l.topicId, l.id).catch((err: any) => {
|
|
console.error(`[repair] generateBranches (stale pending) failed for ${l.id.slice(0, 8)}: ${err?.message ?? err}`);
|
|
});
|
|
}
|
|
}
|
|
} catch (err: any) {
|
|
console.error("[repair] failed to repair stale pending branches:", err?.message ?? err);
|
|
}
|
|
|
|
// ── 6. missing main lesson audio ─────────────────────────────────────────
|
|
try {
|
|
const readyTopics = await db.query.topics.findMany({ where: eq(topics.status, "ready") });
|
|
|
|
for (const topic of readyTopics) {
|
|
const lesson = await db.query.lessons.findFirst({ where: eq(lessons.topicId, topic.id) });
|
|
if (!lesson) continue;
|
|
|
|
let content: { keyConcepts: string[]; analogiesUsed: string[]; steps: any[] };
|
|
try {
|
|
content = JSON.parse(lesson.content);
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
if (!Array.isArray(content.steps)) continue;
|
|
|
|
type Task = () => Promise<void>;
|
|
const tasks: Task[] = [];
|
|
let dirty = false;
|
|
|
|
for (let si = 0; si < content.steps.length; si++) {
|
|
const step = content.steps[si];
|
|
|
|
if (step.type === "concept" || step.type === "example" || step.type === "summary") {
|
|
const text = step.type === "summary"
|
|
? (Array.isArray(step.bullets) ? step.bullets.join(". ") : "")
|
|
: [step.body, step.callout].filter(Boolean).join(" ");
|
|
|
|
if (!text.trim() || step.audioPath) continue;
|
|
|
|
const siCopy = si;
|
|
tasks.push(async () => {
|
|
const r = await generateStepTTS(text, lesson.id, siCopy);
|
|
if (r) {
|
|
content.steps[siCopy].audioPath = r.audioPath;
|
|
content.steps[siCopy].audioChunks = r.audioChunks;
|
|
dirty = true;
|
|
}
|
|
});
|
|
|
|
} else if (step.type === "question") {
|
|
if (step.body?.trim() && !step.questionAudioPath) {
|
|
const siCopy = si;
|
|
const body = step.body;
|
|
tasks.push(async () => {
|
|
const r = await generateQuestionTTS(body, lesson.id, siCopy);
|
|
if (r) {
|
|
content.steps[siCopy].questionAudioPath = r.audioPath;
|
|
content.steps[siCopy].questionAudioChunks = r.audioChunks;
|
|
dirty = true;
|
|
}
|
|
});
|
|
}
|
|
|
|
if (Array.isArray(step.options)) {
|
|
if (!step.optionAudioPaths) {
|
|
content.steps[si].optionAudioPaths = new Array(step.options.length).fill(null);
|
|
}
|
|
|
|
for (let oi = 0; oi < step.options.length; oi++) {
|
|
const optText = step.options[oi];
|
|
if (!optText?.trim()) continue;
|
|
if (content.steps[si].optionAudioPaths?.[oi]) continue;
|
|
|
|
const siCopy = si;
|
|
const oiCopy = oi;
|
|
tasks.push(async () => {
|
|
const r = await generateOptionTTS(optText, lesson.id, siCopy, oiCopy);
|
|
if (r) {
|
|
content.steps[siCopy].optionAudioPaths[oiCopy] = r.audioPath;
|
|
dirty = true;
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (tasks.length === 0) continue;
|
|
|
|
console.log(`[repair] lesson ${lesson.id.slice(0, 8)} — ${tasks.length} missing audio task(s)`);
|
|
|
|
const BATCH = 3;
|
|
for (let i = 0; i < tasks.length; i += BATCH) {
|
|
await Promise.all(tasks.slice(i, i + BATCH).map((fn) => fn()));
|
|
}
|
|
|
|
if (dirty) {
|
|
await db.update(lessons)
|
|
.set({ content: JSON.stringify(content) })
|
|
.where(eq(lessons.id, lesson.id));
|
|
console.log(`[repair] lesson ${lesson.id.slice(0, 8)} audio repaired`);
|
|
}
|
|
}
|
|
} catch (err: any) {
|
|
console.error("[repair] failed during lesson audio repair:", err?.message ?? err);
|
|
}
|
|
|
|
// ── 7. missing branch audio ───────────────────────────────────────────────
|
|
try {
|
|
const readyLessons = await db.query.lessons.findMany({
|
|
where: eq(lessons.branchStatus, "ready"),
|
|
});
|
|
|
|
for (const lesson of readyLessons) {
|
|
let content: { keyConcepts: string[]; analogiesUsed: string[]; steps: any[] };
|
|
try {
|
|
content = JSON.parse(lesson.content);
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
if (!Array.isArray(content.steps)) continue;
|
|
|
|
type Task = () => Promise<void>;
|
|
const tasks: Task[] = [];
|
|
let dirty = false;
|
|
|
|
for (let si = 0; si < content.steps.length; si++) {
|
|
const step = content.steps[si];
|
|
if (step.type !== "question" || !step.branches) continue;
|
|
|
|
const wrongOptions = (step.options as string[]).filter((o: string) => o !== step.answer);
|
|
|
|
for (let bi = 0; bi < wrongOptions.length; bi++) {
|
|
const wrongOpt = wrongOptions[bi];
|
|
const branch = step.branches[wrongOpt];
|
|
if (!branch) continue;
|
|
|
|
|
|
for (let bsi = 0; bsi < (branch.steps ?? []).length; bsi++) {
|
|
const bStep = branch.steps[bsi];
|
|
const text = [bStep.body, bStep.callout].filter(Boolean).join(" ");
|
|
if (!text.trim() || bStep.audioPath) continue;
|
|
|
|
const siCopy = si; const biCopy = bi; const bsiCopy = bsi; const woCopy = wrongOpt;
|
|
tasks.push(async () => {
|
|
const r = await generateTTSToPath(text, lesson.id, `branch_${siCopy}_${biCopy}_step_${bsiCopy}.mp3`);
|
|
if (r) {
|
|
content.steps[siCopy].branches[woCopy].steps[bsiCopy].audioPath = r.audioPath;
|
|
content.steps[siCopy].branches[woCopy].steps[bsiCopy].audioChunks = r.audioChunks;
|
|
dirty = true;
|
|
}
|
|
});
|
|
}
|
|
|
|
const cq = branch.confirmQuestion;
|
|
if (cq?.body?.trim() && !cq.questionAudioPath) {
|
|
const siCopy = si; const biCopy = bi; const woCopy = wrongOpt;
|
|
tasks.push(async () => {
|
|
const r = await generateTTSToPath(cq.body, lesson.id, `branch_${siCopy}_${biCopy}_confirm_q.mp3`);
|
|
if (r) {
|
|
content.steps[siCopy].branches[woCopy].confirmQuestion.questionAudioPath = r.audioPath;
|
|
content.steps[siCopy].branches[woCopy].confirmQuestion.questionAudioChunks = r.audioChunks;
|
|
dirty = true;
|
|
}
|
|
});
|
|
}
|
|
|
|
if (Array.isArray(cq?.options)) {
|
|
if (!cq.optionAudioPaths) {
|
|
branch.confirmQuestion.optionAudioPaths = new Array(cq.options.length).fill(null);
|
|
}
|
|
|
|
for (let oi = 0; oi < cq.options.length; oi++) {
|
|
const optText = cq.options[oi];
|
|
if (!optText?.trim()) continue;
|
|
if (branch.confirmQuestion.optionAudioPaths?.[oi]) continue;
|
|
|
|
const siCopy = si; const biCopy = bi; const oiCopy = oi; const woCopy = wrongOpt;
|
|
tasks.push(async () => {
|
|
const r = await generateTTSToPath(optText, lesson.id, `branch_${siCopy}_${biCopy}_confirm_opt_${oiCopy}.mp3`);
|
|
if (r) {
|
|
content.steps[siCopy].branches[woCopy].confirmQuestion.optionAudioPaths[oiCopy] = r.audioPath;
|
|
dirty = true;
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
if (branch.hardStop?.trim() && !branch.hardStopAudioPath) {
|
|
const siCopy = si; const biCopy = bi; const woCopy = wrongOpt;
|
|
tasks.push(async () => {
|
|
const r = await generateTTSToPath(branch.hardStop, lesson.id, `branch_${siCopy}_${biCopy}_hardstop.mp3`);
|
|
if (r) {
|
|
content.steps[siCopy].branches[woCopy].hardStopAudioPath = r.audioPath;
|
|
dirty = true;
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (tasks.length === 0) continue;
|
|
|
|
console.log(`[repair] lesson ${lesson.id.slice(0, 8)} — ${tasks.length} missing branch audio task(s)`);
|
|
|
|
const BATCH = 3;
|
|
for (let i = 0; i < tasks.length; i += BATCH) {
|
|
await Promise.all(tasks.slice(i, i + BATCH).map((fn) => fn()));
|
|
}
|
|
|
|
if (dirty) {
|
|
await db.update(lessons)
|
|
.set({ content: JSON.stringify(content) })
|
|
.where(eq(lessons.id, lesson.id));
|
|
console.log(`[repair] lesson ${lesson.id.slice(0, 8)} branch audio repaired`);
|
|
}
|
|
}
|
|
} catch (err: any) {
|
|
console.error("[repair] failed during branch audio repair:", err?.message ?? err);
|
|
}
|
|
}
|
|
|
|
export default defineNitroPlugin(() => {
|
|
console.log("[repair] repairBrokenLessons plugin started");
|
|
|
|
repairCycle();
|
|
|
|
setInterval(repairCycle, INTERVAL_MS);
|
|
}); |