import { mkdir, writeFile, access } from "fs/promises"; import { resolve } from "path"; import { askAI } from "./openrouter"; const NARRATION_SYSTEM_PROMPT = `You are a narration script editor for an AI voice actor. Your job is to take educational lesson text and prepare it to be read aloud naturally and engagingly. Rules: - Do NOT change the meaning, facts, or structure of the content. You are not rewriting it. - Fix anything that would sound awkward or robotic when spoken: remove markdown formatting (asterisks, backticks, hashes), spell out acronyms where helpful, rephrase code snippets or technical shorthand into speakable language. - Add square bracket cues to give the voice character and pacing. These are the only ones you may use: [pause], [long pause], [sighs], [laughs], [clears throat], [hesitates]. - Use [pause] at natural breath points — after key ideas, before a new concept, or mid-sentence where a human would pause for effect. Don't overdo it; one every few sentences at most. - Use [sighs] or [laughs] very sparingly — only where a human narrator genuinely would. A [sighs] before a tricky concept, a [laughs] when something is ironic or light. Maybe once or twice per lesson, if at all. - Keep the tone warm, clear, and conversational — like a knowledgeable friend explaining something, not a textbook being read aloud. - Return ONLY the modified narration text. No commentary, no explanation, no quotes around the output.`; async function humaniseTTSText(text: string): Promise { try { const result = await askAI( [ { role: "system", content: NARRATION_SYSTEM_PROMPT }, { role: "user", content: text }, ], { temperature: 0.5, maxTokens: 2048 } ); return result.text.trim(); } catch (err: any) { console.error(`[tts] humanise failed, using raw text: ${err?.message ?? err}`); return text; } } export interface AudioChunk { text: string; start: number; end: number; } export interface TTSResult { audioPath: string; audioChunks: AudioChunk[]; cost: number; } async function callElevenLabs( text: string, apiKey: string, voiceId: string ): Promise<{ audio: Buffer; chunks: AudioChunk[]; cost: number } | null> { const res = await fetch( `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/with-timestamps`, { method: "POST", headers: { "xi-api-key": apiKey, "Content-Type": "application/json", }, body: JSON.stringify({ text, model_id: "eleven_turbo_v2_5", output_format: "mp3_44100_128", }), signal: AbortSignal.timeout(60_000), } ); if (!res.ok) { const errText = await res.text().catch(() => ""); console.error(`[tts] ElevenLabs error ${res.status}: ${errText}`); return null; } const charCost = parseInt(res.headers.get("character-cost") ?? "0", 10); const cost = charCost * 0.0003; const data = await res.json() as { audio_base64: string; alignment: { characters: string[]; character_start_times_seconds: number[]; character_end_times_seconds: number[]; }; }; const audio = Buffer.from(data.audio_base64, "base64"); const { characters, character_start_times_seconds: starts, character_end_times_seconds: ends } = data.alignment; const words: { word: string; charStart: number; charEnd: number }[] = []; let i = 0; while (i < text.length) { while (i < text.length && /\s/.test(text[i])) i++; if (i >= text.length) break; const wordStart = i; while (i < text.length && !/\s/.test(text[i])) i++; words.push({ word: text.slice(wordStart, i), charStart: wordStart, charEnd: i - 1 }); } function timeForChar(charIdx: number, side: "start" | "end"): number { let accumulated = 0; for (let ci = 0; ci < characters.length; ci++) { const c = characters[ci]; if (accumulated + c.length > charIdx) { return side === "start" ? starts[ci] : ends[ci]; } accumulated += c.length; } const last = side === "start" ? starts[starts.length - 1] : ends[ends.length - 1]; return last ?? 0; } const chunks: AudioChunk[] = []; for (let wi = 0; wi < words.length; wi += 3) { const slice = words.slice(wi, wi + 3); chunks.push({ text: slice.map(w => w.word).join(" "), start: timeForChar(slice[0].charStart, "start"), end: timeForChar(slice[slice.length - 1].charEnd, "end"), }); } return { audio, chunks, cost }; } // Fish Audio doesnt return alignment data, so we return empty chunks async function callFishAudio( text: string, apiKey: string, voiceId: string ): Promise<{ audio: Buffer; chunks: AudioChunk[]; cost: number } | null> { const res = await fetch("https://api.fish.audio/v1/tts", { method: "POST", headers: { "Authorization": `Bearer ${apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ text, reference_id: voiceId, format: "mp3", mp3_bitrate: 128, streaming: false, }), signal: AbortSignal.timeout(60_000), }); if (!res.ok) { const errText = await res.text().catch(() => ""); console.error(`[tts] Fish Audio error ${res.status}: ${errText}`); return null; } const audio = Buffer.from(await res.arrayBuffer()); return { audio, chunks: [], cost: 0 }; } function getProvider() { const config = useRuntimeConfig(); return (config.ttsProvider as string | undefined)?.toLowerCase() ?? "elevenlabs"; } async function callTTS( text: string ): Promise<{ audio: Buffer; chunks: AudioChunk[]; cost: number } | null> { const config = useRuntimeConfig(); const provider = getProvider(); text = await humaniseTTSText(text); if (provider === "fishaudio") { const apiKey = config.fishAudioApiKey as string; const voiceId = (config.public.fishAudioVoiceId || config.fishAudioVoiceId) as string; if (!apiKey) return null; return callFishAudio(text, apiKey, voiceId); } const apiKey = config.elevenlabsApiKey as string; const voiceId = (config.public.elevenlabsVoiceId || config.elevenlabsVoiceId) as string; if (!apiKey) return null; return callElevenLabs(text, apiKey, voiceId); } export async function generateStepTTS( text: string, lessonId: string, stepIndex: number ): Promise { try { const result = await callTTS(text); if (!result) return null; const dir = resolve(process.cwd(), `public/audio/lessons/${lessonId}`); await mkdir(dir, { recursive: true }); const filename = `step_${stepIndex}.mp3`; await writeFile(`${dir}/${filename}`, result.audio); const audioPath = `/audio/lessons/${lessonId}/${filename}`; console.log(`[tts] step ${stepIndex} for lesson ${lessonId} — ${result.chunks.length} chunks | $${result.cost.toFixed(4)}`); return { audioPath, audioChunks: result.chunks, cost: result.cost }; } catch (err: any) { console.error(`[tts] step ${stepIndex} for lesson ${lessonId} failed: ${err?.message ?? err}`); return null; } } export async function generateQuestionTTS( text: string, lessonId: string, stepIndex: number ): Promise { try { const result = await callTTS(text); if (!result) return null; const dir = resolve(process.cwd(), `public/audio/lessons/${lessonId}`); await mkdir(dir, { recursive: true }); const filename = `step_${stepIndex}_question.mp3`; await writeFile(`${dir}/${filename}`, result.audio); const audioPath = `/audio/lessons/${lessonId}/${filename}`; return { audioPath, audioChunks: result.chunks, cost: result.cost }; } catch (err: any) { console.error(`[tts] question ${stepIndex} for lesson ${lessonId} failed: ${err?.message ?? err}`); return null; } } export async function generateOptionTTS( text: string, lessonId: string, stepIndex: number, optionIndex: number ): Promise<{ audioPath: string; cost: number } | null> { try { const result = await callTTS(text); if (!result) return null; const dir = resolve(process.cwd(), `public/audio/lessons/${lessonId}`); await mkdir(dir, { recursive: true }); const filename = `step_${stepIndex}_option_${optionIndex}.mp3`; await writeFile(`${dir}/${filename}`, result.audio); const audioPath = `/audio/lessons/${lessonId}/${filename}`; return { audioPath, cost: result.cost }; } catch (err: any) { console.error(`[tts] option ${stepIndex}/${optionIndex} for lesson ${lessonId} failed: ${err?.message ?? err}`); return null; } } // generate a plain audio clip with no timestamp data (used for label clips) // opts are ElevenLabs-specific and are ignored when using Fish Audio export async function generateClip( text: string, outPath: string, apiKey: string, voiceId: string, opts?: { model?: string; voice_settings?: Record } ): Promise<{ cost: number } | null> { const provider = getProvider(); try { let buffer: Buffer; let cost = 0; if (provider === "fishaudio") { const config = useRuntimeConfig(); const fishKey = config.fishAudioApiKey as string; const fishVoice = (config.public.fishAudioVoiceId || config.fishAudioVoiceId as string) || voiceId; const res = await fetch("https://api.fish.audio/v1/tts", { method: "POST", headers: { "Authorization": `Bearer ${fishKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ text, reference_id: fishVoice, format: "mp3", mp3_bitrate: 128, streaming: false, }), signal: AbortSignal.timeout(60_000), }); if (!res.ok) { const errText = await res.text().catch(() => ""); console.error(`[tts] Fish Audio clip error ${res.status}: ${errText}`); return null; } buffer = Buffer.from(await res.arrayBuffer()); } else { const res = await fetch( `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, { method: "POST", headers: { "xi-api-key": apiKey, "Content-Type": "application/json", }, body: JSON.stringify({ text, model_id: opts?.model ?? "eleven_turbo_v2_5", output_format: "mp3_44100_128", ...(opts?.voice_settings ? { voice_settings: opts.voice_settings } : {}), }), signal: AbortSignal.timeout(60_000), } ); if (!res.ok) { const errText = await res.text().catch(() => ""); console.error(`[tts] clip error ${res.status}: ${errText}`); return null; } const charCost = parseInt(res.headers.get("character-cost") ?? "0", 10); cost = charCost * 0.0003; buffer = Buffer.from(await res.arrayBuffer()); } await mkdir(resolve(process.cwd(), "public/audio/labels"), { recursive: true }); await writeFile(outPath, buffer); return { cost }; } catch (err: any) { console.error(`[tts] clip failed for ${outPath}: ${err?.message ?? err}`); return null; } } export async function generateTTSToPath( text: string, lessonId: string, filename: string ): Promise { try { const result = await callTTS(text); if (!result) return null; const dir = resolve(process.cwd(), `public/audio/lessons/${lessonId}`); await mkdir(dir, { recursive: true }); await writeFile(`${dir}/${filename}`, result.audio); const audioPath = `/audio/lessons/${lessonId}/${filename}`; return { audioPath, audioChunks: result.chunks, cost: result.cost }; } catch (err: any) { console.error(`[tts] ${filename} for lesson ${lessonId} failed: ${err?.message ?? err}`); return null; } } export async function fileExists(path: string): Promise { try { await access(path); return true; } catch { return false; } }