diff --git a/scripts/melo_server.py b/scripts/melo_server.py new file mode 100644 index 0000000..2f13c4e --- /dev/null +++ b/scripts/melo_server.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""MeloTTS server for inferon. + +Long-running subprocess: loads the MeloTTS model once, then reads lines of +text on stdin and writes " " per line on stdout. +stderr passes through for logging. + +Protocol: + -> one line of text to synthesize + <- e.g. "/tmp/inferon/tts-1723.wav 3.42" + +Kill with SIGTERM — no cleanup needed, WAVs live in /tmp. +""" + +import sys +import os +import time +import signal +import warnings + +warnings.filterwarnings("ignore") + +MELO_ROOT = os.path.expanduser("~/Projects/MeloTTS") +CKPT = os.path.join(MELO_ROOT, "ckpt/MeloTTS-English-v3/checkpoint.pth") +CONFIG = os.path.join(MELO_ROOT, "ckpt/MeloTTS-English-v3/config.json") +OUT_DIR = "/tmp/inferon" +SPEAKER = "EN-Newest" # v3 English: single speaker (British-accented female) +LANG = "EN" +SPEED = 1.3 # speech rate; 1.0 = natural, higher = faster + +sys.path.insert(0, MELO_ROOT) + + +def main() -> None: + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) + signal.signal(signal.SIGINT, lambda *_: sys.exit(0)) + + os.makedirs(OUT_DIR, exist_ok=True) + + # Heavy imports inside main so signal handlers are installed first. + from melo.api import TTS + + model = TTS(language=LANG, config_path=CONFIG, ckpt_path=CKPT) + + speaker_ids = model.hps.data.spk2id + if SPEAKER not in speaker_ids: + # English v3 fallback: use the first speaker and say so on stderr. + fallback = next(iter(speaker_ids)) + print(f"melo_server: speaker {SPEAKER!r} not in {list(speaker_ids)}; using {fallback!r}", + file=sys.stderr, flush=True) + speaker = fallback + else: + speaker = speaker_ids[SPEAKER] + + # Warm up with a full sentence (longer text paths hit NLTK/num2words, + # which is where missing-data errors surface). Fail before READY. + warmup = "/tmp/inferon/tts-warmup.wav" + model.tts_to_file("Warmup sentence number twelve, spoken on the first of January.", + speaker, warmup, speed=SPEED) + os.unlink(warmup) + + # Ready marker — inferon waits for this before declaring the backend up. + print("READY", flush=True) + + for line in sys.stdin: + text = line.strip() + if not text: + continue + text = text[:2000] # protocol cap — extremely long replies get clipped + + wav_path = os.path.join(OUT_DIR, f"tts-{time.time_ns()}.wav") + try: + start = time.monotonic() + model.tts_to_file(text, speaker, wav_path, speed=SPEED) + dur = time.monotonic() - start + print(f"{wav_path} {dur:.2f}", flush=True) + except Exception as e: # noqa: BLE001 - protocol: report and survive + print(f"melo_server: synthesis failed: {e}", file=sys.stderr, flush=True) + print("ERROR", flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/main.zig b/src/main.zig index 828b01f..e746348 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4,6 +4,7 @@ const std = @import("std"); const Io = std.Io; const whisper = @import("whisper.zig"); +const melo = @import("melo.zig"); const qt6 = @import("libqt6zig"); const QApplication = qt6.QApplication; const QSystemTrayIcon = qt6.QSystemTrayIcon; @@ -98,6 +99,22 @@ fn setState(next: State) void { tray_icon.setToolTip(tip); } +/// Play a WAV via pw-play (blocking; audio length). Fine for v1 — same +/// thread already blocks on whisper/letta. UI-thread decoupling comes later. +fn playWav(path: []const u8) void { + const io = io_ctx orelse return; + var child = std.process.spawn(io, .{ + .argv = &.{ "pw-play", path }, + .stdin = .ignore, + .stdout = .ignore, + .stderr = .ignore, + }) catch |e| { + std.debug.print("inferon: pw-play spawn failed: {s}\n", .{@errorName(e)}); + return; + }; + _ = child.wait(io) catch {}; +} + // --------------------------------------------------------------- capture --- var recorder: ?std.process.Child = null; @@ -170,11 +187,66 @@ fn stopRecording() void { fn onQuit(_: QAction) callconv(.c) void { if (recorder != null) stopRecording(); whisper.stop(); // whisper-server dies with us + melo.stop(); QApplication.quit(); } // -------------------------------------------------------------- handlers --- +/// Delete a temp file if it exists. Best-effort — /tmp cleanup, not critical. +fn cleanupFile(path: []const u8) void { + const io = io_ctx orelse return; + Io.Dir.deleteFileAbsolute(io, path) catch {}; +} + +/// Full pipeline after recording stops: transcribe -> infer -> speak. +/// All blocking on the UI thread for now (same as before); states advance +/// automatically, no manual clicking between stages. +fn processUtterance() void { + const io = io_ctx orelse return; + const wav_in = recording_path orelse return; + defer { + cleanupFile(wav_in); + allocator.free(wav_in); + recording_path = null; + } + + setState(.transcribing); + const text: []u8 = whisper.transcribe(allocator, io, wav_in) catch |e| { + std.debug.print("inferon: transcription failed: {s}\n", .{@errorName(e)}); + setState(.error_backend); + return; + }; + std.debug.print("inferon: transcript: \"{s}\"\n", .{text}); + setLastUserText(text); + + // --- agent --- + setState(.thinking); + const response = letta.infer(io, allocator, LETTA_AGENT_ID, text) catch |err| { + std.log.err("letta failed: {s}", .{@errorName(err)}); + setState(.idle); + return; + }; + defer allocator.free(response); + std.debug.print("[INFERON]\n{s}\n\n", .{response}); + + // --- tts + playback --- + setState(.speaking); + var duration: f64 = 0; + if (melo.speak(allocator, io, response, &duration)) |tts_wav| { + defer allocator.free(tts_wav); + std.debug.print("inferon: tts ready {s} ({d:.2}s synth)\n", .{ tts_wav, duration }); + playWav(tts_wav); + cleanupFile(tts_wav); + } else |e| { + std.debug.print("inferon: tts failed: {s}\n", .{@errorName(e)}); + } + + // After speaking: back to idle-but-in-conversation. User clicks when + // they want to talk again. + setState(.idle); +} + fn onTrayActivated(_: QSystemTrayIcon, reason: i32) callconv(.c) void { // Trigger = 3 (click). Only toggle on left click; context menu handles the rest. if (reason != 3) return; @@ -186,49 +258,10 @@ fn onTrayActivated(_: QSystemTrayIcon, reason: i32) callconv(.c) void { }, .recording => { stopRecording(); - setState(.transcribing); - // TODO: move off UI thread once agent loop lands (blocking call). - if (recording_path) |p| { - const io = io_ctx.?; - if (whisper.transcribe(allocator, io, p)) |text| { - std.debug.print("inferon: transcript: \"{s}\"\n", .{text}); - setLastUserText(text); - setState(.thinking); - } else |e| { - std.debug.print("inferon: transcription failed: {s}\n", .{@errorName(e)}); - setState(.error_backend); - } - } + processUtterance(); // chains transcribe -> infer -> speak -> re-listen }, - // Placeholder transitions until agent wired in: - .transcribing => setState(.thinking), - .thinking => { - const prompt = last_user_text orelse { - std.debug.print("inferon: no transcript to send\n", .{}); - setState(.idle); - return; - }; - - std.debug.print("Sending to agent: {s}\n", .{prompt}); - - const response = letta.infer( - io_ctx.?, - allocator, - LETTA_AGENT_ID, - prompt, - ) catch |err| { - std.log.err("letta failed: {s}", .{@errorName(err)}); - setState(.idle); // adapt to whatever recovery makes sense in your loop - return; - }; - defer allocator.free(response); - - setState(.speaking); - - // TODO: parse `response` (JSON? plain text?) and update UI/state - std.debug.print("[INFERON]\n{s}\n\n", .{response}); - }, - .speaking => if (conversation_active) startRecording() else setState(.idle), + // Clicks during the pipeline are no-ops now — states advance on their own. + .transcribing, .thinking, .speaking => {}, .error_backend => setState(.idle), } } @@ -241,6 +274,42 @@ fn onEndConversation(_: QAction) callconv(.c) void { setState(.idle); } +// --------------------------------------------------------------- signals --- + +/// Async-signal-safe teardown: raw kill() of children only, no Io calls. +/// Children are orphan-reaped by init when we exit right after. +const SigType = @TypeOf(std.posix.SIG.INT); + +fn handleFatalSignal(sig: SigType) callconv(.c) void { + if (whisper.pid()) |p| { + std.posix.kill(p, std.posix.SIG.TERM) catch {}; + } + if (melo.pid()) |p| { + std.posix.kill(p, std.posix.SIG.TERM) catch {}; + } + if (recorder) |c| { + if (c.id) |p| std.posix.kill(p, std.posix.SIG.TERM) catch {}; + } + // Restore default handler and re-raise so the exit status is truthful. + const act = std.posix.Sigaction{ + .handler = .{ .handler = std.posix.SIG.DFL }, + .mask = std.posix.sigemptyset(), + .flags = 0, + }; + std.posix.sigaction(sig, &act, null); + std.posix.raise(sig) catch {}; +} + +fn installSignalHandlers() void { + const act = std.posix.Sigaction{ + .handler = .{ .handler = handleFatalSignal }, + .mask = std.posix.sigemptyset(), + .flags = 0, // no SA_RESTART: let blocking syscalls die too + }; + std.posix.sigaction(std.posix.SIG.INT, &act, null); + std.posix.sigaction(std.posix.SIG.TERM, &act, null); +} + // ------------------------------------------------------------------ main --- pub fn main(init: std.process.Init) !void { @@ -259,6 +328,8 @@ pub fn main(init: std.process.Init) !void { allocator = init.gpa; io_ctx = init.io; + installSignalHandlers(); + const quit_action = QAction.new5("&Quit", QWidget{ .ptr = null }); quit_action.onTriggered(onQuit); @@ -290,7 +361,18 @@ pub fn main(init: std.process.Init) !void { std.debug.print("inferon: whisper-server NOT ready after {d}s\n", .{whisper.READY_TIMEOUT_S}); } + // TTS backend: melo_server.py (fails soft — notifications still work). + melo.start(allocator, init.io) catch |e| { + std.debug.print("inferon: failed to start melo server: {s}\n", .{@errorName(e)}); + }; + if (melo.waitReady(allocator, init.io)) { + std.debug.print("inferon: melo ready\n", .{}); + } else { + std.debug.print("inferon: melo NOT ready\n", .{}); + } + _ = QApplication.exec(); whisper.stop(); + melo.stop(); } diff --git a/src/melo.zig b/src/melo.zig new file mode 100644 index 0000000..f6e0353 --- /dev/null +++ b/src/melo.zig @@ -0,0 +1,149 @@ +//! MeloTTS subprocess lifecycle + line protocol client. +//! +//! inferon owns the melo_server.py process: spawns it on startup (model load +//! takes a few seconds), kills it on exit. Communication is line-based: +//! inferon -> server: "\n" +//! server -> inferon: " \n" | "ERROR\n" +//! The server prints "READY" once the model is loaded. + +const std = @import("std"); +const Io = std.Io; + +// ------------------------------------------------------------- constants --- + +pub const MELO_SCRIPT = "/home/pierre/symbol-inferral/scripts/melo_server.py"; +pub const MELO_PYTHON = "/home/pierre/Projects/MeloTTS/.venv/bin/python"; +pub const READY_TIMEOUT_S: u64 = 120; +const READY_POLL_INTERVAL_MS: u64 = 500; + +// -------------------------------------------------------------- process --- + +var child: ?std.process.Child = null; +var io_ctx: ?Io = null; + +/// Spawn melo_server.py. stdout is a pipe (protocol), stderr inherits. +pub fn start(alloc: std.mem.Allocator, io: Io) !void { + _ = alloc; + io_ctx = io; + + child = try std.process.spawn(io, .{ + .argv = &.{ MELO_PYTHON, MELO_SCRIPT }, + .stdin = .pipe, + .stdout = .pipe, + .stderr = .inherit, + }); + + std.debug.print("inferon/melo: spawned pid {?d}\n", .{child.?.id}); +} + +/// Kill the melo server (quit paths). Uses the Io-aware path. +pub fn stop() void { + const io = io_ctx orelse return; + if (child) |*c| { + c.kill(io); // SIGTERM; melo_server exits cleanly on it + child = null; + } +} + +/// PID for async-signal-safe teardown. +pub fn pid() ?std.process.Child.Id { + return if (child) |c| c.id else null; +} + +/// Wait until the server prints READY on stdout. Blocking read with poll — +/// model load can take a while. +pub fn waitReady(alloc: std.mem.Allocator, io: Io) bool { + _ = alloc; + const c = &(child orelse return false); + const stdout = c.stdout orelse return false; + var acc: [256]u8 = undefined; + var acc_len: usize = 0; + const deadline_ns = Io.Clock.now(.real, io).nanoseconds + + @as(i128, READY_TIMEOUT_S) * std.time.ns_per_s; + + while (Io.Clock.now(.real, io).nanoseconds < deadline_ns) { + var tmp: [64]u8 = undefined; + const n = stdout.readStreaming(io, &.{&tmp}) catch break; + if (n == 0) break; // process died + for (tmp[0..n]) |ch| { + if (acc_len < acc.len) { + acc[acc_len] = ch; + acc_len += 1; + } + if (ch == '\n') { + const line = acc[0 .. acc_len - 1]; + if (std.mem.eql(u8, line, "READY")) return true; + std.debug.print("inferon/melo: {s}\n", .{line}); + acc_len = 0; + } + } + + } + _ = alloc; + return false; +} + +/// Flatten agent text for the line protocol: newlines/marks that would +/// either split the request or be spoken aloud become spaces / vanish. +/// Returns an allocated slice; caller frees. +fn sanitize(alloc: std.mem.Allocator, text: []const u8) ![]u8 { + const out = try alloc.alloc(u8, text.len); + var n: usize = 0; + for (text) |ch| { + const cleaned: u8 = switch (ch) { + '\n', '\r', '\t' => ' ', + '*', '_', '`', '#' => ' ', // markdown emphasis/headers — not speech + else => ch, + }; + out[n] = cleaned; + n += 1; + } + return out; +} + +/// Synthesize text: send one line, read one reply line. +/// Returns the WAV path (allocated, caller frees) and sets `duration_s`. +pub fn speak(alloc: std.mem.Allocator, io: Io, text: []const u8, duration_s: *f64) ![]u8 { + const c = &(child orelse return error.NotRunning); + const stdin = c.stdin orelse return error.NotRunning; + const stdout = c.stdout orelse return error.NotRunning; + + // Send request line (text sanitized: no embedded newlines/markdown). + const clean = try sanitize(alloc, text); + defer alloc.free(clean); + var req_buf: [8192]u8 = undefined; + const req = std.fmt.bufPrint(&req_buf, "{s}\n", .{clean}) catch return error.TextTooLong; + var wbuf: [4096]u8 = undefined; + var writer = stdin.writer(io, &wbuf); + try writer.interface.writeAll(req); + try writer.interface.flush(); + + // Read reply lines until we get a non-READY result line. + var acc: [4096]u8 = undefined; + var acc_len: usize = 0; + while (true) { + var tmp: [128]u8 = undefined; + const n = stdout.readStreaming(io, &.{&tmp}) catch return error.ReadFailed; + if (n == 0) return error.ServerDied; + for (tmp[0..n]) |ch| { + if (ch == '\n') { + const line = acc[0..acc_len]; + acc_len = 0; + if (std.mem.eql(u8, line, "ERROR")) return error.SynthesisFailed; + if (std.mem.startsWith(u8, line, "/tmp/")) { + // " " + const sp = std.mem.lastIndexOfScalar(u8, line, ' ') orelse return error.BadReply; + duration_s.* = std.fmt.parseFloat(f64, line[sp + 1 ..]) catch 0; + return try alloc.dupe(u8, line[0..sp]); + } + // Anything else: log and keep reading. + std.debug.print("inferon/melo: {s}\n", .{line}); + continue; + } + if (acc_len < acc.len) { + acc[acc_len] = ch; + acc_len += 1; + } + } + } +} diff --git a/src/whisper.zig b/src/whisper.zig index f17c7f7..f35a8bc 100644 --- a/src/whisper.zig +++ b/src/whisper.zig @@ -52,6 +52,12 @@ pub fn start(alloc: std.mem.Allocator, io: Io, log_dir: []const u8) !void { if (child) |*c| std.debug.print("inferon/whisper: spawned pid {?d}\n", .{c.id}); } +/// PID of the whisper-server child, or null. For signal handlers that must +/// only use async-signal-safe calls. +pub fn pid() ?std.process.Child.Id { + return if (child) |c| c.id else null; +} + /// Kill the whisper-server child (called on quit paths). pub fn stop() void { const io = io_ctx orelse return;