- overlay.zig: frameless translucent panel (fixed bg + rounded corners, only bubbles scroll), drag handle, hover-to-full-opacity, top-right spawn - chat layout: per-message QLabel bubbles (user purple / agent green / tool steps grey mono), transcript retained across turns, End-conversation resets - agent reply streams into its own growing bubble (re-render per chunk) - basic markdown in all messages: **bold**, *italic*, `code`, HTML-escaped - tool steps truncated to 90 chars, click to expand/collapse - polite autoscroll: pins to bottom unless user scrolled up - letta.zig: streamInfer() — --output-format stream-json, per-event callback (tool calls, returns, reply chunks), full reply still returned for TTS - whisper: --language auto (no more hardcoded English) - fix: stderr_log leak crashing Quit via debug allocator 👾 Generated with [Letta Code](https://letta.com) Co-Authored-By: Letta Code <noreply@letta.com>
488 lines
16 KiB
Zig
488 lines
16 KiB
Zig
//! inferon — voice interface daemon for Letta Code agents.
|
|
//! Phase 2: pw-record subprocess capture wired into the state machine.
|
|
|
|
const std = @import("std");
|
|
const Io = std.Io;
|
|
const whisper = @import("whisper.zig");
|
|
const melo = @import("melo.zig");
|
|
const qt6 = @import("libqt6zig");
|
|
const QApplication = qt6.QApplication;
|
|
const QSystemTrayIcon = qt6.QSystemTrayIcon;
|
|
const QWidget = qt6.QWidget;
|
|
const QMenu = qt6.QMenu;
|
|
const QAction = qt6.QAction;
|
|
const QPixmap = qt6.QPixmap;
|
|
const QIcon = qt6.QIcon;
|
|
const QColor = qt6.QColor;
|
|
const QPainter = qt6.QPainter;
|
|
const QBrush = qt6.QBrush;
|
|
const QRectF = qt6.QRectF;
|
|
const threading = qt6.threading;
|
|
const letta = @import("letta.zig");
|
|
const overlay_mod = @import("overlay.zig");
|
|
|
|
// ------------------------------------------------------------- constants ---
|
|
|
|
const RECORDER_BIN = "pw-record";
|
|
const RECORDINGS_DIR = "/tmp/inferon";
|
|
/// whisper.cpp wants 16 kHz mono S16 LE.
|
|
const RECORD_RATE = "16000";
|
|
const RECORD_CHANNELS = "1";
|
|
const RECORD_FORMAT = "s16";
|
|
const LETTA_AGENT_ID = "agent-local-f69432ca-1bcd-42eb-b7f9-cf18d187282c";
|
|
// ---------------------------------------------------------------- states ---
|
|
|
|
const State = enum {
|
|
idle, // not listening, waiting for user toggle
|
|
recording, // mic capture in progress
|
|
transcribing, // audio -> whisper server
|
|
thinking, // agent processing
|
|
speaking, // TTS playback
|
|
error_backend, // whisper/LLM unreachable
|
|
|
|
fn label(s: State) []const u8 {
|
|
return switch (s) {
|
|
.idle => "idle",
|
|
.recording => "rec",
|
|
.transcribing => "stt",
|
|
.thinking => "think",
|
|
.speaking => "say",
|
|
.error_backend => "err",
|
|
};
|
|
}
|
|
|
|
/// Tray icon color per state (RGB).
|
|
fn color(s: State) [3]u8 {
|
|
return switch (s) {
|
|
.idle => .{ 90, 90, 110 }, // muted grey
|
|
.recording => .{ 220, 60, 60 }, // red
|
|
.transcribing => .{ 220, 160, 60 }, // orange
|
|
.thinking => .{ 90, 160, 230 }, // blue
|
|
.speaking => .{ 120, 210, 110 }, // green
|
|
.error_backend => .{ 180, 40, 180 }, // magenta
|
|
};
|
|
}
|
|
};
|
|
|
|
var state: State = .idle;
|
|
var tray_icon: QSystemTrayIcon = undefined;
|
|
var allocator: std.mem.Allocator = undefined;
|
|
|
|
var conversation_active: bool = false;
|
|
var last_user_text: ?[]u8 = null;
|
|
|
|
fn setLastUserText(text: []u8) void {
|
|
if (last_user_text) |old| allocator.free(old);
|
|
last_user_text = text;
|
|
}
|
|
// ------------------------------------------------------------------ icon ---
|
|
|
|
var current_icon: ?QIcon = null;
|
|
var current_pixmap: ?QPixmap = null;
|
|
|
|
var main_thread_id: std.Thread.Id = 0;
|
|
var pending_state: State = .idle;
|
|
|
|
// --------------------------------------------------------------- overlay ---
|
|
|
|
var overlay: ?overlay_mod.Overlay = null;
|
|
|
|
const OverlayKind = enum { begin, user, step, chunk };
|
|
const OverlayMsg = struct { kind: OverlayKind, text: ?[]u8 };
|
|
|
|
/// Worker-thread side: queue an overlay update onto the Qt main loop.
|
|
fn pushOverlay(kind: OverlayKind, text: ?[]u8) void {
|
|
const msg = allocator.create(OverlayMsg) catch {
|
|
if (text) |t| allocator.free(t);
|
|
return;
|
|
};
|
|
msg.* = .{ .kind = kind, .text = text };
|
|
threading.async(msg, onOverlayMsg);
|
|
}
|
|
|
|
fn onOverlayMsg(ctx: ?*anyopaque) callconv(.c) void {
|
|
const msg: *OverlayMsg = @ptrCast(@alignCast(ctx orelse return));
|
|
defer allocator.destroy(msg);
|
|
if (overlay) |*ov| switch (msg.kind) {
|
|
.begin => ov.begin(),
|
|
.user => ov.user(msg.text orelse ""),
|
|
.step => ov.step(msg.text orelse ""),
|
|
.chunk => ov.chunk(msg.text orelse ""),
|
|
};
|
|
if (msg.text) |t| allocator.free(t);
|
|
}
|
|
|
|
|
|
fn onLettaEvent(_: ?*anyopaque, ev: letta.Event) void {
|
|
switch (ev) {
|
|
.step => |s| {
|
|
const dup = allocator.dupe(u8, s) catch return;
|
|
pushOverlay(.step, dup);
|
|
},
|
|
.chunk => |c| {
|
|
const dup = allocator.dupe(u8, c) catch return;
|
|
pushOverlay(.chunk, dup);
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Must run on the Qt main thread (touches QSystemTrayIcon / QPixmap).
|
|
fn setStateMainThread(next: State) void {
|
|
state = next;
|
|
|
|
// Circle icon drawn with QPainter — a bare pixmap.fill rendered blank
|
|
// on the KDE SNI host.
|
|
const c = next.color();
|
|
const color = QColor.new5(c[0], c[1], c[2]);
|
|
defer color.delete();
|
|
const pen_color = QColor.new5(20, 20, 26);
|
|
defer pen_color.delete();
|
|
const transparent = QColor.new8("transparent");
|
|
defer transparent.delete();
|
|
|
|
const pixmap = QPixmap.new2(128, 128);
|
|
pixmap.fill1(transparent);
|
|
|
|
const brush = QBrush.new3(color);
|
|
defer brush.delete();
|
|
|
|
const painter = QPainter.new();
|
|
_ = painter.begin(pixmap);
|
|
painter.setRenderHint(1); // Antialiasing
|
|
painter.setPen(pen_color);
|
|
painter.setBrush(brush);
|
|
const rect = QRectF.new6(16, 16, 96, 96);
|
|
defer rect.delete();
|
|
painter.drawEllipse(rect);
|
|
_ = painter.end();
|
|
painter.delete();
|
|
|
|
const icon = QIcon.new2(pixmap);
|
|
|
|
if (current_icon) |old| old.delete();
|
|
if (current_pixmap) |old| old.delete();
|
|
current_icon = icon;
|
|
current_pixmap = pixmap;
|
|
|
|
tray_icon.setIcon(icon);
|
|
|
|
const tip = std.fmt.allocPrint(allocator, "inferon — {s}", .{next.label()}) catch return;
|
|
defer allocator.free(tip);
|
|
tray_icon.setToolTip(tip);
|
|
}
|
|
|
|
fn onStateFromWorker(_: ?*anyopaque) callconv(.c) void {
|
|
setStateMainThread(pending_state);
|
|
}
|
|
|
|
/// Thread-safe state transition: runs directly on the main thread, or
|
|
/// marshals through Qt's event loop when called from the pipeline worker.
|
|
fn setState(next: State) void {
|
|
if (std.Thread.getCurrentId() == main_thread_id) {
|
|
setStateMainThread(next);
|
|
} else {
|
|
pending_state = next;
|
|
threading.async(null, onStateFromWorker);
|
|
}
|
|
}
|
|
|
|
/// Play a WAV via pw-play (blocking; audio length). Fine for v1 — same
|
|
/// thread already blocks on whisper/letta. UI-thread decoupling comes later.
|
|
fn playWav(path: []const u8) void {
|
|
const io = io_ctx orelse return;
|
|
var child = std.process.spawn(io, .{
|
|
.argv = &.{ "pw-play", path },
|
|
.stdin = .ignore,
|
|
.stdout = .ignore,
|
|
.stderr = .ignore,
|
|
}) catch |e| {
|
|
std.debug.print("inferon: pw-play spawn failed: {s}\n", .{@errorName(e)});
|
|
return;
|
|
};
|
|
_ = child.wait(io) catch {};
|
|
}
|
|
|
|
// --------------------------------------------------------------- capture ---
|
|
|
|
var recorder: ?std.process.Child = null;
|
|
var recording_path: ?[]const u8 = null;
|
|
var io_ctx: ?std.Io = null;
|
|
|
|
fn startRecording() void {
|
|
const io = io_ctx orelse return;
|
|
Io.Dir.createDirAbsolute(io, RECORDINGS_DIR, .default_dir) catch |e| switch (e) {
|
|
error.PathAlreadyExists => {},
|
|
else => {
|
|
std.debug.print("inferon: cannot create {s}: {s}\n", .{ RECORDINGS_DIR, @errorName(e) });
|
|
setState(.error_backend);
|
|
return;
|
|
},
|
|
};
|
|
|
|
const path = std.fmt.allocPrint(
|
|
allocator,
|
|
"{s}/utterance-{d}.wav",
|
|
.{ RECORDINGS_DIR, Io.Clock.now(.real, io).nanoseconds },
|
|
) catch {
|
|
setState(.error_backend);
|
|
return;
|
|
};
|
|
recording_path = path;
|
|
|
|
// pw-record finalizes the WAV header on SIGTERM — kill() sends it.
|
|
// stderr goes to a log so failures aren't silent.
|
|
const stderr_log = std.fmt.allocPrint(allocator, "{s}/pw-record.log", .{RECORDINGS_DIR}) catch null;
|
|
defer if (stderr_log) |sl| allocator.free(sl);
|
|
const stderr_file: ?Io.File = if (stderr_log) |lp|
|
|
Io.Dir.cwd().createFile(io, lp[1..], .{ .truncate = true }) catch null
|
|
else
|
|
null;
|
|
|
|
recorder = std.process.spawn(io, .{
|
|
.argv = &.{
|
|
RECORDER_BIN,
|
|
"--rate",
|
|
RECORD_RATE,
|
|
"--channels",
|
|
RECORD_CHANNELS,
|
|
"--format",
|
|
RECORD_FORMAT,
|
|
path,
|
|
},
|
|
.stdin = .ignore,
|
|
.stdout = .ignore,
|
|
.stderr = if (stderr_file) |f| .{ .file = f } else .ignore,
|
|
}) catch |e| {
|
|
std.debug.print("inferon: failed to spawn {s}: {s}\n", .{ RECORDER_BIN, @errorName(e) });
|
|
allocator.free(path);
|
|
recording_path = null;
|
|
setState(.error_backend);
|
|
return;
|
|
};
|
|
setState(.recording);
|
|
}
|
|
|
|
/// Stop pw-record. kill() delivers SIGTERM (pw-record finalizes the WAV
|
|
/// header) and reaps the child itself — no separate wait() needed.
|
|
fn stopRecording() void {
|
|
const io = io_ctx orelse return;
|
|
const child = &(recorder orelse return);
|
|
|
|
child.kill(io);
|
|
recorder = null;
|
|
}
|
|
|
|
fn onQuit(_: QAction) callconv(.c) void {
|
|
if (recorder != null) stopRecording();
|
|
whisper.stop(); // whisper-server dies with us
|
|
melo.stop();
|
|
QApplication.quit();
|
|
}
|
|
|
|
// -------------------------------------------------------------- handlers ---
|
|
|
|
/// Delete a temp file if it exists. Best-effort — /tmp cleanup, not critical.
|
|
fn cleanupFile(path: []const u8) void {
|
|
const io = io_ctx orelse return;
|
|
Io.Dir.deleteFileAbsolute(io, path) catch {};
|
|
}
|
|
|
|
/// Full pipeline after recording stops: transcribe -> infer -> speak.
|
|
/// All blocking on the UI thread for now (same as before); states advance
|
|
/// automatically, no manual clicking between stages.
|
|
fn processUtterance() void {
|
|
const io = io_ctx orelse return;
|
|
const wav_in = recording_path orelse return;
|
|
defer {
|
|
cleanupFile(wav_in);
|
|
allocator.free(wav_in);
|
|
recording_path = null;
|
|
}
|
|
|
|
setState(.transcribing);
|
|
const text: []u8 = whisper.transcribe(allocator, io, wav_in) catch |e| {
|
|
std.debug.print("inferon: transcription failed: {s}\n", .{@errorName(e)});
|
|
setState(.error_backend);
|
|
return;
|
|
};
|
|
std.debug.print("inferon: transcript: \"{s}\"\n", .{text});
|
|
setLastUserText(text);
|
|
|
|
// --- agent ---
|
|
setState(.thinking);
|
|
|
|
pushOverlay(.begin, null);
|
|
if (allocator.dupe(u8, text) catch null) |user_dup| pushOverlay(.user, user_dup);
|
|
const response = letta.streamInfer(io, allocator, LETTA_AGENT_ID, text, null, onLettaEvent) catch |err| {
|
|
std.log.err("letta failed: {s}", .{@errorName(err)});
|
|
setState(.idle);
|
|
return;
|
|
};
|
|
defer allocator.free(response);
|
|
std.debug.print("[INFERON]\n{s}\n\n", .{response});
|
|
|
|
// --- tts + playback ---
|
|
setState(.speaking);
|
|
var duration: f64 = 0;
|
|
if (melo.speak(allocator, io, response, &duration)) |tts_wav| {
|
|
defer allocator.free(tts_wav);
|
|
std.debug.print("inferon: tts ready {s} ({d:.2}s synth)\n", .{ tts_wav, duration });
|
|
playWav(tts_wav);
|
|
cleanupFile(tts_wav);
|
|
} else |e| {
|
|
std.debug.print("inferon: tts failed: {s}\n", .{@errorName(e)});
|
|
}
|
|
|
|
// After speaking: back to idle-but-in-conversation. User clicks when
|
|
// they want to talk again.
|
|
setState(.idle);
|
|
}
|
|
|
|
fn onTrayActivated(_: QSystemTrayIcon, reason: i32) callconv(.c) void {
|
|
// Trigger = 3 (click). Only toggle on left click; context menu handles the rest.
|
|
if (reason != 3) return;
|
|
|
|
switch (state) {
|
|
.idle => {
|
|
conversation_active = true;
|
|
startRecording();
|
|
},
|
|
.recording => {
|
|
stopRecording();
|
|
setState(.transcribing); // claim the pipeline slot before spawning
|
|
// Pipeline runs on a worker thread; the Qt event loop stays
|
|
// responsive (icon repaints, menu works) while whisper/letta/melo
|
|
// block. setState marshals back to the main thread.
|
|
const worker = std.Thread.spawn(.{}, processUtterance, .{}) catch |e| {
|
|
std.debug.print("inferon: failed to spawn worker: {s}\n", .{@errorName(e)});
|
|
setState(.error_backend);
|
|
return;
|
|
};
|
|
worker.detach();
|
|
},
|
|
// Clicks during the pipeline are no-ops now — states advance on their own.
|
|
.transcribing, .thinking, .speaking => {},
|
|
.error_backend => setState(.idle),
|
|
}
|
|
}
|
|
|
|
fn onEndConversation(_: QAction) callconv(.c) void {
|
|
if (recorder != null) stopRecording();
|
|
conversation_active = false;
|
|
if (last_user_text) |t| allocator.free(t);
|
|
last_user_text = null;
|
|
if (overlay) |*ov| {
|
|
ov.reset();
|
|
ov.hide();
|
|
}
|
|
setState(.idle);
|
|
}
|
|
|
|
// --------------------------------------------------------------- signals ---
|
|
|
|
/// Async-signal-safe teardown: raw kill() of children only, no Io calls.
|
|
/// Children are orphan-reaped by init when we exit right after.
|
|
const SigType = @TypeOf(std.posix.SIG.INT);
|
|
|
|
fn handleFatalSignal(sig: SigType) callconv(.c) void {
|
|
if (whisper.pid()) |p| {
|
|
std.posix.kill(p, std.posix.SIG.TERM) catch {};
|
|
}
|
|
if (melo.pid()) |p| {
|
|
std.posix.kill(p, std.posix.SIG.TERM) catch {};
|
|
}
|
|
if (recorder) |c| {
|
|
if (c.id) |p| std.posix.kill(p, std.posix.SIG.TERM) catch {};
|
|
}
|
|
// Restore default handler and re-raise so the exit status is truthful.
|
|
const act = std.posix.Sigaction{
|
|
.handler = .{ .handler = std.posix.SIG.DFL },
|
|
.mask = std.posix.sigemptyset(),
|
|
.flags = 0,
|
|
};
|
|
std.posix.sigaction(sig, &act, null);
|
|
std.posix.raise(sig) catch {};
|
|
}
|
|
|
|
fn installSignalHandlers() void {
|
|
const act = std.posix.Sigaction{
|
|
.handler = .{ .handler = handleFatalSignal },
|
|
.mask = std.posix.sigemptyset(),
|
|
.flags = 0, // no SA_RESTART: let blocking syscalls die too
|
|
};
|
|
std.posix.sigaction(std.posix.SIG.INT, &act, null);
|
|
std.posix.sigaction(std.posix.SIG.TERM, &act, null);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ main ---
|
|
|
|
pub fn main(init: std.process.Init) !void {
|
|
const argv = try qt6.init(init.gpa, init.minimal.args);
|
|
defer qt6.deinit(init.gpa, argv);
|
|
var argc: i32 = @intCast(argv.len);
|
|
const qapp: QApplication = .new(init.arena.allocator(), &argc, argv);
|
|
defer qapp.delete();
|
|
|
|
if (!QSystemTrayIcon.isSystemTrayAvailable()) {
|
|
std.debug.print("inferon: no system tray available\n", .{});
|
|
return error.NoSystemTray;
|
|
}
|
|
|
|
QApplication.setQuitOnLastWindowClosed(false);
|
|
allocator = init.gpa;
|
|
io_ctx = init.io;
|
|
|
|
installSignalHandlers();
|
|
|
|
const quit_action = QAction.new5("&Quit", QWidget{ .ptr = null });
|
|
quit_action.onTriggered(onQuit);
|
|
|
|
const end_action = QAction.new5("&End conversation", QWidget{ .ptr = null });
|
|
end_action.onTriggered(onEndConversation);
|
|
|
|
const tray_menu = QMenu.new(QWidget{ .ptr = null });
|
|
tray_menu.addAction(end_action);
|
|
_ = tray_menu.addSeparator();
|
|
tray_menu.addAction(quit_action);
|
|
|
|
tray_icon = .new3(QWidget{ .ptr = null });
|
|
tray_icon.setContextMenu(tray_menu);
|
|
tray_icon.onActivated(onTrayActivated);
|
|
|
|
setState(.idle);
|
|
tray_icon.show();
|
|
main_thread_id = std.Thread.getCurrentId();
|
|
|
|
var ov = overlay_mod.Overlay.create(allocator);
|
|
overlay = ov;
|
|
_ = &ov;
|
|
|
|
// Own the whisper-server lifecycle: spawn AFTER the tray exists (setState
|
|
// touches tray_icon). Create the log dir first.
|
|
Io.Dir.createDirAbsolute(init.io, RECORDINGS_DIR, .default_dir) catch {};
|
|
whisper.start(allocator, init.io, RECORDINGS_DIR) catch |e| {
|
|
std.debug.print("inferon: failed to start whisper-server: {s}\n", .{@errorName(e)});
|
|
setState(.error_backend);
|
|
};
|
|
if (whisper.waitReady(init.io)) {
|
|
std.debug.print("inferon: whisper-server ready on :{d}\n", .{whisper.WHISPER_PORT});
|
|
} else {
|
|
std.debug.print("inferon: whisper-server NOT ready after {d}s\n", .{whisper.READY_TIMEOUT_S});
|
|
}
|
|
|
|
// TTS backend: melo_server.py (fails soft — notifications still work).
|
|
melo.start(allocator, init.io) catch |e| {
|
|
std.debug.print("inferon: failed to start melo server: {s}\n", .{@errorName(e)});
|
|
};
|
|
if (melo.waitReady(allocator, init.io)) {
|
|
std.debug.print("inferon: melo ready\n", .{});
|
|
} else {
|
|
std.debug.print("inferon: melo NOT ready\n", .{});
|
|
}
|
|
|
|
_ = QApplication.exec();
|
|
|
|
whisper.stop();
|
|
melo.stop();
|
|
}
|