commit 3ddb0e77bc0c0a37f6067a993b09700b23185ee6 Author: Pierre De Lancre Date: Mon Aug 24 03:26:43 2026 +0300 holy fuck voice to agent response works kinda sorta diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a08358a --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +zig-pkg +zig-out +.letta +.zig-cache diff --git a/build.zig b/build.zig new file mode 100644 index 0000000..7a25b08 --- /dev/null +++ b/build.zig @@ -0,0 +1,52 @@ +const std = @import("std"); +const configureQtExeRootModule = @import("libqt6zig").configureQtExeRootModule; + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const qt6zig = b.dependency("libqt6zig", .{ + .target = target, + .optimize = optimize, + }); + + // Qt C++ binding libraries we link for the tray daemon. + const qt_libraries = [_][]const u8{ + "qapplication", + "qcoreapplication", + "qguiapplication", + "qaction", + "qcolor", + "qicon", + "qpixmap", + "qmenu", + "qsystemtrayicon", + "qwidget", + }; + + const exe = b.addExecutable(.{ + .name = "inferon", + .root_module = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + .imports = &.{ + .{ .name = "libqt6zig", .module = qt6zig.module("libqt6zig") }, + }, + }), + }); + + for (qt_libraries) |lib| + exe.root_module.linkLibrary(qt6zig.artifact(lib)); + + configureQtExeRootModule(b, exe, .{}) catch @panic("Qt configuration failed"); + + b.installArtifact(exe); + + const run_step = b.step("run", "Run inferon"); + const run_cmd = b.addRunArtifact(exe); + run_step.dependOn(&run_cmd.step); + run_cmd.step.dependOn(b.getInstallStep()); + if (b.args) |args| run_cmd.addArgs(args); +} diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 0000000..bee2717 --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,48 @@ +.{ + // This is the default name used by packages depending on this one. For + // example, when a user runs `zig fetch --save `, this field is used + // as the key in the `dependencies` table. Although the user can choose a + // different name, most users will stick with this provided value. + // + // It is redundant to include "zig" in this name because it is already + // within the Zig package namespace. + .name = .inferon, + // This is a [Semantic Version](https://semver.org/). + // In a future version of Zig it will be used for package deduplication. + .version = "0.0.0", + // Together with name, this represents a globally unique package + // identifier. This field is generated by the Zig toolchain when the + // package is first created, and then *never changes*. This allows + // unambiguous detection of one package being an updated version of + // another. + // + // When forking a Zig project, this id should be regenerated (delete the + // field and run `zig build`) if the upstream project is still maintained. + // Otherwise, the fork is *hostile*, attempting to take control over the + // original project's identity. Thus it is recommended to leave the comment + // on the following line intact, so that it shows up in code reviews that + // modify the field. + .fingerprint = 0xb42cbab79cddafac, // Changing this has security and trust implications. + // Tracks the earliest Zig version that the package considers to be a + // supported use case. + .minimum_zig_version = "0.16.0", + // This field is optional. + // Each dependency must either provide a `url` and `hash`, or a `path`. + // `zig build --fetch` can be used to fetch all dependencies of a package, recursively. + // Once all dependencies are fetched, `zig build` no longer requires + // internet connectivity. + .dependencies = .{ + .libqt6zig = .{ + .url = "git+https://github.com/rcalixte/libqt6zig#c8aa843ae59f3fae372dc4b5aab5c6b0f8623cc8", + .hash = "libqt6zig-6.8.2-OSXtXHXCGx9sMo0pcuOmHLKsOeBYvntRpyp-owxQH8RT", + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "src", + // For example... + //"LICENSE", + //"README.md", + }, +} diff --git a/src/letta.zig b/src/letta.zig new file mode 100644 index 0000000..4bf202b --- /dev/null +++ b/src/letta.zig @@ -0,0 +1,43 @@ +const std = @import("std"); + +const LETTA_PATH = "/home/pierre/.bun/bin/letta"; + +/// Runs `letta -n -p `, captures everything it writes to +/// stdout, and returns it as an owned slice (caller frees it). +pub fn infer( + io: std.Io, + allocator: std.mem.Allocator, + agent: []const u8, + prompt: []const u8, +) ![]u8 { + var child = try std.process.spawn(io, .{ + .argv = &.{ LETTA_PATH, "--agent", agent, "-p", prompt, "--conversation", "default", "--toolset", "default" }, + .stdout = .pipe, + .stderr = .inherit, // letta's error output goes straight to your terminal + }); + + // Non-null because we asked for a pipe. + const stdout_file = child.stdout.?; + + var output: std.ArrayList(u8) = .empty; + errdefer output.deinit(allocator); + + var buf: [16 * 1024]u8 = undefined; + while (true) { + const n = stdout_file.readStreaming(io, &.{&buf}) catch |err| switch (err) { + // EOF in the new Io API: letta closed its stdout. That's success. + error.EndOfStream => break, + else => |e| return e, + }; + if (n == 0) break; // defensive: never spin if a 0 ever comes back + try output.appendSlice(allocator, buf[0..n]); + } + const term = try child.wait(io); + const exited_cleanly = switch (term) { + .exited => |code| code == 0, + else => false, + }; + if (!exited_cleanly) return error.LettaFailed; + + return output.toOwnedSlice(allocator); +} diff --git a/src/main.zig b/src/main.zig new file mode 100644 index 0000000..828b01f --- /dev/null +++ b/src/main.zig @@ -0,0 +1,296 @@ +//! inferon — voice interface daemon for Letta Code agents. +//! Phase 2: pw-record subprocess capture wired into the state machine. + +const std = @import("std"); +const Io = std.Io; +const whisper = @import("whisper.zig"); +const qt6 = @import("libqt6zig"); +const QApplication = qt6.QApplication; +const QSystemTrayIcon = qt6.QSystemTrayIcon; +const QWidget = qt6.QWidget; +const QMenu = qt6.QMenu; +const QAction = qt6.QAction; +const QPixmap = qt6.QPixmap; +const QIcon = qt6.QIcon; +const QColor = qt6.QColor; +const letta = @import("letta.zig"); + +// ------------------------------------------------------------- constants --- + +const RECORDER_BIN = "pw-record"; +const RECORDINGS_DIR = "/tmp/inferon"; +/// whisper.cpp wants 16 kHz mono S16 LE. +const RECORD_RATE = "16000"; +const RECORD_CHANNELS = "1"; +const RECORD_FORMAT = "s16"; +const LETTA_AGENT_ID = "agent-local-f69432ca-1bcd-42eb-b7f9-cf18d187282c"; +// ---------------------------------------------------------------- states --- + +const State = enum { + idle, // not listening, waiting for user toggle + recording, // mic capture in progress + transcribing, // audio -> whisper server + thinking, // agent processing + speaking, // TTS playback + error_backend, // whisper/LLM unreachable + + fn label(s: State) []const u8 { + return switch (s) { + .idle => "idle", + .recording => "rec", + .transcribing => "stt", + .thinking => "think", + .speaking => "say", + .error_backend => "err", + }; + } + + /// Tray icon color per state (RGB). + fn color(s: State) [3]u8 { + return switch (s) { + .idle => .{ 90, 90, 110 }, // muted grey + .recording => .{ 220, 60, 60 }, // red + .transcribing => .{ 220, 160, 60 }, // orange + .thinking => .{ 90, 160, 230 }, // blue + .speaking => .{ 120, 210, 110 }, // green + .error_backend => .{ 180, 40, 180 }, // magenta + }; + } +}; + +var state: State = .idle; +var tray_icon: QSystemTrayIcon = undefined; +var allocator: std.mem.Allocator = undefined; + +var conversation_active: bool = false; +var last_user_text: ?[]u8 = null; + +fn setLastUserText(text: []u8) void { + if (last_user_text) |old| allocator.free(old); + last_user_text = text; +} +// ------------------------------------------------------------------ icon --- + +var current_icon: ?QIcon = null; +var current_pixmap: ?QPixmap = null; + +fn setState(next: State) void { + state = next; + + const c = next.color(); + const color = QColor.new5(c[0], c[1], c[2]); + defer color.delete(); + + const pixmap = QPixmap.new2(128, 128); + pixmap.fill1(color); + + const icon = QIcon.new2(pixmap); + + if (current_icon) |old| old.delete(); + if (current_pixmap) |old| old.delete(); + current_icon = icon; + current_pixmap = pixmap; + + tray_icon.setIcon(icon); + + const tip = std.fmt.allocPrint(allocator, "inferon — {s}", .{next.label()}) catch return; + defer allocator.free(tip); + tray_icon.setToolTip(tip); +} + +// --------------------------------------------------------------- capture --- + +var recorder: ?std.process.Child = null; +var recording_path: ?[]const u8 = null; +var io_ctx: ?std.Io = null; + +fn startRecording() void { + const io = io_ctx orelse return; + Io.Dir.createDirAbsolute(io, RECORDINGS_DIR, .default_dir) catch |e| switch (e) { + error.PathAlreadyExists => {}, + else => { + std.debug.print("inferon: cannot create {s}: {s}\n", .{ RECORDINGS_DIR, @errorName(e) }); + setState(.error_backend); + return; + }, + }; + + const path = std.fmt.allocPrint( + allocator, + "{s}/utterance-{d}.wav", + .{ RECORDINGS_DIR, Io.Clock.now(.real, io).nanoseconds }, + ) catch { + setState(.error_backend); + return; + }; + recording_path = path; + + // pw-record finalizes the WAV header on SIGTERM — kill() sends it. + // stderr goes to a log so failures aren't silent. + const stderr_log = std.fmt.allocPrint(allocator, "{s}/pw-record.log", .{RECORDINGS_DIR}) catch null; + const stderr_file: ?Io.File = if (stderr_log) |lp| + Io.Dir.cwd().createFile(io, lp[1..], .{ .truncate = true }) catch null + else + null; + + recorder = std.process.spawn(io, .{ + .argv = &.{ + RECORDER_BIN, + "--rate", + RECORD_RATE, + "--channels", + RECORD_CHANNELS, + "--format", + RECORD_FORMAT, + path, + }, + .stdin = .ignore, + .stdout = .ignore, + .stderr = if (stderr_file) |f| .{ .file = f } else .ignore, + }) catch |e| { + std.debug.print("inferon: failed to spawn {s}: {s}\n", .{ RECORDER_BIN, @errorName(e) }); + allocator.free(path); + recording_path = null; + setState(.error_backend); + return; + }; + setState(.recording); +} + +/// Stop pw-record. kill() delivers SIGTERM (pw-record finalizes the WAV +/// header) and reaps the child itself — no separate wait() needed. +fn stopRecording() void { + const io = io_ctx orelse return; + const child = &(recorder orelse return); + + child.kill(io); + recorder = null; +} + +fn onQuit(_: QAction) callconv(.c) void { + if (recorder != null) stopRecording(); + whisper.stop(); // whisper-server dies with us + QApplication.quit(); +} + +// -------------------------------------------------------------- handlers --- + +fn onTrayActivated(_: QSystemTrayIcon, reason: i32) callconv(.c) void { + // Trigger = 3 (click). Only toggle on left click; context menu handles the rest. + if (reason != 3) return; + + switch (state) { + .idle => { + conversation_active = true; + startRecording(); + }, + .recording => { + stopRecording(); + setState(.transcribing); + // TODO: move off UI thread once agent loop lands (blocking call). + if (recording_path) |p| { + const io = io_ctx.?; + if (whisper.transcribe(allocator, io, p)) |text| { + std.debug.print("inferon: transcript: \"{s}\"\n", .{text}); + setLastUserText(text); + setState(.thinking); + } else |e| { + std.debug.print("inferon: transcription failed: {s}\n", .{@errorName(e)}); + setState(.error_backend); + } + } + }, + // Placeholder transitions until agent wired in: + .transcribing => setState(.thinking), + .thinking => { + const prompt = last_user_text orelse { + std.debug.print("inferon: no transcript to send\n", .{}); + setState(.idle); + return; + }; + + std.debug.print("Sending to agent: {s}\n", .{prompt}); + + const response = letta.infer( + io_ctx.?, + allocator, + LETTA_AGENT_ID, + prompt, + ) catch |err| { + std.log.err("letta failed: {s}", .{@errorName(err)}); + setState(.idle); // adapt to whatever recovery makes sense in your loop + return; + }; + defer allocator.free(response); + + setState(.speaking); + + // TODO: parse `response` (JSON? plain text?) and update UI/state + std.debug.print("[INFERON]\n{s}\n\n", .{response}); + }, + .speaking => if (conversation_active) startRecording() else setState(.idle), + .error_backend => setState(.idle), + } +} + +fn onEndConversation(_: QAction) callconv(.c) void { + if (recorder != null) stopRecording(); + conversation_active = false; + if (last_user_text) |t| allocator.free(t); + last_user_text = null; + setState(.idle); +} + +// ------------------------------------------------------------------ main --- + +pub fn main(init: std.process.Init) !void { + const argv = try qt6.init(init.gpa, init.minimal.args); + defer qt6.deinit(init.gpa, argv); + var argc: i32 = @intCast(argv.len); + const qapp: QApplication = .new(init.arena.allocator(), &argc, argv); + defer qapp.delete(); + + if (!QSystemTrayIcon.isSystemTrayAvailable()) { + std.debug.print("inferon: no system tray available\n", .{}); + return error.NoSystemTray; + } + + QApplication.setQuitOnLastWindowClosed(false); + allocator = init.gpa; + io_ctx = init.io; + + const quit_action = QAction.new5("&Quit", QWidget{ .ptr = null }); + quit_action.onTriggered(onQuit); + + const end_action = QAction.new5("&End conversation", QWidget{ .ptr = null }); + end_action.onTriggered(onEndConversation); + + const tray_menu = QMenu.new(QWidget{ .ptr = null }); + tray_menu.addAction(end_action); + _ = tray_menu.addSeparator(); + tray_menu.addAction(quit_action); + + tray_icon = .new3(QWidget{ .ptr = null }); + tray_icon.setContextMenu(tray_menu); + tray_icon.onActivated(onTrayActivated); + + setState(.idle); + tray_icon.show(); + + // Own the whisper-server lifecycle: spawn AFTER the tray exists (setState + // touches tray_icon). Create the log dir first. + Io.Dir.createDirAbsolute(init.io, RECORDINGS_DIR, .default_dir) catch {}; + whisper.start(allocator, init.io, RECORDINGS_DIR) catch |e| { + std.debug.print("inferon: failed to start whisper-server: {s}\n", .{@errorName(e)}); + setState(.error_backend); + }; + if (whisper.waitReady(init.io)) { + std.debug.print("inferon: whisper-server ready on :{d}\n", .{whisper.WHISPER_PORT}); + } else { + std.debug.print("inferon: whisper-server NOT ready after {d}s\n", .{whisper.READY_TIMEOUT_S}); + } + + _ = QApplication.exec(); + + whisper.stop(); +} diff --git a/src/whisper.zig b/src/whisper.zig new file mode 100644 index 0000000..f17c7f7 --- /dev/null +++ b/src/whisper.zig @@ -0,0 +1,171 @@ +//! whisper-server lifecycle + HTTP client. +//! +//! inferon owns the whisper-server process: spawns it on startup, kills it on +//! exit. Model load takes a few seconds; `waitReady` polls until the server +//! answers. Transcription is a multipart POST over a raw localhost socket. + +const std = @import("std"); +const Io = std.Io; + +// ------------------------------------------------------------- constants --- + +pub const WHISPER_BIN = "/home/pierre/Projects/whisper.cpp/build/bin/whisper-server"; +pub const WHISPER_MODEL = "/home/pierre/Projects/whisper.cpp/models/ggml-large-v3-turbo-q5_0.bin"; +pub const WHISPER_PORT: u16 = 50153; +const HOST_IP4 = Io.net.Ip4Address.loopback(WHISPER_PORT); +pub const READY_TIMEOUT_S: u64 = 120; // large-v3-turbo q5_0 load can be slow +const READY_POLL_INTERVAL_MS: u64 = 500; + +// -------------------------------------------------------------- process --- + +var child: ?std.process.Child = null; +var io_ctx: ?Io = null; + +/// Spawn whisper-server as a child. stderr -> /whisper-server.log +pub fn start(alloc: std.mem.Allocator, io: Io, log_dir: []const u8) !void { + io_ctx = io; + + const log_path = try std.fmt.allocPrint(alloc, "{s}/whisper-server.log", .{log_dir}); + defer alloc.free(log_path); + const port_str = try std.fmt.allocPrint(alloc, "{d}", .{WHISPER_PORT}); + defer alloc.free(port_str); + + const stderr_file: Io.File = Io.Dir.createFileAbsolute(io, log_path, .{ .truncate = true }) catch |e| { + std.debug.print("inferon/whisper: cannot open log {s}: {s}\n", .{ log_path, @errorName(e) }); + return e; + }; + + child = try std.process.spawn(io, .{ + .argv = &.{ + WHISPER_BIN, + "-m", + WHISPER_MODEL, + "--port", + port_str, + "--convert", + }, + .stdin = .ignore, + .stdout = .ignore, + .stderr = .{ .file = stderr_file }, + }); + + if (child) |*c| std.debug.print("inferon/whisper: spawned pid {?d}\n", .{c.id}); +} + +/// Kill the whisper-server child (called on quit paths). +pub fn stop() void { + const io = io_ctx orelse return; + if (child) |*c| { + c.kill(io); // SIGTERM + child = null; + } +} + +/// Poll until the HTTP server answers. Model load takes seconds. +pub fn waitReady(io: Io) bool { + var waited: u64 = 0; + while (waited < READY_TIMEOUT_S * 1000) : (waited += READY_POLL_INTERVAL_MS) { + io.sleep(.{ .nanoseconds = READY_POLL_INTERVAL_MS * std.time.ns_per_ms }, .real) catch {}; + if (probeAlive(io)) return true; + } + return false; +} + +fn probeAlive(io: Io) bool { + const addr = Io.net.IpAddress{ .ip4 = HOST_IP4 }; + const stream = addr.connect(io, .{ .mode = .stream, .timeout = .none }) catch return false; + stream.close(io); + return true; +} + +// ----------------------------------------------------------------- http --- + +const boundary = "inferonboundary7381a5f2"; + +fn buildMultipart(alloc: std.mem.Allocator, wav: []const u8) !std.ArrayList(u8) { + var body: std.ArrayList(u8) = .empty; + errdefer body.deinit(alloc); + try body.print(alloc, "--{s}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"u.wav\"\r\nContent-Type: audio/wav\r\n\r\n", .{boundary}); + try body.appendSlice(alloc, wav); + try body.print(alloc, "\r\n--{s}\r\nContent-Disposition: form-data; name=\"response_format\"\r\n\r\njson\r\n", .{boundary}); + try body.print(alloc, "--{s}\r\nContent-Disposition: form-data; name=\"temperature\"\r\n\r\n0.0\r\n", .{boundary}); + try body.print(alloc, "--{s}--\r\n", .{boundary}); + return body; +} + +/// POST a WAV file to /inference, return the transcript text (allocated). +/// Caller frees. Minimal HTTP/1.1 over a blocking localhost socket. +pub fn transcribe(alloc: std.mem.Allocator, io: Io, wav_path: []const u8) ![]u8 { + var wav_file = try Io.Dir.openFileAbsolute(io, wav_path, .{}); + defer wav_file.close(io); + const stat = try wav_file.stat(io); + if (stat.size > 64 * 1024 * 1024) return error.FileTooBig; + var file_rbuf: [64 * 1024]u8 = undefined; + var wav_reader = wav_file.reader(io, &file_rbuf); + const wav = try wav_reader.interface.readAlloc(alloc, @intCast(stat.size)); + defer alloc.free(wav); + + var body = try buildMultipart(alloc, wav); + defer body.deinit(alloc); + + const addr = Io.net.IpAddress{ .ip4 = HOST_IP4 }; + const stream = try addr.connect(io, .{ .mode = .stream, .timeout = .none }); + defer stream.close(io); + + var wbuf: [4096]u8 = undefined; + var writer = Io.net.Stream.writer(stream, io, &wbuf); + try writer.interface.print( + "POST /inference HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: multipart/form-data; boundary={s}\r\nContent-Length: {d}\r\nConnection: close\r\n\r\n", + .{ boundary, body.items.len }, + ); + try writer.interface.writeAll(body.items); + try writer.interface.flush(); + + var rbuf: [8192]u8 = undefined; + var reader = Io.net.Stream.reader(stream, io, &rbuf); + + var resp: std.ArrayList(u8) = .empty; + defer resp.deinit(alloc); + // Read loop: stream until EOF (Connection: close). + while (true) { + var tmp: [8192]u8 = undefined; + const n = reader.interface.readSliceShort(&tmp) catch break; + if (n == 0) break; + try resp.appendSlice(alloc, tmp[0..n]); + } + + return parseTranscript(resp.items, alloc); +} + +/// Extract "text" field from the JSON response body. Handles both plain and +/// chunked responses by simply scanning for the key. +fn parseTranscript(raw: []const u8, alloc: std.mem.Allocator) ![]u8 { + const body_start = std.mem.indexOf(u8, raw, "\r\n\r\n") orelse return error.BadResponse; + const json_part = raw[body_start + 4 ..]; + + const key = "\"text\""; + const idx = std.mem.indexOf(u8, json_part, key) orelse return error.BadResponse; + var it = json_part[idx + key.len ..]; + while (it.len > 0 and it[0] != '"') it = it[1..]; + if (it.len == 0) return error.BadResponse; + it = it[1..]; // skip opening quote + + var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(alloc); + while (it.len > 0 and it[0] != '"') { + if (it[0] == '\\' and it.len > 1) { + const esc: u8 = switch (it[1]) { + 'n' => '\n', + 't' => '\t', + '"', '\\' => it[1], + else => it[1], + }; + try out.append(alloc, esc); + it = it[2..]; + } else { + try out.append(alloc, it[0]); + it = it[1..]; + } + } + return out.toOwnedSlice(alloc); +}