holy fuck voice to agent response works kinda sorta

.gitignore

@@ -0,0 +1,4 @@
+zig-pkg
+zig-out
+.letta
+.zig-cache

build.zig

@@ -0,0 +1,52 @@
+const std = @import("std");
+const configureQtExeRootModule = @import("libqt6zig").configureQtExeRootModule;
+
+pub fn build(b: *std.Build) void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const qt6zig = b.dependency("libqt6zig", .{
+        .target = target,
+        .optimize = optimize,
+    });
+
+    // Qt C++ binding libraries we link for the tray daemon.
+    const qt_libraries = [_][]const u8{
+        "qapplication",
+        "qcoreapplication",
+        "qguiapplication",
+        "qaction",
+        "qcolor",
+        "qicon",
+        "qpixmap",
+        "qmenu",
+        "qsystemtrayicon",
+        "qwidget",
+    };
+
+    const exe = b.addExecutable(.{
+        .name = "inferon",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/main.zig"),
+            .target = target,
+            .optimize = optimize,
+            .link_libc = true,
+            .imports = &.{
+                .{ .name = "libqt6zig", .module = qt6zig.module("libqt6zig") },
+            },
+        }),
+    });
+
+    for (qt_libraries) |lib|
+        exe.root_module.linkLibrary(qt6zig.artifact(lib));
+
+    configureQtExeRootModule(b, exe, .{}) catch @panic("Qt configuration failed");
+
+    b.installArtifact(exe);
+
+    const run_step = b.step("run", "Run inferon");
+    const run_cmd = b.addRunArtifact(exe);
+    run_step.dependOn(&run_cmd.step);
+    run_cmd.step.dependOn(b.getInstallStep());
+    if (b.args) |args| run_cmd.addArgs(args);
+}

build.zig.zon

@@ -0,0 +1,48 @@
+.{
+    // This is the default name used by packages depending on this one. For
+    // example, when a user runs `zig fetch --save <url>`, this field is used
+    // as the key in the `dependencies` table. Although the user can choose a
+    // different name, most users will stick with this provided value.
+    //
+    // It is redundant to include "zig" in this name because it is already
+    // within the Zig package namespace.
+    .name = .inferon,
+    // This is a [Semantic Version](https://semver.org/).
+    // In a future version of Zig it will be used for package deduplication.
+    .version = "0.0.0",
+    // Together with name, this represents a globally unique package
+    // identifier. This field is generated by the Zig toolchain when the
+    // package is first created, and then *never changes*. This allows
+    // unambiguous detection of one package being an updated version of
+    // another.
+    //
+    // When forking a Zig project, this id should be regenerated (delete the
+    // field and run `zig build`) if the upstream project is still maintained.
+    // Otherwise, the fork is *hostile*, attempting to take control over the
+    // original project's identity. Thus it is recommended to leave the comment
+    // on the following line intact, so that it shows up in code reviews that
+    // modify the field.
+    .fingerprint = 0xb42cbab79cddafac, // Changing this has security and trust implications.
+    // Tracks the earliest Zig version that the package considers to be a
+    // supported use case.
+    .minimum_zig_version = "0.16.0",
+    // This field is optional.
+    // Each dependency must either provide a `url` and `hash`, or a `path`.
+    // `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
+    // Once all dependencies are fetched, `zig build` no longer requires
+    // internet connectivity.
+    .dependencies = .{
+        .libqt6zig = .{
+            .url = "git+https://github.com/rcalixte/libqt6zig#c8aa843ae59f3fae372dc4b5aab5c6b0f8623cc8",
+            .hash = "libqt6zig-6.8.2-OSXtXHXCGx9sMo0pcuOmHLKsOeBYvntRpyp-owxQH8RT",
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "src",
+        // For example...
+        //"LICENSE",
+        //"README.md",
+    },
+}

src/letta.zig

@@ -0,0 +1,43 @@
+const std = @import("std");
+
+const LETTA_PATH = "/home/pierre/.bun/bin/letta";
+
+/// Runs `letta -n <agent> -p <prompt>`, captures everything it writes to
+/// stdout, and returns it as an owned slice (caller frees it).
+pub fn infer(
+    io: std.Io,
+    allocator: std.mem.Allocator,
+    agent: []const u8,
+    prompt: []const u8,
+) ![]u8 {
+    var child = try std.process.spawn(io, .{
+        .argv = &.{ LETTA_PATH, "--agent", agent, "-p", prompt, "--conversation", "default", "--toolset", "default" },
+        .stdout = .pipe,
+        .stderr = .inherit, // letta's error output goes straight to your terminal
+    });
+
+    // Non-null because we asked for a pipe.
+    const stdout_file = child.stdout.?;
+
+    var output: std.ArrayList(u8) = .empty;
+    errdefer output.deinit(allocator);
+
+    var buf: [16 * 1024]u8 = undefined;
+    while (true) {
+        const n = stdout_file.readStreaming(io, &.{&buf}) catch |err| switch (err) {
+            // EOF in the new Io API: letta closed its stdout. That's success.
+            error.EndOfStream => break,
+            else => |e| return e,
+        };
+        if (n == 0) break; // defensive: never spin if a 0 ever comes back
+        try output.appendSlice(allocator, buf[0..n]);
+    }
+    const term = try child.wait(io);
+    const exited_cleanly = switch (term) {
+        .exited => |code| code == 0,
+        else => false,
+    };
+    if (!exited_cleanly) return error.LettaFailed;
+
+    return output.toOwnedSlice(allocator);
+}

src/main.zig

@@ -0,0 +1,296 @@
+//! inferon — voice interface daemon for Letta Code agents.
+//! Phase 2: pw-record subprocess capture wired into the state machine.
+
+const std = @import("std");
+const Io = std.Io;
+const whisper = @import("whisper.zig");
+const qt6 = @import("libqt6zig");
+const QApplication = qt6.QApplication;
+const QSystemTrayIcon = qt6.QSystemTrayIcon;
+const QWidget = qt6.QWidget;
+const QMenu = qt6.QMenu;
+const QAction = qt6.QAction;
+const QPixmap = qt6.QPixmap;
+const QIcon = qt6.QIcon;
+const QColor = qt6.QColor;
+const letta = @import("letta.zig");
+
+// ------------------------------------------------------------- constants ---
+
+const RECORDER_BIN = "pw-record";
+const RECORDINGS_DIR = "/tmp/inferon";
+/// whisper.cpp wants 16 kHz mono S16 LE.
+const RECORD_RATE = "16000";
+const RECORD_CHANNELS = "1";
+const RECORD_FORMAT = "s16";
+const LETTA_AGENT_ID = "agent-local-f69432ca-1bcd-42eb-b7f9-cf18d187282c";
+// ---------------------------------------------------------------- states ---
+
+const State = enum {
+    idle, // not listening, waiting for user toggle
+    recording, // mic capture in progress
+    transcribing, // audio -> whisper server
+    thinking, // agent processing
+    speaking, // TTS playback
+    error_backend, // whisper/LLM unreachable
+
+    fn label(s: State) []const u8 {
+        return switch (s) {
+            .idle => "idle",
+            .recording => "rec",
+            .transcribing => "stt",
+            .thinking => "think",
+            .speaking => "say",
+            .error_backend => "err",
+        };
+    }
+
+    /// Tray icon color per state (RGB).
+    fn color(s: State) [3]u8 {
+        return switch (s) {
+            .idle => .{ 90, 90, 110 }, // muted grey
+            .recording => .{ 220, 60, 60 }, // red
+            .transcribing => .{ 220, 160, 60 }, // orange
+            .thinking => .{ 90, 160, 230 }, // blue
+            .speaking => .{ 120, 210, 110 }, // green
+            .error_backend => .{ 180, 40, 180 }, // magenta
+        };
+    }
+};
+
+var state: State = .idle;
+var tray_icon: QSystemTrayIcon = undefined;
+var allocator: std.mem.Allocator = undefined;
+
+var conversation_active: bool = false;
+var last_user_text: ?[]u8 = null;
+
+fn setLastUserText(text: []u8) void {
+    if (last_user_text) |old| allocator.free(old);
+    last_user_text = text;
+}
+// ------------------------------------------------------------------ icon ---
+
+var current_icon: ?QIcon = null;
+var current_pixmap: ?QPixmap = null;
+
+fn setState(next: State) void {
+    state = next;
+
+    const c = next.color();
+    const color = QColor.new5(c[0], c[1], c[2]);
+    defer color.delete();
+
+    const pixmap = QPixmap.new2(128, 128);
+    pixmap.fill1(color);
+
+    const icon = QIcon.new2(pixmap);
+
+    if (current_icon) |old| old.delete();
+    if (current_pixmap) |old| old.delete();
+    current_icon = icon;
+    current_pixmap = pixmap;
+
+    tray_icon.setIcon(icon);
+
+    const tip = std.fmt.allocPrint(allocator, "inferon — {s}", .{next.label()}) catch return;
+    defer allocator.free(tip);
+    tray_icon.setToolTip(tip);
+}
+
+// --------------------------------------------------------------- capture ---
+
+var recorder: ?std.process.Child = null;
+var recording_path: ?[]const u8 = null;
+var io_ctx: ?std.Io = null;
+
+fn startRecording() void {
+    const io = io_ctx orelse return;
+    Io.Dir.createDirAbsolute(io, RECORDINGS_DIR, .default_dir) catch |e| switch (e) {
+        error.PathAlreadyExists => {},
+        else => {
+            std.debug.print("inferon: cannot create {s}: {s}\n", .{ RECORDINGS_DIR, @errorName(e) });
+            setState(.error_backend);
+            return;
+        },
+    };
+
+    const path = std.fmt.allocPrint(
+        allocator,
+        "{s}/utterance-{d}.wav",
+        .{ RECORDINGS_DIR, Io.Clock.now(.real, io).nanoseconds },
+    ) catch {
+        setState(.error_backend);
+        return;
+    };
+    recording_path = path;
+
+    // pw-record finalizes the WAV header on SIGTERM — kill() sends it.
+    // stderr goes to a log so failures aren't silent.
+    const stderr_log = std.fmt.allocPrint(allocator, "{s}/pw-record.log", .{RECORDINGS_DIR}) catch null;
+    const stderr_file: ?Io.File = if (stderr_log) |lp|
+        Io.Dir.cwd().createFile(io, lp[1..], .{ .truncate = true }) catch null
+    else
+        null;
+
+    recorder = std.process.spawn(io, .{
+        .argv = &.{
+            RECORDER_BIN,
+            "--rate",
+            RECORD_RATE,
+            "--channels",
+            RECORD_CHANNELS,
+            "--format",
+            RECORD_FORMAT,
+            path,
+        },
+        .stdin = .ignore,
+        .stdout = .ignore,
+        .stderr = if (stderr_file) |f| .{ .file = f } else .ignore,
+    }) catch |e| {
+        std.debug.print("inferon: failed to spawn {s}: {s}\n", .{ RECORDER_BIN, @errorName(e) });
+        allocator.free(path);
+        recording_path = null;
+        setState(.error_backend);
+        return;
+    };
+    setState(.recording);
+}
+
+/// Stop pw-record. kill() delivers SIGTERM (pw-record finalizes the WAV
+/// header) and reaps the child itself — no separate wait() needed.
+fn stopRecording() void {
+    const io = io_ctx orelse return;
+    const child = &(recorder orelse return);
+
+    child.kill(io);
+    recorder = null;
+}
+
+fn onQuit(_: QAction) callconv(.c) void {
+    if (recorder != null) stopRecording();
+    whisper.stop(); // whisper-server dies with us
+    QApplication.quit();
+}
+
+// -------------------------------------------------------------- handlers ---
+
+fn onTrayActivated(_: QSystemTrayIcon, reason: i32) callconv(.c) void {
+    // Trigger = 3 (click). Only toggle on left click; context menu handles the rest.
+    if (reason != 3) return;
+
+    switch (state) {
+        .idle => {
+            conversation_active = true;
+            startRecording();
+        },
+        .recording => {
+            stopRecording();
+            setState(.transcribing);
+            // TODO: move off UI thread once agent loop lands (blocking call).
+            if (recording_path) |p| {
+                const io = io_ctx.?;
+                if (whisper.transcribe(allocator, io, p)) |text| {
+                    std.debug.print("inferon: transcript: \"{s}\"\n", .{text});
+                    setLastUserText(text);
+                    setState(.thinking);
+                } else |e| {
+                    std.debug.print("inferon: transcription failed: {s}\n", .{@errorName(e)});
+                    setState(.error_backend);
+                }
+            }
+        },
+        // Placeholder transitions until agent wired in:
+        .transcribing => setState(.thinking),
+        .thinking => {
+            const prompt = last_user_text orelse {
+                std.debug.print("inferon: no transcript to send\n", .{});
+                setState(.idle);
+                return;
+            };
+
+            std.debug.print("Sending to agent: {s}\n", .{prompt});
+
+            const response = letta.infer(
+                io_ctx.?,
+                allocator,
+                LETTA_AGENT_ID,
+                prompt,
+            ) catch |err| {
+                std.log.err("letta failed: {s}", .{@errorName(err)});
+                setState(.idle); // adapt to whatever recovery makes sense in your loop
+                return;
+            };
+            defer allocator.free(response);
+
+            setState(.speaking);
+
+            // TODO: parse `response` (JSON? plain text?) and update UI/state
+            std.debug.print("[INFERON]\n{s}\n\n", .{response});
+        },
+        .speaking => if (conversation_active) startRecording() else setState(.idle),
+        .error_backend => setState(.idle),
+    }
+}
+
+fn onEndConversation(_: QAction) callconv(.c) void {
+    if (recorder != null) stopRecording();
+    conversation_active = false;
+    if (last_user_text) |t| allocator.free(t);
+    last_user_text = null;
+    setState(.idle);
+}
+
+// ------------------------------------------------------------------ main ---
+
+pub fn main(init: std.process.Init) !void {
+    const argv = try qt6.init(init.gpa, init.minimal.args);
+    defer qt6.deinit(init.gpa, argv);
+    var argc: i32 = @intCast(argv.len);
+    const qapp: QApplication = .new(init.arena.allocator(), &argc, argv);
+    defer qapp.delete();
+
+    if (!QSystemTrayIcon.isSystemTrayAvailable()) {
+        std.debug.print("inferon: no system tray available\n", .{});
+        return error.NoSystemTray;
+    }
+
+    QApplication.setQuitOnLastWindowClosed(false);
+    allocator = init.gpa;
+    io_ctx = init.io;
+
+    const quit_action = QAction.new5("&Quit", QWidget{ .ptr = null });
+    quit_action.onTriggered(onQuit);
+
+    const end_action = QAction.new5("&End conversation", QWidget{ .ptr = null });
+    end_action.onTriggered(onEndConversation);
+
+    const tray_menu = QMenu.new(QWidget{ .ptr = null });
+    tray_menu.addAction(end_action);
+    _ = tray_menu.addSeparator();
+    tray_menu.addAction(quit_action);
+
+    tray_icon = .new3(QWidget{ .ptr = null });
+    tray_icon.setContextMenu(tray_menu);
+    tray_icon.onActivated(onTrayActivated);
+
+    setState(.idle);
+    tray_icon.show();
+
+    // Own the whisper-server lifecycle: spawn AFTER the tray exists (setState
+    // touches tray_icon). Create the log dir first.
+    Io.Dir.createDirAbsolute(init.io, RECORDINGS_DIR, .default_dir) catch {};
+    whisper.start(allocator, init.io, RECORDINGS_DIR) catch |e| {
+        std.debug.print("inferon: failed to start whisper-server: {s}\n", .{@errorName(e)});
+        setState(.error_backend);
+    };
+    if (whisper.waitReady(init.io)) {
+        std.debug.print("inferon: whisper-server ready on :{d}\n", .{whisper.WHISPER_PORT});
+    } else {
+        std.debug.print("inferon: whisper-server NOT ready after {d}s\n", .{whisper.READY_TIMEOUT_S});
+    }
+
+    _ = QApplication.exec();
+
+    whisper.stop();
+}

src/whisper.zig

@@ -0,0 +1,171 @@
+//! whisper-server lifecycle + HTTP client.
+//!
+//! inferon owns the whisper-server process: spawns it on startup, kills it on
+//! exit. Model load takes a few seconds; `waitReady` polls until the server
+//! answers. Transcription is a multipart POST over a raw localhost socket.
+
+const std = @import("std");
+const Io = std.Io;
+
+// ------------------------------------------------------------- constants ---
+
+pub const WHISPER_BIN = "/home/pierre/Projects/whisper.cpp/build/bin/whisper-server";
+pub const WHISPER_MODEL = "/home/pierre/Projects/whisper.cpp/models/ggml-large-v3-turbo-q5_0.bin";
+pub const WHISPER_PORT: u16 = 50153;
+const HOST_IP4 = Io.net.Ip4Address.loopback(WHISPER_PORT);
+pub const READY_TIMEOUT_S: u64 = 120; // large-v3-turbo q5_0 load can be slow
+const READY_POLL_INTERVAL_MS: u64 = 500;
+
+// -------------------------------------------------------------- process ---
+
+var child: ?std.process.Child = null;
+var io_ctx: ?Io = null;
+
+/// Spawn whisper-server as a child. stderr -> <log_dir>/whisper-server.log
+pub fn start(alloc: std.mem.Allocator, io: Io, log_dir: []const u8) !void {
+    io_ctx = io;
+
+    const log_path = try std.fmt.allocPrint(alloc, "{s}/whisper-server.log", .{log_dir});
+    defer alloc.free(log_path);
+    const port_str = try std.fmt.allocPrint(alloc, "{d}", .{WHISPER_PORT});
+    defer alloc.free(port_str);
+
+    const stderr_file: Io.File = Io.Dir.createFileAbsolute(io, log_path, .{ .truncate = true }) catch |e| {
+        std.debug.print("inferon/whisper: cannot open log {s}: {s}\n", .{ log_path, @errorName(e) });
+        return e;
+    };
+
+    child = try std.process.spawn(io, .{
+        .argv = &.{
+            WHISPER_BIN,
+            "-m",
+            WHISPER_MODEL,
+            "--port",
+            port_str,
+            "--convert",
+        },
+        .stdin = .ignore,
+        .stdout = .ignore,
+        .stderr = .{ .file = stderr_file },
+    });
+
+    if (child) |*c| std.debug.print("inferon/whisper: spawned pid {?d}\n", .{c.id});
+}
+
+/// Kill the whisper-server child (called on quit paths).
+pub fn stop() void {
+    const io = io_ctx orelse return;
+    if (child) |*c| {
+        c.kill(io); // SIGTERM
+        child = null;
+    }
+}
+
+/// Poll until the HTTP server answers. Model load takes seconds.
+pub fn waitReady(io: Io) bool {
+    var waited: u64 = 0;
+    while (waited < READY_TIMEOUT_S * 1000) : (waited += READY_POLL_INTERVAL_MS) {
+        io.sleep(.{ .nanoseconds = READY_POLL_INTERVAL_MS * std.time.ns_per_ms }, .real) catch {};
+        if (probeAlive(io)) return true;
+    }
+    return false;
+}
+
+fn probeAlive(io: Io) bool {
+    const addr = Io.net.IpAddress{ .ip4 = HOST_IP4 };
+    const stream = addr.connect(io, .{ .mode = .stream, .timeout = .none }) catch return false;
+    stream.close(io);
+    return true;
+}
+
+// ----------------------------------------------------------------- http ---
+
+const boundary = "inferonboundary7381a5f2";
+
+fn buildMultipart(alloc: std.mem.Allocator, wav: []const u8) !std.ArrayList(u8) {
+    var body: std.ArrayList(u8) = .empty;
+    errdefer body.deinit(alloc);
+    try body.print(alloc, "--{s}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"u.wav\"\r\nContent-Type: audio/wav\r\n\r\n", .{boundary});
+    try body.appendSlice(alloc, wav);
+    try body.print(alloc, "\r\n--{s}\r\nContent-Disposition: form-data; name=\"response_format\"\r\n\r\njson\r\n", .{boundary});
+    try body.print(alloc, "--{s}\r\nContent-Disposition: form-data; name=\"temperature\"\r\n\r\n0.0\r\n", .{boundary});
+    try body.print(alloc, "--{s}--\r\n", .{boundary});
+    return body;
+}
+
+/// POST a WAV file to /inference, return the transcript text (allocated).
+/// Caller frees. Minimal HTTP/1.1 over a blocking localhost socket.
+pub fn transcribe(alloc: std.mem.Allocator, io: Io, wav_path: []const u8) ![]u8 {
+    var wav_file = try Io.Dir.openFileAbsolute(io, wav_path, .{});
+    defer wav_file.close(io);
+    const stat = try wav_file.stat(io);
+    if (stat.size > 64 * 1024 * 1024) return error.FileTooBig;
+    var file_rbuf: [64 * 1024]u8 = undefined;
+    var wav_reader = wav_file.reader(io, &file_rbuf);
+    const wav = try wav_reader.interface.readAlloc(alloc, @intCast(stat.size));
+    defer alloc.free(wav);
+
+    var body = try buildMultipart(alloc, wav);
+    defer body.deinit(alloc);
+
+    const addr = Io.net.IpAddress{ .ip4 = HOST_IP4 };
+    const stream = try addr.connect(io, .{ .mode = .stream, .timeout = .none });
+    defer stream.close(io);
+
+    var wbuf: [4096]u8 = undefined;
+    var writer = Io.net.Stream.writer(stream, io, &wbuf);
+    try writer.interface.print(
+        "POST /inference HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: multipart/form-data; boundary={s}\r\nContent-Length: {d}\r\nConnection: close\r\n\r\n",
+        .{ boundary, body.items.len },
+    );
+    try writer.interface.writeAll(body.items);
+    try writer.interface.flush();
+
+    var rbuf: [8192]u8 = undefined;
+    var reader = Io.net.Stream.reader(stream, io, &rbuf);
+
+    var resp: std.ArrayList(u8) = .empty;
+    defer resp.deinit(alloc);
+    // Read loop: stream until EOF (Connection: close).
+    while (true) {
+        var tmp: [8192]u8 = undefined;
+        const n = reader.interface.readSliceShort(&tmp) catch break;
+        if (n == 0) break;
+        try resp.appendSlice(alloc, tmp[0..n]);
+    }
+
+    return parseTranscript(resp.items, alloc);
+}
+
+/// Extract "text" field from the JSON response body. Handles both plain and
+/// chunked responses by simply scanning for the key.
+fn parseTranscript(raw: []const u8, alloc: std.mem.Allocator) ![]u8 {
+    const body_start = std.mem.indexOf(u8, raw, "\r\n\r\n") orelse return error.BadResponse;
+    const json_part = raw[body_start + 4 ..];
+
+    const key = "\"text\"";
+    const idx = std.mem.indexOf(u8, json_part, key) orelse return error.BadResponse;
+    var it = json_part[idx + key.len ..];
+    while (it.len > 0 and it[0] != '"') it = it[1..];
+    if (it.len == 0) return error.BadResponse;
+    it = it[1..]; // skip opening quote
+
+    var out: std.ArrayList(u8) = .empty;
+    errdefer out.deinit(alloc);
+    while (it.len > 0 and it[0] != '"') {
+        if (it[0] == '\\' and it.len > 1) {
+            const esc: u8 = switch (it[1]) {
+                'n' => '\n',
+                't' => '\t',
+                '"', '\\' => it[1],
+                else => it[1],
+            };
+            try out.append(alloc, esc);
+            it = it[2..];
+        } else {
+            try out.append(alloc, it[0]);
+            it = it[1..];
+        }
+    }
+    return out.toOwnedSlice(alloc);
+}