Daemon side of the 2026-09-08 incident (spec Addenda 3, 5, 6): - ws-lifecycle: createWsLifecycle() returns the handle synchronously with ready exposed separately. A first hello-ack timeout used to REJECT the connect promise while the helper kept reconnecting on its own, so DaemonBrokerClient.lifecycle stayed null forever and every RPC gated on it (listPeers, send, ...) short-circuited: peer list answered [] on all 9 meshes for as long as the daemon lived. Ack timeout 5 s → 15 s (a timeout is now just a reconnect). 1000 session_replaced is terminal for a lifecycle (two daemons no longer fight over the same sessions). - drain: 24 h outbox TTL (expired), session_gone after 10 min, attempt cap on the not-ready path, drain_session_ws_not_ready logged on attempt 1 and every 60th (was every tick: 4 rows × ~130 k attempts = a 1 GB log). - Rotating log sink (20 MB × 5, total budget) behind an explicit --log-sink flag that the launchd/systemd unit and the detached spawn pass; units point StandardOut/Err at daemon.launchd.log and pin ThrottleInterval 10. - Exit forensics: daemon_exit / daemon_uncaught on every exit path; SIGHUP is a clean shutdown. - Singleton hardening: the CLI stale-probe no longer unlinks a live daemon's pid/socket on a slow probe (only ECONNREFUSED/ENOENT or a dead pid is stale); the daemon probes the socket for another live daemon before taking the lock and re-asserts its pid file every 30 s. - MCP SSE: resubscribe on every termination event with a 45 s idle watchdog; the daemon ends open SSE streams before exiting. - Operator signal: GET /v1/diagnostics; daemon status shows per-mesh and per-session WS state; new daemon restart; doctor adds version match, member/session WS open, ghost detector (every registered session must appear in the broker peer list), log size + stale unit; peer list warns when the daemon is not connected to the broker for a mesh. Tests: ws-lifecycle (timeout → same handle, close mid-connect, session_replaced terminal), daemon-broker-lost-handle (Addendum 3 regression), drain-expiry, log-sink, sse-resubscribe (socket destroyed without end), daemon-probe-classify. Local drills against the fixed broker: SIGKILL + respawn → sessions visible at t+1 s, second daemon refused, pid file self-healed. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
111 lines
4.1 KiB
TypeScript
111 lines
4.1 KiB
TypeScript
/**
|
|
* DaemonBrokerClient — Addendum 3 regression (2026-09-08 mesh blackout).
|
|
*
|
|
* The broker, busy with a reconnect storm, took >5s to ack the daemon's
|
|
* member `hello`. The old `connect()` did `this.lifecycle = await
|
|
* connectWsWithBackoff(...)`; the timeout REJECTED that promise so
|
|
* `lifecycle` stayed null forever, while the helper's own reconnect flipped
|
|
* `_status` to "open". `listPeers()` then short-circuited on
|
|
* `!this.lifecycle` and returned [] in ~30ms on every mesh — `peer list`
|
|
* showed 0 peers for as long as the daemon lived.
|
|
*
|
|
* Now: first hello times out, second is acked, and `listPeers()` returns
|
|
* the broker's list.
|
|
*/
|
|
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
import { WebSocketServer, type WebSocket } from "ws";
|
|
import type { AddressInfo } from "node:net";
|
|
import sodium from "libsodium-wrappers";
|
|
|
|
import { DaemonBrokerClient } from "~/daemon/broker.js";
|
|
import type { JoinedMesh } from "~/services/config/schemas.js";
|
|
|
|
const silent = () => { /* quiet */ };
|
|
|
|
describe("DaemonBrokerClient keeps its lifecycle handle across a first-ack timeout", () => {
|
|
let wss: WebSocketServer | null = null;
|
|
let client: DaemonBrokerClient | null = null;
|
|
const sockets: WebSocket[] = [];
|
|
|
|
afterEach(async () => {
|
|
await client?.close();
|
|
client = null;
|
|
for (const s of sockets) { try { s.terminate(); } catch { /* ignore */ } }
|
|
sockets.length = 0;
|
|
await new Promise<void>((r) => (wss ? wss.close(() => r()) : r()));
|
|
wss = null;
|
|
});
|
|
|
|
it("listPeers() returns the broker's peers after the second hello is acked", async () => {
|
|
await sodium.ready;
|
|
const kp = sodium.crypto_sign_keypair();
|
|
|
|
let hellos = 0;
|
|
let listPeersSeen = 0;
|
|
wss = new WebSocketServer({ port: 0 });
|
|
wss.on("connection", (sock) => {
|
|
sockets.push(sock);
|
|
sock.on("message", (raw) => {
|
|
const msg = JSON.parse(raw.toString()) as Record<string, unknown>;
|
|
if (msg.type === "hello") {
|
|
hellos++;
|
|
if (hellos === 1) return; // simulate a broker stalled under load
|
|
sock.send(JSON.stringify({ type: "hello_ack", presenceId: "p1", memberDisplayName: "d" }));
|
|
return;
|
|
}
|
|
if (msg.type === "list_peers") {
|
|
listPeersSeen++;
|
|
sock.send(JSON.stringify({
|
|
type: "peers_list",
|
|
_reqId: msg._reqId,
|
|
peers: [
|
|
{ pubkey: "a".repeat(64), displayName: "alpha", status: "idle", summary: null, groups: [], sessionId: "s1", connectedAt: new Date().toISOString(), peerRole: "session" },
|
|
{ pubkey: "b".repeat(64), displayName: "beta", status: "idle", summary: null, groups: [], sessionId: "s2", connectedAt: new Date().toISOString(), peerRole: "session" },
|
|
],
|
|
}));
|
|
}
|
|
});
|
|
});
|
|
await new Promise<void>((r) => wss!.on("listening", () => r()));
|
|
const { port } = wss.address() as AddressInfo;
|
|
|
|
const mesh: JoinedMesh = {
|
|
meshId: "mesh_test",
|
|
memberId: "member_test",
|
|
slug: "test",
|
|
name: "test",
|
|
pubkey: sodium.to_hex(kp.publicKey),
|
|
secretKey: sodium.to_hex(kp.privateKey),
|
|
brokerUrl: `ws://127.0.0.1:${port}`,
|
|
joinedAt: new Date().toISOString(),
|
|
};
|
|
|
|
const statuses: string[] = [];
|
|
client = new DaemonBrokerClient(mesh, {
|
|
onStatusChange: (s) => statuses.push(s),
|
|
log: silent,
|
|
// Real client-side hello-ack timeout path, shortened for the test.
|
|
lifecycle: { helloAckTimeoutMs: 150, backoffCapsMs: [50] },
|
|
});
|
|
|
|
await client.connect(); // resolves once the SECOND hello is acked
|
|
|
|
expect(statuses).toEqual(["connecting", "reconnecting", "connecting", "open"]);
|
|
|
|
expect(hellos).toBe(2);
|
|
expect(client.status).toBe("open");
|
|
expect(client.isOpen()).toBe(true);
|
|
|
|
const peers = await client.listPeers(2_000);
|
|
expect(listPeersSeen).toBe(1);
|
|
expect(peers.map((p) => p.displayName)).toEqual(["alpha", "beta"]);
|
|
|
|
const d = client.diagnostics();
|
|
expect(d.status).toBe("open");
|
|
expect(d.isOpen).toBe(true);
|
|
expect(d.reconnects).toBe(1);
|
|
expect(d.lastAckAt).not.toBeNull();
|
|
}, 10_000);
|
|
});
|