feat(api+web): stream topic chat live over server-sent events
Some checks failed
CI / Lint (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Broker tests (Postgres) (push) Has been cancelled
CI / Docker build (linux/amd64) (push) Has been cancelled

GET /v1/topics/:name/stream opens an SSE firehose, polled server-side
every 2s and streamed as `message` events. Forward-only — clients
hit /messages once for backfill, then live from connect-time onward.
Heartbeats every 30s keep the connection through proxies.

Web chat panel reads the stream via fetch + ReadableStream so the
bearer token stays in the Authorization header (EventSource can't
set custom headers, which would force token-in-URL leaks). Auto-
reconnect with exponential backoff. setInterval polling removed.

Vercel maxDuration bumped to 300s on the catch-all API route so
streams aren't cut at the 10s default.

drizzle migrations/meta/ deleted — superseded by the filename-
tracked custom runner in apps/broker/src/migrate.ts (c2cd67a).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
2026-05-02 19:02:38 +01:00
parent d7cef45640
commit 7e71a61db4
16 changed files with 288 additions and 36155 deletions

View File

@@ -6,20 +6,24 @@
* for mesh A cannot read or write mesh B.
*
* Endpoints (v0.2.0 minimum):
* POST /v1/messages — send to a topic
* GET /v1/topics — list topics in the key's mesh
* GET /v1/topics/:name/messages — fetch topic history (paginated)
* GET /v1/peers — list peers in the mesh
* POST /v1/messages — send to a topic
* GET /v1/topics — list topics in the key's mesh
* GET /v1/topics/:name/messages — fetch topic history (paginated)
* GET /v1/topics/:name/stream — SSE: live message firehose for a topic
* PATCH /v1/topics/:name/read — mark a topic read up to now
* GET /v1/peers — list peers in the mesh
*
* Live delivery: writes to mesh.message_queue + mesh.topic_message. The
* broker's existing pendingTimer drains the queue and pushes to live
* peers. Latency = polling interval (~2s today). Real-time push from
* REST writes is a follow-up.
* peers. The /stream endpoint server-side polls topic_message every
* 2s and pushes new rows as SSE events — clients see new messages
* within 2s without burning a poll-per-tab.
*
* Spec: .artifacts/specs/2026-05-02-v0.2.0-scope.md
*/
import { Hono } from "hono";
import { streamSSE } from "hono/streaming";
import { z } from "zod";
import { db } from "@turbostarter/db/server";
@@ -32,7 +36,7 @@ import {
messageQueue,
presence,
} from "@turbostarter/db/schema/mesh";
import { and, asc, desc, eq, isNull, lt } from "drizzle-orm";
import { and, asc, desc, eq, gt, isNull, lt } from "drizzle-orm";
import { validate } from "../../middleware";
import {
@@ -239,6 +243,120 @@ export const v1Router = new Hono<Env>()
},
)
// GET /v1/topics/:name/stream — live SSE firehose for a topic.
//
// Server-side polls mesh.topic_message every STREAM_POLL_MS for rows
// newer than the last seen createdAt and pushes each as an SSE
// `message` event. First connection sample establishes the watermark
// (no historical replay — clients fetch /messages for that). The
// stream ends when the client disconnects or the topic is archived.
//
// Heartbeats every 30s as SSE comments (`:keep-alive`) keep the
// connection through proxies that drop idle TCP. Postgres LISTEN/
// NOTIFY is the obvious upgrade path when message volume grows; the
// poll loop here is fine for v0.2.0's low write rate.
.get("/topics/:name/stream", async (c) => {
const key = c.var.apiKey;
requireCapability(key, "read");
const name = c.req.param("name");
requireTopicScope(key, name);
const [topic] = await db
.select({ id: meshTopic.id })
.from(meshTopic)
.where(
and(
eq(meshTopic.meshId, key.meshId),
eq(meshTopic.name, name),
isNull(meshTopic.archivedAt),
),
);
if (!topic) {
return c.json({ error: "topic_not_found", topic: name }, 404);
}
const STREAM_POLL_MS = 2000;
const HEARTBEAT_MS = 30_000;
return streamSSE(c, async (stream) => {
// Watermark: skip messages older than connect time so we don't
// replay history. Clients backfill via GET /messages.
let cursor = new Date();
let lastHeartbeat = Date.now();
let aborted = false;
stream.onAbort(() => {
aborted = true;
});
// Initial hello so clients know the stream is alive.
await stream.writeSSE({
event: "ready",
data: JSON.stringify({
topic: name,
topicId: topic.id,
connectedAt: cursor.toISOString(),
}),
});
while (!aborted) {
try {
const rows = await db
.select({
id: meshTopicMessage.id,
senderPubkey: meshMember.peerPubkey,
senderName: meshMember.displayName,
nonce: meshTopicMessage.nonce,
ciphertext: meshTopicMessage.ciphertext,
createdAt: meshTopicMessage.createdAt,
})
.from(meshTopicMessage)
.innerJoin(
meshMember,
eq(meshTopicMessage.senderMemberId, meshMember.id),
)
.where(
and(
eq(meshTopicMessage.topicId, topic.id),
gt(meshTopicMessage.createdAt, cursor),
),
)
.orderBy(asc(meshTopicMessage.createdAt))
.limit(100);
for (const r of rows) {
await stream.writeSSE({
event: "message",
id: r.id,
data: JSON.stringify({
id: r.id,
senderPubkey: r.senderPubkey,
senderName: r.senderName,
nonce: r.nonce,
ciphertext: r.ciphertext,
createdAt: r.createdAt.toISOString(),
}),
});
if (r.createdAt > cursor) cursor = r.createdAt;
}
if (Date.now() - lastHeartbeat > HEARTBEAT_MS) {
await stream.writeSSE({ event: "heartbeat", data: String(Date.now()) });
lastHeartbeat = Date.now();
}
} catch (e) {
await stream.writeSSE({
event: "error",
data: JSON.stringify({
error: e instanceof Error ? e.message : String(e),
}),
});
}
await stream.sleep(STREAM_POLL_MS);
}
});
})
// GET /v1/peers — connected peers in the key's mesh
// Dedupe by memberId — a member can have multiple active presence
// rows (one per session). Status reflects the most recent presence;

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,90 +0,0 @@
{
"version": "7",
"dialect": "postgresql",
"entries": [
{
"idx": 0,
"version": "7",
"when": 1775336269295,
"tag": "0000_living_namora",
"breakpoints": true
},
{
"idx": 1,
"version": "7",
"when": 1775339743477,
"tag": "0001_demonic_karnak",
"breakpoints": true
},
{
"idx": 2,
"version": "7",
"when": 1775340519054,
"tag": "0002_vengeful_enchantress",
"breakpoints": true
},
{
"idx": 3,
"version": "7",
"when": 1775463897329,
"tag": "0003_add-presence-summary",
"breakpoints": true
},
{
"idx": 4,
"version": "7",
"when": 1775468683383,
"tag": "0004_add-presence-display-name",
"breakpoints": true
},
{
"idx": 5,
"version": "7",
"when": 1775470435032,
"tag": "0005_add-presence-session-pubkey",
"breakpoints": true
},
{
"idx": 6,
"version": "7",
"when": 1775470979207,
"tag": "0006_add-sender-session-pubkey",
"breakpoints": true
},
{
"idx": 7,
"version": "7",
"when": 1775476994511,
"tag": "0007_add-presence-groups",
"breakpoints": true
},
{
"idx": 8,
"version": "7",
"when": 1775477883426,
"tag": "0008_add-state-and-memory",
"breakpoints": true
},
{
"idx": 9,
"version": "7",
"when": 1775480008546,
"tag": "0009_add-file-tables",
"breakpoints": true
},
{
"idx": 10,
"version": "7",
"when": 1775480729014,
"tag": "0010_add-context-and-tasks",
"breakpoints": true
},
{
"idx": 11,
"version": "7",
"when": 1775481222701,
"tag": "0011_add-streams",
"breakpoints": true
}
]
}