feat(db): mesh data model — meshes, members, invites, audit log
- pgSchema "mesh" with 4 tables isolating the peer mesh domain - Enums: visibility, transport, tier, role - audit_log is metadata-only (E2E encryption enforced at broker/client) - Cascade on mesh delete, soft-delete via archivedAt/revokedAt Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
81
packages/api/tests/check-document.ts
Normal file
81
packages/api/tests/check-document.ts
Normal file
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* Script to check a specific document and its embeddings
|
||||
* Run: pnpm with-env npx tsx packages/api/tests/check-document.ts
|
||||
*/
|
||||
import { sql } from "@turbostarter/db";
|
||||
import { db } from "@turbostarter/db/server";
|
||||
|
||||
const DOC_ID = "JnOv9Z7JK2ZWU92OrFFFuuCY1TSksaWv";
|
||||
|
||||
async function check() {
|
||||
console.log(`\n=== Checking document: ${DOC_ID} ===\n`);
|
||||
|
||||
// 1. Check if document exists in pdf.document
|
||||
const docResult = await db.execute<{
|
||||
id: string;
|
||||
path: string;
|
||||
name: string | null;
|
||||
chat_id: string;
|
||||
created_at: Date;
|
||||
}>(sql`
|
||||
SELECT id, path, name, chat_id, created_at
|
||||
FROM pdf.document
|
||||
WHERE id = ${DOC_ID}
|
||||
`);
|
||||
|
||||
console.log("Document record:", docResult);
|
||||
|
||||
if (!Array.isArray(docResult) || docResult.length === 0) {
|
||||
console.log("❌ Document NOT FOUND in pdf.document table!");
|
||||
|
||||
// Check if it might be in a different table or with different ID
|
||||
const allDocs = await db.execute<{
|
||||
id: string;
|
||||
path: string;
|
||||
chat_id: string;
|
||||
}>(sql`SELECT id, path, chat_id FROM pdf.document ORDER BY created_at DESC LIMIT 10`);
|
||||
|
||||
console.log("\nRecent documents in pdf.document:");
|
||||
for (const doc of allDocs as { id: string; path: string }[]) {
|
||||
console.log(` - ${doc.id} | ${doc.path}`);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const doc = docResult[0]!;
|
||||
console.log(`✅ Document found: ${doc.name ?? doc.path}`);
|
||||
|
||||
// 2. Count embeddings for this document
|
||||
const embeddingCount = await db.execute<{ count: number }>(sql`
|
||||
SELECT COUNT(*)::int as count
|
||||
FROM pdf.embedding
|
||||
WHERE document_id = ${DOC_ID}
|
||||
`);
|
||||
|
||||
const count = (embeddingCount as { count: number }[])[0]?.count ?? 0;
|
||||
console.log(`📊 Embedding count: ${count}`);
|
||||
|
||||
if (count === 0) {
|
||||
console.log("❌ Document has 0 embeddings - needs regeneration!");
|
||||
console.log(` Path: ${doc.path}`);
|
||||
} else {
|
||||
console.log("✅ Document has embeddings");
|
||||
|
||||
// Show sample embeddings
|
||||
const samples = await db.execute(sql`
|
||||
SELECT id, LEFT(content, 80) as preview, page_number
|
||||
FROM pdf.embedding
|
||||
WHERE document_id = ${DOC_ID}
|
||||
LIMIT 3
|
||||
`);
|
||||
console.log("\nSample embeddings:", samples);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
check().catch((e) => {
|
||||
console.error("Error:", e);
|
||||
process.exit(1);
|
||||
});
|
||||
181
packages/api/tests/credits/credits-schema.test.ts
Normal file
181
packages/api/tests/credits/credits-schema.test.ts
Normal file
@@ -0,0 +1,181 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import { updateCreditsSchema, getTransactionsSchema } from "../../src/schema/admin";
|
||||
|
||||
describe("updateCreditsSchema", () => {
|
||||
describe("action field", () => {
|
||||
it("should accept 'set' action", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "set",
|
||||
amount: 100,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept 'add' action", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "add",
|
||||
amount: 100,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("should accept 'deduct' action", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "deduct",
|
||||
amount: 100,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("should reject invalid action", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "invalid",
|
||||
amount: 100,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("amount field", () => {
|
||||
it("should accept positive integers", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "add",
|
||||
amount: 500,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("should reject zero", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "add",
|
||||
amount: 0,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject negative numbers", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "add",
|
||||
amount: -100,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("should reject non-integers", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "add",
|
||||
amount: 10.5,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("reason field", () => {
|
||||
it("should accept optional reason", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "add",
|
||||
amount: 100,
|
||||
reason: "Promotional credit",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.data.reason).toBe("Promotional credit");
|
||||
}
|
||||
});
|
||||
|
||||
it("should accept missing reason", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "add",
|
||||
amount: 100,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.data.reason).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("should reject reason over 500 characters", () => {
|
||||
const result = updateCreditsSchema.safeParse({
|
||||
action: "add",
|
||||
amount: 100,
|
||||
reason: "a".repeat(501),
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("getTransactionsSchema", () => {
|
||||
it("should require customerId", () => {
|
||||
const result = getTransactionsSchema.safeParse({
|
||||
page: 1,
|
||||
perPage: 20,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("should accept valid input with defaults", () => {
|
||||
const result = getTransactionsSchema.safeParse({
|
||||
customerId: "cust-123",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.data.page).toBe(1);
|
||||
expect(result.data.perPage).toBe(20);
|
||||
}
|
||||
});
|
||||
|
||||
it("should accept pagination parameters", () => {
|
||||
const result = getTransactionsSchema.safeParse({
|
||||
customerId: "cust-123",
|
||||
page: 3,
|
||||
perPage: 50,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.data.page).toBe(3);
|
||||
expect(result.data.perPage).toBe(50);
|
||||
}
|
||||
});
|
||||
|
||||
it("should reject perPage over 100", () => {
|
||||
const result = getTransactionsSchema.safeParse({
|
||||
customerId: "cust-123",
|
||||
perPage: 150,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
describe("type filter", () => {
|
||||
const validTypes = [
|
||||
"signup",
|
||||
"purchase",
|
||||
"usage",
|
||||
"admin_grant",
|
||||
"admin_deduct",
|
||||
"refund",
|
||||
"promo",
|
||||
"referral",
|
||||
"expiry",
|
||||
] as const;
|
||||
|
||||
validTypes.forEach((type) => {
|
||||
it(`should accept type '${type}'`, () => {
|
||||
const result = getTransactionsSchema.safeParse({
|
||||
customerId: "cust-123",
|
||||
type,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
it("should reject invalid type", () => {
|
||||
const result = getTransactionsSchema.safeParse({
|
||||
customerId: "cust-123",
|
||||
type: "invalid",
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
91
packages/api/tests/credits/credits-utils.test.ts
Normal file
91
packages/api/tests/credits/credits-utils.test.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import {
|
||||
Credits,
|
||||
hasEnoughCredits,
|
||||
getCreditsLevel,
|
||||
getCreditsProgress,
|
||||
} from "@turbostarter/ai/credits/utils";
|
||||
|
||||
describe("Credits constants", () => {
|
||||
it("should have default balance of 100", () => {
|
||||
expect(Credits.BALANCE).toBe(100);
|
||||
});
|
||||
|
||||
it("should have FREE cost of 0", () => {
|
||||
expect(Credits.COST.FREE).toBe(0);
|
||||
});
|
||||
|
||||
it("should have DEFAULT cost of 5", () => {
|
||||
expect(Credits.COST.DEFAULT).toBe(5);
|
||||
});
|
||||
|
||||
it("should have HIGH cost of 10", () => {
|
||||
expect(Credits.COST.HIGH).toBe(10);
|
||||
});
|
||||
});
|
||||
|
||||
describe("hasEnoughCredits", () => {
|
||||
it("should return true when credits are sufficient", () => {
|
||||
expect(hasEnoughCredits(100, 50)).toBe(true);
|
||||
expect(hasEnoughCredits(50, 50)).toBe(true);
|
||||
expect(hasEnoughCredits(100, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("should return false when credits are insufficient", () => {
|
||||
expect(hasEnoughCredits(50, 100)).toBe(false);
|
||||
expect(hasEnoughCredits(0, 1)).toBe(false);
|
||||
expect(hasEnoughCredits(99, 100)).toBe(false);
|
||||
});
|
||||
|
||||
it("should handle edge cases", () => {
|
||||
expect(hasEnoughCredits(0, 0)).toBe(true);
|
||||
expect(hasEnoughCredits(1, 1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getCreditsLevel", () => {
|
||||
it("should return 'high' when credits > 50%", () => {
|
||||
expect(getCreditsLevel(60, 100)).toBe("high");
|
||||
expect(getCreditsLevel(100, 100)).toBe("high");
|
||||
expect(getCreditsLevel(51, 100)).toBe("high");
|
||||
});
|
||||
|
||||
it("should return 'medium' when credits between 15% and 50%", () => {
|
||||
expect(getCreditsLevel(50, 100)).toBe("medium");
|
||||
expect(getCreditsLevel(16, 100)).toBe("medium");
|
||||
expect(getCreditsLevel(30, 100)).toBe("medium");
|
||||
});
|
||||
|
||||
it("should return 'low' when credits <= 15%", () => {
|
||||
expect(getCreditsLevel(15, 100)).toBe("low");
|
||||
expect(getCreditsLevel(10, 100)).toBe("low");
|
||||
expect(getCreditsLevel(0, 100)).toBe("low");
|
||||
expect(getCreditsLevel(1, 100)).toBe("low");
|
||||
});
|
||||
|
||||
it("should use default max of 100 when not specified", () => {
|
||||
expect(getCreditsLevel(60)).toBe("high");
|
||||
expect(getCreditsLevel(30)).toBe("medium");
|
||||
expect(getCreditsLevel(10)).toBe("low");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getCreditsProgress", () => {
|
||||
it("should calculate correct progress ratio", () => {
|
||||
expect(getCreditsProgress(50, 100)).toBe(0.5);
|
||||
expect(getCreditsProgress(25, 100)).toBe(0.25);
|
||||
expect(getCreditsProgress(100, 100)).toBe(1);
|
||||
expect(getCreditsProgress(0, 100)).toBe(0);
|
||||
});
|
||||
|
||||
it("should use default max of 100", () => {
|
||||
expect(getCreditsProgress(50)).toBe(0.5);
|
||||
expect(getCreditsProgress(100)).toBe(1);
|
||||
});
|
||||
|
||||
it("should handle custom max values", () => {
|
||||
expect(getCreditsProgress(500, 1000)).toBe(0.5);
|
||||
expect(getCreditsProgress(250, 500)).toBe(0.5);
|
||||
});
|
||||
});
|
||||
183
packages/api/tests/credits/update-credits.test.ts
Normal file
183
packages/api/tests/credits/update-credits.test.ts
Normal file
@@ -0,0 +1,183 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
// Mock the database module before importing the mutation
|
||||
vi.mock("@turbostarter/db/server", () => ({
|
||||
db: {
|
||||
transaction: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@turbostarter/shared/utils", () => ({
|
||||
generateId: vi.fn(() => "test-transaction-id"),
|
||||
HttpException: class HttpException extends Error {
|
||||
constructor(public statusCode: number, public body?: { code: string; message?: string }) {
|
||||
super(body?.message ?? "HttpException");
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
import { db } from "@turbostarter/db/server";
|
||||
|
||||
import { updateCustomerCredits } from "../../src/modules/admin/customers/mutations";
|
||||
|
||||
describe("updateCustomerCredits", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("action: add", () => {
|
||||
it("should add credits to customer balance", async () => {
|
||||
const mockCustomer = { id: "cust-1", credits: 100 };
|
||||
const mockTx = {
|
||||
select: vi.fn().mockReturnThis(),
|
||||
from: vi.fn().mockReturnThis(),
|
||||
where: vi.fn().mockResolvedValue([mockCustomer]),
|
||||
update: vi.fn().mockReturnThis(),
|
||||
set: vi.fn().mockReturnThis(),
|
||||
insert: vi.fn().mockReturnThis(),
|
||||
values: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument, @typescript-eslint/unbound-method
|
||||
vi.mocked(db.transaction).mockImplementation((callback) => callback(mockTx as any));
|
||||
|
||||
const result = await updateCustomerCredits(
|
||||
"cust-1",
|
||||
{ action: "add", amount: 50 },
|
||||
"admin-1"
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
previousBalance: 100,
|
||||
newBalance: 150,
|
||||
action: "add",
|
||||
amount: 50,
|
||||
});
|
||||
expect(mockTx.insert).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("action: deduct", () => {
|
||||
it("should deduct credits from customer balance", async () => {
|
||||
const mockCustomer = { id: "cust-1", credits: 100 };
|
||||
const mockTx = {
|
||||
select: vi.fn().mockReturnThis(),
|
||||
from: vi.fn().mockReturnThis(),
|
||||
where: vi.fn().mockResolvedValue([mockCustomer]),
|
||||
update: vi.fn().mockReturnThis(),
|
||||
set: vi.fn().mockReturnThis(),
|
||||
insert: vi.fn().mockReturnThis(),
|
||||
values: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument, @typescript-eslint/unbound-method
|
||||
vi.mocked(db.transaction).mockImplementation((callback) => callback(mockTx as any));
|
||||
|
||||
const result = await updateCustomerCredits(
|
||||
"cust-1",
|
||||
{ action: "deduct", amount: 30 },
|
||||
"admin-1"
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
previousBalance: 100,
|
||||
newBalance: 70,
|
||||
action: "deduct",
|
||||
amount: 30,
|
||||
});
|
||||
});
|
||||
|
||||
it("should throw error when deducting more than available", async () => {
|
||||
const mockCustomer = { id: "cust-1", credits: 20 };
|
||||
const mockTx = {
|
||||
select: vi.fn().mockReturnThis(),
|
||||
from: vi.fn().mockReturnThis(),
|
||||
where: vi.fn().mockResolvedValue([mockCustomer]),
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument, @typescript-eslint/unbound-method
|
||||
vi.mocked(db.transaction).mockImplementation((callback) => callback(mockTx as any));
|
||||
|
||||
await expect(
|
||||
updateCustomerCredits("cust-1", { action: "deduct", amount: 50 }, "admin-1")
|
||||
).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("action: set", () => {
|
||||
it("should set credits to exact amount (increase)", async () => {
|
||||
const mockCustomer = { id: "cust-1", credits: 100 };
|
||||
const mockTx = {
|
||||
select: vi.fn().mockReturnThis(),
|
||||
from: vi.fn().mockReturnThis(),
|
||||
where: vi.fn().mockResolvedValue([mockCustomer]),
|
||||
update: vi.fn().mockReturnThis(),
|
||||
set: vi.fn().mockReturnThis(),
|
||||
insert: vi.fn().mockReturnThis(),
|
||||
values: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument, @typescript-eslint/unbound-method
|
||||
vi.mocked(db.transaction).mockImplementation((callback) => callback(mockTx as any));
|
||||
|
||||
const result = await updateCustomerCredits(
|
||||
"cust-1",
|
||||
{ action: "set", amount: 200 },
|
||||
"admin-1"
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
previousBalance: 100,
|
||||
newBalance: 200,
|
||||
action: "set",
|
||||
amount: 200,
|
||||
});
|
||||
});
|
||||
|
||||
it("should set credits to exact amount (decrease)", async () => {
|
||||
const mockCustomer = { id: "cust-1", credits: 100 };
|
||||
const mockTx = {
|
||||
select: vi.fn().mockReturnThis(),
|
||||
from: vi.fn().mockReturnThis(),
|
||||
where: vi.fn().mockResolvedValue([mockCustomer]),
|
||||
update: vi.fn().mockReturnThis(),
|
||||
set: vi.fn().mockReturnThis(),
|
||||
insert: vi.fn().mockReturnThis(),
|
||||
values: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument, @typescript-eslint/unbound-method
|
||||
vi.mocked(db.transaction).mockImplementation((callback) => callback(mockTx as any));
|
||||
|
||||
const result = await updateCustomerCredits(
|
||||
"cust-1",
|
||||
{ action: "set", amount: 50 },
|
||||
"admin-1"
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
previousBalance: 100,
|
||||
newBalance: 50,
|
||||
action: "set",
|
||||
amount: 50,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("error cases", () => {
|
||||
it("should throw when customer not found", async () => {
|
||||
const mockTx = {
|
||||
select: vi.fn().mockReturnThis(),
|
||||
from: vi.fn().mockReturnThis(),
|
||||
where: vi.fn().mockResolvedValue([]),
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument, @typescript-eslint/unbound-method
|
||||
vi.mocked(db.transaction).mockImplementation((callback) => callback(mockTx as any));
|
||||
|
||||
await expect(
|
||||
updateCustomerCredits("nonexistent", { action: "add", amount: 100 }, "admin-1")
|
||||
).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
55
packages/api/tests/find-working-chat.ts
Normal file
55
packages/api/tests/find-working-chat.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Find the chat with the working sample PDF
|
||||
* Run: pnpm with-env npx tsx packages/api/tests/find-working-chat.ts
|
||||
*/
|
||||
import { sql } from "@turbostarter/db";
|
||||
import { db } from "@turbostarter/db/server";
|
||||
|
||||
async function find() {
|
||||
// Get the most recent sample-local-pdf document
|
||||
const doc = await db.execute<{
|
||||
id: string;
|
||||
chat_id: string;
|
||||
path: string;
|
||||
}>(sql`
|
||||
SELECT id, chat_id, path
|
||||
FROM pdf.document
|
||||
WHERE name = 'sample-local-pdf'
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
`);
|
||||
|
||||
const docRecord = (doc as { id: string; chat_id: string; path: string }[])[0];
|
||||
if (!docRecord) {
|
||||
console.log("No sample-local-pdf found");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`\n📄 Most recent sample-local-pdf:`);
|
||||
console.log(` Document ID: ${docRecord.id}`);
|
||||
console.log(` Chat ID: ${docRecord.chat_id}`);
|
||||
console.log(`\n🔗 URL: /pdf/${docRecord.chat_id}`);
|
||||
|
||||
// Check embeddings content
|
||||
const embeddings = await db.execute<{
|
||||
content: string;
|
||||
page_number: number;
|
||||
}>(sql`
|
||||
SELECT LEFT(content, 100) as content, page_number
|
||||
FROM pdf.embedding
|
||||
WHERE document_id = ${docRecord.id}
|
||||
LIMIT 3
|
||||
`);
|
||||
|
||||
console.log(`\n📊 Sample embedding content:`);
|
||||
for (const emb of embeddings as { content: string; page_number: number }[]) {
|
||||
console.log(` Page ${emb.page_number}: "${emb.content.replace(/\n/g, " ")}"`);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
find().catch((e) => {
|
||||
console.error("Error:", e);
|
||||
process.exit(1);
|
||||
});
|
||||
99
packages/api/tests/fix-document-embeddings.ts
Normal file
99
packages/api/tests/fix-document-embeddings.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Script to fix embeddings for a specific document with corrupted embeddings
|
||||
* Run: pnpm with-env npx tsx packages/api/tests/fix-document-embeddings.ts
|
||||
*/
|
||||
import { generateDocumentEmbeddings } from "@turbostarter/ai/pdf/embeddings";
|
||||
import { sql } from "@turbostarter/db";
|
||||
import { pdfEmbedding } from "@turbostarter/db/schema/pdf";
|
||||
import { db } from "@turbostarter/db/server";
|
||||
|
||||
const DOC_ID = "JnOv9Z7JK2ZWU92OrFFFuuCY1TSksaWv";
|
||||
|
||||
async function fix() {
|
||||
console.log(`\n=== Fixing embeddings for document: ${DOC_ID} ===\n`);
|
||||
|
||||
// 1. Get the document path
|
||||
const docResult = await db.execute<{
|
||||
id: string;
|
||||
path: string;
|
||||
name: string | null;
|
||||
}>(sql`
|
||||
SELECT id, path, name
|
||||
FROM pdf.document
|
||||
WHERE id = ${DOC_ID}
|
||||
`);
|
||||
|
||||
const docs = Array.isArray(docResult) ? docResult : [];
|
||||
if (docs.length === 0) {
|
||||
console.log("❌ Document not found!");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const doc = docs[0]!;
|
||||
console.log(`📄 Document: ${doc.name ?? doc.id}`);
|
||||
console.log(`📁 Path: ${doc.path}`);
|
||||
|
||||
// 2. Delete existing (corrupted) embeddings
|
||||
console.log("\n🗑️ Deleting corrupted embeddings...");
|
||||
await db.execute(sql`
|
||||
DELETE FROM pdf.embedding
|
||||
WHERE document_id = ${DOC_ID}
|
||||
`);
|
||||
console.log(" Done.");
|
||||
|
||||
// 3. Regenerate embeddings from the actual PDF
|
||||
console.log("\n📊 Generating new embeddings from PDF...");
|
||||
try {
|
||||
const generated = await generateDocumentEmbeddings(doc.path);
|
||||
console.log(` Generated ${generated.length} chunks`);
|
||||
|
||||
if (generated.length === 0) {
|
||||
console.log("⚠️ No chunks generated - PDF may be empty or unreadable");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Show sample of new embeddings
|
||||
console.log("\n Sample content:");
|
||||
for (let i = 0; i < Math.min(3, generated.length); i++) {
|
||||
const chunk = generated[i]!;
|
||||
console.log(` Page ${chunk.metadata.pageNumber}: "${chunk.content.substring(0, 60)}..."`);
|
||||
}
|
||||
|
||||
// 4. Insert new embeddings
|
||||
console.log("\n💾 Inserting new embeddings...");
|
||||
await db
|
||||
.insert(pdfEmbedding)
|
||||
.values(
|
||||
generated.map((chunk) => ({
|
||||
content: chunk.content,
|
||||
documentId: DOC_ID,
|
||||
embedding: chunk.embedding,
|
||||
pageNumber: chunk.metadata.pageNumber,
|
||||
charStart: chunk.metadata.charStart,
|
||||
charEnd: chunk.metadata.charEnd,
|
||||
sectionTitle: chunk.metadata.sectionTitle,
|
||||
})),
|
||||
);
|
||||
|
||||
console.log(` ✅ Inserted ${generated.length} embeddings`);
|
||||
|
||||
// 5. Verify
|
||||
const countResult = await db.execute<{ count: number }>(sql`
|
||||
SELECT COUNT(*)::int as count
|
||||
FROM pdf.embedding
|
||||
WHERE document_id = ${DOC_ID}
|
||||
`);
|
||||
const count = (countResult as { count: number }[])[0]?.count ?? 0;
|
||||
console.log(`\n✅ Verification: Document now has ${count} embeddings`);
|
||||
} catch (error) {
|
||||
console.error("\n❌ Error generating embeddings:", error);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
fix().catch((e) => {
|
||||
console.error("Fatal error:", e);
|
||||
process.exit(1);
|
||||
});
|
||||
39
packages/api/tests/list-documents.ts
Normal file
39
packages/api/tests/list-documents.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* List recent documents and their embedding counts
|
||||
* Run: pnpm with-env npx tsx packages/api/tests/list-documents.ts
|
||||
*/
|
||||
import { sql } from "@turbostarter/db";
|
||||
import { db } from "@turbostarter/db/server";
|
||||
|
||||
async function list() {
|
||||
const docs = await db.execute<{
|
||||
id: string;
|
||||
name: string | null;
|
||||
path: string;
|
||||
embedding_count: number;
|
||||
created_at: Date;
|
||||
}>(sql`
|
||||
SELECT d.id, d.name, d.path, d.created_at,
|
||||
(SELECT COUNT(*)::int FROM pdf.embedding e WHERE e.document_id = d.id) as embedding_count
|
||||
FROM pdf.document d
|
||||
ORDER BY d.created_at DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
console.log("\nRecent documents:\n");
|
||||
for (const doc of docs as { id: string; name: string | null; path: string; embedding_count: number; created_at: Date }[]) {
|
||||
console.log(` 📄 ${doc.name ?? "unnamed"}`);
|
||||
console.log(` ID: ${doc.id}`);
|
||||
console.log(` Path: ${doc.path}`);
|
||||
console.log(` Embeddings: ${doc.embedding_count}`);
|
||||
console.log(` Created: ${String(doc.created_at)}`);
|
||||
console.log("");
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
list().catch((e) => {
|
||||
console.error("Error:", e);
|
||||
process.exit(1);
|
||||
});
|
||||
63
packages/api/tests/regenerate-embeddings.ts
Normal file
63
packages/api/tests/regenerate-embeddings.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Script to regenerate embeddings for documents with 0 embeddings
|
||||
* Run: pnpm with-env npx tsx packages/api/tests/regenerate-embeddings.ts
|
||||
*/
|
||||
import { generateDocumentEmbeddings } from "@turbostarter/ai/pdf/embeddings";
|
||||
import { sql } from "@turbostarter/db";
|
||||
import { pdfEmbedding } from "@turbostarter/db/schema/pdf";
|
||||
import { db } from "@turbostarter/db/server";
|
||||
|
||||
async function regenerate() {
|
||||
// Find documents with 0 embeddings
|
||||
const orphans = await db.execute<{
|
||||
id: string;
|
||||
path: string;
|
||||
chat_id: string;
|
||||
}>(sql`
|
||||
SELECT d.id, d.path, d.chat_id
|
||||
FROM pdf.document d
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM pdf.embedding e WHERE e.document_id = d.id
|
||||
)
|
||||
`);
|
||||
|
||||
console.log(`Found ${orphans.length} documents without embeddings`);
|
||||
|
||||
for (const doc of orphans) {
|
||||
console.log(`\nProcessing document: ${doc.id}`);
|
||||
console.log(` Path: ${doc.path}`);
|
||||
|
||||
try {
|
||||
const generated = await generateDocumentEmbeddings(doc.path);
|
||||
console.log(` Generated ${generated.length} chunks`);
|
||||
|
||||
if (generated.length > 0) {
|
||||
await db
|
||||
.insert(pdfEmbedding)
|
||||
.values(
|
||||
generated.map((chunk) => ({
|
||||
content: chunk.content,
|
||||
documentId: doc.id,
|
||||
embedding: chunk.embedding,
|
||||
pageNumber: chunk.metadata.pageNumber,
|
||||
charStart: chunk.metadata.charStart,
|
||||
charEnd: chunk.metadata.charEnd,
|
||||
sectionTitle: chunk.metadata.sectionTitle,
|
||||
})),
|
||||
)
|
||||
.onConflictDoNothing();
|
||||
console.log(` ✅ Inserted embeddings`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(` ❌ Error:`, error instanceof Error ? error.message : error);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\nDone!");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
regenerate().catch((e) => {
|
||||
console.error("Fatal error:", e);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user