Fix JobDevTools contrast + log normalization, add Platform Spec

- Fix contrast issues in JobDevTools (level badges, text colors, timestamps)
- Make log normalization more robust (handles old/new formats, edge cases)
- Add ReviewIQ Platform Spec v1.2 defining:
  - Multi-tenant scraping-as-a-service architecture
  - Requester metadata, batches, webhooks, priority
  - Scraper versioning with A/B testing (stable/beta/canary)
  - API endpoints for job types, dashboard, admin
  - Output schemas for external service integration
  - Project structure reorganization plan

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
2026-01-24 15:13:19 +00:00
parent 1e5401a9d1
commit 12d37e350b
3 changed files with 825 additions and 82 deletions

View File

@@ -47,58 +47,85 @@ function extractBusinessName(job: JobStatus): string {
}
}
// Valid categories for structured logs
const VALID_CATEGORIES: StructuredLog['category'][] = ['scraper', 'browser', 'network', 'system'];
// Valid log levels
const VALID_LEVELS: StructuredLog['level'][] = ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL'];
/**
* Check if a log entry is in the old format (has 'source' property)
* or new structured format (has 'category' property)
* Map source/category strings to valid category values
*/
function isOldLogFormat(log: OldLogEntry | StructuredLog): log is OldLogEntry {
return 'source' in log && !('category' in log);
function mapToCategory(source: string | undefined | null): StructuredLog['category'] {
if (!source) return 'scraper';
const lower = source.toLowerCase();
if (lower === 'browser' || lower === 'navigation' || lower === 'page') return 'browser';
if (lower === 'network' || lower === 'api') return 'network';
if (lower === 'system' || lower === 'memory' || lower === 'chrome') return 'system';
if (lower === 'scraper') return 'scraper';
return 'scraper'; // Default to scraper for unknown sources
}
/**
* Convert old log format to new StructuredLog format
* Map level strings to valid level values
*/
function convertOldToStructured(oldLog: OldLogEntry): StructuredLog {
// Map old source to new category
const categoryMap: Record<string, StructuredLog['category']> = {
browser: 'browser',
scraper: 'scraper',
network: 'network',
system: 'system',
};
function mapToLevel(level: string | undefined | null): StructuredLog['level'] {
if (!level) return 'INFO';
const upper = level.toUpperCase();
if (upper === 'WARNING') return 'WARN';
if (VALID_LEVELS.includes(upper as StructuredLog['level'])) {
return upper as StructuredLog['level'];
}
return 'INFO';
}
// Map old level to new level
const levelMap: Record<string, StructuredLog['level']> = {
DEBUG: 'DEBUG',
INFO: 'INFO',
WARNING: 'WARN',
WARN: 'WARN',
ERROR: 'ERROR',
FATAL: 'FATAL',
};
/**
* Normalize any log entry to StructuredLog format
* Handles: new format, old format with 'source', logs without category, edge cases
*/
function normalizeLog(log: Record<string, unknown>): StructuredLog {
// Get timestamp
const timestamp = (log.timestamp as string) || new Date().toISOString();
const timestampMs = (log.timestamp_ms as number) || new Date(timestamp).getTime() || Date.now();
const timestamp = oldLog.timestamp;
const timestampMs = new Date(timestamp).getTime();
// Get message
const message = (log.message as string) || '';
// Determine category: prefer 'category' field, fall back to 'source' field
let category: StructuredLog['category'];
if (log.category && VALID_CATEGORIES.includes(log.category as StructuredLog['category'])) {
category = log.category as StructuredLog['category'];
} else {
category = mapToCategory((log.category as string) || (log.source as string));
}
// Determine level
const level = mapToLevel(log.level as string);
return {
timestamp,
timestamp_ms: timestampMs || Date.now(),
level: levelMap[oldLog.level?.toUpperCase()] || 'INFO',
category: categoryMap[oldLog.source] || 'system',
message: oldLog.message,
timestamp_ms: timestampMs,
level,
category,
message,
metrics: log.metrics as Record<string, unknown> | undefined,
network: log.network as Record<string, unknown> | undefined,
};
}
/**
* Convert array of logs to structured format if needed
* Convert array of logs to structured format
* Robust handling of various log formats (old, new, malformed)
*/
function normalizeLogsTOStructured(logs: (OldLogEntry | StructuredLog)[]): StructuredLog[] {
return logs.map((log) => {
if (isOldLogFormat(log)) {
return convertOldToStructured(log);
}
return log as StructuredLog;
});
function normalizeLogsTOStructured(logs: unknown[]): StructuredLog[] {
if (!Array.isArray(logs)) return [];
return logs
.filter((log): log is Record<string, unknown> => {
// Filter out non-objects and nulls
return log != null && typeof log === 'object' && !Array.isArray(log);
})
.map(normalizeLog);
}
export default function JobDetailPage() {
@@ -190,17 +217,7 @@ export default function JobDetailPage() {
const data = JSON.parse(event.data);
// Handle {"type": "log", "data": {...}} format
const logData = data.data || data;
const newLog: StructuredLog = {
timestamp: logData.timestamp || new Date().toISOString(),
timestamp_ms: logData.timestamp_ms || Date.now(),
level: logData.level || 'INFO',
category: logData.category || 'system',
message: logData.message || '',
metrics: logData.metrics,
network: logData.network,
};
const newLog = normalizeLog(logData);
setStructuredLogs((prev) => [...prev, newLog]);
} catch (err) {
console.error('Failed to parse log event:', err);
@@ -347,15 +364,7 @@ export default function JobDetailPage() {
// Check for type field to route to correct handler
if (data.type === 'log') {
const logData = data.data || data;
const newLog: StructuredLog = {
timestamp: logData.timestamp || new Date().toISOString(),
timestamp_ms: logData.timestamp_ms || Date.now(),
level: logData.level || 'INFO',
category: logData.category || 'system',
message: logData.message || '',
metrics: logData.metrics,
network: logData.network,
};
const newLog = normalizeLog(logData);
setStructuredLogs((prev) => [...prev, newLog]);
} else if (data.type === 'metrics') {
const metricsPayload = data.data || data;