Fix JobDevTools contrast + log normalization, add Platform Spec
- Fix contrast issues in JobDevTools (level badges, text colors, timestamps) - Make log normalization more robust (handles old/new formats, edge cases) - Add ReviewIQ Platform Spec v1.2 defining: - Multi-tenant scraping-as-a-service architecture - Requester metadata, batches, webhooks, priority - Scraper versioning with A/B testing (stable/beta/canary) - API endpoints for job types, dashboard, admin - Output schemas for external service integration - Project structure reorganization plan Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -47,58 +47,85 @@ function extractBusinessName(job: JobStatus): string {
|
||||
}
|
||||
}
|
||||
|
||||
// Valid categories for structured logs
|
||||
const VALID_CATEGORIES: StructuredLog['category'][] = ['scraper', 'browser', 'network', 'system'];
|
||||
|
||||
// Valid log levels
|
||||
const VALID_LEVELS: StructuredLog['level'][] = ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL'];
|
||||
|
||||
/**
|
||||
* Check if a log entry is in the old format (has 'source' property)
|
||||
* or new structured format (has 'category' property)
|
||||
* Map source/category strings to valid category values
|
||||
*/
|
||||
function isOldLogFormat(log: OldLogEntry | StructuredLog): log is OldLogEntry {
|
||||
return 'source' in log && !('category' in log);
|
||||
function mapToCategory(source: string | undefined | null): StructuredLog['category'] {
|
||||
if (!source) return 'scraper';
|
||||
const lower = source.toLowerCase();
|
||||
if (lower === 'browser' || lower === 'navigation' || lower === 'page') return 'browser';
|
||||
if (lower === 'network' || lower === 'api') return 'network';
|
||||
if (lower === 'system' || lower === 'memory' || lower === 'chrome') return 'system';
|
||||
if (lower === 'scraper') return 'scraper';
|
||||
return 'scraper'; // Default to scraper for unknown sources
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert old log format to new StructuredLog format
|
||||
* Map level strings to valid level values
|
||||
*/
|
||||
function convertOldToStructured(oldLog: OldLogEntry): StructuredLog {
|
||||
// Map old source to new category
|
||||
const categoryMap: Record<string, StructuredLog['category']> = {
|
||||
browser: 'browser',
|
||||
scraper: 'scraper',
|
||||
network: 'network',
|
||||
system: 'system',
|
||||
};
|
||||
function mapToLevel(level: string | undefined | null): StructuredLog['level'] {
|
||||
if (!level) return 'INFO';
|
||||
const upper = level.toUpperCase();
|
||||
if (upper === 'WARNING') return 'WARN';
|
||||
if (VALID_LEVELS.includes(upper as StructuredLog['level'])) {
|
||||
return upper as StructuredLog['level'];
|
||||
}
|
||||
return 'INFO';
|
||||
}
|
||||
|
||||
// Map old level to new level
|
||||
const levelMap: Record<string, StructuredLog['level']> = {
|
||||
DEBUG: 'DEBUG',
|
||||
INFO: 'INFO',
|
||||
WARNING: 'WARN',
|
||||
WARN: 'WARN',
|
||||
ERROR: 'ERROR',
|
||||
FATAL: 'FATAL',
|
||||
};
|
||||
/**
|
||||
* Normalize any log entry to StructuredLog format
|
||||
* Handles: new format, old format with 'source', logs without category, edge cases
|
||||
*/
|
||||
function normalizeLog(log: Record<string, unknown>): StructuredLog {
|
||||
// Get timestamp
|
||||
const timestamp = (log.timestamp as string) || new Date().toISOString();
|
||||
const timestampMs = (log.timestamp_ms as number) || new Date(timestamp).getTime() || Date.now();
|
||||
|
||||
const timestamp = oldLog.timestamp;
|
||||
const timestampMs = new Date(timestamp).getTime();
|
||||
// Get message
|
||||
const message = (log.message as string) || '';
|
||||
|
||||
// Determine category: prefer 'category' field, fall back to 'source' field
|
||||
let category: StructuredLog['category'];
|
||||
if (log.category && VALID_CATEGORIES.includes(log.category as StructuredLog['category'])) {
|
||||
category = log.category as StructuredLog['category'];
|
||||
} else {
|
||||
category = mapToCategory((log.category as string) || (log.source as string));
|
||||
}
|
||||
|
||||
// Determine level
|
||||
const level = mapToLevel(log.level as string);
|
||||
|
||||
return {
|
||||
timestamp,
|
||||
timestamp_ms: timestampMs || Date.now(),
|
||||
level: levelMap[oldLog.level?.toUpperCase()] || 'INFO',
|
||||
category: categoryMap[oldLog.source] || 'system',
|
||||
message: oldLog.message,
|
||||
timestamp_ms: timestampMs,
|
||||
level,
|
||||
category,
|
||||
message,
|
||||
metrics: log.metrics as Record<string, unknown> | undefined,
|
||||
network: log.network as Record<string, unknown> | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert array of logs to structured format if needed
|
||||
* Convert array of logs to structured format
|
||||
* Robust handling of various log formats (old, new, malformed)
|
||||
*/
|
||||
function normalizeLogsTOStructured(logs: (OldLogEntry | StructuredLog)[]): StructuredLog[] {
|
||||
return logs.map((log) => {
|
||||
if (isOldLogFormat(log)) {
|
||||
return convertOldToStructured(log);
|
||||
}
|
||||
return log as StructuredLog;
|
||||
});
|
||||
function normalizeLogsTOStructured(logs: unknown[]): StructuredLog[] {
|
||||
if (!Array.isArray(logs)) return [];
|
||||
|
||||
return logs
|
||||
.filter((log): log is Record<string, unknown> => {
|
||||
// Filter out non-objects and nulls
|
||||
return log != null && typeof log === 'object' && !Array.isArray(log);
|
||||
})
|
||||
.map(normalizeLog);
|
||||
}
|
||||
|
||||
export default function JobDetailPage() {
|
||||
@@ -190,17 +217,7 @@ export default function JobDetailPage() {
|
||||
const data = JSON.parse(event.data);
|
||||
// Handle {"type": "log", "data": {...}} format
|
||||
const logData = data.data || data;
|
||||
|
||||
const newLog: StructuredLog = {
|
||||
timestamp: logData.timestamp || new Date().toISOString(),
|
||||
timestamp_ms: logData.timestamp_ms || Date.now(),
|
||||
level: logData.level || 'INFO',
|
||||
category: logData.category || 'system',
|
||||
message: logData.message || '',
|
||||
metrics: logData.metrics,
|
||||
network: logData.network,
|
||||
};
|
||||
|
||||
const newLog = normalizeLog(logData);
|
||||
setStructuredLogs((prev) => [...prev, newLog]);
|
||||
} catch (err) {
|
||||
console.error('Failed to parse log event:', err);
|
||||
@@ -347,15 +364,7 @@ export default function JobDetailPage() {
|
||||
// Check for type field to route to correct handler
|
||||
if (data.type === 'log') {
|
||||
const logData = data.data || data;
|
||||
const newLog: StructuredLog = {
|
||||
timestamp: logData.timestamp || new Date().toISOString(),
|
||||
timestamp_ms: logData.timestamp_ms || Date.now(),
|
||||
level: logData.level || 'INFO',
|
||||
category: logData.category || 'system',
|
||||
message: logData.message || '',
|
||||
metrics: logData.metrics,
|
||||
network: logData.network,
|
||||
};
|
||||
const newLog = normalizeLog(logData);
|
||||
setStructuredLogs((prev) => [...prev, newLog]);
|
||||
} else if (data.type === 'metrics') {
|
||||
const metricsPayload = data.data || data;
|
||||
|
||||
Reference in New Issue
Block a user