Fix JobDevTools contrast + log normalization, add Platform Spec
- Fix contrast issues in JobDevTools (level badges, text colors, timestamps) - Make log normalization more robust (handles old/new formats, edge cases) - Add ReviewIQ Platform Spec v1.2 defining: - Multi-tenant scraping-as-a-service architecture - Requester metadata, batches, webhooks, priority - Scraper versioning with A/B testing (stable/beta/canary) - API endpoints for job types, dashboard, admin - Output schemas for external service integration - Project structure reorganization plan Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
@@ -47,58 +47,85 @@ function extractBusinessName(job: JobStatus): string {
|
||||
}
|
||||
}
|
||||
|
||||
// Valid categories for structured logs
|
||||
const VALID_CATEGORIES: StructuredLog['category'][] = ['scraper', 'browser', 'network', 'system'];
|
||||
|
||||
// Valid log levels
|
||||
const VALID_LEVELS: StructuredLog['level'][] = ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL'];
|
||||
|
||||
/**
|
||||
* Check if a log entry is in the old format (has 'source' property)
|
||||
* or new structured format (has 'category' property)
|
||||
* Map source/category strings to valid category values
|
||||
*/
|
||||
function isOldLogFormat(log: OldLogEntry | StructuredLog): log is OldLogEntry {
|
||||
return 'source' in log && !('category' in log);
|
||||
function mapToCategory(source: string | undefined | null): StructuredLog['category'] {
|
||||
if (!source) return 'scraper';
|
||||
const lower = source.toLowerCase();
|
||||
if (lower === 'browser' || lower === 'navigation' || lower === 'page') return 'browser';
|
||||
if (lower === 'network' || lower === 'api') return 'network';
|
||||
if (lower === 'system' || lower === 'memory' || lower === 'chrome') return 'system';
|
||||
if (lower === 'scraper') return 'scraper';
|
||||
return 'scraper'; // Default to scraper for unknown sources
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert old log format to new StructuredLog format
|
||||
* Map level strings to valid level values
|
||||
*/
|
||||
function convertOldToStructured(oldLog: OldLogEntry): StructuredLog {
|
||||
// Map old source to new category
|
||||
const categoryMap: Record<string, StructuredLog['category']> = {
|
||||
browser: 'browser',
|
||||
scraper: 'scraper',
|
||||
network: 'network',
|
||||
system: 'system',
|
||||
};
|
||||
function mapToLevel(level: string | undefined | null): StructuredLog['level'] {
|
||||
if (!level) return 'INFO';
|
||||
const upper = level.toUpperCase();
|
||||
if (upper === 'WARNING') return 'WARN';
|
||||
if (VALID_LEVELS.includes(upper as StructuredLog['level'])) {
|
||||
return upper as StructuredLog['level'];
|
||||
}
|
||||
return 'INFO';
|
||||
}
|
||||
|
||||
// Map old level to new level
|
||||
const levelMap: Record<string, StructuredLog['level']> = {
|
||||
DEBUG: 'DEBUG',
|
||||
INFO: 'INFO',
|
||||
WARNING: 'WARN',
|
||||
WARN: 'WARN',
|
||||
ERROR: 'ERROR',
|
||||
FATAL: 'FATAL',
|
||||
};
|
||||
/**
|
||||
* Normalize any log entry to StructuredLog format
|
||||
* Handles: new format, old format with 'source', logs without category, edge cases
|
||||
*/
|
||||
function normalizeLog(log: Record<string, unknown>): StructuredLog {
|
||||
// Get timestamp
|
||||
const timestamp = (log.timestamp as string) || new Date().toISOString();
|
||||
const timestampMs = (log.timestamp_ms as number) || new Date(timestamp).getTime() || Date.now();
|
||||
|
||||
const timestamp = oldLog.timestamp;
|
||||
const timestampMs = new Date(timestamp).getTime();
|
||||
// Get message
|
||||
const message = (log.message as string) || '';
|
||||
|
||||
// Determine category: prefer 'category' field, fall back to 'source' field
|
||||
let category: StructuredLog['category'];
|
||||
if (log.category && VALID_CATEGORIES.includes(log.category as StructuredLog['category'])) {
|
||||
category = log.category as StructuredLog['category'];
|
||||
} else {
|
||||
category = mapToCategory((log.category as string) || (log.source as string));
|
||||
}
|
||||
|
||||
// Determine level
|
||||
const level = mapToLevel(log.level as string);
|
||||
|
||||
return {
|
||||
timestamp,
|
||||
timestamp_ms: timestampMs || Date.now(),
|
||||
level: levelMap[oldLog.level?.toUpperCase()] || 'INFO',
|
||||
category: categoryMap[oldLog.source] || 'system',
|
||||
message: oldLog.message,
|
||||
timestamp_ms: timestampMs,
|
||||
level,
|
||||
category,
|
||||
message,
|
||||
metrics: log.metrics as Record<string, unknown> | undefined,
|
||||
network: log.network as Record<string, unknown> | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert array of logs to structured format if needed
|
||||
* Convert array of logs to structured format
|
||||
* Robust handling of various log formats (old, new, malformed)
|
||||
*/
|
||||
function normalizeLogsTOStructured(logs: (OldLogEntry | StructuredLog)[]): StructuredLog[] {
|
||||
return logs.map((log) => {
|
||||
if (isOldLogFormat(log)) {
|
||||
return convertOldToStructured(log);
|
||||
}
|
||||
return log as StructuredLog;
|
||||
});
|
||||
function normalizeLogsTOStructured(logs: unknown[]): StructuredLog[] {
|
||||
if (!Array.isArray(logs)) return [];
|
||||
|
||||
return logs
|
||||
.filter((log): log is Record<string, unknown> => {
|
||||
// Filter out non-objects and nulls
|
||||
return log != null && typeof log === 'object' && !Array.isArray(log);
|
||||
})
|
||||
.map(normalizeLog);
|
||||
}
|
||||
|
||||
export default function JobDetailPage() {
|
||||
@@ -190,17 +217,7 @@ export default function JobDetailPage() {
|
||||
const data = JSON.parse(event.data);
|
||||
// Handle {"type": "log", "data": {...}} format
|
||||
const logData = data.data || data;
|
||||
|
||||
const newLog: StructuredLog = {
|
||||
timestamp: logData.timestamp || new Date().toISOString(),
|
||||
timestamp_ms: logData.timestamp_ms || Date.now(),
|
||||
level: logData.level || 'INFO',
|
||||
category: logData.category || 'system',
|
||||
message: logData.message || '',
|
||||
metrics: logData.metrics,
|
||||
network: logData.network,
|
||||
};
|
||||
|
||||
const newLog = normalizeLog(logData);
|
||||
setStructuredLogs((prev) => [...prev, newLog]);
|
||||
} catch (err) {
|
||||
console.error('Failed to parse log event:', err);
|
||||
@@ -347,15 +364,7 @@ export default function JobDetailPage() {
|
||||
// Check for type field to route to correct handler
|
||||
if (data.type === 'log') {
|
||||
const logData = data.data || data;
|
||||
const newLog: StructuredLog = {
|
||||
timestamp: logData.timestamp || new Date().toISOString(),
|
||||
timestamp_ms: logData.timestamp_ms || Date.now(),
|
||||
level: logData.level || 'INFO',
|
||||
category: logData.category || 'system',
|
||||
message: logData.message || '',
|
||||
metrics: logData.metrics,
|
||||
network: logData.network,
|
||||
};
|
||||
const newLog = normalizeLog(logData);
|
||||
setStructuredLogs((prev) => [...prev, newLog]);
|
||||
} else if (data.type === 'metrics') {
|
||||
const metricsPayload = data.data || data;
|
||||
|
||||
@@ -60,19 +60,19 @@ const TAB_CONFIG: { id: TabType; label: string; icon: typeof Bug; category?: Str
|
||||
];
|
||||
|
||||
const LEVEL_COLORS: Record<LogLevel, { bg: string; text: string; border: string }> = {
|
||||
DEBUG: { bg: 'bg-gray-700', text: 'text-gray-300', border: 'border-gray-600' },
|
||||
INFO: { bg: 'bg-blue-900', text: 'text-blue-300', border: 'border-blue-700' },
|
||||
WARN: { bg: 'bg-yellow-900', text: 'text-yellow-300', border: 'border-yellow-700' },
|
||||
ERROR: { bg: 'bg-red-900', text: 'text-red-300', border: 'border-red-700' },
|
||||
FATAL: { bg: 'bg-purple-900', text: 'text-purple-300', border: 'border-purple-700' },
|
||||
DEBUG: { bg: 'bg-gray-900', text: 'text-gray-200', border: 'border-gray-700' },
|
||||
INFO: { bg: 'bg-gray-900', text: 'text-gray-100', border: 'border-gray-700' },
|
||||
WARN: { bg: 'bg-gray-900', text: 'text-amber-200', border: 'border-gray-700' },
|
||||
ERROR: { bg: 'bg-gray-900', text: 'text-red-200', border: 'border-gray-700' },
|
||||
FATAL: { bg: 'bg-gray-900', text: 'text-fuchsia-200', border: 'border-gray-700' },
|
||||
};
|
||||
|
||||
const LEVEL_BADGE_COLORS: Record<LogLevel, string> = {
|
||||
DEBUG: 'bg-gray-600 text-gray-200',
|
||||
INFO: 'bg-blue-600 text-blue-100',
|
||||
WARN: 'bg-yellow-600 text-yellow-100',
|
||||
ERROR: 'bg-red-600 text-red-100',
|
||||
FATAL: 'bg-purple-600 text-purple-100',
|
||||
DEBUG: 'bg-gray-500 text-white',
|
||||
INFO: 'bg-blue-500 text-white',
|
||||
WARN: 'bg-amber-500 text-gray-900',
|
||||
ERROR: 'bg-red-500 text-white',
|
||||
FATAL: 'bg-fuchsia-500 text-white',
|
||||
};
|
||||
|
||||
export default function JobDevTools({
|
||||
@@ -263,11 +263,11 @@ export default function JobDevTools({
|
||||
{/* Log entries - scrollable area */}
|
||||
<div className="flex-1 overflow-y-auto min-h-[250px] max-h-[500px] font-mono text-sm">
|
||||
{filteredLogs.length === 0 ? (
|
||||
<div className="flex items-center justify-center h-full text-gray-500">
|
||||
<div className="flex items-center justify-center h-full text-gray-400">
|
||||
<div className="text-center">
|
||||
<Bug className="w-8 h-8 mx-auto mb-2 opacity-50" />
|
||||
<Bug className="w-8 h-8 mx-auto mb-2 opacity-60" />
|
||||
<p>No logs to display</p>
|
||||
<p className="text-xs mt-1">
|
||||
<p className="text-xs mt-1 text-gray-500">
|
||||
{logs.length > 0
|
||||
? 'Try adjusting your filters'
|
||||
: 'Logs will appear here during job execution'}
|
||||
@@ -281,23 +281,23 @@ export default function JobDevTools({
|
||||
return (
|
||||
<div
|
||||
key={`${log.timestamp_ms}-${index}`}
|
||||
className={`px-4 py-2 hover:bg-gray-800 transition-colors ${levelStyle.bg} bg-opacity-20`}
|
||||
className="px-4 py-2 hover:bg-gray-800 transition-colors"
|
||||
>
|
||||
<div className="flex items-start gap-3">
|
||||
{/* Timestamp */}
|
||||
<span className="text-gray-500 text-xs whitespace-nowrap pt-0.5">
|
||||
<span className="text-gray-400 text-xs whitespace-nowrap pt-0.5 font-mono">
|
||||
{formatTimestamp(log.timestamp)}
|
||||
</span>
|
||||
|
||||
{/* Level badge */}
|
||||
<span
|
||||
className={`px-1.5 py-0.5 text-xs font-semibold rounded ${LEVEL_BADGE_COLORS[log.level]} whitespace-nowrap`}
|
||||
className={`px-1.5 py-0.5 text-xs font-bold rounded ${LEVEL_BADGE_COLORS[log.level]} whitespace-nowrap`}
|
||||
>
|
||||
{log.level}
|
||||
</span>
|
||||
|
||||
{/* Category badge */}
|
||||
<span className="px-1.5 py-0.5 text-xs font-medium rounded bg-gray-700 text-gray-300 whitespace-nowrap">
|
||||
<span className="px-1.5 py-0.5 text-xs font-medium rounded bg-gray-600 text-gray-100 whitespace-nowrap">
|
||||
{log.category}
|
||||
</span>
|
||||
|
||||
@@ -309,14 +309,14 @@ export default function JobDevTools({
|
||||
|
||||
{/* Additional data (metrics/network) */}
|
||||
{(log.metrics || log.network) && (
|
||||
<div className="mt-1 ml-[72px] text-xs text-gray-500">
|
||||
<div className="mt-1 ml-[88px] text-xs text-gray-300 font-mono">
|
||||
{log.metrics && (
|
||||
<span className="mr-4">
|
||||
metrics: {JSON.stringify(log.metrics)}
|
||||
<span className="text-gray-500">metrics:</span> {JSON.stringify(log.metrics)}
|
||||
</span>
|
||||
)}
|
||||
{log.network && (
|
||||
<span>network: {JSON.stringify(log.network)}</span>
|
||||
<span><span className="text-gray-500">network:</span> {JSON.stringify(log.network)}</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
@@ -329,30 +329,30 @@ export default function JobDevTools({
|
||||
|
||||
{/* Reserved space for metrics/session panels (footer) */}
|
||||
<div className="border-t border-gray-700 bg-gray-800 px-4 py-3 rounded-b-xl">
|
||||
<div className="flex items-center justify-between text-xs text-gray-400">
|
||||
<div className="flex items-center justify-between text-xs text-gray-300">
|
||||
<div className="flex items-center gap-4">
|
||||
{metrics && (
|
||||
<>
|
||||
{metrics.duration_ms !== undefined && (
|
||||
<span>Duration: {(metrics.duration_ms / 1000).toFixed(2)}s</span>
|
||||
<span><span className="text-gray-500">Duration:</span> {(metrics.duration_ms / 1000).toFixed(2)}s</span>
|
||||
)}
|
||||
{metrics.reviews_scraped !== undefined && (
|
||||
<span>Reviews: {metrics.reviews_scraped}</span>
|
||||
<span><span className="text-gray-500">Reviews:</span> {metrics.reviews_scraped}</span>
|
||||
)}
|
||||
{metrics.memory_mb !== undefined && (
|
||||
<span>Memory: {metrics.memory_mb.toFixed(1)}MB</span>
|
||||
<span><span className="text-gray-500">Memory:</span> {metrics.memory_mb.toFixed(1)}MB</span>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-4">
|
||||
{sessionFingerprint && (
|
||||
<span className="text-gray-500">
|
||||
<span className="text-gray-400">
|
||||
Session: {sessionFingerprint.session_id?.slice(0, 8)}...
|
||||
</span>
|
||||
)}
|
||||
{crashReport && (
|
||||
<span className="text-red-400 font-medium">
|
||||
<span className="text-red-300 font-medium">
|
||||
Crash: {crashReport.error_type}
|
||||
</span>
|
||||
)}
|
||||
|
||||
Reference in New Issue
Block a user