'use client'; import { useEffect, useState, useRef, useCallback } from 'react'; import { useParams, useRouter } from 'next/navigation'; import Link from 'next/link'; import { useJobs } from '@/contexts/JobsContext'; import { JobStatus } from '@/components/ScraperTest'; import JobDevTools, { StructuredLog, MetricsData, SessionFingerprint as DevToolsSessionFingerprint, } from '@/components/JobDevTools'; import CrashReport, { CrashReportData } from '@/components/JobDevTools/CrashReport'; import SessionPanel, { SessionFingerprint as DetailedSessionFingerprint } from '@/components/JobDevTools/SessionPanel'; import MetricsDashboard, { MetricsSample } from '@/components/JobDevTools/MetricsDashboard'; interface OldLogEntry { timestamp: string; level: string; message: string; source: string; } interface JobLogs { job_id: string; status: string; error_message: string | null; logs: OldLogEntry[] | StructuredLog[]; log_count: number; } function formatDuration(seconds: number): string { if (seconds < 60) return `${seconds.toFixed(1)}s`; const mins = Math.floor(seconds / 60); const secs = Math.round(seconds % 60); return `${mins}m ${secs}s`; } function extractBusinessName(job: JobStatus): string { if (job.business_name) return job.business_name; try { const urlObj = new URL(job.url); const query = urlObj.searchParams.get('query'); return query ? decodeURIComponent(query) : 'Unknown Business'; } catch { return 'Unknown Business'; } } // Valid categories for structured logs const VALID_CATEGORIES: StructuredLog['category'][] = ['scraper', 'browser', 'network', 'system']; // Valid log levels const VALID_LEVELS: StructuredLog['level'][] = ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL']; /** * Map source/category strings to valid category values */ function mapToCategory(source: string | undefined | null): StructuredLog['category'] { if (!source) return 'scraper'; const lower = source.toLowerCase(); if (lower === 'browser' || lower === 'navigation' || lower === 'page') return 'browser'; if (lower === 'network' || lower === 'api') return 'network'; if (lower === 'system' || lower === 'memory' || lower === 'chrome') return 'system'; if (lower === 'scraper') return 'scraper'; return 'scraper'; // Default to scraper for unknown sources } /** * Map level strings to valid level values */ function mapToLevel(level: string | undefined | null): StructuredLog['level'] { if (!level) return 'INFO'; const upper = level.toUpperCase(); if (upper === 'WARNING') return 'WARN'; if (VALID_LEVELS.includes(upper as StructuredLog['level'])) { return upper as StructuredLog['level']; } return 'INFO'; } /** * Normalize any log entry to StructuredLog format * Handles: new format, old format with 'source', logs without category, edge cases */ function normalizeLog(log: Record): StructuredLog { // Get timestamp const timestamp = (log.timestamp as string) || new Date().toISOString(); const timestampMs = (log.timestamp_ms as number) || new Date(timestamp).getTime() || Date.now(); // Get message const message = (log.message as string) || ''; // Determine category: prefer 'category' field, fall back to 'source' field let category: StructuredLog['category']; if (log.category && VALID_CATEGORIES.includes(log.category as StructuredLog['category'])) { category = log.category as StructuredLog['category']; } else { category = mapToCategory((log.category as string) || (log.source as string)); } // Determine level const level = mapToLevel(log.level as string); return { timestamp, timestamp_ms: timestampMs, level, category, message, metrics: log.metrics as Record | undefined, network: log.network as Record | undefined, }; } /** * Convert array of logs to structured format * Robust handling of various log formats (old, new, malformed) */ function normalizeLogsTOStructured(logs: unknown[]): StructuredLog[] { if (!Array.isArray(logs)) return []; return logs .filter((log): log is Record => { // Filter out non-objects and nulls return log != null && typeof log === 'object' && !Array.isArray(log); }) .map(normalizeLog); } export default function JobDetailPage() { const params = useParams(); const router = useRouter(); const { jobs, refreshJobs } = useJobs(); const [job, setJob] = useState(null); const [structuredLogs, setStructuredLogs] = useState([]); const [metricsData, setMetricsData] = useState(undefined); const [metricsHistory, setMetricsHistory] = useState([]); const [crashReport, setCrashReport] = useState(null); const [sessionFingerprint, setSessionFingerprint] = useState(undefined); const [isStreaming, setIsStreaming] = useState(false); const [isLoadingLogs, setIsLoadingLogs] = useState(false); const [isLoadingCrashReport, setIsLoadingCrashReport] = useState(false); const [isDeleting, setIsDeleting] = useState(false); const [retryFeedback, setRetryFeedback] = useState<{ type: 'success' | 'error'; message: string } | null>(null); const jobId = params.id as string; const eventSourceRef = useRef(null); // Find job from context or fetch it useEffect(() => { const foundJob = jobs.find((j) => j.job_id === jobId); if (foundJob) { setJob(foundJob); } else { // Fetch job directly if not in context fetch(`/api/jobs/${jobId}`) .then((res) => res.json()) .then((data) => setJob(data)) .catch(console.error); } }, [jobId, jobs]); // Fetch initial logs when job is loaded useEffect(() => { if (!jobId) return; setIsLoadingLogs(true); fetch(`/api/jobs/${jobId}/logs`) .then((res) => res.json()) .then((data: JobLogs) => { if (data.logs && data.logs.length > 0) { const normalized = normalizeLogsTOStructured(data.logs); setStructuredLogs(normalized); } }) .catch(console.error) .finally(() => setIsLoadingLogs(false)); }, [jobId]); // Connect to SSE stream for running jobs useEffect(() => { if (!job || job.status !== 'running') { // Close any existing connection for non-running jobs if (eventSourceRef.current) { eventSourceRef.current.close(); eventSourceRef.current = null; setIsStreaming(false); } return; } // Create SSE connection const eventSource = new EventSource(`/api/jobs/${jobId}/stream`); eventSourceRef.current = eventSource; eventSource.onopen = () => { console.log('SSE connected for job:', jobId); setIsStreaming(true); }; eventSource.onerror = (err) => { console.error('SSE error:', err); setIsStreaming(false); // Try to reconnect after a delay setTimeout(() => { if (eventSourceRef.current === eventSource) { eventSource.close(); // Will reconnect on next render cycle if job is still running } }, 3000); }; // Handle structured log events eventSource.addEventListener('log', (event) => { try { const data = JSON.parse(event.data); // Handle {"type": "log", "data": {...}} format const logData = data.data || data; const newLog = normalizeLog(logData); setStructuredLogs((prev) => [...prev, newLog]); } catch (err) { console.error('Failed to parse log event:', err); } }); // Handle metrics events eventSource.addEventListener('metrics', (event) => { try { const data = JSON.parse(event.data); // Handle {"type": "metrics", "data": {...}} format const metricsPayload = data.data || data; setMetricsData({ cpu_percent: metricsPayload.cpu_percent, memory_mb: metricsPayload.memory_mb, duration_ms: metricsPayload.duration_ms, requests_made: metricsPayload.requests_made, reviews_scraped: metricsPayload.reviews_extracted || metricsPayload.reviews_scraped, }); // Add to metrics history for charts const sample: MetricsSample = { timestamp_ms: metricsPayload.timestamp_ms || Date.now(), reviews_extracted: metricsPayload.reviews_extracted || 0, scroll_count: metricsPayload.scroll_count || 0, memory_mb: metricsPayload.memory_mb || 0, extraction_rate: metricsPayload.extraction_rate || 0, }; setMetricsHistory((prev) => [...prev, sample]); } catch (err) { console.error('Failed to parse metrics event:', err); } }); // Handle job progress events (from existing SSE format) eventSource.addEventListener('job_progress', (event) => { try { const data = JSON.parse(event.data); setJob((prev) => prev ? { ...prev, reviews_count: data.reviews_count, total_reviews: data.total_reviews, scrape_time: data.scrape_time, } : prev ); } catch (err) { console.error('Failed to parse job_progress event:', err); } }); // Handle job completed eventSource.addEventListener('job_completed', (event) => { try { const data = JSON.parse(event.data); setJob((prev) => prev ? { ...prev, status: 'completed', reviews_count: data.reviews_count, total_reviews: data.total_reviews, scrape_time: data.scrape_time, } : prev ); eventSource.close(); setIsStreaming(false); refreshJobs(); } catch (err) { console.error('Failed to parse job_completed event:', err); } }); // Handle job failed eventSource.addEventListener('job_failed', (event) => { try { const data = JSON.parse(event.data); setJob((prev) => prev ? { ...prev, status: 'failed', error_message: data.error || data.error_message, } : prev ); eventSource.close(); setIsStreaming(false); refreshJobs(); // Fetch crash report when job fails fetchCrashReport(); } catch (err) { console.error('Failed to parse job_failed event:', err); } }); // Handle job partial eventSource.addEventListener('job_partial', (event) => { try { const data = JSON.parse(event.data); setJob((prev) => prev ? { ...prev, status: 'partial', reviews_count: data.reviews_count, error_message: data.error || data.error_message, } : prev ); eventSource.close(); setIsStreaming(false); refreshJobs(); // Fetch crash report for partial jobs too fetchCrashReport(); } catch (err) { console.error('Failed to parse job_partial event:', err); } }); // Handle initial state (all current logs) eventSource.addEventListener('initial_state', (event) => { try { const data = JSON.parse(event.data); if (data.logs && data.logs.length > 0) { const normalized = normalizeLogsTOStructured(data.logs); setStructuredLogs(normalized); } } catch (err) { console.error('Failed to parse initial_state event:', err); } }); // Handle generic message events eventSource.onmessage = (event) => { try { const data = JSON.parse(event.data); // Check for type field to route to correct handler if (data.type === 'log') { const logData = data.data || data; const newLog = normalizeLog(logData); setStructuredLogs((prev) => [...prev, newLog]); } else if (data.type === 'metrics') { const metricsPayload = data.data || data; setMetricsData({ cpu_percent: metricsPayload.cpu_percent, memory_mb: metricsPayload.memory_mb, duration_ms: metricsPayload.duration_ms, requests_made: metricsPayload.requests_made, reviews_scraped: metricsPayload.reviews_extracted || metricsPayload.reviews_scraped, }); } } catch { // Ignore non-JSON messages } }; return () => { eventSource.close(); eventSourceRef.current = null; setIsStreaming(false); }; }, [job?.status, jobId, refreshJobs]); // Fetch crash report when job status is failed or partial const fetchCrashReport = useCallback(async () => { if (!jobId) return; setIsLoadingCrashReport(true); try { const response = await fetch(`/api/jobs/${jobId}/crash-report`); if (response.ok) { const data = await response.json(); setCrashReport(data); } } catch (err) { console.error('Failed to fetch crash report:', err); } finally { setIsLoadingCrashReport(false); } }, [jobId]); // Fetch crash report if job is failed or partial on load useEffect(() => { if (job && (job.status === 'failed' || job.status === 'partial')) { fetchCrashReport(); } }, [job?.status, fetchCrashReport]); // Extract session fingerprint from job metadata useEffect(() => { if (!job) return; // Try to get session fingerprint from job metadata fetch(`/api/jobs/${jobId}`) .then((res) => res.json()) .then((fullJob) => { if (fullJob.metadata) { const metadata = typeof fullJob.metadata === 'string' ? JSON.parse(fullJob.metadata) : fullJob.metadata; if (metadata.session_fingerprint) { setSessionFingerprint(metadata.session_fingerprint); } else if (metadata.browser_fingerprint) { // Convert browser fingerprint to session fingerprint format const bf = metadata.browser_fingerprint; setSessionFingerprint({ user_agent: bf.userAgent || '', platform: bf.platform || '', language: bf.language || '', languages: bf.languages || [bf.language || ''], timezone: bf.timezone || '', screen: { width: bf.viewport?.width || 1920, height: bf.viewport?.height || 1080, colorDepth: 24, }, viewport: bf.viewport || { width: 1920, height: 1080 }, webgl_vendor: '', webgl_renderer: '', canvas_fingerprint: '', hardware_concurrency: 4, device_memory: 8, bot_detection_tests: { webdriver_hidden: true, chrome_runtime: true, permissions_query: true, }, captured_at: new Date().toISOString(), }); } } }) .catch(console.error); }, [job, jobId]); const handleDelete = async () => { if (!confirm('Are you sure you want to delete this job?')) return; setIsDeleting(true); try { await fetch(`/api/jobs/${jobId}`, { method: 'DELETE' }); await refreshJobs(); router.push('/jobs'); } catch (err) { console.error('Failed to delete job:', err); } finally { setIsDeleting(false); } }; const handleRetry = useCallback( (applyFix: boolean) => { setRetryFeedback({ type: 'success', message: applyFix ? 'Retrying job with auto-fix applied...' : 'Retrying job without modifications...', }); // Refresh jobs to pick up the new job setTimeout(() => { refreshJobs(); setRetryFeedback(null); }, 2000); }, [refreshJobs] ); if (!job) { return (
); } const businessName = extractBusinessName(job); const canViewAnalytics = job.reviews_count && job.reviews_count > 0; const showCrashReport = (job.status === 'failed' || job.status === 'partial') && crashReport; return (
{/* Breadcrumb */}
Jobs / {jobId.slice(0, 8)}...
{/* Header */}

{businessName}

{job.business_address && (

{job.business_address}

)}
{job.status === 'running' && (
)} {job.status.charAt(0).toUpperCase() + job.status.slice(1)}
{/* Stats Grid */}
{job.reviews_count !== null && (
{job.reviews_count.toLocaleString()}
Reviews
)} {job.scrape_time !== null && (
{formatDuration(job.scrape_time)}
Duration
)} {job.rating_snapshot != null && (
{job.rating_snapshot.toFixed(1)}
Rating
)}
{new Date(job.created_at).toLocaleDateString()}
{new Date(job.created_at).toLocaleTimeString()}
Created
{/* Actions */}
{canViewAnalytics && ( View Analytics )} Open in Maps
{/* Error Message (legacy - shown when no crash report) */} {job.error_message && !showCrashReport && (

Error

{job.error_message}

)} {/* Retry Feedback */} {retryFeedback && (
{retryFeedback.type === 'success' ? ( ) : ( )} {retryFeedback.message}
)}
{/* Crash Report Section (for failed/partial jobs) */} {showCrashReport && (
)} {/* Metrics Dashboard (for running jobs) */} {job.status === 'running' && metricsHistory.length > 0 && (
)} {/* Session Panel (if fingerprint available) */} {sessionFingerprint && (
)} {/* Job DevTools - Main Log Viewer */}
{isLoadingLogs ? (

Loading logs...

) : ( )}
{/* Loading Crash Report Indicator */} {isLoadingCrashReport && (
Loading crash report...
)}
); }