Files
whyrating-engine-legacy/web/app/jobs/[id]/page.tsx
2026-01-24 13:15:14 +00:00

785 lines
27 KiB
TypeScript

'use client';
import { useEffect, useState, useRef, useCallback } from 'react';
import { useParams, useRouter } from 'next/navigation';
import Link from 'next/link';
import { useJobs } from '@/contexts/JobsContext';
import { JobStatus } from '@/components/ScraperTest';
import JobDevTools, {
StructuredLog,
MetricsData,
SessionFingerprint as DevToolsSessionFingerprint,
} from '@/components/JobDevTools';
import CrashReport, { CrashReportData } from '@/components/JobDevTools/CrashReport';
import SessionPanel, { SessionFingerprint as DetailedSessionFingerprint } from '@/components/JobDevTools/SessionPanel';
import MetricsDashboard, { MetricsSample } from '@/components/JobDevTools/MetricsDashboard';
interface OldLogEntry {
timestamp: string;
level: string;
message: string;
source: string;
}
interface JobLogs {
job_id: string;
status: string;
error_message: string | null;
logs: OldLogEntry[] | StructuredLog[];
log_count: number;
}
function formatDuration(seconds: number): string {
if (seconds < 60) return `${seconds.toFixed(1)}s`;
const mins = Math.floor(seconds / 60);
const secs = Math.round(seconds % 60);
return `${mins}m ${secs}s`;
}
function extractBusinessName(job: JobStatus): string {
if (job.business_name) return job.business_name;
try {
const urlObj = new URL(job.url);
const query = urlObj.searchParams.get('query');
return query ? decodeURIComponent(query) : 'Unknown Business';
} catch {
return 'Unknown Business';
}
}
/**
* Check if a log entry is in the old format (has 'source' property)
* or new structured format (has 'category' property)
*/
function isOldLogFormat(log: OldLogEntry | StructuredLog): log is OldLogEntry {
return 'source' in log && !('category' in log);
}
/**
* Convert old log format to new StructuredLog format
*/
function convertOldToStructured(oldLog: OldLogEntry): StructuredLog {
// Map old source to new category
const categoryMap: Record<string, StructuredLog['category']> = {
browser: 'browser',
scraper: 'scraper',
network: 'network',
system: 'system',
};
// Map old level to new level
const levelMap: Record<string, StructuredLog['level']> = {
DEBUG: 'DEBUG',
INFO: 'INFO',
WARNING: 'WARN',
WARN: 'WARN',
ERROR: 'ERROR',
FATAL: 'FATAL',
};
const timestamp = oldLog.timestamp;
const timestampMs = new Date(timestamp).getTime();
return {
timestamp,
timestamp_ms: timestampMs || Date.now(),
level: levelMap[oldLog.level?.toUpperCase()] || 'INFO',
category: categoryMap[oldLog.source] || 'system',
message: oldLog.message,
};
}
/**
* Convert array of logs to structured format if needed
*/
function normalizeLogsTOStructured(logs: (OldLogEntry | StructuredLog)[]): StructuredLog[] {
return logs.map((log) => {
if (isOldLogFormat(log)) {
return convertOldToStructured(log);
}
return log as StructuredLog;
});
}
export default function JobDetailPage() {
const params = useParams();
const router = useRouter();
const { jobs, refreshJobs } = useJobs();
const [job, setJob] = useState<JobStatus | null>(null);
const [structuredLogs, setStructuredLogs] = useState<StructuredLog[]>([]);
const [metricsData, setMetricsData] = useState<MetricsData | undefined>(undefined);
const [metricsHistory, setMetricsHistory] = useState<MetricsSample[]>([]);
const [crashReport, setCrashReport] = useState<CrashReportData | null>(null);
const [sessionFingerprint, setSessionFingerprint] = useState<DetailedSessionFingerprint | undefined>(undefined);
const [isStreaming, setIsStreaming] = useState(false);
const [isLoadingLogs, setIsLoadingLogs] = useState(false);
const [isLoadingCrashReport, setIsLoadingCrashReport] = useState(false);
const [isDeleting, setIsDeleting] = useState(false);
const [retryFeedback, setRetryFeedback] = useState<{ type: 'success' | 'error'; message: string } | null>(null);
const jobId = params.id as string;
const eventSourceRef = useRef<EventSource | null>(null);
// Find job from context or fetch it
useEffect(() => {
const foundJob = jobs.find((j) => j.job_id === jobId);
if (foundJob) {
setJob(foundJob);
} else {
// Fetch job directly if not in context
fetch(`/api/jobs/${jobId}`)
.then((res) => res.json())
.then((data) => setJob(data))
.catch(console.error);
}
}, [jobId, jobs]);
// Fetch initial logs when job is loaded
useEffect(() => {
if (!jobId) return;
setIsLoadingLogs(true);
fetch(`/api/jobs/${jobId}/logs`)
.then((res) => res.json())
.then((data: JobLogs) => {
if (data.logs && data.logs.length > 0) {
const normalized = normalizeLogsTOStructured(data.logs);
setStructuredLogs(normalized);
}
})
.catch(console.error)
.finally(() => setIsLoadingLogs(false));
}, [jobId]);
// Connect to SSE stream for running jobs
useEffect(() => {
if (!job || job.status !== 'running') {
// Close any existing connection for non-running jobs
if (eventSourceRef.current) {
eventSourceRef.current.close();
eventSourceRef.current = null;
setIsStreaming(false);
}
return;
}
// Create SSE connection
const eventSource = new EventSource(`/api/jobs/${jobId}/stream`);
eventSourceRef.current = eventSource;
eventSource.onopen = () => {
console.log('SSE connected for job:', jobId);
setIsStreaming(true);
};
eventSource.onerror = (err) => {
console.error('SSE error:', err);
setIsStreaming(false);
// Try to reconnect after a delay
setTimeout(() => {
if (eventSourceRef.current === eventSource) {
eventSource.close();
// Will reconnect on next render cycle if job is still running
}
}, 3000);
};
// Handle structured log events
eventSource.addEventListener('log', (event) => {
try {
const data = JSON.parse(event.data);
// Handle {"type": "log", "data": {...}} format
const logData = data.data || data;
const newLog: StructuredLog = {
timestamp: logData.timestamp || new Date().toISOString(),
timestamp_ms: logData.timestamp_ms || Date.now(),
level: logData.level || 'INFO',
category: logData.category || 'system',
message: logData.message || '',
metrics: logData.metrics,
network: logData.network,
};
setStructuredLogs((prev) => [...prev, newLog]);
} catch (err) {
console.error('Failed to parse log event:', err);
}
});
// Handle metrics events
eventSource.addEventListener('metrics', (event) => {
try {
const data = JSON.parse(event.data);
// Handle {"type": "metrics", "data": {...}} format
const metricsPayload = data.data || data;
setMetricsData({
cpu_percent: metricsPayload.cpu_percent,
memory_mb: metricsPayload.memory_mb,
duration_ms: metricsPayload.duration_ms,
requests_made: metricsPayload.requests_made,
reviews_scraped: metricsPayload.reviews_extracted || metricsPayload.reviews_scraped,
});
// Add to metrics history for charts
const sample: MetricsSample = {
timestamp_ms: metricsPayload.timestamp_ms || Date.now(),
reviews_extracted: metricsPayload.reviews_extracted || 0,
scroll_count: metricsPayload.scroll_count || 0,
memory_mb: metricsPayload.memory_mb || 0,
extraction_rate: metricsPayload.extraction_rate || 0,
};
setMetricsHistory((prev) => [...prev, sample]);
} catch (err) {
console.error('Failed to parse metrics event:', err);
}
});
// Handle job progress events (from existing SSE format)
eventSource.addEventListener('job_progress', (event) => {
try {
const data = JSON.parse(event.data);
setJob((prev) =>
prev
? {
...prev,
reviews_count: data.reviews_count,
total_reviews: data.total_reviews,
scrape_time: data.scrape_time,
}
: prev
);
} catch (err) {
console.error('Failed to parse job_progress event:', err);
}
});
// Handle job completed
eventSource.addEventListener('job_completed', (event) => {
try {
const data = JSON.parse(event.data);
setJob((prev) =>
prev
? {
...prev,
status: 'completed',
reviews_count: data.reviews_count,
total_reviews: data.total_reviews,
scrape_time: data.scrape_time,
}
: prev
);
eventSource.close();
setIsStreaming(false);
refreshJobs();
} catch (err) {
console.error('Failed to parse job_completed event:', err);
}
});
// Handle job failed
eventSource.addEventListener('job_failed', (event) => {
try {
const data = JSON.parse(event.data);
setJob((prev) =>
prev
? {
...prev,
status: 'failed',
error_message: data.error || data.error_message,
}
: prev
);
eventSource.close();
setIsStreaming(false);
refreshJobs();
// Fetch crash report when job fails
fetchCrashReport();
} catch (err) {
console.error('Failed to parse job_failed event:', err);
}
});
// Handle job partial
eventSource.addEventListener('job_partial', (event) => {
try {
const data = JSON.parse(event.data);
setJob((prev) =>
prev
? {
...prev,
status: 'partial',
reviews_count: data.reviews_count,
error_message: data.error || data.error_message,
}
: prev
);
eventSource.close();
setIsStreaming(false);
refreshJobs();
// Fetch crash report for partial jobs too
fetchCrashReport();
} catch (err) {
console.error('Failed to parse job_partial event:', err);
}
});
// Handle initial state (all current logs)
eventSource.addEventListener('initial_state', (event) => {
try {
const data = JSON.parse(event.data);
if (data.logs && data.logs.length > 0) {
const normalized = normalizeLogsTOStructured(data.logs);
setStructuredLogs(normalized);
}
} catch (err) {
console.error('Failed to parse initial_state event:', err);
}
});
// Handle generic message events
eventSource.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
// Check for type field to route to correct handler
if (data.type === 'log') {
const logData = data.data || data;
const newLog: StructuredLog = {
timestamp: logData.timestamp || new Date().toISOString(),
timestamp_ms: logData.timestamp_ms || Date.now(),
level: logData.level || 'INFO',
category: logData.category || 'system',
message: logData.message || '',
metrics: logData.metrics,
network: logData.network,
};
setStructuredLogs((prev) => [...prev, newLog]);
} else if (data.type === 'metrics') {
const metricsPayload = data.data || data;
setMetricsData({
cpu_percent: metricsPayload.cpu_percent,
memory_mb: metricsPayload.memory_mb,
duration_ms: metricsPayload.duration_ms,
requests_made: metricsPayload.requests_made,
reviews_scraped: metricsPayload.reviews_extracted || metricsPayload.reviews_scraped,
});
}
} catch {
// Ignore non-JSON messages
}
};
return () => {
eventSource.close();
eventSourceRef.current = null;
setIsStreaming(false);
};
}, [job?.status, jobId, refreshJobs]);
// Fetch crash report when job status is failed or partial
const fetchCrashReport = useCallback(async () => {
if (!jobId) return;
setIsLoadingCrashReport(true);
try {
const response = await fetch(`/api/jobs/${jobId}/crash-report`);
if (response.ok) {
const data = await response.json();
setCrashReport(data);
}
} catch (err) {
console.error('Failed to fetch crash report:', err);
} finally {
setIsLoadingCrashReport(false);
}
}, [jobId]);
// Fetch crash report if job is failed or partial on load
useEffect(() => {
if (job && (job.status === 'failed' || job.status === 'partial')) {
fetchCrashReport();
}
}, [job?.status, fetchCrashReport]);
// Extract session fingerprint from job metadata
useEffect(() => {
if (!job) return;
// Try to get session fingerprint from job metadata
fetch(`/api/jobs/${jobId}`)
.then((res) => res.json())
.then((fullJob) => {
if (fullJob.metadata) {
const metadata =
typeof fullJob.metadata === 'string'
? JSON.parse(fullJob.metadata)
: fullJob.metadata;
if (metadata.session_fingerprint) {
setSessionFingerprint(metadata.session_fingerprint);
} else if (metadata.browser_fingerprint) {
// Convert browser fingerprint to session fingerprint format
const bf = metadata.browser_fingerprint;
setSessionFingerprint({
user_agent: bf.userAgent || '',
platform: bf.platform || '',
language: bf.language || '',
languages: bf.languages || [bf.language || ''],
timezone: bf.timezone || '',
screen: {
width: bf.viewport?.width || 1920,
height: bf.viewport?.height || 1080,
colorDepth: 24,
},
viewport: bf.viewport || { width: 1920, height: 1080 },
webgl_vendor: '',
webgl_renderer: '',
canvas_fingerprint: '',
hardware_concurrency: 4,
device_memory: 8,
bot_detection_tests: {
webdriver_hidden: true,
chrome_runtime: true,
permissions_query: true,
},
captured_at: new Date().toISOString(),
});
}
}
})
.catch(console.error);
}, [job, jobId]);
const handleDelete = async () => {
if (!confirm('Are you sure you want to delete this job?')) return;
setIsDeleting(true);
try {
await fetch(`/api/jobs/${jobId}`, { method: 'DELETE' });
await refreshJobs();
router.push('/jobs');
} catch (err) {
console.error('Failed to delete job:', err);
} finally {
setIsDeleting(false);
}
};
const handleRetry = useCallback(
(applyFix: boolean) => {
setRetryFeedback({
type: 'success',
message: applyFix
? 'Retrying job with auto-fix applied...'
: 'Retrying job without modifications...',
});
// Refresh jobs to pick up the new job
setTimeout(() => {
refreshJobs();
setRetryFeedback(null);
}, 2000);
},
[refreshJobs]
);
if (!job) {
return (
<div className="h-full flex items-center justify-center">
<div className="w-8 h-8 border-4 border-blue-500 border-t-transparent rounded-full animate-spin" />
</div>
);
}
const businessName = extractBusinessName(job);
const canViewAnalytics = job.reviews_count && job.reviews_count > 0;
const showCrashReport = (job.status === 'failed' || job.status === 'partial') && crashReport;
return (
<div className="h-full overflow-y-auto p-6">
{/* Breadcrumb */}
<div className="mb-4 flex items-center gap-2 text-sm text-gray-500">
<Link href="/jobs" className="hover:text-blue-600">
Jobs
</Link>
<span>/</span>
<span className="text-gray-900 font-medium">{jobId.slice(0, 8)}...</span>
</div>
{/* Header */}
<div className="bg-white border-2 border-gray-200 rounded-xl p-6 mb-6">
<div className="flex items-start justify-between mb-4">
<div>
<h1 className="text-2xl font-bold text-gray-900 mb-1">{businessName}</h1>
{job.business_address && (
<p className="text-gray-500 text-sm">{job.business_address}</p>
)}
</div>
<span
className={`inline-flex items-center gap-1.5 px-3 py-1.5 rounded-full text-sm font-semibold ${
job.status === 'completed'
? 'bg-green-100 text-green-800'
: job.status === 'partial'
? 'bg-orange-100 text-orange-800'
: job.status === 'running'
? 'bg-blue-100 text-blue-800'
: job.status === 'failed'
? 'bg-red-100 text-red-800'
: 'bg-gray-100 text-gray-800'
}`}
>
{job.status === 'running' && (
<div className="w-2 h-2 border border-current border-t-transparent rounded-full animate-spin" />
)}
{job.status.charAt(0).toUpperCase() + job.status.slice(1)}
</span>
</div>
{/* Stats Grid */}
<div className="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
{job.reviews_count !== null && (
<div className="p-4 bg-blue-50 border border-blue-200 rounded-lg">
<div className="text-2xl font-bold text-blue-800">
{job.reviews_count.toLocaleString()}
</div>
<div className="text-xs font-medium text-blue-600">Reviews</div>
</div>
)}
{job.scrape_time !== null && (
<div className="p-4 bg-green-50 border border-green-200 rounded-lg">
<div className="text-2xl font-bold text-green-800">
{formatDuration(job.scrape_time)}
</div>
<div className="text-xs font-medium text-green-600">Duration</div>
</div>
)}
{job.rating_snapshot != null && (
<div className="p-4 bg-yellow-50 border border-yellow-200 rounded-lg">
<div className="text-2xl font-bold text-yellow-800 flex items-center gap-1">
{job.rating_snapshot.toFixed(1)}
<svg className="w-5 h-5" fill="currentColor" viewBox="0 0 20 20">
<path d="M9.049 2.927c.3-.921 1.603-.921 1.902 0l1.07 3.292a1 1 0 00.95.69h3.462c.969 0 1.371 1.24.588 1.81l-2.8 2.034a1 1 0 00-.364 1.118l1.07 3.292c.3.921-.755 1.688-1.54 1.118l-2.8-2.034a1 1 0 00-1.175 0l-2.8 2.034c-.784.57-1.838-.197-1.539-1.118l1.07-3.292a1 1 0 00-.364-1.118L2.98 8.72c-.783-.57-.38-1.81.588-1.81h3.461a1 1 0 00.951-.69l1.07-3.292z" />
</svg>
</div>
<div className="text-xs font-medium text-yellow-600">Rating</div>
</div>
)}
<div className="p-4 bg-gray-50 border border-gray-200 rounded-lg">
<div className="text-sm font-bold text-gray-800">
{new Date(job.created_at).toLocaleDateString()}
</div>
<div className="text-xs text-gray-500">
{new Date(job.created_at).toLocaleTimeString()}
</div>
<div className="text-xs font-medium text-gray-600 mt-1">Created</div>
</div>
</div>
{/* Actions */}
<div className="flex gap-3">
{canViewAnalytics && (
<Link
href={`/analytics/${jobId}`}
className="flex-1 py-3 bg-blue-600 hover:bg-blue-700 text-white rounded-xl font-semibold transition-colors flex items-center justify-center gap-2"
>
<svg
className="w-5 h-5"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M9 19v-6a2 2 0 00-2-2H5a2 2 0 00-2 2v6a2 2 0 002 2h2a2 2 0 002-2zm0 0V9a2 2 0 012-2h2a2 2 0 012 2v10m-6 0a2 2 0 002 2h2a2 2 0 002-2m0 0V5a2 2 0 012-2h2a2 2 0 012 2v14a2 2 0 01-2 2h-2a2 2 0 01-2-2z"
/>
</svg>
View Analytics
</Link>
)}
<a
href={job.url}
target="_blank"
rel="noopener noreferrer"
className="px-6 py-3 bg-gray-100 hover:bg-gray-200 text-gray-700 rounded-xl font-semibold transition-colors flex items-center justify-center gap-2"
>
<svg
className="w-5 h-5"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14"
/>
</svg>
Open in Maps
</a>
<button
onClick={handleDelete}
disabled={isDeleting || job.status === 'running'}
className="px-6 py-3 bg-red-100 hover:bg-red-200 text-red-700 rounded-xl font-semibold transition-colors flex items-center justify-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed"
>
{isDeleting ? (
<div className="w-5 h-5 border-2 border-red-500 border-t-transparent rounded-full animate-spin" />
) : (
<svg
className="w-5 h-5"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
/>
</svg>
)}
Delete
</button>
</div>
{/* Error Message (legacy - shown when no crash report) */}
{job.error_message && !showCrashReport && (
<div className="mt-4 p-4 bg-red-50 border border-red-200 rounded-lg">
<div className="flex items-start gap-2">
<svg
className="w-5 h-5 text-red-600 flex-shrink-0 mt-0.5"
fill="currentColor"
viewBox="0 0 20 20"
>
<path
fillRule="evenodd"
d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z"
clipRule="evenodd"
/>
</svg>
<div>
<p className="font-semibold text-red-800">Error</p>
<p className="text-sm text-red-700">{job.error_message}</p>
</div>
</div>
</div>
)}
{/* Retry Feedback */}
{retryFeedback && (
<div
className={`mt-4 p-4 rounded-lg ${
retryFeedback.type === 'success'
? 'bg-green-50 border border-green-200 text-green-800'
: 'bg-red-50 border border-red-200 text-red-800'
}`}
>
<div className="flex items-center gap-2">
{retryFeedback.type === 'success' ? (
<svg className="w-5 h-5 text-green-600" fill="currentColor" viewBox="0 0 20 20">
<path
fillRule="evenodd"
d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z"
clipRule="evenodd"
/>
</svg>
) : (
<svg className="w-5 h-5 text-red-600" fill="currentColor" viewBox="0 0 20 20">
<path
fillRule="evenodd"
d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z"
clipRule="evenodd"
/>
</svg>
)}
<span className="font-medium">{retryFeedback.message}</span>
</div>
</div>
)}
</div>
{/* Crash Report Section (for failed/partial jobs) */}
{showCrashReport && (
<div className="mb-6">
<CrashReport
jobId={jobId}
crashReport={crashReport}
onRetry={handleRetry}
/>
</div>
)}
{/* Metrics Dashboard (for running jobs) */}
{job.status === 'running' && metricsHistory.length > 0 && (
<div className="bg-gray-900 rounded-xl border-2 border-gray-700 p-6 mb-6">
<MetricsDashboard
metricsHistory={metricsHistory}
currentMetrics={metricsHistory[metricsHistory.length - 1]}
isStreaming={isStreaming}
/>
</div>
)}
{/* Session Panel (if fingerprint available) */}
{sessionFingerprint && (
<div className="mb-6">
<SessionPanel fingerprint={sessionFingerprint} />
</div>
)}
{/* Job DevTools - Main Log Viewer */}
<div className="mb-6">
{isLoadingLogs ? (
<div className="bg-gray-900 rounded-xl border-2 border-gray-700 p-8 flex items-center justify-center">
<div className="text-center">
<div className="w-8 h-8 border-4 border-blue-500 border-t-transparent rounded-full animate-spin mx-auto mb-3" />
<p className="text-gray-400">Loading logs...</p>
</div>
</div>
) : (
<JobDevTools
logs={structuredLogs}
metrics={metricsData}
crashReport={
crashReport
? {
error_type: crashReport.crash_type,
error_message: crashReport.error_message,
stack_trace: crashReport.analysis?.description,
timestamp: crashReport.created_at,
context: crashReport.analysis?.auto_fix_params,
}
: undefined
}
sessionFingerprint={
sessionFingerprint
? {
session_id: jobId,
browser_version: sessionFingerprint.user_agent,
proxy_used: false,
locale: sessionFingerprint.language,
viewport: sessionFingerprint.viewport,
}
: undefined
}
isStreaming={isStreaming}
/>
)}
</div>
{/* Loading Crash Report Indicator */}
{isLoadingCrashReport && (
<div className="fixed bottom-4 right-4 bg-gray-900 text-white px-4 py-2 rounded-lg shadow-lg flex items-center gap-2">
<div className="w-4 h-4 border-2 border-white border-t-transparent rounded-full animate-spin" />
Loading crash report...
</div>
)}
</div>
);
}