From 2637d982e0ab2e480f20429e3ac62570f64a991c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Guti=C3=A9rrez?= <35082514+alezmad@users.noreply.github.com> Date: Sat, 24 Jan 2026 12:37:56 +0000 Subject: [PATCH] Wave 4: JobDevTools UI components and crash report API - Task #5: Create JobDevTools container component (tabs: All/Scraper/Browser/Network/System, level filters, count badges) - Task #11: Add crash report API endpoints (GET /jobs/{id}/crash-report, POST /jobs/{id}/retry?apply_fix=true, GET /crashes/stats) - Task #14: Create SessionPanel component (fingerprint display, bot detection indicators, collapsible sections) - Task #15: Create MetricsDashboard with recharts (extraction rate, cumulative reviews, memory usage, scroll progress) Co-Authored-By: Claude Opus 4.5 --- api_server_production.py | 370 +++++++++++++++++ .../JobDevTools/MetricsDashboard.tsx | 386 ++++++++++++++++++ web/components/JobDevTools/SessionPanel.tsx | 211 ++++++++++ web/components/JobDevTools/index.tsx | 364 +++++++++++++++++ 4 files changed, 1331 insertions(+) create mode 100644 web/components/JobDevTools/MetricsDashboard.tsx create mode 100644 web/components/JobDevTools/SessionPanel.tsx create mode 100644 web/components/JobDevTools/index.tsx diff --git a/api_server_production.py b/api_server_production.py index 03d3ced..f7b0733 100644 --- a/api_server_production.py +++ b/api_server_production.py @@ -11,6 +11,7 @@ import logging import os import time from contextlib import asynccontextmanager +from datetime import datetime, timedelta from typing import Optional, List, Dict, Any from uuid import UUID @@ -23,6 +24,7 @@ from modules.database import DatabaseManager, JobStatus from modules.webhooks import WebhookDispatcher, WebhookManager from modules.health_checks import HealthCheckSystem from modules.scraper_clean import fast_scrape_reviews, LogCapture, get_business_card_info # Clean scraper +from modules.crash_analyzer import analyze_crash, summarize_crash_patterns, apply_auto_fix from modules.structured_logger import StructuredLogger, LogEntry from modules.chrome_pool import ( start_worker_pools, @@ -207,6 +209,51 @@ class StatsResponse(BaseModel): total_reviews: Optional[int] = None +class CrashAnalysisModel(BaseModel): + """Crash analysis details""" + pattern: str = Field(..., description="Identified crash pattern type") + confidence: float = Field(..., description="Confidence score 0.0 to 1.0") + description: str = Field(..., description="Description of the crash cause") + suggested_fix: str = Field(..., description="Recommended fix action") + auto_fix_params: Optional[Dict[str, Any]] = Field(None, description="Parameters for auto-fix") + + +class CrashReportResponse(BaseModel): + """Response model for crash report""" + crash_id: str + job_id: str + crash_type: str + error_message: Optional[str] = None + analysis: Optional[CrashAnalysisModel] = None + metrics_history: Optional[List[Dict[str, Any]]] = None + logs_before_crash: Optional[List[Dict[str, Any]]] = None + screenshot_url: Optional[str] = None + created_at: str + + +class RetryJobResponse(BaseModel): + """Response model for retry job""" + job_id: str + status: str + message: str + applied_fixes: Optional[Dict[str, Any]] = None + + +class CrashPatternStats(BaseModel): + """Statistics for a single crash pattern""" + count: int + percentage: float + avg_confidence: float + + +class CrashStatsResponse(BaseModel): + """Response model for aggregate crash statistics""" + total_crashes: int + patterns: Dict[str, CrashPatternStats] + most_common: Optional[str] = None + recommendations: List[Dict[str, Any]] + + # ==================== API Endpoints ==================== @app.get("/", summary="API Health Check") @@ -946,6 +993,329 @@ async def pool_stats(): return await asyncio.to_thread(get_pool_stats) +# ==================== Crash Report Endpoints ==================== + +@app.get("/jobs/{job_id}/crash-report", response_model=CrashReportResponse, summary="Get Crash Report") +async def get_crash_report(job_id: UUID): + """ + Get the crash report for a failed or partial job. + + Returns detailed crash analysis including: + - Crash pattern identification (memory_exhaustion, rate_limited, etc.) + - Confidence score for the pattern match + - Suggested fixes and auto-fix parameters + - Metrics history and logs before the crash + """ + if not db: + raise HTTPException(status_code=500, detail="Database not initialized") + + # Verify job exists + job = await db.get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + # Only failed or partial jobs have crash reports + if job['status'] not in ['failed', 'partial']: + raise HTTPException( + status_code=400, + detail=f"Job status is '{job['status']}' - crash reports only available for failed or partial jobs" + ) + + # Get crash report from database + crash_report = await db.get_crash_report(str(job_id)) + + if not crash_report: + # No stored crash report - generate one from job data + # Build crash report from job data for analysis + scrape_logs = job.get('scrape_logs') + if isinstance(scrape_logs, str): + try: + scrape_logs = json.loads(scrape_logs) + except: + scrape_logs = [] + + # Get metrics_history if available + metrics_history = job.get('metrics_history') + if isinstance(metrics_history, str): + try: + metrics_history = json.loads(metrics_history) + except: + metrics_history = [] + + crash_data = { + 'error_message': job.get('error_message', 'Unknown error'), + 'metrics_history': metrics_history or [], + 'logs_before_crash': scrape_logs or [], + 'state': { + 'reviews_extracted': job.get('reviews_count', 0), + 'total_reviews': job.get('total_reviews') + } + } + + # Analyze the crash + analysis = analyze_crash(crash_data) + + # Build response from job data and analysis + return CrashReportResponse( + crash_id=str(job_id), # Use job_id as crash_id when no stored report + job_id=str(job_id), + crash_type=analysis.pattern, + error_message=job.get('error_message'), + analysis=CrashAnalysisModel( + pattern=analysis.pattern, + confidence=analysis.confidence, + description=analysis.description, + suggested_fix=analysis.suggested_fix, + auto_fix_params=analysis.auto_fix_params + ), + metrics_history=metrics_history, + logs_before_crash=scrape_logs, + screenshot_url=None, + created_at=job['completed_at'].isoformat() if job.get('completed_at') else job['created_at'].isoformat() + ) + + # Parse JSONB fields if needed + metrics_history = crash_report.get('metrics_history') + if isinstance(metrics_history, str): + try: + metrics_history = json.loads(metrics_history) + except: + metrics_history = [] + + logs_before_crash = crash_report.get('logs_before_crash') + if isinstance(logs_before_crash, str): + try: + logs_before_crash = json.loads(logs_before_crash) + except: + logs_before_crash = [] + + stored_analysis = crash_report.get('analysis') + if isinstance(stored_analysis, str): + try: + stored_analysis = json.loads(stored_analysis) + except: + stored_analysis = None + + # If no analysis stored, generate one + if not stored_analysis: + crash_data = { + 'error_message': crash_report.get('error_message', ''), + 'metrics_history': metrics_history or [], + 'logs_before_crash': logs_before_crash or [], + 'crash_type': crash_report.get('crash_type'), + 'state': crash_report.get('state', {}) + } + analysis = analyze_crash(crash_data) + stored_analysis = { + 'pattern': analysis.pattern, + 'confidence': analysis.confidence, + 'description': analysis.description, + 'suggested_fix': analysis.suggested_fix, + 'auto_fix_params': analysis.auto_fix_params + } + + return CrashReportResponse( + crash_id=crash_report['crash_id'], + job_id=crash_report['job_id'], + crash_type=crash_report['crash_type'], + error_message=crash_report.get('error_message'), + analysis=CrashAnalysisModel(**stored_analysis) if stored_analysis else None, + metrics_history=metrics_history, + logs_before_crash=logs_before_crash, + screenshot_url=crash_report.get('screenshot_url'), + created_at=crash_report['created_at'].isoformat() + ) + + +@app.post("/jobs/{job_id}/retry", response_model=RetryJobResponse, summary="Retry Failed Job") +async def retry_job( + job_id: UUID, + apply_fix: bool = Query(False, description="Apply auto-fix parameters based on crash analysis") +): + """ + Retry a failed or partial job, optionally applying auto-fix parameters. + + When apply_fix=true: + - Analyzes the crash pattern from the original job + - Applies recommended parameter adjustments (e.g., reduced batch size for memory issues) + - Creates a new job with the adjusted parameters + + Returns the new job ID for tracking. + """ + if not db: + raise HTTPException(status_code=500, detail="Database not initialized") + + # Get original job + original_job = await db.get_job(job_id) + if not original_job: + raise HTTPException(status_code=404, detail="Job not found") + + # Can only retry failed or partial jobs + if original_job['status'] not in ['failed', 'partial']: + raise HTTPException( + status_code=400, + detail=f"Cannot retry job with status '{original_job['status']}' - only failed or partial jobs can be retried" + ) + + # Parse original metadata + original_metadata = original_job.get('metadata') + if isinstance(original_metadata, str): + try: + original_metadata = json.loads(original_metadata) + except: + original_metadata = {} + original_metadata = original_metadata or {} + + applied_fixes = None + + if apply_fix: + # Get crash analysis to determine fixes + scrape_logs = original_job.get('scrape_logs') + if isinstance(scrape_logs, str): + try: + scrape_logs = json.loads(scrape_logs) + except: + scrape_logs = [] + + metrics_history = original_job.get('metrics_history') + if isinstance(metrics_history, str): + try: + metrics_history = json.loads(metrics_history) + except: + metrics_history = [] + + crash_data = { + 'error_message': original_job.get('error_message', 'Unknown error'), + 'metrics_history': metrics_history or [], + 'logs_before_crash': scrape_logs or [], + 'state': { + 'reviews_extracted': original_job.get('reviews_count', 0), + 'total_reviews': original_job.get('total_reviews') + } + } + + analysis = analyze_crash(crash_data) + + if analysis.auto_fix_params: + # Get current scraper params from metadata or use defaults + current_params = original_metadata.get('scraper_params', {}) + + # Apply the auto-fix parameters + fixed_params = apply_auto_fix(analysis.pattern, current_params) + + # Store applied fixes in metadata + original_metadata['scraper_params'] = fixed_params + original_metadata['retry_info'] = { + 'original_job_id': str(job_id), + 'crash_pattern': analysis.pattern, + 'applied_fixes': analysis.auto_fix_params + } + + applied_fixes = analysis.auto_fix_params + log.info(f"Applying auto-fix for pattern '{analysis.pattern}': {applied_fixes}") + + # Create new job with same URL and (possibly modified) metadata + new_job_id = await db.create_job( + url=original_job['url'], + webhook_url=original_job.get('webhook_url'), + webhook_secret=original_job.get('webhook_secret'), + metadata=original_metadata + ) + + # Start the new scraping job + asyncio.create_task(run_scraping_job(new_job_id)) + + log.info(f"Created retry job {new_job_id} for original job {job_id}") + + return RetryJobResponse( + job_id=str(new_job_id), + status="started", + message=f"Retry job created from original job {job_id}", + applied_fixes=applied_fixes + ) + + +@app.get("/crashes/stats", response_model=CrashStatsResponse, summary="Get Crash Statistics") +async def get_crash_stats( + days: int = Query(7, description="Number of days to look back", ge=1, le=90) +): + """ + Get aggregate crash statistics and pattern analysis. + + Analyzes all crash reports from the specified time period to identify: + - Most common crash patterns + - Confidence scores for pattern detection + - Recommended fixes based on recurring patterns + + Use this to identify systemic issues and optimize scraper configuration. + """ + if not db: + raise HTTPException(status_code=500, detail="Database not initialized") + + # Get basic crash stats from database + basic_stats = await db.get_crash_stats(days=days) + + # Get all failed/partial jobs for deeper analysis + failed_jobs = await db.list_jobs(status=JobStatus.FAILED, limit=500) + partial_jobs = await db.list_jobs(status=JobStatus.PARTIAL, limit=500) + + all_crash_jobs = failed_jobs + partial_jobs + + # Filter by time if needed (list_jobs doesn't have date filter) + cutoff = datetime.now() - timedelta(days=days) + recent_crash_jobs = [ + job for job in all_crash_jobs + if job.get('completed_at') and job['completed_at'] > cutoff + ] + + if not recent_crash_jobs: + return CrashStatsResponse( + total_crashes=0, + patterns={}, + most_common=None, + recommendations=[] + ) + + # Build crash reports for analysis + crash_reports = [] + for job in recent_crash_jobs: + scrape_logs = job.get('scrape_logs') + if isinstance(scrape_logs, str): + try: + scrape_logs = json.loads(scrape_logs) + except: + scrape_logs = [] + + crash_reports.append({ + 'error_message': job.get('error_message', ''), + 'metrics_history': [], # Not stored in job list query + 'logs_before_crash': scrape_logs or [], + 'state': { + 'reviews_extracted': job.get('reviews_count', 0), + 'total_reviews': job.get('total_reviews') + } + }) + + # Use summarize_crash_patterns for deep analysis + summary = summarize_crash_patterns(crash_reports) + + # Convert patterns to response model format + patterns_response = {} + for pattern_name, stats in summary.get('patterns', {}).items(): + patterns_response[pattern_name] = CrashPatternStats( + count=stats['count'], + percentage=stats['percentage'], + avg_confidence=stats['avg_confidence'] + ) + + return CrashStatsResponse( + total_crashes=summary.get('total_crashes', 0), + patterns=patterns_response, + most_common=summary.get('most_common'), + recommendations=summary.get('recommendations', []) + ) + + # ==================== Health Check Endpoints ==================== @app.get("/health/live", summary="Liveness Probe") diff --git a/web/components/JobDevTools/MetricsDashboard.tsx b/web/components/JobDevTools/MetricsDashboard.tsx new file mode 100644 index 0000000..d4b4e81 --- /dev/null +++ b/web/components/JobDevTools/MetricsDashboard.tsx @@ -0,0 +1,386 @@ +'use client'; + +import { useMemo } from 'react'; +import { + LineChart, + Line, + AreaChart, + Area, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + ResponsiveContainer, + ReferenceLine, +} from 'recharts'; +import { Activity, TrendingUp, HardDrive, Scroll } from 'lucide-react'; + +/** + * Represents a single metrics sample collected during job execution + */ +export interface MetricsSample { + timestamp_ms: number; + reviews_extracted: number; + scroll_count: number; + memory_mb: number; + extraction_rate: number; // reviews per second +} + +interface MetricsDashboardProps { + metricsHistory: MetricsSample[]; + currentMetrics?: MetricsSample; + isStreaming: boolean; +} + +/** + * Formats a timestamp (in ms) to a relative time string + * e.g., "0s", "30s", "1m", "1m 30s", etc. + */ +function formatRelativeTime(timestampMs: number, startMs: number): string { + const elapsedMs = timestampMs - startMs; + const totalSeconds = Math.floor(elapsedMs / 1000); + + if (totalSeconds < 60) { + return `${totalSeconds}s`; + } + + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + + if (seconds === 0) { + return `${minutes}m`; + } + + return `${minutes}m ${seconds}s`; +} + +/** + * MetricsDashboard displays real-time metrics during job execution + * with charts for extraction rate, cumulative reviews, and memory usage. + */ +export default function MetricsDashboard({ + metricsHistory, + currentMetrics, + isStreaming, +}: MetricsDashboardProps) { + // Determine the starting timestamp for relative time calculations + const startTimestamp = useMemo(() => { + if (metricsHistory.length > 0) { + return metricsHistory[0].timestamp_ms; + } + return currentMetrics?.timestamp_ms ?? Date.now(); + }, [metricsHistory, currentMetrics]); + + // Transform metrics history for charts with relative time labels + const chartData = useMemo(() => { + return metricsHistory.map((sample) => ({ + ...sample, + time: formatRelativeTime(sample.timestamp_ms, startTimestamp), + timeMs: sample.timestamp_ms - startTimestamp, + })); + }, [metricsHistory, startTimestamp]); + + // Get the latest metrics (either current or last from history) + const latestMetrics = currentMetrics ?? metricsHistory[metricsHistory.length - 1]; + + // Memory warning threshold + const MEMORY_WARNING_MB = 1500; + + // Check if memory is above warning threshold + const isMemoryWarning = latestMetrics && latestMetrics.memory_mb >= MEMORY_WARNING_MB; + + // Custom tooltip style + const tooltipStyle = { + backgroundColor: '#1f2937', + border: '1px solid #374151', + borderRadius: '8px', + padding: '8px 12px', + }; + + return ( +
+ {/* Header with Live Indicator */} +
+

Real-Time Metrics

+ {isStreaming && ( +
+ + + + + Live +
+ )} +
+ + {/* Progress Summary - Current Stats */} +
+ {/* Total Reviews */} +
+
+ + Total Reviews +
+
+ {latestMetrics?.reviews_extracted ?? 0} +
+
+ + {/* Scroll Count */} +
+
+ + Scrolls +
+
+ {latestMetrics?.scroll_count ?? 0} +
+
+ + {/* Extraction Rate */} +
+
+ + Rate (r/s) +
+
+ {latestMetrics?.extraction_rate?.toFixed(2) ?? '0.00'} +
+
+ + {/* Memory Usage */} +
+
+ + Memory (MB) +
+
+ {latestMetrics?.memory_mb?.toFixed(0) ?? '0'} +
+ {isMemoryWarning && ( +
Warning: High memory
+ )} +
+
+ + {/* Charts Grid - 2x2 on desktop, stacked on mobile */} +
+ {/* Extraction Rate Chart */} +
+

+ Extraction Rate (reviews/second) +

+
+ {chartData.length > 0 ? ( + + + + + + [`${(value as number)?.toFixed(2) ?? '0.00'} r/s`, 'Rate']} + labelFormatter={(label) => `Time: ${label}`} + /> + + + + ) : ( +
+ No data yet +
+ )} +
+
+ + {/* Cumulative Reviews Chart */} +
+

+ Cumulative Reviews Extracted +

+
+ {chartData.length > 0 ? ( + + + + + + [`${value ?? 0} reviews`, 'Total']} + labelFormatter={(label) => `Time: ${label}`} + /> + + + + + + + + + + ) : ( +
+ No data yet +
+ )} +
+
+ + {/* Memory Usage Chart */} +
+

+ Memory Usage (MB) + Warning threshold: {MEMORY_WARNING_MB}MB +

+
+ {chartData.length > 0 ? ( + + + + + Math.max(dataMax * 1.1, MEMORY_WARNING_MB * 1.2)]} + /> + [`${(value as number)?.toFixed(0) ?? '0'} MB`, 'Memory']} + labelFormatter={(label) => `Time: ${label}`} + /> + + + + + ) : ( +
+ No data yet +
+ )} +
+
+ + {/* Scroll Count Chart */} +
+

+ Scroll Progress +

+
+ {chartData.length > 0 ? ( + + + + + + [`${value ?? 0} scrolls`, 'Count']} + labelFormatter={(label) => `Time: ${label}`} + /> + + + + + + + + + + ) : ( +
+ No data yet +
+ )} +
+
+
+
+ ); +} diff --git a/web/components/JobDevTools/SessionPanel.tsx b/web/components/JobDevTools/SessionPanel.tsx new file mode 100644 index 0000000..7d431e1 --- /dev/null +++ b/web/components/JobDevTools/SessionPanel.tsx @@ -0,0 +1,211 @@ +'use client'; + +import { useState } from 'react'; +import { ChevronDown, ChevronRight, User, Globe, Monitor, Cpu, Shield, Check, X, AlertTriangle } from 'lucide-react'; + +export interface SessionFingerprint { + user_agent: string; + platform: string; + language: string; + languages: string[]; + timezone: string; + screen: { width: number; height: number; colorDepth: number }; + viewport: { width: number; height: number }; + webgl_vendor: string; + webgl_renderer: string; + canvas_fingerprint: string; + hardware_concurrency: number; + device_memory: number; + bot_detection_tests: { + webdriver_hidden: boolean; + chrome_runtime: boolean; + permissions_query: boolean; + }; + captured_at: string; +} + +interface SessionPanelProps { + fingerprint: SessionFingerprint; +} + +function BotTestIndicator({ passed, label }: { passed: boolean | null | undefined; label: string }) { + if (passed === null || passed === undefined) { + return ( +
+ + {label} + UNKNOWN +
+ ); + } + + if (passed) { + return ( +
+ + {label} + PASSED +
+ ); + } + + return ( +
+ + {label} + FAILED +
+ ); +} + +function SectionHeader({ icon: Icon, title }: { icon: React.ElementType; title: string }) { + return ( +
+ +

{title}

+
+ ); +} + +function DataRow({ label, value, mono = true }: { label: string; value: string | number; mono?: boolean }) { + return ( +
+ {label} + {value} +
+ ); +} + +export default function SessionPanel({ fingerprint }: SessionPanelProps) { + const [isExpanded, setIsExpanded] = useState(false); + + // Calculate overall bot detection status + const tests = fingerprint.bot_detection_tests; + const testResults = [tests.webdriver_hidden, tests.chrome_runtime, tests.permissions_query]; + const passedCount = testResults.filter(t => t === true).length; + const failedCount = testResults.filter(t => t === false).length; + const unknownCount = testResults.filter(t => t === null || t === undefined).length; + + const overallStatus = failedCount > 0 ? 'warning' : unknownCount > 0 ? 'partial' : 'success'; + const statusColors = { + success: 'bg-green-900/30 border-green-700/50 text-green-400', + partial: 'bg-yellow-900/30 border-yellow-700/50 text-yellow-400', + warning: 'bg-red-900/30 border-red-700/50 text-red-400', + }; + + return ( +
+ {/* Collapsible Header */} + + + {/* Collapsible Content */} + {isExpanded && ( +
+
+ {/* Identity Section */} +
+ +
+ + + + +
+
+ + {/* Geolocation Section */} +
+ +
+ + +
+
+ + {/* Display Section */} +
+ +
+ + + +
+
+ + {/* Hardware Section */} +
+ +
+ + + + + +
+
+
+ + {/* Bot Detection Section - Full Width */} +
+ +
+ + + +
+
+

+ Green checkmark = Test passed (bot detection evaded) +

+

+ Red X = Test failed (may have been detected as a bot) +

+

+ Yellow warning = Test result unknown +

+
+
+
+ )} +
+ ); +} diff --git a/web/components/JobDevTools/index.tsx b/web/components/JobDevTools/index.tsx new file mode 100644 index 0000000..c4854cf --- /dev/null +++ b/web/components/JobDevTools/index.tsx @@ -0,0 +1,364 @@ +'use client'; + +import { useState, useMemo } from 'react'; +import { Bug, Globe, Network, Cpu, Filter, ChevronDown, ChevronUp } from 'lucide-react'; + +// Type definitions +export interface StructuredLog { + timestamp: string; + timestamp_ms: number; + level: 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' | 'FATAL'; + category: 'scraper' | 'browser' | 'network' | 'system'; + message: string; + metrics?: Record; + network?: Record; +} + +export interface MetricsData { + cpu_percent?: number; + memory_mb?: number; + duration_ms?: number; + requests_made?: number; + reviews_scraped?: number; + [key: string]: any; +} + +export interface CrashReport { + error_type: string; + error_message: string; + stack_trace?: string; + timestamp: string; + context?: Record; +} + +export interface SessionFingerprint { + session_id: string; + browser_version?: string; + proxy_used?: boolean; + locale?: string; + viewport?: { width: number; height: number }; + [key: string]: any; +} + +export interface JobDevToolsProps { + logs: StructuredLog[]; + metrics?: MetricsData; + crashReport?: CrashReport; + sessionFingerprint?: SessionFingerprint; + isStreaming?: boolean; +} + +type TabType = 'all' | 'scraper' | 'browser' | 'network' | 'system'; +type LogLevel = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' | 'FATAL'; + +const TAB_CONFIG: { id: TabType; label: string; icon: typeof Bug; category?: StructuredLog['category'] }[] = [ + { id: 'all', label: 'All', icon: Filter }, + { id: 'scraper', label: 'Scraper', icon: Bug, category: 'scraper' }, + { id: 'browser', label: 'Browser', icon: Globe, category: 'browser' }, + { id: 'network', label: 'Network', icon: Network, category: 'network' }, + { id: 'system', label: 'System', icon: Cpu, category: 'system' }, +]; + +const LEVEL_COLORS: Record = { + DEBUG: { bg: 'bg-gray-700', text: 'text-gray-300', border: 'border-gray-600' }, + INFO: { bg: 'bg-blue-900', text: 'text-blue-300', border: 'border-blue-700' }, + WARN: { bg: 'bg-yellow-900', text: 'text-yellow-300', border: 'border-yellow-700' }, + ERROR: { bg: 'bg-red-900', text: 'text-red-300', border: 'border-red-700' }, + FATAL: { bg: 'bg-purple-900', text: 'text-purple-300', border: 'border-purple-700' }, +}; + +const LEVEL_BADGE_COLORS: Record = { + DEBUG: 'bg-gray-600 text-gray-200', + INFO: 'bg-blue-600 text-blue-100', + WARN: 'bg-yellow-600 text-yellow-100', + ERROR: 'bg-red-600 text-red-100', + FATAL: 'bg-purple-600 text-purple-100', +}; + +export default function JobDevTools({ + logs, + metrics, + crashReport, + sessionFingerprint, + isStreaming = false, +}: JobDevToolsProps) { + const [activeTab, setActiveTab] = useState('all'); + const [enabledLevels, setEnabledLevels] = useState>( + new Set(['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL']) + ); + const [showLevelFilter, setShowLevelFilter] = useState(false); + + // Calculate counts per category + const categoryCounts = useMemo(() => { + const counts: Record = { + all: logs.length, + scraper: 0, + browser: 0, + network: 0, + system: 0, + }; + + logs.forEach((log) => { + if (log.category in counts) { + counts[log.category as keyof typeof counts]++; + } + }); + + return counts; + }, [logs]); + + // Calculate counts per level + const levelCounts = useMemo(() => { + const counts: Record = { + DEBUG: 0, + INFO: 0, + WARN: 0, + ERROR: 0, + FATAL: 0, + }; + + logs.forEach((log) => { + if (log.level in counts) { + counts[log.level]++; + } + }); + + return counts; + }, [logs]); + + // Filter logs by active tab and enabled levels + const filteredLogs = useMemo(() => { + return logs.filter((log) => { + const matchesTab = activeTab === 'all' || log.category === activeTab; + const matchesLevel = enabledLevels.has(log.level); + return matchesTab && matchesLevel; + }); + }, [logs, activeTab, enabledLevels]); + + const toggleLevel = (level: LogLevel) => { + setEnabledLevels((prev) => { + const next = new Set(prev); + if (next.has(level)) { + // Don't allow deselecting all levels + if (next.size > 1) { + next.delete(level); + } + } else { + next.add(level); + } + return next; + }); + }; + + const formatTimestamp = (timestamp: string) => { + try { + const date = new Date(timestamp); + return date.toLocaleTimeString('en-US', { + hour12: false, + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + fractionalSecondDigits: 3, + }); + } catch { + return timestamp; + } + }; + + return ( +
+ {/* Header with streaming indicator */} +
+
+ + Job DevTools + {isStreaming && ( + + + Streaming + + )} +
+ + {filteredLogs.length} / {logs.length} logs + +
+ + {/* Tab bar */} +
+ {TAB_CONFIG.map((tab) => { + const Icon = tab.icon; + const count = categoryCounts[tab.id]; + const isActive = activeTab === tab.id; + + return ( + + ); + })} + + {/* Level filter toggle */} +
+ + + {/* Level filter dropdown */} + {showLevelFilter && ( +
+ {(Object.keys(LEVEL_BADGE_COLORS) as LogLevel[]).map((level) => ( + + ))} +
+ )} +
+
+ + {/* Log entries - scrollable area */} +
+ {filteredLogs.length === 0 ? ( +
+
+ +

No logs to display

+

+ {logs.length > 0 + ? 'Try adjusting your filters' + : 'Logs will appear here during job execution'} +

+
+
+ ) : ( +
+ {filteredLogs.map((log, index) => { + const levelStyle = LEVEL_COLORS[log.level]; + return ( +
+
+ {/* Timestamp */} + + {formatTimestamp(log.timestamp)} + + + {/* Level badge */} + + {log.level} + + + {/* Category badge */} + + {log.category} + + + {/* Message */} + + {log.message} + +
+ + {/* Additional data (metrics/network) */} + {(log.metrics || log.network) && ( +
+ {log.metrics && ( + + metrics: {JSON.stringify(log.metrics)} + + )} + {log.network && ( + network: {JSON.stringify(log.network)} + )} +
+ )} +
+ ); + })} +
+ )} +
+ + {/* Reserved space for metrics/session panels (footer) */} +
+
+
+ {metrics && ( + <> + {metrics.duration_ms !== undefined && ( + Duration: {(metrics.duration_ms / 1000).toFixed(2)}s + )} + {metrics.reviews_scraped !== undefined && ( + Reviews: {metrics.reviews_scraped} + )} + {metrics.memory_mb !== undefined && ( + Memory: {metrics.memory_mb.toFixed(1)}MB + )} + + )} +
+
+ {sessionFingerprint && ( + + Session: {sessionFingerprint.session_id?.slice(0, 8)}... + + )} + {crashReport && ( + + Crash: {crashReport.error_type} + + )} +
+
+
+
+ ); +}