'use client'; import React, { useState, useCallback, useMemo, useEffect } from 'react'; // API base URL const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000'; // Types from API interface ScraperStats { total_jobs: number; success_rate: number; avg_duration: number; } interface ApiScraperVersion { id: string; job_type: string; version: string; variant: 'stable' | 'beta' | 'canary'; is_default: boolean; traffic_pct: number; module_path: string; function_name: string | null; deprecated_at: string | null; stats: ScraperStats; } // Internal UI type interface ScraperVersion { id: string; version: string; variant: 'stable' | 'beta' | 'canary'; traffic_percentage: number; jobs_24h: number; success_rate: number; avg_duration: number; status: 'active' | 'deprecated' | 'inactive'; module_path: string; function_name: string; created_at: string; promoted_at?: string; is_default?: boolean; } // Transform API response to internal format function transformApiScraper(api: ApiScraperVersion): ScraperVersion { let status: 'active' | 'deprecated' | 'inactive' = 'inactive'; if (api.deprecated_at) { status = 'deprecated'; } else if (api.traffic_pct > 0) { status = 'active'; } return { id: api.id, version: api.version, variant: api.variant, traffic_percentage: api.traffic_pct, jobs_24h: api.stats.total_jobs, success_rate: api.stats.success_rate, avg_duration: api.stats.avg_duration, status, module_path: api.module_path, function_name: api.function_name || 'scrape', created_at: new Date().toISOString(), is_default: api.is_default, }; } export default function ScrapersPage() { const [versions, setVersions] = useState([]); const [isLoadingVersions, setIsLoadingVersions] = useState(true); const [loadError, setLoadError] = useState(null); const [showAddForm, setShowAddForm] = useState(false); const [editingTraffic, setEditingTraffic] = useState(null); const [trafficValues, setTrafficValues] = useState>({}); const [isUpdatingTraffic, setIsUpdatingTraffic] = useState(false); // Fetch scrapers from API const fetchScrapers = useCallback(async () => { try { setLoadError(null); const response = await fetch(`${API_BASE}/api/admin/scrapers`); if (!response.ok) { throw new Error(`Failed to fetch scrapers: ${response.statusText}`); } const data: ApiScraperVersion[] = await response.json(); setVersions(data.map(transformApiScraper)); } catch (error) { console.error('Failed to fetch scrapers:', error); setLoadError(error instanceof Error ? error.message : 'Failed to load scrapers'); } finally { setIsLoadingVersions(false); } }, []); useEffect(() => { fetchScrapers(); }, [fetchScrapers]); // Confirmation modal state const [confirmAction, setConfirmAction] = useState<{ type: 'promote' | 'deprecate' | 'delete'; version: ScraperVersion; } | null>(null); const [isProcessing, setIsProcessing] = useState(false); // New version form state const [newVersion, setNewVersion] = useState({ version: '', variant: 'beta' as 'stable' | 'beta' | 'canary', module_path: '', function_name: '', traffic_percentage: 0, }); // Calculate total traffic for active versions const totalTraffic = useMemo(() => { return versions .filter(v => v.status === 'active') .reduce((sum, v) => sum + (trafficValues[v.id] ?? v.traffic_percentage), 0); }, [versions, trafficValues]); // Initialize traffic values when editing starts const startEditingTraffic = useCallback(() => { const initial: Record = {}; versions.forEach(v => { initial[v.id] = v.traffic_percentage; }); setTrafficValues(initial); setEditingTraffic('all'); }, [versions]); // Update traffic allocation const handleTrafficUpdate = async () => { if (totalTraffic !== 100) { alert('Traffic allocation must equal 100%'); return; } setIsUpdatingTraffic(true); try { // Update each scraper's traffic via API const updates = Object.entries(trafficValues).map(async ([id, pct]) => { const response = await fetch(`${API_BASE}/api/admin/scrapers/${id}/traffic`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ traffic_pct: pct }), }); if (!response.ok) { throw new Error(`Failed to update traffic for ${id}`); } }); await Promise.all(updates); // Refresh data from server await fetchScrapers(); setEditingTraffic(null); } catch (error) { console.error('Failed to update traffic:', error); alert('Failed to update traffic allocation. Please try again.'); } finally { setIsUpdatingTraffic(false); } }; // Promote version to stable const promoteVersion = async (version: ScraperVersion) => { setIsProcessing(true); try { const response = await fetch(`${API_BASE}/api/admin/scrapers/${version.id}/promote?traffic_pct=80`, { method: 'POST', }); if (!response.ok) { const error = await response.json(); throw new Error(error.detail || 'Failed to promote version'); } // Refresh data from server await fetchScrapers(); setConfirmAction(null); } catch (error) { console.error('Failed to promote version:', error); alert(error instanceof Error ? error.message : 'Failed to promote version'); } finally { setIsProcessing(false); } }; // Deprecate version const deprecateVersion = async (version: ScraperVersion) => { setIsProcessing(true); try { const response = await fetch(`${API_BASE}/api/admin/scrapers/${version.id}/deprecate`, { method: 'POST', }); if (!response.ok) { const error = await response.json(); throw new Error(error.detail || 'Failed to deprecate version'); } // Refresh data from server await fetchScrapers(); setConfirmAction(null); } catch (error) { console.error('Failed to deprecate version:', error); alert(error instanceof Error ? error.message : 'Failed to deprecate version'); } finally { setIsProcessing(false); } }; // Add new version const handleAddVersion = async (e: React.FormEvent) => { e.preventDefault(); try { const response = await fetch(`${API_BASE}/api/admin/scrapers`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ job_type: 'google_reviews', version: newVersion.version, variant: newVersion.variant, module_path: newVersion.module_path, function_name: newVersion.function_name, traffic_pct: newVersion.traffic_percentage, }), }); if (!response.ok) { const error = await response.json(); throw new Error(error.detail || 'Failed to add version'); } // Refresh data from server await fetchScrapers(); setNewVersion({ version: '', variant: 'beta', module_path: '', function_name: '', traffic_percentage: 0, }); setShowAddForm(false); } catch (error) { console.error('Failed to add version:', error); alert(error instanceof Error ? error.message : 'Failed to add version'); } }; // Variant badge styling const getVariantStyle = (variant: string) => { switch (variant) { case 'stable': return 'bg-green-100 text-green-800 border-green-300'; case 'beta': return 'bg-blue-100 text-blue-800 border-blue-300'; case 'canary': return 'bg-yellow-100 text-yellow-800 border-yellow-300'; default: return 'bg-gray-100 text-gray-800 border-gray-300'; } }; // Status badge styling const getStatusStyle = (status: string) => { switch (status) { case 'active': return 'bg-green-100 text-green-800'; case 'deprecated': return 'bg-red-100 text-red-800'; case 'inactive': return 'bg-gray-100 text-gray-600'; default: return 'bg-gray-100 text-gray-800'; } }; // Loading state if (isLoadingVersions) { return (

Loading scrapers...

); } // Error state if (loadError) { return (

Failed to load scrapers

{loadError}

); } return (
{/* Header */}

Scraper Versions

Manage scraper versions and A/B testing

{/* Add New Version Form */} {showAddForm && (

Add New Version

{/* Version */}
setNewVersion(prev => ({ ...prev, version: e.target.value }))} placeholder="e.g., 1.2.0" className="w-full px-3 py-2 border-2 border-gray-200 rounded-lg focus:border-blue-500 focus:outline-none text-sm" required />
{/* Variant */}
{/* Initial Traffic */}
setNewVersion(prev => ({ ...prev, traffic_percentage: parseInt(e.target.value) || 0 }))} className="w-full px-3 py-2 border-2 border-gray-200 rounded-lg focus:border-blue-500 focus:outline-none text-sm" required />
{/* Module Path */}
setNewVersion(prev => ({ ...prev, module_path: e.target.value }))} placeholder="e.g., scrapers.google_reviews" className="w-full px-3 py-2 border-2 border-gray-200 rounded-lg focus:border-blue-500 focus:outline-none text-sm font-mono" required />
{/* Function Name */}
setNewVersion(prev => ({ ...prev, function_name: e.target.value }))} placeholder="e.g., scrape_reviews" className="w-full px-3 py-2 border-2 border-gray-200 rounded-lg focus:border-blue-500 focus:outline-none text-sm font-mono" required />
)} {/* Traffic Allocation Section */}

Traffic Allocation

{editingTraffic ? (
Total: {totalTraffic}%
) : ( )}
{/* Traffic Bar Visualization */}
{versions .filter(v => v.status === 'active') .sort((a, b) => b.traffic_percentage - a.traffic_percentage) .map((version) => { const percentage = trafficValues[version.id] ?? version.traffic_percentage; if (percentage === 0) return null; return (
{percentage >= 10 && `${version.version} (${percentage}%)`}
); })}
0% 50% 100%
{/* Traffic Sliders (when editing) */} {editingTraffic && (
{versions .filter(v => v.status === 'active') .map((version) => (
{version.variant}
v{version.version} setTrafficValues(prev => ({ ...prev, [version.id]: parseInt(e.target.value), }))} className="flex-1 h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-600" /> setTrafficValues(prev => ({ ...prev, [version.id]: parseInt(e.target.value) || 0, }))} className="w-20 px-2 py-1 border-2 border-gray-200 rounded-lg text-sm text-center focus:border-blue-500 focus:outline-none" /> %
))}
)}
{/* Scraper Versions Table */}
{versions.map((version) => ( {/* Version */} {/* Variant */} {/* Traffic % */} {/* Jobs (24h) */} {/* Success Rate */} {/* Avg Duration */} {/* Status */} {/* Actions */} ))}
Version Variant Traffic % Jobs (24h) Success Rate Avg Duration Status Actions
{version.version} {version.promoted_at && (
Promoted {new Date(version.promoted_at).toLocaleDateString()}
)}
{version.variant}
{version.traffic_percentage}%
{version.jobs_24h.toLocaleString()} = 95 ? 'text-green-600' : version.success_rate >= 80 ? 'text-yellow-600' : version.success_rate > 0 ? 'text-red-600' : 'text-gray-400' }`}> {version.success_rate > 0 ? `${version.success_rate.toFixed(1)}%` : '-'} {version.avg_duration > 0 ? `${version.avg_duration}s` : '-'} {version.status === 'active' && ( )} {version.status.charAt(0).toUpperCase() + version.status.slice(1)}
{/* Promote Button (for beta/canary) */} {version.variant !== 'stable' && version.status === 'active' && ( )} {/* Deprecate Button */} {version.status === 'active' && ( )} {/* Edit Button */} {/* View Details */}
{/* Empty State */} {versions.length === 0 && (

No Scraper Versions

Add your first scraper version to get started

)}
{/* Confirmation Modal */} {confirmAction && (
setConfirmAction(null)} >
e.stopPropagation()} >
{confirmAction.type === 'promote' && ( )} {confirmAction.type === 'deprecate' && ( )}

{confirmAction.type === 'promote' ? 'Promote to Stable' : confirmAction.type === 'deprecate' ? 'Deprecate Version' : 'Delete Version'}

v{confirmAction.version.version} ({confirmAction.version.variant})

{confirmAction.type === 'promote' && (

This will promote v{confirmAction.version.version} to the stable channel.

Note: The current stable version will be demoted to beta with reduced traffic.

)} {confirmAction.type === 'deprecate' && (

This will deprecate v{confirmAction.version.version} and redistribute its traffic.

Warning: This version will no longer receive traffic. Any in-progress jobs will complete, but no new jobs will use this version.

)}
)}
); }