'use client'; import { useState, useEffect, useCallback } from 'react'; import Link from 'next/link'; const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000'; interface ScraperInfo { job_type: string; version: string; variant: string; traffic_pct: number; deprecated_at: string | null; } interface ScraperTypeCard { job_type: string; name: string; description: string; icon: React.ReactNode; route: string; color: string; available: boolean; versions: string[]; } // Define scraper type metadata (icons, descriptions, routes) const SCRAPER_METADATA: Record> = { 'google-reviews': { name: 'Google Reviews', description: 'Extract reviews from Google Maps business listings. Supports any business with a Google Maps presence.', icon: ( ), route: '/new/google-reviews', color: 'from-blue-500 to-indigo-600', }, 'google_reviews': { name: 'Google Reviews', description: 'Extract reviews from Google Maps business listings. Supports any business with a Google Maps presence.', icon: ( ), route: '/new/google-reviews', color: 'from-blue-500 to-indigo-600', }, 'yelp-reviews': { name: 'Yelp Reviews', description: 'Extract reviews from Yelp business pages. Perfect for restaurants, services, and local businesses.', icon: ( ), route: '/new/yelp-reviews', color: 'from-red-500 to-rose-600', }, 'tripadvisor-reviews': { name: 'TripAdvisor Reviews', description: 'Extract reviews from TripAdvisor. Ideal for hotels, restaurants, and tourist attractions.', icon: ( ), route: '/new/tripadvisor-reviews', color: 'from-green-500 to-emerald-600', }, }; // Fallback for unknown scraper types const DEFAULT_METADATA = { name: 'Unknown Scraper', description: 'A scraper for extracting reviews.', icon: ( ), route: '/new', color: 'from-gray-500 to-gray-600', }; export default function NewScrapePage() { const [scrapers, setScrapers] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const fetchScrapers = useCallback(async () => { try { const response = await fetch(`${API_BASE}/api/admin/scrapers`); if (!response.ok) throw new Error('Failed to fetch scrapers'); const data: ScraperInfo[] = await response.json(); // Group by job_type and collect versions const scrapersByType = data.reduce((acc, scraper) => { const key = scraper.job_type; if (!acc[key]) { acc[key] = { job_type: key, versions: [], hasActive: false, }; } acc[key].versions.push(`v${scraper.version}${scraper.variant !== 'stable' ? ` (${scraper.variant})` : ''}`); if (!scraper.deprecated_at && scraper.traffic_pct > 0) { acc[key].hasActive = true; } return acc; }, {} as Record); // Transform to ScraperTypeCard array const cards: ScraperTypeCard[] = Object.values(scrapersByType).map(({ job_type, versions, hasActive }) => { const metadata = SCRAPER_METADATA[job_type] || { ...DEFAULT_METADATA, name: job_type.split(/[-_]/).map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' '), route: `/new/${job_type}`, }; return { job_type, ...metadata, available: hasActive, versions, }; }); setScrapers(cards); } catch (err) { console.error('Failed to fetch scrapers:', err); setError('Failed to load available scrapers'); // Fallback to showing Google Reviews as available setScrapers([{ job_type: 'google-reviews', ...SCRAPER_METADATA['google-reviews'], available: true, versions: ['v1.0.0'], }]); } finally { setLoading(false); } }, []); useEffect(() => { fetchScrapers(); }, [fetchScrapers]); // Coming soon scrapers (not in registry yet) const comingSoonScrapers: ScraperTypeCard[] = [ { job_type: 'yelp-reviews', ...SCRAPER_METADATA['yelp-reviews'], available: false, versions: [], }, { job_type: 'tripadvisor-reviews', ...SCRAPER_METADATA['tripadvisor-reviews'], available: false, versions: [], }, ].filter(s => !scrapers.some(existing => existing.job_type === s.job_type)); return (
{/* Header */}

New Scrape Job

Select a scraper type to start extracting reviews

{/* Error State */} {error && (
{error}
)} {/* Loading State */} {loading ? (
{[1, 2].map(i => (
))}
) : ( <> {/* Available Scrapers */}

Available Scrapers

{scrapers.filter(s => s.available).map(scraper => (
{scraper.icon}

{scraper.name}

Active

{scraper.description}

{scraper.versions.slice(0, 2).map(v => ( {v} ))} {scraper.versions.length > 2 && ( +{scraper.versions.length - 2} more )}
))}
{/* Coming Soon Scrapers */} {comingSoonScrapers.length > 0 && (

Coming Soon

{comingSoonScrapers.map(scraper => (
{scraper.icon}

{scraper.name}

Coming Soon

{scraper.description}

))}
)} )} {/* Help Section */}

Need a different scraper?

We're constantly adding new scrapers. If you need reviews from a platform not listed here,{' '} let us know.

); }