diff --git a/web/components/ScraperTest.tsx b/web/components/ScraperTest.tsx index bac2f00..dc92dce 100644 --- a/web/components/ScraperTest.tsx +++ b/web/components/ScraperTest.tsx @@ -1,8 +1,17 @@ 'use client'; -import { useState, useEffect, useRef } from 'react'; +import { useState, useEffect, useRef, useCallback } from 'react'; import ReviewAnalytics from './ReviewAnalytics'; +const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000'; + +interface ScraperType { + job_type: string; + version: string; + variant: string; + label: string; +} + interface Review { author: string; rating: number; @@ -59,6 +68,11 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe const [businessRating, setBusinessRating] = useState(null); const [businessImage, setBusinessImage] = useState(null); const [businessCategory, setBusinessCategory] = useState(null); + + // Scraper type selection + const [availableScrapers, setAvailableScrapers] = useState([]); + const [selectedScraper, setSelectedScraper] = useState(null); + const [scrapersLoading, setScrapersLoading] = useState(true); const [userFingerprint, setUserFingerprint] = useState<{ geolocation?: {lat: number, lng: number}, userAgent?: string, @@ -117,6 +131,49 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe collectFingerprint(); }, []); + + // Fetch available scraper types on mount + const fetchScrapers = useCallback(async () => { + try { + const response = await fetch(`${API_BASE}/api/admin/scrapers`); + if (response.ok) { + const data = await response.json(); + // Transform to ScraperType format and filter to active scrapers + const scrapers: ScraperType[] = data + .filter((s: { deprecated_at: string | null; traffic_pct: number }) => !s.deprecated_at && s.traffic_pct > 0) + .map((s: { job_type: string; version: string; variant: string }) => ({ + job_type: s.job_type, + version: s.version, + variant: s.variant, + // Format job_type nicely: google_reviews or google-reviews -> "Google Reviews" + label: `${s.job_type.split(/[-_]/).map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')} v${s.version}${s.variant !== 'stable' ? ` (${s.variant})` : ''}`, + })); + setAvailableScrapers(scrapers); + // Auto-select first scraper (usually google-reviews stable) + if (scrapers.length > 0 && !selectedScraper) { + setSelectedScraper(scrapers[0]); + } + } + } catch (err) { + console.error('Failed to fetch scrapers:', err); + // Fallback to default google-reviews + const defaultScraper: ScraperType = { + job_type: 'google-reviews', + version: '1.0.0', + variant: 'stable', + label: 'Google Reviews v1.0.0', + }; + setAvailableScrapers([defaultScraper]); + setSelectedScraper(defaultScraper); + } finally { + setScrapersLoading(false); + } + }, [selectedScraper]); + + useEffect(() => { + fetchScrapers(); + }, [fetchScrapers]); + const pollingIntervals = useRef>(new Map()); const abortControllerRef = useRef(null); @@ -322,6 +379,11 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe const url = `https://www.google.com/maps/search/?api=1&query=${encodeURIComponent(searchedQuery)}&hl=en`; try { + // Use selected scraper or default to google-reviews + const jobType = selectedScraper?.job_type || 'google-reviews'; + const scraperVersion = selectedScraper?.version; + const scraperVariant = selectedScraper?.variant; + const response = await fetch('/api/scrape', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -333,6 +395,10 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe total_reviews_snapshot: availableReviewCount, geolocation: userFingerprint.geolocation, browser_fingerprint: userFingerprint, // Pass full fingerprint + // Include scraper selection + job_type: jobType, + scraper_version: scraperVersion, + scraper_variant: scraperVariant, }), }); @@ -419,26 +485,13 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe } }; - const embedUrl = searchedQuery - ? `https://maps.google.com/maps?q=${encodeURIComponent(searchedQuery)}&output=embed&z=15` + // Google Maps link for opening in new tab + const googleMapsUrl = searchedQuery + ? `https://www.google.com/maps/search/?api=1&query=${encodeURIComponent(searchedQuery)}` : ''; - const [mapClicked, setMapClicked] = useState(false); const searchInputRef = useRef(null); - const handleMapClick = () => { - setMapClicked(true); - }; - - const closeModal = () => { - setMapClicked(false); - }; - - const focusSearchBar = () => { - setMapClicked(false); - searchInputRef.current?.focus(); - }; - // Test URLs at different scales const testUrls = [ { name: '🏪 Small (~79)', query: 'R. Fleitas Peluqueros Gran Canaria' }, @@ -449,6 +502,68 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe return (
+ {/* Scraper Type Selection */} +
+
+
+
+ + + +
+
+ +

Select the type of data to scrape

+
+
+ + {scrapersLoading ? ( +
+
+ Loading... +
+ ) : ( + + )} +
+ + {/* Show selected scraper info */} + {selectedScraper && ( +
+ + {selectedScraper.job_type.split(/[-_]/).map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')} + + + v{selectedScraper.version} + + {selectedScraper.variant !== 'stable' && ( + + {selectedScraper.variant} + + )} +
+ )} +
+ {/* Test URL Quick Select */}
@@ -536,78 +651,52 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
- {/* Map Preview with Click Overlay */} -
+ {/* Map Preview Area */} +
{searchedQuery ? ( - <> -