feat: Add scraper version selector to frontend
- Add version selector dropdown in scrape confirmation modal - Default to v1.1.0 (Multi-Sort) which bypasses ~1000 review limit - Pass scraper_version through API proxy to backend - Update /new page fallback to show v1.1.0 as available - Show version description explaining multi-sort benefits Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,23 +5,25 @@ const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const body = await request.json();
|
||||
const { url, business_name, business_address, rating_snapshot, total_reviews_snapshot } = body;
|
||||
const { url, business_name, business_address, rating_snapshot, total_reviews_snapshot, scraper_version } = body;
|
||||
|
||||
if (!url) {
|
||||
return NextResponse.json({ error: 'URL is required' }, { status: 400 });
|
||||
}
|
||||
|
||||
// Call the containerized scraper API with business metadata
|
||||
// Call the containerized scraper API with business metadata and version
|
||||
const response = await fetch(`${API_BASE_URL}/scrape`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url,
|
||||
scraper_version, // Pass version to backend for routing
|
||||
metadata: {
|
||||
business_name,
|
||||
business_address,
|
||||
rating_snapshot,
|
||||
total_reviews_snapshot,
|
||||
scraper_version, // Also store in metadata for job tracking
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -134,12 +134,12 @@ export default function NewScrapePage() {
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch scrapers:', err);
|
||||
setError('Failed to load available scrapers');
|
||||
// Fallback to showing Google Reviews as available
|
||||
// Fallback to showing Google Reviews as available with both versions
|
||||
setScrapers([{
|
||||
job_type: 'google-reviews',
|
||||
...SCRAPER_METADATA['google-reviews'],
|
||||
available: true,
|
||||
versions: ['v1.0.0'],
|
||||
versions: ['v1.1.0 (Multi-Sort)', 'v1.0.0'],
|
||||
}]);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
|
||||
@@ -64,6 +64,13 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
||||
const [businessImage, setBusinessImage] = useState<string | null>(null);
|
||||
const [businessCategory, setBusinessCategory] = useState<string | null>(null);
|
||||
|
||||
// Scraper version selection - v1.1.0 is default (multi-sort enabled)
|
||||
const AVAILABLE_VERSIONS = [
|
||||
{ value: '1.1.0', label: 'v1.1.0 (Multi-Sort)', description: 'Bypasses ~1000 review limit' },
|
||||
{ value: '1.0.0', label: 'v1.0.0 (Standard)', description: 'Original scraper' },
|
||||
];
|
||||
const [scraperVersion, setScraperVersion] = useState('1.1.0');
|
||||
|
||||
const [userFingerprint, setUserFingerprint] = useState<{
|
||||
geolocation?: {lat: number, lng: number},
|
||||
userAgent?: string,
|
||||
@@ -471,6 +478,7 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
||||
browser_fingerprint: userFingerprint, // Pass full fingerprint
|
||||
// Google Reviews scraper (this component is specific to Google Reviews)
|
||||
job_type: 'google-reviews',
|
||||
scraper_version: scraperVersion, // Selected scraper version
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -1304,6 +1312,30 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
||||
<p className="text-sm text-green-700 mt-1">{businessAddress}</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Scraper Version Selector */}
|
||||
<div className="mb-4">
|
||||
<label className="block text-sm font-medium text-gray-700 mb-2">
|
||||
Scraper Version
|
||||
</label>
|
||||
<select
|
||||
value={scraperVersion}
|
||||
onChange={(e) => setScraperVersion(e.target.value)}
|
||||
className="w-full px-3 py-2 border-2 border-gray-200 rounded-lg focus:border-green-500 focus:ring-2 focus:ring-green-200 transition-all bg-white"
|
||||
>
|
||||
{AVAILABLE_VERSIONS.map((v) => (
|
||||
<option key={v.value} value={v.value}>
|
||||
{v.label} - {v.description}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
{scraperVersion === '1.1.0'
|
||||
? '✨ Multi-sort enabled: Can collect more than 1000 reviews'
|
||||
: 'Standard scraping (up to ~1000 reviews)'}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<p className="text-sm text-gray-600">
|
||||
The scraping job will run in the background. You can monitor progress below.
|
||||
</p>
|
||||
|
||||
Reference in New Issue
Block a user