feat: Add scraper version selector to frontend
- Add version selector dropdown in scrape confirmation modal - Default to v1.1.0 (Multi-Sort) which bypasses ~1000 review limit - Pass scraper_version through API proxy to backend - Update /new page fallback to show v1.1.0 as available - Show version description explaining multi-sort benefits Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,23 +5,25 @@ const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
|
|||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
const body = await request.json();
|
||||||
const { url, business_name, business_address, rating_snapshot, total_reviews_snapshot } = body;
|
const { url, business_name, business_address, rating_snapshot, total_reviews_snapshot, scraper_version } = body;
|
||||||
|
|
||||||
if (!url) {
|
if (!url) {
|
||||||
return NextResponse.json({ error: 'URL is required' }, { status: 400 });
|
return NextResponse.json({ error: 'URL is required' }, { status: 400 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Call the containerized scraper API with business metadata
|
// Call the containerized scraper API with business metadata and version
|
||||||
const response = await fetch(`${API_BASE_URL}/scrape`, {
|
const response = await fetch(`${API_BASE_URL}/scrape`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
url,
|
url,
|
||||||
|
scraper_version, // Pass version to backend for routing
|
||||||
metadata: {
|
metadata: {
|
||||||
business_name,
|
business_name,
|
||||||
business_address,
|
business_address,
|
||||||
rating_snapshot,
|
rating_snapshot,
|
||||||
total_reviews_snapshot,
|
total_reviews_snapshot,
|
||||||
|
scraper_version, // Also store in metadata for job tracking
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -134,12 +134,12 @@ export default function NewScrapePage() {
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to fetch scrapers:', err);
|
console.error('Failed to fetch scrapers:', err);
|
||||||
setError('Failed to load available scrapers');
|
setError('Failed to load available scrapers');
|
||||||
// Fallback to showing Google Reviews as available
|
// Fallback to showing Google Reviews as available with both versions
|
||||||
setScrapers([{
|
setScrapers([{
|
||||||
job_type: 'google-reviews',
|
job_type: 'google-reviews',
|
||||||
...SCRAPER_METADATA['google-reviews'],
|
...SCRAPER_METADATA['google-reviews'],
|
||||||
available: true,
|
available: true,
|
||||||
versions: ['v1.0.0'],
|
versions: ['v1.1.0 (Multi-Sort)', 'v1.0.0'],
|
||||||
}]);
|
}]);
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
|
|||||||
@@ -64,6 +64,13 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
|||||||
const [businessImage, setBusinessImage] = useState<string | null>(null);
|
const [businessImage, setBusinessImage] = useState<string | null>(null);
|
||||||
const [businessCategory, setBusinessCategory] = useState<string | null>(null);
|
const [businessCategory, setBusinessCategory] = useState<string | null>(null);
|
||||||
|
|
||||||
|
// Scraper version selection - v1.1.0 is default (multi-sort enabled)
|
||||||
|
const AVAILABLE_VERSIONS = [
|
||||||
|
{ value: '1.1.0', label: 'v1.1.0 (Multi-Sort)', description: 'Bypasses ~1000 review limit' },
|
||||||
|
{ value: '1.0.0', label: 'v1.0.0 (Standard)', description: 'Original scraper' },
|
||||||
|
];
|
||||||
|
const [scraperVersion, setScraperVersion] = useState('1.1.0');
|
||||||
|
|
||||||
const [userFingerprint, setUserFingerprint] = useState<{
|
const [userFingerprint, setUserFingerprint] = useState<{
|
||||||
geolocation?: {lat: number, lng: number},
|
geolocation?: {lat: number, lng: number},
|
||||||
userAgent?: string,
|
userAgent?: string,
|
||||||
@@ -471,6 +478,7 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
|||||||
browser_fingerprint: userFingerprint, // Pass full fingerprint
|
browser_fingerprint: userFingerprint, // Pass full fingerprint
|
||||||
// Google Reviews scraper (this component is specific to Google Reviews)
|
// Google Reviews scraper (this component is specific to Google Reviews)
|
||||||
job_type: 'google-reviews',
|
job_type: 'google-reviews',
|
||||||
|
scraper_version: scraperVersion, // Selected scraper version
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1304,6 +1312,30 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
|||||||
<p className="text-sm text-green-700 mt-1">{businessAddress}</p>
|
<p className="text-sm text-green-700 mt-1">{businessAddress}</p>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Scraper Version Selector */}
|
||||||
|
<div className="mb-4">
|
||||||
|
<label className="block text-sm font-medium text-gray-700 mb-2">
|
||||||
|
Scraper Version
|
||||||
|
</label>
|
||||||
|
<select
|
||||||
|
value={scraperVersion}
|
||||||
|
onChange={(e) => setScraperVersion(e.target.value)}
|
||||||
|
className="w-full px-3 py-2 border-2 border-gray-200 rounded-lg focus:border-green-500 focus:ring-2 focus:ring-green-200 transition-all bg-white"
|
||||||
|
>
|
||||||
|
{AVAILABLE_VERSIONS.map((v) => (
|
||||||
|
<option key={v.value} value={v.value}>
|
||||||
|
{v.label} - {v.description}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
<p className="mt-1 text-xs text-gray-500">
|
||||||
|
{scraperVersion === '1.1.0'
|
||||||
|
? '✨ Multi-sort enabled: Can collect more than 1000 reviews'
|
||||||
|
: 'Standard scraping (up to ~1000 reviews)'}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
<p className="text-sm text-gray-600">
|
<p className="text-sm text-gray-600">
|
||||||
The scraping job will run in the background. You can monitor progress below.
|
The scraping job will run in the background. You can monitor progress below.
|
||||||
</p>
|
</p>
|
||||||
|
|||||||
Reference in New Issue
Block a user