Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
34 lines
1005 B
TypeScript
34 lines
1005 B
TypeScript
import { NextRequest, NextResponse } from 'next/server';
|
|
|
|
const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
|
|
|
|
export async function GET(
|
|
request: NextRequest,
|
|
{ params }: { params: Promise<{ jobId: string }> }
|
|
) {
|
|
try {
|
|
const { jobId } = await params;
|
|
const { searchParams } = new URL(request.url);
|
|
const limit = searchParams.get('limit') || '1000';
|
|
|
|
const response = await fetch(`${API_BASE_URL}/jobs/${jobId}/reviews?limit=${limit}`);
|
|
|
|
if (!response.ok) {
|
|
return NextResponse.json(
|
|
{ error: 'Failed to get reviews' },
|
|
{ status: response.status }
|
|
);
|
|
}
|
|
|
|
const data = await response.json();
|
|
// API returns { job_id, reviews: [...], count }, we just need the reviews array
|
|
return NextResponse.json({ reviews: data.reviews || [] });
|
|
} catch (error) {
|
|
console.error('Reviews API error:', error);
|
|
return NextResponse.json(
|
|
{ error: 'Failed to get reviews' },
|
|
{ status: 500 }
|
|
);
|
|
}
|
|
}
|