Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
55
test_validation_local.py
Normal file
55
test_validation_local.py
Normal file
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script for validating review detection on search results pages.
|
||||
Tests the check_reviews_available() function locally.
|
||||
"""
|
||||
import sys
|
||||
import logging
|
||||
from modules.fast_scraper import check_reviews_available
|
||||
|
||||
# Setup logging to see all debug info
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
|
||||
def test_validation(search_query: str):
|
||||
"""Test validation for a search query."""
|
||||
# Convert search query to Google Maps search URL
|
||||
url = f"https://www.google.com/maps/search/?api=1&query={search_query.replace(' ', '+')}"
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"Testing validation for: {search_query}")
|
||||
print(f"URL: {url}")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Run the check
|
||||
result = check_reviews_available(url, headless=False)
|
||||
|
||||
# Display results
|
||||
print(f"\n{'='*80}")
|
||||
print(f"RESULTS:")
|
||||
print(f"{'='*80}")
|
||||
print(f"Success: {result['success']}")
|
||||
print(f"Has Reviews: {result['has_reviews']}")
|
||||
print(f"Review Count: {result['review_count']}")
|
||||
print(f"Business Name: {result['business_name']}")
|
||||
if result.get('error'):
|
||||
print(f"Error: {result['error']}")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
return result
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test with the problematic search query
|
||||
test_cases = [
|
||||
"soho vilnius club",
|
||||
"google dublin office", # Known business with many reviews
|
||||
]
|
||||
|
||||
for query in test_cases:
|
||||
result = test_validation(query)
|
||||
|
||||
# Pause between tests
|
||||
if query != test_cases[-1]:
|
||||
input("\nPress Enter to continue to next test...")
|
||||
Reference in New Issue
Block a user