Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
73
test_english_dates_simple.py
Normal file
73
test_english_dates_simple.py
Normal file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test if English locale exposes better date formats
|
||||
"""
|
||||
import json
|
||||
from modules.fast_scraper import fast_scrape_reviews
|
||||
|
||||
# Try both Spanish and English URLs
|
||||
urls = {
|
||||
'spanish': "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2F1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=es&rclk=1",
|
||||
'english': "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2F1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=en&rclk=1"
|
||||
}
|
||||
|
||||
results = {}
|
||||
|
||||
for lang, url in urls.items():
|
||||
print(f"\n{'='*80}")
|
||||
print(f"Testing: {lang.upper()}")
|
||||
print('='*80)
|
||||
|
||||
result = fast_scrape_reviews(url, headless=True)
|
||||
reviews = result.get('reviews', [])
|
||||
|
||||
print(f"Extracted {len(reviews)} reviews")
|
||||
|
||||
if reviews:
|
||||
# Show first 5 review dates
|
||||
sample = []
|
||||
for i, rev in enumerate(reviews[:5], 1):
|
||||
date_info = {
|
||||
'author': rev.get('author'),
|
||||
'date_text': rev.get('date_text'),
|
||||
'debug_date_info': rev.get('debug_date_info')
|
||||
}
|
||||
sample.append(date_info)
|
||||
print(f"\nReview {i}:")
|
||||
print(f" Author: {date_info['author']}")
|
||||
print(f" Date: {date_info['date_text']}")
|
||||
|
||||
if date_info.get('debug_date_info'):
|
||||
date_attrs = date_info['debug_date_info'].get('date_elem_attrs', {})
|
||||
print(f" Date element attributes: {date_attrs}")
|
||||
|
||||
results[lang] = {
|
||||
'count': len(reviews),
|
||||
'sample': sample
|
||||
}
|
||||
|
||||
# Save comparison
|
||||
with open('/tmp/date_format_comparison.json', 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("COMPARISON SAVED TO: /tmp/date_format_comparison.json")
|
||||
print('='*80)
|
||||
|
||||
# Quick comparison
|
||||
if 'spanish' in results and 'english' in results:
|
||||
print("\n📊 SPANISH vs ENGLISH DATE FORMATS:")
|
||||
print("-" * 80)
|
||||
sp_sample = results['spanish'].get('sample', [])
|
||||
en_sample = results['english'].get('sample', [])
|
||||
|
||||
for i in range(min(len(sp_sample), len(en_sample))):
|
||||
sp_date = sp_sample[i].get('date_text', 'N/A')
|
||||
en_date = en_sample[i].get('date_text', 'N/A')
|
||||
|
||||
# Check if formats are different
|
||||
marker = "🔄" if sp_date != en_date else "="
|
||||
print(f" {marker} Review {i+1}:")
|
||||
print(f" ES: '{sp_date}'")
|
||||
print(f" EN: '{en_date}'")
|
||||
print()
|
||||
Reference in New Issue
Block a user