Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
136
test_english_dates.py
Normal file
136
test_english_dates.py
Normal file
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test if English locale exposes better date formats
|
||||
"""
|
||||
import json
|
||||
from seleniumbase import Driver
|
||||
import time
|
||||
|
||||
# Try both Spanish and English URLs
|
||||
urls = {
|
||||
'spanish': "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2F1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=es&rclk=1",
|
||||
'english': "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2G1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=en&rclk=1"
|
||||
}
|
||||
|
||||
results = {}
|
||||
|
||||
for lang, url in urls.items():
|
||||
print(f"\n{'='*80}")
|
||||
print(f"Testing: {lang.upper()}")
|
||||
print('='*80)
|
||||
|
||||
# Configure browser for English
|
||||
chrome_options = []
|
||||
if lang == 'english':
|
||||
chrome_options = [
|
||||
'--lang=en-US',
|
||||
'--accept-lang=en-US,en;q=0.9'
|
||||
]
|
||||
|
||||
driver = Driver(uc=True, headless=False, chromium_arg=','.join(chrome_options) if chrome_options else None)
|
||||
|
||||
try:
|
||||
driver.get(url)
|
||||
time.sleep(5)
|
||||
|
||||
# Click on reviews tab if needed
|
||||
try:
|
||||
reviews_button = driver.find_element("css selector", "button[aria-label*='eviews'], button[aria-label*='eseñas']")
|
||||
reviews_button.click()
|
||||
time.sleep(3)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Scroll to load reviews
|
||||
try:
|
||||
scrollable_pane = driver.find_element("css selector", "div[role='main']")
|
||||
driver.execute_script("arguments[0].scrollBy(0, 500);", scrollable_pane)
|
||||
time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Extract first 3 review dates
|
||||
extract_script = """
|
||||
const reviews = [];
|
||||
const elements = document.querySelectorAll('div.jftiEf.fontBodyMedium');
|
||||
|
||||
for (let i = 0; i < Math.min(3, elements.length); i++) {
|
||||
const elem = elements[i];
|
||||
const review = {};
|
||||
|
||||
// Author
|
||||
const authorElem = elem.querySelector('div.d4r55');
|
||||
review.author = authorElem ? authorElem.textContent.trim() : null;
|
||||
|
||||
// Date element
|
||||
const dateElem = elem.querySelector('span.rsqaWe');
|
||||
if (dateElem) {
|
||||
review.date_text = dateElem.textContent.trim();
|
||||
|
||||
// Check ALL attributes
|
||||
const attrs = {};
|
||||
for (let attr of dateElem.attributes) {
|
||||
attrs[attr.name] = attr.value;
|
||||
}
|
||||
review.date_attrs = attrs;
|
||||
|
||||
// Check for datetime, aria-label, title, data-*
|
||||
review.datetime = dateElem.getAttribute('datetime');
|
||||
review.aria_label = dateElem.getAttribute('aria-label');
|
||||
review.title = dateElem.getAttribute('title');
|
||||
review.data_timestamp = dateElem.getAttribute('data-timestamp');
|
||||
review.data_time = dateElem.getAttribute('data-time');
|
||||
|
||||
// Check parent elements
|
||||
let parent = dateElem.parentElement;
|
||||
if (parent) {
|
||||
review.parent_tag = parent.tagName;
|
||||
review.parent_class = parent.className;
|
||||
const parentAttrs = {};
|
||||
for (let attr of parent.attributes) {
|
||||
if (attr.name.includes('time') || attr.name.includes('date') || attr.name.includes('data-')) {
|
||||
parentAttrs[attr.name] = attr.value;
|
||||
}
|
||||
}
|
||||
review.parent_attrs = parentAttrs;
|
||||
}
|
||||
}
|
||||
|
||||
reviews.push(review);
|
||||
}
|
||||
|
||||
return reviews;
|
||||
"""
|
||||
|
||||
reviews = driver.execute_script(extract_script)
|
||||
results[lang] = reviews
|
||||
|
||||
print(f"\nExtracted {len(reviews)} reviews")
|
||||
for i, rev in enumerate(reviews, 1):
|
||||
print(f"\nReview {i}:")
|
||||
print(f" Author: {rev.get('author')}")
|
||||
print(f" Date Text: {rev.get('date_text')}")
|
||||
print(f" Datetime attr: {rev.get('datetime')}")
|
||||
print(f" Aria-label: {rev.get('aria_label')}")
|
||||
print(f" Title: {rev.get('title')}")
|
||||
print(f" Data-timestamp: {rev.get('data_timestamp')}")
|
||||
print(f" Parent attrs: {rev.get('parent_attrs')}")
|
||||
|
||||
finally:
|
||||
driver.quit()
|
||||
|
||||
# Save comparison
|
||||
with open('/tmp/date_format_comparison.json', 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("COMPARISON SAVED TO: /tmp/date_format_comparison.json")
|
||||
print('='*80)
|
||||
|
||||
# Quick comparison
|
||||
if 'spanish' in results and 'english' in results:
|
||||
print("\nSPANISH vs ENGLISH:")
|
||||
for i in range(min(len(results['spanish']), len(results['english']))):
|
||||
sp = results['spanish'][i].get('date_text', 'N/A')
|
||||
en = results['english'][i].get('date_text', 'N/A')
|
||||
print(f" Review {i+1}: '{sp}' vs '{en}'")
|
||||
Reference in New Issue
Block a user