Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
61
dump_api_response.py
Normal file
61
dump_api_response.py
Normal file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Quick script to dump API responses for debugging
|
||||
"""
|
||||
import json
|
||||
from modules.api_interceptor import GoogleMapsAPIInterceptor
|
||||
from seleniumbase import SB
|
||||
|
||||
url = "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2F1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=es&rclk=1"
|
||||
|
||||
with SB(uc=True, headless=False) as sb:
|
||||
# Set up interceptor BEFORE loading page
|
||||
interceptor = GoogleMapsAPIInterceptor(sb.driver)
|
||||
|
||||
sb.open(url)
|
||||
sb.sleep(2)
|
||||
|
||||
# Inject interceptor early
|
||||
interceptor.inject_response_interceptor()
|
||||
sb.sleep(2)
|
||||
|
||||
# Click reviews tab
|
||||
try:
|
||||
sb.click('.LRkQ2:contains("Reseñas")', timeout=5)
|
||||
except:
|
||||
try:
|
||||
sb.click('.LRkQ2:contains("Reviews")', timeout=5)
|
||||
except:
|
||||
pass
|
||||
|
||||
print("Waiting for reviews to load...")
|
||||
sb.sleep(5)
|
||||
|
||||
# Scroll to trigger more requests
|
||||
print("Scrolling to load more...")
|
||||
for i in range(5):
|
||||
sb.execute_script("window.scrollBy(0, 800)")
|
||||
sb.sleep(2)
|
||||
print(f" Scroll {i+1}/5...")
|
||||
|
||||
print("\nCollecting responses...")
|
||||
|
||||
# Get responses
|
||||
responses = interceptor.get_intercepted_responses()
|
||||
|
||||
print(f"\nCaptured {len(responses)} responses")
|
||||
|
||||
# Dump to files
|
||||
for i, resp in enumerate(responses):
|
||||
filename = f"api_response_{i}.json"
|
||||
with open(filename, 'w', encoding='utf-8') as f:
|
||||
json.dump(resp, f, indent=2, ensure_ascii=False)
|
||||
print(f"Saved: {filename} ({len(resp.get('body', ''))} bytes)")
|
||||
|
||||
# Also save just the body for easier viewing
|
||||
body_file = f"api_response_{i}_body.txt"
|
||||
with open(body_file, 'w', encoding='utf-8') as f:
|
||||
f.write(resp.get('body', ''))
|
||||
print(f"Saved body: {body_file}")
|
||||
|
||||
print("\nDone! Check api_response_*.json files")
|
||||
Reference in New Issue
Block a user