Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
70
test_extract_app_state.py
Normal file
70
test_extract_app_state.py
Normal file
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Extract Google Maps APP_INITIALIZATION_STATE to find timestamps
|
||||
"""
|
||||
import json
|
||||
from seleniumbase import Driver
|
||||
import time
|
||||
|
||||
url = "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2F1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=es&rclk=1"
|
||||
|
||||
print("Starting browser...")
|
||||
driver = Driver(uc=True, headless=False)
|
||||
|
||||
try:
|
||||
print(f"Loading URL: {url}")
|
||||
driver.get(url)
|
||||
time.sleep(8) # Wait for page to fully load
|
||||
|
||||
# Extract global state objects
|
||||
extract_script = """
|
||||
const results = {};
|
||||
|
||||
// Get APP_INITIALIZATION_STATE
|
||||
if (window.APP_INITIALIZATION_STATE) {
|
||||
results.app_init_state = window.APP_INITIALIZATION_STATE;
|
||||
}
|
||||
|
||||
// Get APP_OPTIONS
|
||||
if (window.APP_OPTIONS) {
|
||||
results.app_options = window.APP_OPTIONS;
|
||||
}
|
||||
|
||||
// Get WIZ_global_data
|
||||
if (window.WIZ_global_data) {
|
||||
results.wiz_data = window.WIZ_global_data;
|
||||
}
|
||||
|
||||
return results;
|
||||
"""
|
||||
|
||||
print("Extracting global state...")
|
||||
state_data = driver.execute_script(extract_script)
|
||||
|
||||
print(f"\nFound keys: {list(state_data.keys())}")
|
||||
|
||||
# Save to file
|
||||
with open('/tmp/google_maps_app_state.json', 'w') as f:
|
||||
json.dump(state_data, f, indent=2, default=str)
|
||||
|
||||
print("\nApp state saved to: /tmp/google_maps_app_state.json")
|
||||
|
||||
# Try to find review data in the state
|
||||
state_str = json.dumps(state_data)
|
||||
if '"Hace' in state_str:
|
||||
print("\n✅ Found 'Hace' in app state - reviews data is there!")
|
||||
else:
|
||||
print("\n❌ No 'Hace' found in app state")
|
||||
|
||||
# Check for timestamp-like numbers (Unix timestamps are 10-13 digits)
|
||||
import re
|
||||
timestamps = re.findall(r'\b\d{10,13}\b', state_str)
|
||||
if timestamps:
|
||||
print(f"\n✅ Found {len(timestamps)} potential timestamps (10-13 digit numbers)")
|
||||
print(f"Sample: {timestamps[:5]}")
|
||||
else:
|
||||
print("\n❌ No timestamp-like numbers found")
|
||||
|
||||
finally:
|
||||
driver.quit()
|
||||
print("\nBrowser closed")
|
||||
Reference in New Issue
Block a user