Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
@@ -1420,14 +1420,65 @@ class GoogleReviewsScraper:
|
||||
try:
|
||||
responses = self.api_interceptor.get_intercepted_responses()
|
||||
if responses:
|
||||
log.debug(f"Collected {len(responses)} network responses from browser")
|
||||
|
||||
# Dump first few responses for analysis
|
||||
if not hasattr(self, '_dumped_responses'):
|
||||
self._dumped_responses = 0
|
||||
|
||||
if self._dumped_responses < 5: # Dump first 5 responses
|
||||
from pathlib import Path
|
||||
import json
|
||||
output_dir = Path("api_response_samples")
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
|
||||
for resp in responses:
|
||||
if self._dumped_responses >= 5:
|
||||
break
|
||||
|
||||
idx = self._dumped_responses
|
||||
body = resp.get('body', '')
|
||||
|
||||
# Save full response
|
||||
full_file = output_dir / f"response_{idx:02d}_full.json"
|
||||
with open(full_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(resp, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Save body
|
||||
body_file = output_dir / f"response_{idx:02d}_body.txt"
|
||||
with open(body_file, 'w', encoding='utf-8') as f:
|
||||
f.write(body)
|
||||
|
||||
# Try to parse and save
|
||||
clean_body = body[4:].strip() if body.startswith(")]}'") else body
|
||||
try:
|
||||
parsed_data = json.loads(clean_body)
|
||||
parsed_file = output_dir / f"response_{idx:02d}_parsed.json"
|
||||
with open(parsed_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(parsed_data, f, indent=2, ensure_ascii=False)
|
||||
log.info(f"Dumped API response {idx} to {output_dir}/ ({len(body)} bytes)")
|
||||
except:
|
||||
log.debug(f"Response {idx} is not JSON")
|
||||
|
||||
self._dumped_responses += 1
|
||||
|
||||
parsed = self.api_interceptor.parse_reviews_from_responses(responses)
|
||||
log.debug(f"Parsed {len(parsed)} reviews from responses")
|
||||
for intercepted in parsed:
|
||||
if intercepted.review_id and intercepted.review_id not in api_reviews:
|
||||
api_reviews[intercepted.review_id] = self.api_interceptor.convert_to_raw_review_format(intercepted)
|
||||
if parsed:
|
||||
log.debug(f"API interceptor captured {len(parsed)} reviews (total unique: {len(api_reviews)})")
|
||||
log.info(f"API interceptor captured {len(parsed)} reviews (total unique API: {len(api_reviews)})")
|
||||
|
||||
# Log stats every 10 iterations
|
||||
if attempts % 10 == 0:
|
||||
stats = self.api_interceptor.get_interceptor_stats()
|
||||
if stats:
|
||||
log.debug(f"Interceptor stats - Fetch: {stats.get('totalFetch', 0)}/{stats.get('capturedFetch', 0)}, "
|
||||
f"XHR: {stats.get('totalXHR', 0)}/{stats.get('capturedXHR', 0)}, "
|
||||
f"Last: {stats.get('lastCapture', 'never')}")
|
||||
except Exception as api_err:
|
||||
log.debug(f"API interception error: {api_err}")
|
||||
log.warning(f"API interception error: {api_err}", exc_info=True)
|
||||
|
||||
# Dynamic sleep: sleep less when processing many reviews, more when finding none
|
||||
if len(fresh_cards) > 5:
|
||||
@@ -1470,6 +1521,35 @@ class GoogleReviewsScraper:
|
||||
if key not in existing or not existing.get(key):
|
||||
existing[key] = value
|
||||
log.info(f"After merge: {len(docs)} total reviews")
|
||||
elif self.enable_api_intercept:
|
||||
# Log final stats even if no reviews captured
|
||||
if self.api_interceptor:
|
||||
stats = self.api_interceptor.get_interceptor_stats()
|
||||
if stats:
|
||||
log.warning(f"⚠️ API interception was enabled but captured 0 reviews. "
|
||||
f"Network stats - Fetch requests: {stats.get('capturedFetch', 0)}/{stats.get('totalFetch', 0)}, "
|
||||
f"XHR requests: {stats.get('capturedXHR', 0)}/{stats.get('totalXHR', 0)}")
|
||||
|
||||
# Get browser console logs for debugging
|
||||
console_logs = self.api_interceptor.get_browser_console_logs()
|
||||
api_logs = [log_entry for log_entry in console_logs
|
||||
if 'API Interceptor' in log_entry.get('message', '')]
|
||||
if api_logs:
|
||||
log.info(f"Found {len(api_logs)} API interceptor console messages")
|
||||
for entry in api_logs[:10]: # Show first 10
|
||||
log.debug(f" Console: {entry.get('message', '')[:200]}")
|
||||
else:
|
||||
log.debug("No API interceptor console messages found")
|
||||
|
||||
# In debug mode, try to dump any responses that were collected
|
||||
if log.level <= logging.DEBUG:
|
||||
all_responses = self.api_interceptor.get_intercepted_responses()
|
||||
if all_responses:
|
||||
dump_path = self.api_interceptor.dump_responses_to_file(all_responses)
|
||||
if dump_path:
|
||||
log.info(f"Raw responses dumped to: {dump_path}")
|
||||
else:
|
||||
log.warning("API interceptor stats not available")
|
||||
|
||||
# Save to MongoDB if enabled
|
||||
if self.use_mongodb and self.mongodb:
|
||||
|
||||
Reference in New Issue
Block a user