Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
57
test_docker_chrome.py
Normal file
57
test_docker_chrome.py
Normal file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script to verify Chrome + fast_scraper works inside Docker container.
|
||||
"""
|
||||
import sys
|
||||
sys.path.insert(0, '/app')
|
||||
|
||||
from modules.fast_scraper import fast_scrape_reviews
|
||||
|
||||
def test_chrome_in_container():
|
||||
"""Test Chrome with fast_scraper in container"""
|
||||
print("=" * 70)
|
||||
print("Testing Chrome + Fast Scraper in Docker Container")
|
||||
print("=" * 70)
|
||||
|
||||
# Known good URL
|
||||
url = "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2F1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=es&rclk=1"
|
||||
|
||||
print("\nRunning fast_scrape_reviews()...")
|
||||
print("-" * 70)
|
||||
|
||||
try:
|
||||
result = fast_scrape_reviews(url=url, headless=False, max_scrolls=30)
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
if result['success'] and result['count'] > 0:
|
||||
print("✅ SUCCESS! Container scraping works!")
|
||||
print("=" * 70)
|
||||
print(f"Reviews scraped: {result['count']}")
|
||||
print(f"Time: {result['time']:.1f}s")
|
||||
print(f"Speed: {result['count']/result['time']:.1f} reviews/sec")
|
||||
|
||||
print(f"\nFirst 3 reviews:")
|
||||
for i, review in enumerate(result['reviews'][:3], 1):
|
||||
author = review.get('author', 'N/A')
|
||||
rating = review.get('rating', 'N/A')
|
||||
print(f"{i}. {author} - {rating}⭐")
|
||||
|
||||
print("\n✅ Container is production-ready!")
|
||||
return True
|
||||
else:
|
||||
print("⚠️ Scraping didn't work as expected")
|
||||
print("=" * 70)
|
||||
print(f"Success: {result['success']}")
|
||||
print(f"Reviews: {result['count']}")
|
||||
print(f"Error: {result.get('error', 'None')}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Test failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = test_chrome_in_container()
|
||||
sys.exit(0 if success else 1)
|
||||
Reference in New Issue
Block a user