Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
352
hybrid_api_scraper.py
Normal file
352
hybrid_api_scraper.py
Normal file
@@ -0,0 +1,352 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Hybrid API scraper - Capture session from browser, then use direct API calls.
|
||||
This combines the best of both worlds:
|
||||
1. Browser establishes authentic session with Google
|
||||
2. We capture ALL headers from real XHR requests
|
||||
3. Replay those headers in direct API calls
|
||||
4. No scrolling needed - just fast API pagination
|
||||
|
||||
Expected speed: 10-25x faster than traditional browser scrolling.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import List, Optional, Tuple, Dict
|
||||
import requests
|
||||
from seleniumbase import SB
|
||||
from modules.api_interceptor import GoogleMapsAPIInterceptor, InterceptedReview
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HybridAPIScraper:
|
||||
"""
|
||||
Capture session from browser, then scrape via direct API calls.
|
||||
"""
|
||||
|
||||
def __init__(self, url: str, headless: bool = False):
|
||||
"""
|
||||
Initialize the hybrid scraper.
|
||||
|
||||
Args:
|
||||
url: Google Maps place URL
|
||||
headless: Run browser in headless mode
|
||||
"""
|
||||
self.url = url
|
||||
self.headless = headless
|
||||
self.captured_headers = None
|
||||
self.place_id = None
|
||||
self.session = requests.Session()
|
||||
|
||||
# Initialize parser
|
||||
self.interceptor = GoogleMapsAPIInterceptor(None)
|
||||
|
||||
def capture_session_from_browser(self) -> bool:
|
||||
"""
|
||||
Start a browser session, capture headers from actual API requests.
|
||||
|
||||
Returns:
|
||||
True if session captured successfully
|
||||
"""
|
||||
log.info("Starting browser to capture session headers...")
|
||||
|
||||
try:
|
||||
with SB(uc=True, headless=self.headless) as sb:
|
||||
# Navigate to the place
|
||||
log.info(f"Navigating to: {self.url[:80]}...")
|
||||
sb.open(self.url)
|
||||
sb.sleep(3)
|
||||
|
||||
# Dismiss cookie consent
|
||||
try:
|
||||
sb.click('button[aria-label*="Accept" i],button[aria-label*="Aceptar" i]', timeout=5)
|
||||
log.info("Cookie dialog dismissed")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Click reviews tab
|
||||
log.info("Opening reviews...")
|
||||
try:
|
||||
sb.click('.LRkQ2', timeout=5)
|
||||
sb.sleep(3)
|
||||
except:
|
||||
log.warning("Could not click reviews tab")
|
||||
|
||||
# Enable Chrome DevTools Protocol for network monitoring
|
||||
log.info("Enabling network interception...")
|
||||
sb.driver.execute_cdp_cmd('Network.enable', {})
|
||||
|
||||
# Store captured requests
|
||||
captured_requests = []
|
||||
|
||||
# Create event listener for network requests
|
||||
def add_request_listener():
|
||||
"""Inject JS to capture fetch/XHR requests with headers."""
|
||||
script = """
|
||||
window.__capturedRequests = [];
|
||||
|
||||
// Capture fetch
|
||||
const originalFetch = window.fetch;
|
||||
window.fetch = function(...args) {
|
||||
const url = args[0].toString();
|
||||
if (url.includes('listugcposts')) {
|
||||
console.log('[CAPTURE] Fetch to:', url);
|
||||
// Can't easily get headers from fetch without cloning
|
||||
}
|
||||
return originalFetch.apply(this, args);
|
||||
};
|
||||
|
||||
// Capture XHR (more reliable for headers)
|
||||
const originalXHR = window.XMLHttpRequest;
|
||||
window.XMLHttpRequest = function() {
|
||||
const xhr = new originalXHR();
|
||||
const originalOpen = xhr.open;
|
||||
const originalSetRequestHeader = xhr.setRequestHeader;
|
||||
const headers = {};
|
||||
|
||||
xhr.setRequestHeader = function(name, value) {
|
||||
headers[name.toLowerCase()] = value;
|
||||
return originalSetRequestHeader.apply(this, arguments);
|
||||
};
|
||||
|
||||
xhr.open = function(method, url, ...rest) {
|
||||
if (url.includes('listugcposts')) {
|
||||
console.log('[CAPTURE] XHR to:', url);
|
||||
window.__capturedRequests.push({
|
||||
url: url,
|
||||
method: method,
|
||||
headers: {...headers}
|
||||
});
|
||||
}
|
||||
return originalOpen.apply(this, [method, url, ...rest]);
|
||||
};
|
||||
|
||||
return xhr;
|
||||
};
|
||||
|
||||
console.log('[CAPTURE] Request capture initialized');
|
||||
"""
|
||||
sb.execute_script(script)
|
||||
|
||||
add_request_listener()
|
||||
|
||||
# Scroll to trigger an API call
|
||||
log.info("Scrolling to trigger API request...")
|
||||
for i in range(5):
|
||||
sb.execute_script("window.scrollBy(0, 800)")
|
||||
sb.sleep(1.5)
|
||||
|
||||
# Check captured requests
|
||||
captured_requests = sb.execute_script("return window.__capturedRequests || []")
|
||||
if captured_requests:
|
||||
log.info(f"✓ Captured {len(captured_requests)} API request(s)!")
|
||||
break
|
||||
|
||||
captured_request = captured_requests[0] if captured_requests else {}
|
||||
|
||||
if not captured_request:
|
||||
log.error("Failed to capture API request")
|
||||
return False
|
||||
|
||||
# Extract place ID from URL
|
||||
if 'place_id:' in self.url:
|
||||
self.place_id = self.url.split('place_id:')[1].split('&')[0].split('/')[0]
|
||||
elif '!1s' in captured_request['url']:
|
||||
# Extract from pb parameter
|
||||
import urllib.parse
|
||||
parsed = urllib.parse.urlparse(captured_request['url'])
|
||||
params = urllib.parse.parse_qs(parsed.query)
|
||||
pb = params.get('pb', [''])[0]
|
||||
if '!1s' in pb:
|
||||
self.place_id = pb.split('!1s')[1].split('!')[0]
|
||||
|
||||
# Store captured headers
|
||||
self.captured_headers = captured_request['headers']
|
||||
|
||||
# Also get cookies from browser
|
||||
cookies = sb.driver.get_cookies()
|
||||
for cookie in cookies:
|
||||
self.session.cookies.set(cookie['name'], cookie['value'], domain=cookie.get('domain'))
|
||||
|
||||
log.info(f"\n{'='*60}")
|
||||
log.info("✅ Session captured successfully!")
|
||||
log.info(f"{'='*60}")
|
||||
log.info(f"Place ID: {self.place_id}")
|
||||
log.info(f"Headers captured: {len(self.captured_headers)}")
|
||||
log.info(f"Cookies captured: {len(cookies)}")
|
||||
log.info(f"{'='*60}\n")
|
||||
|
||||
# Print sample headers for debugging
|
||||
log.debug("Sample headers:")
|
||||
for key in ['cookie', 'x-goog-api-key', 'authorization', 'user-agent']:
|
||||
if key in self.captured_headers:
|
||||
value = self.captured_headers[key]
|
||||
preview = value[:50] + '...' if len(value) > 50 else value
|
||||
log.debug(f" {key}: {preview}")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Failed to capture session: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
def fetch_reviews_page(self, continuation_token: Optional[str] = None) -> Tuple[List[InterceptedReview], Optional[str]]:
|
||||
"""
|
||||
Fetch reviews page using captured session.
|
||||
|
||||
Args:
|
||||
continuation_token: Pagination token
|
||||
|
||||
Returns:
|
||||
Tuple of (reviews, next_token)
|
||||
"""
|
||||
# Build pb parameter
|
||||
if continuation_token:
|
||||
pb = f"!1m6!1s{self.place_id}!6m4!4m1!1e1!4m1!1e3!2m2!1i10!2s{continuation_token}!5m2!1sByJsaaTKLK-bi-gPiqKAiQE!7e81!8m9!2b1!3b1!5b1!7b1!12m4!1b1!2b1!4m1!1e1!11m4!1e3!2e1!6m1!1i2!13m1!1e1"
|
||||
else:
|
||||
pb = f"!1m6!1s{self.place_id}!6m4!4m1!1e1!4m1!1e3!2m2!1i10!5m2!1sByJsaaTKLK-bi-gPiqKAiQE!7e81!8m9!2b1!3b1!5b1!7b1!12m4!1b1!2b1!4m1!1e1!11m4!1e3!2e1!6m1!1i2!13m1!1e1"
|
||||
|
||||
params = {
|
||||
'authuser': '0',
|
||||
'hl': 'es',
|
||||
'gl': 'es',
|
||||
'pb': pb
|
||||
}
|
||||
|
||||
try:
|
||||
log.info(f"Fetching page (token: {'initial' if not continuation_token else 'paginated'})...")
|
||||
|
||||
# Make request with captured headers
|
||||
url = 'https://www.google.com/maps/rpc/listugcposts'
|
||||
response = self.session.get(url, params=params, headers=self.captured_headers, timeout=10)
|
||||
|
||||
log.debug(f"Response status: {response.status_code}")
|
||||
|
||||
if response.status_code != 200:
|
||||
log.error(f"API error {response.status_code}: {response.text[:500]}")
|
||||
return [], None
|
||||
|
||||
# Parse response
|
||||
body = response.text
|
||||
if body.startswith(")]}'"):
|
||||
body = body[4:].strip()
|
||||
|
||||
data = json.loads(body)
|
||||
|
||||
# Extract reviews
|
||||
reviews = self.interceptor._parse_listugcposts_response(data)
|
||||
|
||||
# Get next token
|
||||
next_token = None
|
||||
if isinstance(data, list) and len(data) > 1 and isinstance(data[1], str):
|
||||
next_token = data[1]
|
||||
|
||||
log.info(f"✓ Extracted {len(reviews)} reviews")
|
||||
|
||||
return reviews, next_token
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"API request failed: {e}")
|
||||
return [], None
|
||||
|
||||
def scrape_all_reviews(self, max_pages: int = 100, delay: float = 0.3) -> List[dict]:
|
||||
"""
|
||||
Scrape all reviews using hybrid approach.
|
||||
|
||||
Args:
|
||||
max_pages: Maximum pages to fetch
|
||||
delay: Delay between API calls
|
||||
|
||||
Returns:
|
||||
List of review dictionaries
|
||||
"""
|
||||
# Step 1: Capture session from browser
|
||||
if not self.capture_session_from_browser():
|
||||
log.error("Failed to capture session - aborting")
|
||||
return []
|
||||
|
||||
# Step 2: Fetch all reviews via API
|
||||
log.info("\nStarting API-based scraping (no browser needed!)...")
|
||||
start_time = time.time()
|
||||
|
||||
all_reviews = []
|
||||
seen_ids = set()
|
||||
continuation_token = None
|
||||
page = 0
|
||||
|
||||
while page < max_pages:
|
||||
page += 1
|
||||
|
||||
reviews, continuation_token = self.fetch_reviews_page(continuation_token)
|
||||
|
||||
if not reviews:
|
||||
log.info("No more reviews found")
|
||||
break
|
||||
|
||||
# Deduplicate
|
||||
for review in reviews:
|
||||
review_id = review.review_id or f"{review.author}_{review.date_text}"
|
||||
if review_id not in seen_ids:
|
||||
seen_ids.add(review_id)
|
||||
all_reviews.append({
|
||||
'review_id': review.review_id,
|
||||
'author': review.author,
|
||||
'rating': review.rating,
|
||||
'text': review.text,
|
||||
'date_text': review.date_text,
|
||||
'avatar_url': review.avatar_url,
|
||||
'profile_url': review.profile_url,
|
||||
})
|
||||
|
||||
log.info(f"Page {page}: {len(all_reviews)} total unique reviews")
|
||||
|
||||
if not continuation_token:
|
||||
log.info("No continuation token - finished")
|
||||
break
|
||||
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
log.info(f"\n{'='*60}")
|
||||
log.info(f"✅ API SCRAPING COMPLETED!")
|
||||
log.info(f"{'='*60}")
|
||||
log.info(f"Total reviews: {len(all_reviews)}")
|
||||
log.info(f"API calls: {page}")
|
||||
log.info(f"Time (API only): {elapsed:.2f} seconds")
|
||||
log.info(f"Speed: {len(all_reviews)/elapsed:.1f} reviews/second")
|
||||
log.info(f"{'='*60}\n")
|
||||
|
||||
return all_reviews
|
||||
|
||||
|
||||
def main():
|
||||
"""Example usage."""
|
||||
url = "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2F1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=es&rclk=1"
|
||||
|
||||
scraper = HybridAPIScraper(url, headless=False)
|
||||
reviews = scraper.scrape_all_reviews(max_pages=50, delay=0.3)
|
||||
|
||||
# Save results
|
||||
output_file = 'hybrid_api_reviews.json'
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(reviews, f, indent=2, ensure_ascii=False)
|
||||
|
||||
log.info(f"Saved {len(reviews)} reviews to {output_file}")
|
||||
|
||||
# Show sample
|
||||
if reviews:
|
||||
log.info("\nSample review:")
|
||||
sample = reviews[0]
|
||||
log.info(f" Author: {sample['author']}")
|
||||
log.info(f" Rating: {sample['rating']}★")
|
||||
log.info(f" Text: {sample['text'][:80]}..." if sample['text'] else " Text: (none)")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user