Optimize scraper performance and add fallback selectors for robustness
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
198
reverse_engineer_date_formatter.py
Normal file
198
reverse_engineer_date_formatter.py
Normal file
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Reverse-engineer Google's date formatting library to understand:
|
||||
1. What library they use
|
||||
2. All possible date format patterns
|
||||
3. Time range boundaries for each pattern
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
from seleniumbase import Driver
|
||||
import time
|
||||
|
||||
url = "https://www.google.com/maps/place/Soho+Club/data=!4m7!3m6!1s0x46dd947294b213bf:0x864c7a232527adb4!8m2!3d54.67869!4d25.2667181!16s%2Fg%2F1thhj5ml!19sChIJvxOylHKU3UYRtK0nJSN6TIY?authuser=0&hl=en&rclk=1"
|
||||
|
||||
print("Starting browser...")
|
||||
driver = Driver(uc=True, headless=False)
|
||||
|
||||
try:
|
||||
print(f"Loading URL: {url}")
|
||||
driver.get(url)
|
||||
time.sleep(8)
|
||||
|
||||
# Script to find date formatting function
|
||||
find_formatter_script = """
|
||||
const results = {
|
||||
scripts: [],
|
||||
potential_formatters: [],
|
||||
date_strings: []
|
||||
};
|
||||
|
||||
// 1. Search all script tags for date-related code
|
||||
const scriptTags = document.querySelectorAll('script');
|
||||
let scriptContent = '';
|
||||
|
||||
scriptTags.forEach((script, idx) => {
|
||||
const content = script.textContent || script.innerText;
|
||||
if (content) {
|
||||
scriptContent += content + '\\n';
|
||||
|
||||
// Look for date formatting patterns
|
||||
if (content.includes('ago') || content.includes('month') || content.includes('year')) {
|
||||
const snippet = content.substring(0, 500);
|
||||
results.scripts.push({
|
||||
index: idx,
|
||||
snippet: snippet,
|
||||
length: content.length
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 2. Search for common date formatting library signatures
|
||||
const librarySignatures = [
|
||||
'moment',
|
||||
'date-fns',
|
||||
'dayjs',
|
||||
'luxon',
|
||||
'timeago',
|
||||
'formatRelative',
|
||||
'relativeTime',
|
||||
'fromNow'
|
||||
];
|
||||
|
||||
librarySignatures.forEach(sig => {
|
||||
if (scriptContent.includes(sig)) {
|
||||
results.potential_formatters.push(sig);
|
||||
}
|
||||
});
|
||||
|
||||
// 3. Try to find the actual formatting function by injecting test dates
|
||||
// Look for Google's internal date formatter
|
||||
const googleFormatters = [];
|
||||
for (let key in window) {
|
||||
if (typeof window[key] === 'function') {
|
||||
const funcStr = window[key].toString();
|
||||
if (funcStr.includes('ago') && funcStr.includes('month')) {
|
||||
googleFormatters.push({
|
||||
name: key,
|
||||
signature: funcStr.substring(0, 200)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
results.google_formatters = googleFormatters;
|
||||
|
||||
// 4. Extract all "X ago" patterns from the page
|
||||
const pageText = document.body.innerText;
|
||||
const agoPatterns = pageText.match(/\\d+\\s+(second|minute|hour|day|week|month|year)s?\\s+ago/gi) || [];
|
||||
const singlePatterns = pageText.match(/a\\s+(second|minute|hour|day|week|month|year)\\s+ago/gi) || [];
|
||||
|
||||
results.date_strings = [...new Set([...agoPatterns, ...singlePatterns])];
|
||||
|
||||
return results;
|
||||
"""
|
||||
|
||||
print("Searching for date formatting code...")
|
||||
formatter_info = driver.execute_script(find_formatter_script)
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("FINDINGS:")
|
||||
print("="*80)
|
||||
|
||||
print(f"\n1. Scripts with date-related code: {len(formatter_info.get('scripts', []))}")
|
||||
|
||||
print(f"\n2. Potential libraries detected: {formatter_info.get('potential_formatters', [])}")
|
||||
|
||||
print(f"\n3. Google formatter functions found: {len(formatter_info.get('google_formatters', []))}")
|
||||
for gf in formatter_info.get('google_formatters', [])[:3]:
|
||||
print(f" - {gf['name']}: {gf['signature'][:100]}...")
|
||||
|
||||
print(f"\n4. Date patterns found on page:")
|
||||
date_strings = formatter_info.get('date_strings', [])
|
||||
for ds in sorted(set(date_strings))[:20]:
|
||||
print(f" - '{ds}'")
|
||||
|
||||
# Now let's test different timestamps to understand the boundaries
|
||||
print("\n" + "="*80)
|
||||
print("TESTING TIME RANGE BOUNDARIES:")
|
||||
print("="*80)
|
||||
|
||||
# We need to inject JavaScript that can format dates like Google does
|
||||
# Let's search the actual DOM for the pattern
|
||||
boundary_test_script = """
|
||||
// Collect all unique date strings from reviews
|
||||
const dateElements = document.querySelectorAll('span.rsqaWe');
|
||||
const dateStrings = new Set();
|
||||
|
||||
dateElements.forEach(elem => {
|
||||
const text = elem.textContent.trim();
|
||||
if (text) {
|
||||
dateStrings.add(text);
|
||||
}
|
||||
});
|
||||
|
||||
return Array.from(dateStrings).sort();
|
||||
"""
|
||||
|
||||
all_date_strings = driver.execute_script(boundary_test_script)
|
||||
|
||||
print(f"\nFound {len(all_date_strings)} unique date formats:")
|
||||
for ds in all_date_strings[:30]:
|
||||
print(f" - '{ds}'")
|
||||
|
||||
# Analyze the patterns
|
||||
print("\n" + "="*80)
|
||||
print("PATTERN ANALYSIS:")
|
||||
print("="*80)
|
||||
|
||||
patterns = {
|
||||
'seconds': [],
|
||||
'minutes': [],
|
||||
'hours': [],
|
||||
'days': [],
|
||||
'weeks': [],
|
||||
'months': [],
|
||||
'years': []
|
||||
}
|
||||
|
||||
for ds in all_date_strings:
|
||||
ds_lower = ds.lower()
|
||||
if 'second' in ds_lower:
|
||||
patterns['seconds'].append(ds)
|
||||
elif 'minute' in ds_lower:
|
||||
patterns['minutes'].append(ds)
|
||||
elif 'hour' in ds_lower:
|
||||
patterns['hours'].append(ds)
|
||||
elif 'day' in ds_lower:
|
||||
patterns['days'].append(ds)
|
||||
elif 'week' in ds_lower:
|
||||
patterns['weeks'].append(ds)
|
||||
elif 'month' in ds_lower:
|
||||
patterns['months'].append(ds)
|
||||
elif 'year' in ds_lower:
|
||||
patterns['years'].append(ds)
|
||||
|
||||
for unit, examples in patterns.items():
|
||||
if examples:
|
||||
print(f"\n{unit.upper()}:")
|
||||
for ex in examples[:5]:
|
||||
print(f" - '{ex}'")
|
||||
|
||||
# Save all data
|
||||
output = {
|
||||
'formatter_info': formatter_info,
|
||||
'all_date_strings': all_date_strings,
|
||||
'pattern_analysis': {k: v for k, v in patterns.items() if v}
|
||||
}
|
||||
|
||||
with open('/tmp/google_date_formatter_analysis.json', 'w') as f:
|
||||
json.dump(output, f, indent=2)
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("Full analysis saved to: /tmp/google_date_formatter_analysis.json")
|
||||
print("="*80)
|
||||
|
||||
finally:
|
||||
driver.quit()
|
||||
print("\nBrowser closed")
|
||||
Reference in New Issue
Block a user