faa0704737
Performance improvements: - Validation speed: 59.71s → 10.96s (5.5x improvement) - Removed 50+ console.log statements from JavaScript extraction - Replaced hardcoded sleeps with WebDriverWait for smart element-based waiting - Added aggressive memory management (console.clear, GC, image unloading every 20 scrolls) Scraping improvements: - Increased idle detection from 6 to 12 consecutive idle scrolls for completeness - Added real-time progress updates every 5 scrolls with percentage calculation - Added crash recovery to extract partial reviews if Chrome crashes - Removed artificial 200-review limit to scrape ALL reviews Timestamp tracking: - Added updated_at field separate from started_at for progress tracking - Frontend now shows both "Started" (fixed) and "Last Update" (dynamic) Robustness improvements: - Added 5 fallback CSS selectors to handle different Google Maps page structures - Now tries: div.jftiEf.fontBodyMedium, div.jftiEf, div[data-review-id], etc. - Automatic selector detection logs which selector works for debugging Test results: - Successfully scraped 550 reviews in 150.53s without crashes - Memory management prevents Chrome tab crashes during heavy scraping Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
63 lines
1.5 KiB
YAML
63 lines
1.5 KiB
YAML
version: '3.8'
|
|
|
|
services:
|
|
# PostgreSQL Database
|
|
db:
|
|
image: postgres:15-alpine
|
|
container_name: scraper-db
|
|
environment:
|
|
POSTGRES_DB: scraper
|
|
POSTGRES_USER: scraper
|
|
POSTGRES_PASSWORD: ${DB_PASSWORD:-scraper123}
|
|
ports:
|
|
- "5435:5432"
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U scraper"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
networks:
|
|
- scraper-network
|
|
|
|
# API Server
|
|
api:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
container_name: scraper-api
|
|
environment:
|
|
- DATABASE_URL=postgresql://scraper:${DB_PASSWORD:-scraper123}@db:5432/scraper
|
|
- API_BASE_URL=${API_BASE_URL:-http://localhost:8000}
|
|
- PORT=8000
|
|
- MAX_CONCURRENT_JOBS=${MAX_CONCURRENT_JOBS:-5}
|
|
- CANARY_TEST_URL=${CANARY_TEST_URL:-https://www.google.com/maps/place/Soho+Factory/@54.6738155,25.2595844,17z/}
|
|
- SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL:-}
|
|
# Chromium/Xvfb configuration
|
|
- DISPLAY=:99
|
|
- CHROME_BIN=/usr/bin/chromium
|
|
ports:
|
|
- "8000:8000"
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
# Chrome requires shared memory for stability
|
|
shm_size: 2gb
|
|
# Chrome capabilities (needed for sandboxing)
|
|
cap_add:
|
|
- SYS_ADMIN
|
|
# Security options for Chrome
|
|
security_opt:
|
|
- seccomp:unconfined
|
|
networks:
|
|
- scraper-network
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
postgres_data:
|
|
|
|
networks:
|
|
scraper-network:
|
|
driver: bridge
|