Clean up project root - remove 51 obsolete files

Deleted:
- 26 old markdown summary/documentation files
- 16 debug/test Python scripts (debug_*, test_*, diagnose_*)
- 10 untracked JSON files from api_response_samples
- terms-of-usage.md, pane_not_found.png

Also includes pending web app changes:
- Jobs management UI (JobsView, Sidebar components)
- API routes for job streaming and comparison
- Enhanced ReviewAnalytics and ScraperTest components

Final clean structure:
├── api_server_production.py  (main entry)
├── modules/                  (core Python)
├── web/                      (Next.js frontend)
├── tests/                    (test suite)
├── docs/                     (documentation)
└── examples/                 (usage examples)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

web/app/api/jobs/[jobId]/compare/route.ts

@@ -0,0 +1,86 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+// GET /api/jobs/[jobId]/compare?previous=<previousJobId>
+// Returns reviews from current job with a flag indicating if they're new
+export async function GET(
+  request: NextRequest,
+  { params }: { params: Promise<{ jobId: string }> }
+) {
+  try {
+    const { jobId } = await params;
+    const { searchParams } = new URL(request.url);
+    const previousJobId = searchParams.get('previous');
+
+    // Fetch current job reviews
+    const currentResponse = await fetch(`${API_BASE_URL}/jobs/${jobId}/reviews?limit=10000`);
+    if (!currentResponse.ok) {
+      return NextResponse.json(
+        { error: 'Failed to get current job reviews' },
+        { status: currentResponse.status }
+      );
+    }
+    const currentData = await currentResponse.json();
+    const currentReviews = currentData.reviews || [];
+
+    // If no previous job to compare, all reviews are "new"
+    if (!previousJobId) {
+      const reviewsWithStatus = currentReviews.map((review: Record<string, unknown>) => ({
+        ...review,
+        is_new: true,
+      }));
+      return NextResponse.json({
+        reviews: reviewsWithStatus,
+        total_count: reviewsWithStatus.length,
+        new_count: reviewsWithStatus.length,
+        previous_job_id: null,
+      });
+    }
+
+    // Fetch previous job reviews
+    const previousResponse = await fetch(`${API_BASE_URL}/jobs/${previousJobId}/reviews?limit=10000`);
+    if (!previousResponse.ok) {
+      // Previous job not found, treat all as new
+      const reviewsWithStatus = currentReviews.map((review: Record<string, unknown>) => ({
+        ...review,
+        is_new: true,
+      }));
+      return NextResponse.json({
+        reviews: reviewsWithStatus,
+        total_count: reviewsWithStatus.length,
+        new_count: reviewsWithStatus.length,
+        previous_job_id: previousJobId,
+      });
+    }
+
+    const previousData = await previousResponse.json();
+    const previousReviews = previousData.reviews || [];
+
+    // Create a Set of previous review IDs for O(1) lookup
+    const previousReviewIds = new Set(
+      previousReviews.map((r: { review_id: string }) => r.review_id)
+    );
+
+    // Mark reviews as new if they weren't in the previous job
+    const reviewsWithStatus = currentReviews.map((review: { review_id: string }) => ({
+      ...review,
+      is_new: !previousReviewIds.has(review.review_id),
+    }));
+
+    const newCount = reviewsWithStatus.filter((r: { is_new: boolean }) => r.is_new).length;
+
+    return NextResponse.json({
+      reviews: reviewsWithStatus,
+      total_count: reviewsWithStatus.length,
+      new_count: newCount,
+      previous_job_id: previousJobId,
+    });
+  } catch (error) {
+    console.error('Compare API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to compare reviews' },
+      { status: 500 }
+    );
+  }
+}

web/app/api/jobs/[jobId]/logs/route.ts

@@ -0,0 +1,30 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+export async function GET(
+  request: NextRequest,
+  { params }: { params: Promise<{ jobId: string }> }
+) {
+  try {
+    const { jobId } = await params;
+
+    const response = await fetch(`${API_BASE_URL}/jobs/${jobId}/logs`);
+
+    if (!response.ok) {
+      return NextResponse.json(
+        { error: 'Failed to get logs' },
+        { status: response.status }
+      );
+    }
+
+    const data = await response.json();
+    return NextResponse.json(data);
+  } catch (error) {
+    console.error('Logs API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to get logs' },
+      { status: 500 }
+    );
+  }
+}

web/app/api/jobs/[jobId]/route.ts

@@ -28,3 +28,32 @@ export async function GET(
     );
   }
 }
+
+export async function DELETE(
+  request: NextRequest,
+  { params }: { params: Promise<{ jobId: string }> }
+) {
+  try {
+    const { jobId } = await params;
+
+    const response = await fetch(`${API_BASE_URL}/jobs/${jobId}`, {
+      method: 'DELETE',
+    });
+
+    if (!response.ok) {
+      const data = await response.json();
+      return NextResponse.json(
+        { error: data.detail || 'Failed to delete job' },
+        { status: response.status }
+      );
+    }
+
+    return NextResponse.json({ success: true, message: 'Job deleted successfully' });
+  } catch (error) {
+    console.error('Delete job API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to delete job' },
+      { status: 500 }
+    );
+  }
+}

web/app/api/jobs/[jobId]/stream/route.ts

@@ -0,0 +1,57 @@
+import { NextRequest } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET(
+  request: NextRequest,
+  { params }: { params: Promise<{ jobId: string }> }
+) {
+  const { jobId } = await params;
+  const encoder = new TextEncoder();
+
+  const stream = new ReadableStream({
+    async start(controller) {
+      try {
+        const response = await fetch(`${API_BASE_URL}/jobs/${jobId}/stream`, {
+          headers: {
+            'Accept': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+          },
+        });
+
+        if (!response.ok || !response.body) {
+          controller.enqueue(encoder.encode(`event: error\ndata: {"error": "Failed to connect to backend"}\n\n`));
+          controller.close();
+          return;
+        }
+
+        const reader = response.body.getReader();
+
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) {
+            break;
+          }
+          // Forward the SSE data as-is
+          controller.enqueue(value);
+        }
+      } catch (error) {
+        console.error('SSE stream error:', error);
+        controller.enqueue(encoder.encode(`event: error\ndata: {"error": "Stream connection failed"}\n\n`));
+      } finally {
+        controller.close();
+      }
+    },
+  });
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache, no-transform',
+      'Connection': 'keep-alive',
+      'X-Accel-Buffering': 'no',
+    },
+  });
+}

web/app/api/jobs/route.ts

@@ -0,0 +1,30 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+export async function GET(request: NextRequest) {
+  try {
+    const { searchParams } = new URL(request.url);
+    const limit = searchParams.get('limit') || '100';
+
+    const response = await fetch(`${API_BASE_URL}/jobs?limit=${limit}`);
+
+    if (!response.ok) {
+      return NextResponse.json(
+        { error: 'Failed to get jobs' },
+        { status: response.status }
+      );
+    }
+
+    const data = await response.json();
+    // Backend returns array directly, not { jobs: [...] }
+    const jobs = Array.isArray(data) ? data : (data.jobs || []);
+    return NextResponse.json({ jobs });
+  } catch (error) {
+    console.error('Jobs API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to get jobs' },
+      { status: 500 }
+    );
+  }
+}

web/app/api/jobs/stream/route.ts

@@ -0,0 +1,54 @@
+import { NextRequest } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET(request: NextRequest) {
+  const encoder = new TextEncoder();
+
+  const stream = new ReadableStream({
+    async start(controller) {
+      try {
+        const response = await fetch(`${API_BASE_URL}/jobs/stream`, {
+          headers: {
+            'Accept': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+          },
+        });
+
+        if (!response.ok || !response.body) {
+          controller.enqueue(encoder.encode(`event: error\ndata: {"error": "Failed to connect to backend"}\n\n`));
+          controller.close();
+          return;
+        }
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) {
+            break;
+          }
+          // Forward the SSE data as-is
+          controller.enqueue(value);
+        }
+      } catch (error) {
+        console.error('SSE stream error:', error);
+        controller.enqueue(encoder.encode(`event: error\ndata: {"error": "Stream connection failed"}\n\n`));
+      } finally {
+        controller.close();
+      }
+    },
+  });
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache, no-transform',
+      'Connection': 'keep-alive',
+      'X-Accel-Buffering': 'no',
+    },
+  });
+}

web/app/api/scrape/route.ts

@@ -4,17 +4,26 @@ const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
 
 export async function POST(request: NextRequest) {
   try {
-    const { url } = await request.json();
+    const body = await request.json();
+    const { url, business_name, business_address, rating_snapshot, total_reviews_snapshot } = body;
 
     if (!url) {
       return NextResponse.json({ error: 'URL is required' }, { status: 400 });
     }
 
-    // Call the containerized scraper API
+    // Call the containerized scraper API with business metadata
     const response = await fetch(`${API_BASE_URL}/scrape`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ url }),
+      body: JSON.stringify({
+        url,
+        metadata: {
+          business_name,
+          business_address,
+          rating_snapshot,
+          total_reviews_snapshot,
+        },
+      }),
     });
 
     const data = await response.json();

web/app/page.tsx

@@ -1,38 +1,259 @@
-import ScraperTest from '@/components/ScraperTest';
+'use client';
+
+import ScraperTest, { JobStatus } from '@/components/ScraperTest';
+import ReviewAnalytics from '@/components/ReviewAnalytics';
+import Sidebar from '@/components/Sidebar';
+import JobsView from '@/components/JobsView';
+import { useState, useCallback, useEffect } from 'react';
+
+interface Review {
+  author: string;
+  rating: number;
+  text: string | null;
+  date_text: string;
+  avatar_url: string | null;
+  profile_url: string | null;
+  review_id: string;
+}
+
+interface ReviewWithNew extends Review {
+  is_new?: boolean;
+}
+
+interface SelectedJob {
+  reviews: ReviewWithNew[];
+  businessName: string;
+  businessUrl: string;
+  jobId: string;
+  newCount?: number;
+  previousJobId?: string;
+}
+
+type ViewType = 'newScrape' | 'jobs' | 'reports';
 
 export default function Home() {
+  const [activeView, setActiveView] = useState<ViewType>('newScrape');
+  const [jobs, setJobs] = useState<JobStatus[]>([]);
+  const [selectedJob, setSelectedJob] = useState<SelectedJob | null>(null);
+  const [isLoadingJob, setIsLoadingJob] = useState<string | null>(null);
+
+  // Load jobs from API
+  const refreshJobs = useCallback(async () => {
+    try {
+      const response = await fetch('/api/jobs?limit=100');
+      if (response.ok) {
+        const data = await response.json();
+        if (data.jobs) {
+          setJobs(data.jobs);
+        }
+      }
+    } catch (err) {
+      console.error('Failed to load jobs:', err);
+    }
+  }, []);
+
+  // Load jobs from API on mount
+  useEffect(() => {
+    refreshJobs();
+  }, [refreshJobs]);
+
+  const handleJobsChange = useCallback((newJobs: JobStatus[]) => {
+    setJobs(prev => {
+      // Merge new jobs with existing, updating duplicates
+      const jobMap = new Map(prev.map(j => [j.job_id, j]));
+      newJobs.forEach(job => jobMap.set(job.job_id, job));
+      return Array.from(jobMap.values());
+    });
+  }, []);
+
+  const handleSelectReviews = useCallback((reviews: Review[], businessName: string, jobId: string, businessUrl?: string) => {
+    setSelectedJob({ reviews, businessName, businessUrl: businessUrl || '', jobId });
+    setActiveView('reports');
+  }, []);
+
+  const loadJobReviews = async (job: JobStatus, previousJob?: JobStatus) => {
+    if (job.status !== 'completed' || !job.reviews_count) return;
+
+    setIsLoadingJob(job.job_id);
+    try {
+      // Use compare API if we have a previous job
+      const url = previousJob
+        ? `/api/jobs/${job.job_id}/compare?previous=${previousJob.job_id}`
+        : `/api/jobs/${job.job_id}/reviews?limit=10000`;
+
+      const response = await fetch(url);
+      if (!response.ok) throw new Error('Failed to fetch reviews');
+      const data = await response.json();
+
+      const reviews = data.reviews || [];
+      if (reviews.length > 0) {
+        // Extract business name from URL query param as fallback
+        let businessName = job.business_name;
+        if (!businessName) {
+          try {
+            const urlObj = new URL(job.url);
+            const query = urlObj.searchParams.get('query');
+            businessName = query ? decodeURIComponent(query) : 'Unknown Business';
+          } catch {
+            businessName = 'Unknown Business';
+          }
+        }
+
+        setSelectedJob({
+          reviews,
+          businessName,
+          businessUrl: job.url,
+          jobId: job.job_id,
+          newCount: data.new_count,
+          previousJobId: previousJob?.job_id,
+        });
+        setActiveView('reports');
+      }
+    } catch (err) {
+      console.error('Failed to load job reviews:', err);
+    } finally {
+      setIsLoadingJob(null);
+    }
+  };
+
+  const renderMainContent = () => {
+    switch (activeView) {
+      case 'newScrape':
+        return (
+          <div className="h-full overflow-y-auto p-6">
+            <ScraperTest onJobsChange={handleJobsChange} onSelectReviews={handleSelectReviews} />
+          </div>
+        );
+
+      case 'jobs':
+        return (
+          <JobsView
+            jobs={jobs}
+            onSelectJob={loadJobReviews}
+            isLoadingJob={isLoadingJob}
+            onRefresh={refreshJobs}
+          />
+        );
+
+      case 'reports': {
+        // Get completed jobs with reviews
+        const completedJobs = jobs
+          .filter(j => j.status === 'completed' && j.reviews_count && j.reviews_count > 0)
+          .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
+
+        return selectedJob ? (
+          <div className="h-full overflow-y-auto p-6">
+            <div className="mb-4 flex items-center justify-between">
+              <h2 className="text-xl font-bold text-gray-900">Analytics</h2>
+              <button
+                onClick={() => setSelectedJob(null)}
+                className="px-4 py-2 bg-gray-200 hover:bg-gray-300 text-gray-700 rounded-lg font-medium transition-colors flex items-center gap-2"
+              >
+                <svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                  <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M10 19l-7-7m0 0l7-7m-7 7h18" />
+                </svg>
+                Back to Reports
+              </button>
+            </div>
+            <ReviewAnalytics reviews={selectedJob.reviews} businessName={selectedJob.businessName} businessUrl={selectedJob.businessUrl} newCount={selectedJob.newCount} />
+          </div>
+        ) : (
+          <div className="h-full overflow-y-auto p-6">
+            <div className="mb-6">
+              <h2 className="text-2xl font-bold text-gray-900">Reports</h2>
+              <p className="text-sm text-gray-600 mt-1">
+                {completedJobs.length} completed {completedJobs.length === 1 ? 'scrape' : 'scrapes'} with reviews
+              </p>
+            </div>
+
+            {completedJobs.length === 0 ? (
+              <div className="flex flex-col items-center justify-center py-16 text-gray-500">
+                <svg className="w-20 h-20 mb-4 opacity-30" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                  <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={1.5} d="M9 19v-6a2 2 0 00-2-2H5a2 2 0 00-2 2v6a2 2 0 002 2h2a2 2 0 002-2zm0 0V9a2 2 0 012-2h2a2 2 0 012 2v10m-6 0a2 2 0 002 2h2a2 2 0 002-2m0 0V5a2 2 0 012-2h2a2 2 0 012 2v14a2 2 0 01-2 2h-2a2 2 0 01-2-2z" />
+                </svg>
+                <h3 className="text-xl font-semibold text-gray-700 mb-2">No Reports Yet</h3>
+                <p className="text-sm text-gray-500 mb-4">Complete a scrape job to see analytics reports</p>
+                <button
+                  onClick={() => setActiveView('newScrape')}
+                  className="px-4 py-2 bg-blue-600 text-white rounded-lg font-medium hover:bg-blue-700 transition-colors"
+                >
+                  Start New Scrape
+                </button>
+              </div>
+            ) : (
+              <div className="grid gap-4 md:grid-cols-2 lg:grid-cols-3">
+                {completedJobs.map(job => {
+                  // Extract business name from URL as fallback
+                  let businessName = job.business_name;
+                  if (!businessName) {
+                    try {
+                      const urlObj = new URL(job.url);
+                      const query = urlObj.searchParams.get('query');
+                      businessName = query ? decodeURIComponent(query) : 'Unknown Business';
+                    } catch {
+                      businessName = 'Unknown Business';
+                    }
+                  }
+
+                  return (
+                    <div
+                      key={job.job_id}
+                      onClick={() => loadJobReviews(job)}
+                      className="bg-white rounded-xl border-2 border-gray-200 p-5 cursor-pointer hover:border-blue-400 hover:shadow-lg transition-all"
+                    >
+                      <div className="flex items-start justify-between mb-3">
+                        <h3 className="font-bold text-gray-900 truncate flex-1" title={businessName}>
+                          {businessName}
+                        </h3>
+                        {job.rating_snapshot && (
+                          <span className="flex items-center gap-1 text-yellow-600 font-semibold ml-2">
+                            <svg className="w-4 h-4" fill="currentColor" viewBox="0 0 20 20">
+                              <path d="M9.049 2.927c.3-.921 1.603-.921 1.902 0l1.07 3.292a1 1 0 00.95.69h3.462c.969 0 1.371 1.24.588 1.81l-2.8 2.034a1 1 0 00-.364 1.118l1.07 3.292c.3.921-.755 1.688-1.54 1.118l-2.8-2.034a1 1 0 00-1.175 0l-2.8 2.034c-.784.57-1.838-.197-1.539-1.118l1.07-3.292a1 1 0 00-.364-1.118L2.98 8.72c-.783-.57-.38-1.81.588-1.81h3.461a1 1 0 00.951-.69l1.07-3.292z" />
+                            </svg>
+                            {job.rating_snapshot.toFixed(1)}
+                          </span>
+                        )}
+                      </div>
+
+                      <div className="flex items-center gap-4 text-sm text-gray-600 mb-3">
+                        <span className="font-semibold text-blue-700">{job.reviews_count} reviews</span>
+                        {job.scrape_time && <span>{job.scrape_time.toFixed(1)}s</span>}
+                      </div>
+
+                      <div className="text-xs text-gray-500">
+                        {new Date(job.created_at).toLocaleDateString()} at {new Date(job.created_at).toLocaleTimeString()}
+                      </div>
+
+                      {isLoadingJob === job.job_id && (
+                        <div className="mt-3 flex items-center gap-2 text-blue-600">
+                          <div className="w-4 h-4 border-2 border-blue-500 border-t-transparent rounded-full animate-spin" />
+                          <span className="text-sm font-medium">Loading...</span>
+                        </div>
+                      )}
+                    </div>
+                  );
+                })}
+              </div>
+            )}
+          </div>
+        );
+      }
+    }
+  };
+
   return (
-    <div className="min-h-screen bg-gradient-to-br from-blue-600 to-indigo-700 py-12 px-4">
-      <main className="max-w-5xl mx-auto">
-        <div className="text-center mb-10">
-          <h1 className="text-4xl md:text-5xl font-bold text-white mb-3">
-            Google Reviews Scraper
-          </h1>
-          <p className="text-blue-100 text-lg">
-            Test the containerized scraper API
-          </p>
-          <div className="mt-4 inline-flex items-center gap-2 px-4 py-2 bg-blue-500/30 rounded-lg text-blue-100 text-sm">
-            <div className="w-2 h-2 bg-green-400 rounded-full animate-pulse"></div>
-            Powered by SeleniumBase UC Mode
-          </div>
-        </div>
+    <div className="h-screen w-screen overflow-hidden flex">
+      {/* Sidebar */}
+      <Sidebar
+        activeView={activeView}
+        onViewChange={setActiveView}
+        jobCount={jobs.length}
+      />
 
-        <div className="bg-white rounded-2xl shadow-2xl p-6 md:p-8">
-          <ScraperTest />
-        </div>
-
-        <div className="mt-8 text-center text-blue-100 text-sm space-y-2">
-          <p className="font-medium">💡 Example URLs to test:</p>
-          <div className="space-y-1 text-xs">
-            <p className="font-mono bg-blue-500/20 rounded px-3 py-1 inline-block">
-              https://www.google.com/maps/place/Soho+Club/...
-            </p>
-          </div>
-          <p className="mt-4 text-blue-200">
-            API running at: <span className="font-mono">localhost:8000</span>
-          </p>
-        </div>
-      </main>
+      {/* Main Content */}
+      <div className="flex-1 bg-gray-50 overflow-hidden">
+        {renderMainContent()}
+      </div>
     </div>
   );
 }