Clean up project root - remove 51 obsolete files

Deleted:
- 26 old markdown summary/documentation files
- 16 debug/test Python scripts (debug_*, test_*, diagnose_*)
- 10 untracked JSON files from api_response_samples
- terms-of-usage.md, pane_not_found.png

Also includes pending web app changes:
- Jobs management UI (JobsView, Sidebar components)
- API routes for job streaming and comparison
- Enhanced ReviewAnalytics and ScraperTest components

Final clean structure:
├── api_server_production.py  (main entry)
├── modules/                  (core Python)
├── web/                      (Next.js frontend)
├── tests/                    (test suite)
├── docs/                     (documentation)
└── examples/                 (usage examples)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

web/app/api/jobs/[jobId]/compare/route.ts

@@ -0,0 +1,86 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+// GET /api/jobs/[jobId]/compare?previous=<previousJobId>
+// Returns reviews from current job with a flag indicating if they're new
+export async function GET(
+  request: NextRequest,
+  { params }: { params: Promise<{ jobId: string }> }
+) {
+  try {
+    const { jobId } = await params;
+    const { searchParams } = new URL(request.url);
+    const previousJobId = searchParams.get('previous');
+
+    // Fetch current job reviews
+    const currentResponse = await fetch(`${API_BASE_URL}/jobs/${jobId}/reviews?limit=10000`);
+    if (!currentResponse.ok) {
+      return NextResponse.json(
+        { error: 'Failed to get current job reviews' },
+        { status: currentResponse.status }
+      );
+    }
+    const currentData = await currentResponse.json();
+    const currentReviews = currentData.reviews || [];
+
+    // If no previous job to compare, all reviews are "new"
+    if (!previousJobId) {
+      const reviewsWithStatus = currentReviews.map((review: Record<string, unknown>) => ({
+        ...review,
+        is_new: true,
+      }));
+      return NextResponse.json({
+        reviews: reviewsWithStatus,
+        total_count: reviewsWithStatus.length,
+        new_count: reviewsWithStatus.length,
+        previous_job_id: null,
+      });
+    }
+
+    // Fetch previous job reviews
+    const previousResponse = await fetch(`${API_BASE_URL}/jobs/${previousJobId}/reviews?limit=10000`);
+    if (!previousResponse.ok) {
+      // Previous job not found, treat all as new
+      const reviewsWithStatus = currentReviews.map((review: Record<string, unknown>) => ({
+        ...review,
+        is_new: true,
+      }));
+      return NextResponse.json({
+        reviews: reviewsWithStatus,
+        total_count: reviewsWithStatus.length,
+        new_count: reviewsWithStatus.length,
+        previous_job_id: previousJobId,
+      });
+    }
+
+    const previousData = await previousResponse.json();
+    const previousReviews = previousData.reviews || [];
+
+    // Create a Set of previous review IDs for O(1) lookup
+    const previousReviewIds = new Set(
+      previousReviews.map((r: { review_id: string }) => r.review_id)
+    );
+
+    // Mark reviews as new if they weren't in the previous job
+    const reviewsWithStatus = currentReviews.map((review: { review_id: string }) => ({
+      ...review,
+      is_new: !previousReviewIds.has(review.review_id),
+    }));
+
+    const newCount = reviewsWithStatus.filter((r: { is_new: boolean }) => r.is_new).length;
+
+    return NextResponse.json({
+      reviews: reviewsWithStatus,
+      total_count: reviewsWithStatus.length,
+      new_count: newCount,
+      previous_job_id: previousJobId,
+    });
+  } catch (error) {
+    console.error('Compare API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to compare reviews' },
+      { status: 500 }
+    );
+  }
+}

web/app/api/jobs/[jobId]/logs/route.ts

@@ -0,0 +1,30 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+export async function GET(
+  request: NextRequest,
+  { params }: { params: Promise<{ jobId: string }> }
+) {
+  try {
+    const { jobId } = await params;
+
+    const response = await fetch(`${API_BASE_URL}/jobs/${jobId}/logs`);
+
+    if (!response.ok) {
+      return NextResponse.json(
+        { error: 'Failed to get logs' },
+        { status: response.status }
+      );
+    }
+
+    const data = await response.json();
+    return NextResponse.json(data);
+  } catch (error) {
+    console.error('Logs API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to get logs' },
+      { status: 500 }
+    );
+  }
+}

web/app/api/jobs/[jobId]/route.ts

@@ -28,3 +28,32 @@ export async function GET(
     );
   }
 }
+
+export async function DELETE(
+  request: NextRequest,
+  { params }: { params: Promise<{ jobId: string }> }
+) {
+  try {
+    const { jobId } = await params;
+
+    const response = await fetch(`${API_BASE_URL}/jobs/${jobId}`, {
+      method: 'DELETE',
+    });
+
+    if (!response.ok) {
+      const data = await response.json();
+      return NextResponse.json(
+        { error: data.detail || 'Failed to delete job' },
+        { status: response.status }
+      );
+    }
+
+    return NextResponse.json({ success: true, message: 'Job deleted successfully' });
+  } catch (error) {
+    console.error('Delete job API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to delete job' },
+      { status: 500 }
+    );
+  }
+}

web/app/api/jobs/[jobId]/stream/route.ts

@@ -0,0 +1,57 @@
+import { NextRequest } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET(
+  request: NextRequest,
+  { params }: { params: Promise<{ jobId: string }> }
+) {
+  const { jobId } = await params;
+  const encoder = new TextEncoder();
+
+  const stream = new ReadableStream({
+    async start(controller) {
+      try {
+        const response = await fetch(`${API_BASE_URL}/jobs/${jobId}/stream`, {
+          headers: {
+            'Accept': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+          },
+        });
+
+        if (!response.ok || !response.body) {
+          controller.enqueue(encoder.encode(`event: error\ndata: {"error": "Failed to connect to backend"}\n\n`));
+          controller.close();
+          return;
+        }
+
+        const reader = response.body.getReader();
+
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) {
+            break;
+          }
+          // Forward the SSE data as-is
+          controller.enqueue(value);
+        }
+      } catch (error) {
+        console.error('SSE stream error:', error);
+        controller.enqueue(encoder.encode(`event: error\ndata: {"error": "Stream connection failed"}\n\n`));
+      } finally {
+        controller.close();
+      }
+    },
+  });
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache, no-transform',
+      'Connection': 'keep-alive',
+      'X-Accel-Buffering': 'no',
+    },
+  });
+}

web/app/api/jobs/route.ts

@@ -0,0 +1,30 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+export async function GET(request: NextRequest) {
+  try {
+    const { searchParams } = new URL(request.url);
+    const limit = searchParams.get('limit') || '100';
+
+    const response = await fetch(`${API_BASE_URL}/jobs?limit=${limit}`);
+
+    if (!response.ok) {
+      return NextResponse.json(
+        { error: 'Failed to get jobs' },
+        { status: response.status }
+      );
+    }
+
+    const data = await response.json();
+    // Backend returns array directly, not { jobs: [...] }
+    const jobs = Array.isArray(data) ? data : (data.jobs || []);
+    return NextResponse.json({ jobs });
+  } catch (error) {
+    console.error('Jobs API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to get jobs' },
+      { status: 500 }
+    );
+  }
+}

web/app/api/jobs/stream/route.ts

@@ -0,0 +1,54 @@
+import { NextRequest } from 'next/server';
+
+const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET(request: NextRequest) {
+  const encoder = new TextEncoder();
+
+  const stream = new ReadableStream({
+    async start(controller) {
+      try {
+        const response = await fetch(`${API_BASE_URL}/jobs/stream`, {
+          headers: {
+            'Accept': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+          },
+        });
+
+        if (!response.ok || !response.body) {
+          controller.enqueue(encoder.encode(`event: error\ndata: {"error": "Failed to connect to backend"}\n\n`));
+          controller.close();
+          return;
+        }
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) {
+            break;
+          }
+          // Forward the SSE data as-is
+          controller.enqueue(value);
+        }
+      } catch (error) {
+        console.error('SSE stream error:', error);
+        controller.enqueue(encoder.encode(`event: error\ndata: {"error": "Stream connection failed"}\n\n`));
+      } finally {
+        controller.close();
+      }
+    },
+  });
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache, no-transform',
+      'Connection': 'keep-alive',
+      'X-Accel-Buffering': 'no',
+    },
+  });
+}