Wire frontend to real API endpoints

Dashboard page:
- Fetch top clients from /api/dashboard/by-client
- Show loading state while fetching
- Display empty state when no client data
- Show real client_id, job count, and success rate

Scrapers page:
- Fetch versions from /api/admin/scrapers
- Wire promote/deprecate buttons to real API calls
- Wire add version form to POST /api/admin/scrapers
- Wire traffic allocation to PUT /api/admin/scrapers/{id}/traffic
- Add loading and error states

Dockerfile:
- Add COPY commands for new directories (api/, core/, scrapers/, etc.)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Dockerfile

@@ -55,6 +55,12 @@ RUN pip install --no-cache-dir -r requirements-production.txt
 
 # Copy application code
 COPY modules/ ./modules/
+COPY api/ ./api/
+COPY core/ ./core/
+COPY scrapers/ ./scrapers/
+COPY services/ ./services/
+COPY utils/ ./utils/
+COPY workers/ ./workers/
 COPY api_server_production.py .
 COPY config.yaml .
 

web/app/dashboard/page.tsx

@@ -1,18 +1,23 @@
 'use client';
 
 import Link from 'next/link';
-import { useState, useMemo } from 'react';
+import { useState, useMemo, useEffect } from 'react';
 import { useJobs } from '@/contexts/JobsContext';
 import { JobStatus } from '@/components/ScraperTest';
 
-// Mock data for initial development - will be replaced with API data
+// API base URL
-const MOCK_CLIENTS = [
+const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
-  { client_id: 'client-001', job_count: 45, success_rate: 94.2 },
+
-  { client_id: 'client-002', job_count: 38, success_rate: 89.5 },
+// Client stats type from API
-  { client_id: 'client-003', job_count: 27, success_rate: 96.3 },
+interface ClientStats {
-  { client_id: 'client-004', job_count: 19, success_rate: 84.2 },
+  client_id: string;
-  { client_id: 'client-005', job_count: 12, success_rate: 91.7 },
+  source: string | null;
-];
+  total_jobs: number;
+  completed: number;
+  failed: number;
+  success_rate: number;
+  total_reviews: number;
+}
 
 function formatDuration(seconds: number): string {
   if (seconds < 60) return `${seconds.toFixed(1)}s`;
@@ -55,6 +60,26 @@ function getErrorType(errorMessage: string | null): string {
 export default function DashboardPage() {
   const { jobs, isLoading } = useJobs();
   const [currentDate] = useState(new Date());
+  const [clients, setClients] = useState<ClientStats[]>([]);
+  const [clientsLoading, setClientsLoading] = useState(true);
+
+  // Fetch client stats from API
+  useEffect(() => {
+    async function fetchClients() {
+      try {
+        const response = await fetch(`${API_BASE}/api/dashboard/by-client?limit=5`);
+        if (response.ok) {
+          const data = await response.json();
+          setClients(data);
+        }
+      } catch (error) {
+        console.error('Failed to fetch client stats:', error);
+      } finally {
+        setClientsLoading(false);
+      }
+    }
+    fetchClients();
+  }, []);
 
   // Calculate stats from jobs data
   const stats = useMemo(() => {
@@ -453,8 +478,31 @@ export default function DashboardPage() {
               View all
             </Link>
           </div>
+          {clientsLoading ? (
+            <div className="py-8 flex justify-center">
+              <div className="w-6 h-6 border-2 border-blue-500 border-t-transparent rounded-full animate-spin" />
+            </div>
+          ) : clients.length === 0 ? (
+            <div className="py-8 text-center text-gray-500">
+              <svg
+                className="w-12 h-12 mx-auto mb-3 opacity-30"
+                fill="none"
+                stroke="currentColor"
+                viewBox="0 0 24 24"
+              >
+                <path
+                  strokeLinecap="round"
+                  strokeLinejoin="round"
+                  strokeWidth={1.5}
+                  d="M17 20h5v-2a3 3 0 00-5.356-1.857M17 20H7m10 0v-2c0-.656-.126-1.283-.356-1.857M7 20H2v-2a3 3 0 015.356-1.857M7 20v-2c0-.656.126-1.283.356-1.857m0 0a5.002 5.002 0 019.288 0M15 7a3 3 0 11-6 0 3 3 0 016 0zm6 3a2 2 0 11-4 0 2 2 0 014 0zM7 10a2 2 0 11-4 0 2 2 0 014 0z"
+                />
+              </svg>
+              <p className="font-medium">No client data yet</p>
+              <p className="text-sm mt-1">Run jobs with requester metadata to see stats</p>
+            </div>
+          ) : (
             <div className="space-y-3">
-            {MOCK_CLIENTS.map((client, index) => (
+              {clients.map((client, index) => (
                 <div
                   key={client.client_id}
                   className="flex items-center justify-between p-3 bg-gray-50 rounded-lg"
@@ -468,7 +516,8 @@ export default function DashboardPage() {
                         {client.client_id}
                       </p>
                       <p className="text-xs text-gray-500">
-                      {client.job_count} jobs
+                        {client.total_jobs} jobs
+                        {client.source && ` · ${client.source}`}
                       </p>
                     </div>
                   </div>
@@ -482,13 +531,14 @@ export default function DashboardPage() {
                           : 'text-red-600'
                       }`}
                     >
-                    {client.success_rate}%
+                      {client.success_rate.toFixed(1)}%
                     </p>
                     <p className="text-xs text-gray-500">success</p>
                   </div>
                 </div>
               ))}
             </div>
+          )}
         </div>
       </div>
 

web/app/dashboard/scrapers/page.tsx

@@ -1,8 +1,31 @@
 'use client';
 
-import React, { useState, useCallback, useMemo } from 'react';
+import React, { useState, useCallback, useMemo, useEffect } from 'react';
 
-// Types
+// API base URL
+const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+// Types from API
+interface ScraperStats {
+  total_jobs: number;
+  success_rate: number;
+  avg_duration: number;
+}
+
+interface ApiScraperVersion {
+  id: string;
+  job_type: string;
+  version: string;
+  variant: 'stable' | 'beta' | 'canary';
+  is_default: boolean;
+  traffic_pct: number;
+  module_path: string;
+  function_name: string | null;
+  deprecated_at: string | null;
+  stats: ScraperStats;
+}
+
+// Internal UI type
 interface ScraperVersion {
   id: string;
   version: string;
@@ -16,59 +39,65 @@ interface ScraperVersion {
   function_name: string;
   created_at: string;
   promoted_at?: string;
+  is_default?: boolean;
 }
 
-// Mock data - replace with API calls
+// Transform API response to internal format
-const mockScraperVersions: ScraperVersion[] = [
+function transformApiScraper(api: ApiScraperVersion): ScraperVersion {
-  {
+  let status: 'active' | 'deprecated' | 'inactive' = 'inactive';
-    id: '1',
+  if (api.deprecated_at) {
-    version: '1.0.0',
+    status = 'deprecated';
-    variant: 'stable',
+  } else if (api.traffic_pct > 0) {
-    traffic_percentage: 90,
+    status = 'active';
-    jobs_24h: 150,
+  }
-    success_rate: 95.2,
+
-    avg_duration: 42,
+  return {
-    status: 'active',
+    id: api.id,
-    module_path: 'scrapers.google_reviews',
+    version: api.version,
-    function_name: 'scrape_reviews_v1',
+    variant: api.variant,
-    created_at: '2024-01-01T00:00:00Z',
+    traffic_percentage: api.traffic_pct,
-    promoted_at: '2024-01-15T00:00:00Z',
+    jobs_24h: api.stats.total_jobs,
-  },
+    success_rate: api.stats.success_rate,
-  {
+    avg_duration: api.stats.avg_duration,
-    id: '2',
+    status,
-    version: '1.1.0',
+    module_path: api.module_path,
-    variant: 'beta',
+    function_name: api.function_name || 'scrape',
-    traffic_percentage: 10,
+    created_at: new Date().toISOString(),
-    jobs_24h: 15,
+    is_default: api.is_default,
-    success_rate: 97.1,
+  };
-    avg_duration: 38,
+}
-    status: 'active',
-    module_path: 'scrapers.google_reviews_v2',
-    function_name: 'scrape_reviews_v2',
-    created_at: '2024-01-20T00:00:00Z',
-  },
-  {
-    id: '3',
-    version: '1.2.0-alpha',
-    variant: 'canary',
-    traffic_percentage: 0,
-    jobs_24h: 0,
-    success_rate: 0,
-    avg_duration: 0,
-    status: 'inactive',
-    module_path: 'scrapers.google_reviews_v3',
-    function_name: 'scrape_reviews_v3',
-    created_at: '2024-01-22T00:00:00Z',
-  },
-];
 
 export default function ScrapersPage() {
-  const [versions, setVersions] = useState<ScraperVersion[]>(mockScraperVersions);
+  const [versions, setVersions] = useState<ScraperVersion[]>([]);
+  const [isLoadingVersions, setIsLoadingVersions] = useState(true);
+  const [loadError, setLoadError] = useState<string | null>(null);
   const [showAddForm, setShowAddForm] = useState(false);
   const [editingTraffic, setEditingTraffic] = useState<string | null>(null);
   const [trafficValues, setTrafficValues] = useState<Record<string, number>>({});
   const [isUpdatingTraffic, setIsUpdatingTraffic] = useState(false);
 
+  // Fetch scrapers from API
+  const fetchScrapers = useCallback(async () => {
+    try {
+      setLoadError(null);
+      const response = await fetch(`${API_BASE}/api/admin/scrapers`);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch scrapers: ${response.statusText}`);
+      }
+      const data: ApiScraperVersion[] = await response.json();
+      setVersions(data.map(transformApiScraper));
+    } catch (error) {
+      console.error('Failed to fetch scrapers:', error);
+      setLoadError(error instanceof Error ? error.message : 'Failed to load scrapers');
+    } finally {
+      setIsLoadingVersions(false);
+    }
+  }, []);
+
+  useEffect(() => {
+    fetchScrapers();
+  }, [fetchScrapers]);
+
   // Confirmation modal state
   const [confirmAction, setConfirmAction] = useState<{
     type: 'promote' | 'deprecate' | 'delete';
@@ -111,16 +140,26 @@ export default function ScrapersPage() {
 
     setIsUpdatingTraffic(true);
     try {
-      // Mock API call
+      // Update each scraper's traffic via API
-      await new Promise(resolve => setTimeout(resolve, 500));
+      const updates = Object.entries(trafficValues).map(async ([id, pct]) => {
+        const response = await fetch(`${API_BASE}/api/admin/scrapers/${id}/traffic`, {
+          method: 'PUT',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ traffic_pct: pct }),
+        });
+        if (!response.ok) {
+          throw new Error(`Failed to update traffic for ${id}`);
+        }
+      });
 
-      setVersions(prev => prev.map(v => ({
+      await Promise.all(updates);
-        ...v,
+
-        traffic_percentage: trafficValues[v.id] ?? v.traffic_percentage,
+      // Refresh data from server
-      })));
+      await fetchScrapers();
       setEditingTraffic(null);
     } catch (error) {
       console.error('Failed to update traffic:', error);
+      alert('Failed to update traffic allocation. Please try again.');
     } finally {
       setIsUpdatingTraffic(false);
     }
@@ -130,26 +169,21 @@ export default function ScrapersPage() {
   const promoteVersion = async (version: ScraperVersion) => {
     setIsProcessing(true);
     try {
-      // Mock API call
+      const response = await fetch(`${API_BASE}/api/admin/scrapers/${version.id}/promote?traffic_pct=80`, {
-      await new Promise(resolve => setTimeout(resolve, 500));
+        method: 'POST',
+      });
 
-      setVersions(prev => prev.map(v => {
+      if (!response.ok) {
-        if (v.variant === 'stable') {
+        const error = await response.json();
-          return { ...v, variant: 'beta' as const, traffic_percentage: 10 };
+        throw new Error(error.detail || 'Failed to promote version');
       }
-        if (v.id === version.id) {
+
-          return {
+      // Refresh data from server
-            ...v,
+      await fetchScrapers();
-            variant: 'stable' as const,
-            traffic_percentage: 90,
-            promoted_at: new Date().toISOString(),
-          };
-        }
-        return v;
-      }));
       setConfirmAction(null);
     } catch (error) {
       console.error('Failed to promote version:', error);
+      alert(error instanceof Error ? error.message : 'Failed to promote version');
     } finally {
       setIsProcessing(false);
     }
@@ -159,25 +193,21 @@ export default function ScrapersPage() {
   const deprecateVersion = async (version: ScraperVersion) => {
     setIsProcessing(true);
     try {
-      // Mock API call
+      const response = await fetch(`${API_BASE}/api/admin/scrapers/${version.id}/deprecate`, {
-      await new Promise(resolve => setTimeout(resolve, 500));
+        method: 'POST',
+      });
 
-      // Redistribute traffic
+      if (!response.ok) {
-      const activeVersions = versions.filter(v => v.status === 'active' && v.id !== version.id);
+        const error = await response.json();
-      const redistributedTraffic = version.traffic_percentage / activeVersions.length;
+        throw new Error(error.detail || 'Failed to deprecate version');
+      }
 
-      setVersions(prev => prev.map(v => {
+      // Refresh data from server
-        if (v.id === version.id) {
+      await fetchScrapers();
-          return { ...v, status: 'deprecated' as const, traffic_percentage: 0 };
-        }
-        if (v.status === 'active') {
-          return { ...v, traffic_percentage: v.traffic_percentage + redistributedTraffic };
-        }
-        return v;
-      }));
       setConfirmAction(null);
     } catch (error) {
       console.error('Failed to deprecate version:', error);
+      alert(error instanceof Error ? error.message : 'Failed to deprecate version');
     } finally {
       setIsProcessing(false);
     }
@@ -187,21 +217,28 @@ export default function ScrapersPage() {
   const handleAddVersion = async (e: React.FormEvent) => {
     e.preventDefault();
 
-    const newScraperVersion: ScraperVersion = {
+    try {
-      id: Date.now().toString(),
+      const response = await fetch(`${API_BASE}/api/admin/scrapers`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          job_type: 'google_reviews',
           version: newVersion.version,
           variant: newVersion.variant,
-      traffic_percentage: newVersion.traffic_percentage,
-      jobs_24h: 0,
-      success_rate: 0,
-      avg_duration: 0,
-      status: newVersion.traffic_percentage > 0 ? 'active' : 'inactive',
           module_path: newVersion.module_path,
           function_name: newVersion.function_name,
-      created_at: new Date().toISOString(),
+          traffic_pct: newVersion.traffic_percentage,
-    };
+        }),
+      });
+
+      if (!response.ok) {
+        const error = await response.json();
+        throw new Error(error.detail || 'Failed to add version');
+      }
+
+      // Refresh data from server
+      await fetchScrapers();
 
-    setVersions(prev => [...prev, newScraperVersion]);
       setNewVersion({
         version: '',
         variant: 'beta',
@@ -210,6 +247,10 @@ export default function ScrapersPage() {
         traffic_percentage: 0,
       });
       setShowAddForm(false);
+    } catch (error) {
+      console.error('Failed to add version:', error);
+      alert(error instanceof Error ? error.message : 'Failed to add version');
+    }
   };
 
   // Variant badge styling
@@ -240,6 +281,41 @@ export default function ScrapersPage() {
     }
   };
 
+  // Loading state
+  if (isLoadingVersions) {
+    return (
+      <div className="h-full flex items-center justify-center">
+        <div className="text-center">
+          <div className="w-8 h-8 border-4 border-blue-500 border-t-transparent rounded-full animate-spin mx-auto mb-4" />
+          <p className="text-gray-600">Loading scrapers...</p>
+        </div>
+      </div>
+    );
+  }
+
+  // Error state
+  if (loadError) {
+    return (
+      <div className="h-full flex items-center justify-center">
+        <div className="text-center">
+          <div className="w-16 h-16 bg-red-100 rounded-full flex items-center justify-center mx-auto mb-4">
+            <svg className="w-8 h-8 text-red-600" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+              <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
+            </svg>
+          </div>
+          <h3 className="text-lg font-semibold text-gray-800 mb-2">Failed to load scrapers</h3>
+          <p className="text-gray-600 mb-4">{loadError}</p>
+          <button
+            onClick={() => { setIsLoadingVersions(true); fetchScrapers(); }}
+            className="px-4 py-2 bg-blue-600 text-white rounded-lg font-semibold hover:bg-blue-700 transition-colors"
+          >
+            Try Again
+          </button>
+        </div>
+      </div>
+    );
+  }
+
   return (
     <div className="h-full overflow-y-auto p-6">
       {/* Header */}