Wire frontend to real API endpoints

Dashboard page: - Fetch top clients from /api/dashboard/by-client - Show loading state while fetching - Display empty state when no client data - Show real client_id, job count, and success rate Scrapers page: - Fetch versions from /api/admin/scrapers - Wire promote/deprecate buttons to real API calls - Wire add version form to POST /api/admin/scrapers - Wire traffic allocation to PUT /api/admin/scrapers/{id}/traffic - Add loading and error states Dockerfile: - Add COPY commands for new directories (api/, core/, scrapers/, etc.) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-24 16:05:29 +00:00
parent 39c80fc8be
commit 3317553658
3 changed files with 277 additions and 145 deletions
--- a/6
+++ b/6
@@ -55,6 +55,12 @@ RUN pip install --no-cache-dir -r requirements-production.txt

 # Copy application code
 COPY modules/ ./modules/
+COPY api/ ./api/
+COPY core/ ./core/
+COPY scrapers/ ./scrapers/
+COPY services/ ./services/
+COPY utils/ ./utils/
+COPY workers/ ./workers/
 COPY api_server_production.py .
 COPY config.yaml .

--- a/web/app/dashboard/page.tsx
+++ b/web/app/dashboard/page.tsx
@@ -1,18 +1,23 @@
 'use client';

 import Link from 'next/link';
-import { useState, useMemo } from 'react';
+import { useState, useMemo, useEffect } from 'react';
 import { useJobs } from '@/contexts/JobsContext';
 import { JobStatus } from '@/components/ScraperTest';

-// Mock data for initial development - will be replaced with API data
-const MOCK_CLIENTS = [
-  { client_id: 'client-001', job_count: 45, success_rate: 94.2 },
-  { client_id: 'client-002', job_count: 38, success_rate: 89.5 },
-  { client_id: 'client-003', job_count: 27, success_rate: 96.3 },
-  { client_id: 'client-004', job_count: 19, success_rate: 84.2 },
-  { client_id: 'client-005', job_count: 12, success_rate: 91.7 },
-];
+// API base URL
+const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+// Client stats type from API
+interface ClientStats {
+  client_id: string;
+  source: string | null;
+  total_jobs: number;
+  completed: number;
+  failed: number;
+  success_rate: number;
+  total_reviews: number;
+}

 function formatDuration(seconds: number): string {
  if (seconds < 60) return `${seconds.toFixed(1)}s`;
@@ -55,6 +60,26 @@ function getErrorType(errorMessage: string | null): string {
 export default function DashboardPage() {
  const { jobs, isLoading } = useJobs();
  const [currentDate] = useState(new Date());
+  const [clients, setClients] = useState<ClientStats[]>([]);
+  const [clientsLoading, setClientsLoading] = useState(true);
+
+  // Fetch client stats from API
+  useEffect(() => {
+    async function fetchClients() {
+      try {
+        const response = await fetch(`${API_BASE}/api/dashboard/by-client?limit=5`);
+        if (response.ok) {
+          const data = await response.json();
+          setClients(data);
+        }
+      } catch (error) {
+        console.error('Failed to fetch client stats:', error);
+      } finally {
+        setClientsLoading(false);
+      }
+    }
+    fetchClients();
+  }, []);

  // Calculate stats from jobs data
  const stats = useMemo(() => {
@@ -453,42 +478,67 @@ export default function DashboardPage() {
              View all
            </Link>
          </div>
-          <div className="space-y-3">
-            {MOCK_CLIENTS.map((client, index) => (
-              <div
-                key={client.client_id}
-                className="flex items-center justify-between p-3 bg-gray-50 rounded-lg"
+          {clientsLoading ? (
+            <div className="py-8 flex justify-center">
+              <div className="w-6 h-6 border-2 border-blue-500 border-t-transparent rounded-full animate-spin" />
+            </div>
+          ) : clients.length === 0 ? (
+            <div className="py-8 text-center text-gray-500">
+              <svg
+                className="w-12 h-12 mx-auto mb-3 opacity-30"
+                fill="none"
+                stroke="currentColor"
+                viewBox="0 0 24 24"
              >
-                <div className="flex items-center gap-3">
-                  <span className="w-6 h-6 bg-blue-100 text-blue-700 rounded-full flex items-center justify-center text-xs font-bold">
-                    {index + 1}
-                  </span>
-                  <div>
-                    <p className="font-medium text-gray-800">
-                      {client.client_id}
-                    </p>
-                    <p className="text-xs text-gray-500">
-                      {client.job_count} jobs
+                <path
+                  strokeLinecap="round"
+                  strokeLinejoin="round"
+                  strokeWidth={1.5}
+                  d="M17 20h5v-2a3 3 0 00-5.356-1.857M17 20H7m10 0v-2c0-.656-.126-1.283-.356-1.857M7 20H2v-2a3 3 0 015.356-1.857M7 20v-2c0-.656.126-1.283.356-1.857m0 0a5.002 5.002 0 019.288 0M15 7a3 3 0 11-6 0 3 3 0 016 0zm6 3a2 2 0 11-4 0 2 2 0 014 0zM7 10a2 2 0 11-4 0 2 2 0 014 0z"
+                />
+              </svg>
+              <p className="font-medium">No client data yet</p>
+              <p className="text-sm mt-1">Run jobs with requester metadata to see stats</p>
+            </div>
+          ) : (
+            <div className="space-y-3">
+              {clients.map((client, index) => (
+                <div
+                  key={client.client_id}
+                  className="flex items-center justify-between p-3 bg-gray-50 rounded-lg"
+                >
+                  <div className="flex items-center gap-3">
+                    <span className="w-6 h-6 bg-blue-100 text-blue-700 rounded-full flex items-center justify-center text-xs font-bold">
+                      {index + 1}
+                    </span>
+                    <div>
+                      <p className="font-medium text-gray-800">
+                        {client.client_id}
+                      </p>
+                      <p className="text-xs text-gray-500">
+                        {client.total_jobs} jobs
+                        {client.source && ` · ${client.source}`}
+                      </p>
+                    </div>
+                  </div>
+                  <div className="text-right">
+                    <p
+                      className={`font-semibold ${
+                        client.success_rate >= 90
+                          ? 'text-green-600'
+                          : client.success_rate >= 80
+                          ? 'text-yellow-600'
+                          : 'text-red-600'
+                      }`}
+                    >
+                      {client.success_rate.toFixed(1)}%
                    </p>
+                    <p className="text-xs text-gray-500">success</p>
                  </div>
                </div>
-                <div className="text-right">
-                  <p
-                    className={`font-semibold ${
-                      client.success_rate >= 90
-                        ? 'text-green-600'
-                        : client.success_rate >= 80
-                        ? 'text-yellow-600'
-                        : 'text-red-600'
-                    }`}
-                  >
-                    {client.success_rate}%
-                  </p>
-                  <p className="text-xs text-gray-500">success</p>
-                </div>
-              </div>
-            ))}
-          </div>
+              ))}
+            </div>
+          )}
        </div>
      </div>

--- a/web/app/dashboard/scrapers/page.tsx
+++ b/web/app/dashboard/scrapers/page.tsx
@@ -1,8 +1,31 @@
 'use client';

-import React, { useState, useCallback, useMemo } from 'react';
+import React, { useState, useCallback, useMemo, useEffect } from 'react';

-// Types
+// API base URL
+const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
+
+// Types from API
+interface ScraperStats {
+  total_jobs: number;
+  success_rate: number;
+  avg_duration: number;
+}
+
+interface ApiScraperVersion {
+  id: string;
+  job_type: string;
+  version: string;
+  variant: 'stable' | 'beta' | 'canary';
+  is_default: boolean;
+  traffic_pct: number;
+  module_path: string;
+  function_name: string | null;
+  deprecated_at: string | null;
+  stats: ScraperStats;
+}
+
+// Internal UI type
 interface ScraperVersion {
  id: string;
  version: string;
@@ -16,59 +39,65 @@ interface ScraperVersion {
  function_name: string;
  created_at: string;
  promoted_at?: string;
+  is_default?: boolean;
 }

-// Mock data - replace with API calls
-const mockScraperVersions: ScraperVersion[] = [
-  {
-    id: '1',
-    version: '1.0.0',
-    variant: 'stable',
-    traffic_percentage: 90,
-    jobs_24h: 150,
-    success_rate: 95.2,
-    avg_duration: 42,
-    status: 'active',
-    module_path: 'scrapers.google_reviews',
-    function_name: 'scrape_reviews_v1',
-    created_at: '2024-01-01T00:00:00Z',
-    promoted_at: '2024-01-15T00:00:00Z',
-  },
-  {
-    id: '2',
-    version: '1.1.0',
-    variant: 'beta',
-    traffic_percentage: 10,
-    jobs_24h: 15,
-    success_rate: 97.1,
-    avg_duration: 38,
-    status: 'active',
-    module_path: 'scrapers.google_reviews_v2',
-    function_name: 'scrape_reviews_v2',
-    created_at: '2024-01-20T00:00:00Z',
-  },
-  {
-    id: '3',
-    version: '1.2.0-alpha',
-    variant: 'canary',
-    traffic_percentage: 0,
-    jobs_24h: 0,
-    success_rate: 0,
-    avg_duration: 0,
-    status: 'inactive',
-    module_path: 'scrapers.google_reviews_v3',
-    function_name: 'scrape_reviews_v3',
-    created_at: '2024-01-22T00:00:00Z',
-  },
-];
+// Transform API response to internal format
+function transformApiScraper(api: ApiScraperVersion): ScraperVersion {
+  let status: 'active' | 'deprecated' | 'inactive' = 'inactive';
+  if (api.deprecated_at) {
+    status = 'deprecated';
+  } else if (api.traffic_pct > 0) {
+    status = 'active';
+  }
+
+  return {
+    id: api.id,
+    version: api.version,
+    variant: api.variant,
+    traffic_percentage: api.traffic_pct,
+    jobs_24h: api.stats.total_jobs,
+    success_rate: api.stats.success_rate,
+    avg_duration: api.stats.avg_duration,
+    status,
+    module_path: api.module_path,
+    function_name: api.function_name || 'scrape',
+    created_at: new Date().toISOString(),
+    is_default: api.is_default,
+  };
+}

 export default function ScrapersPage() {
-  const [versions, setVersions] = useState<ScraperVersion[]>(mockScraperVersions);
+  const [versions, setVersions] = useState<ScraperVersion[]>([]);
+  const [isLoadingVersions, setIsLoadingVersions] = useState(true);
+  const [loadError, setLoadError] = useState<string | null>(null);
  const [showAddForm, setShowAddForm] = useState(false);
  const [editingTraffic, setEditingTraffic] = useState<string | null>(null);
  const [trafficValues, setTrafficValues] = useState<Record<string, number>>({});
  const [isUpdatingTraffic, setIsUpdatingTraffic] = useState(false);

+  // Fetch scrapers from API
+  const fetchScrapers = useCallback(async () => {
+    try {
+      setLoadError(null);
+      const response = await fetch(`${API_BASE}/api/admin/scrapers`);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch scrapers: ${response.statusText}`);
+      }
+      const data: ApiScraperVersion[] = await response.json();
+      setVersions(data.map(transformApiScraper));
+    } catch (error) {
+      console.error('Failed to fetch scrapers:', error);
+      setLoadError(error instanceof Error ? error.message : 'Failed to load scrapers');
+    } finally {
+      setIsLoadingVersions(false);
+    }
+  }, []);
+
+  useEffect(() => {
+    fetchScrapers();
+  }, [fetchScrapers]);
+
  // Confirmation modal state
  const [confirmAction, setConfirmAction] = useState<{
    type: 'promote' | 'deprecate' | 'delete';
@@ -111,16 +140,26 @@ export default function ScrapersPage() {

    setIsUpdatingTraffic(true);
    try {
-      // Mock API call
-      await new Promise(resolve => setTimeout(resolve, 500));
+      // Update each scraper's traffic via API
+      const updates = Object.entries(trafficValues).map(async ([id, pct]) => {
+        const response = await fetch(`${API_BASE}/api/admin/scrapers/${id}/traffic`, {
+          method: 'PUT',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ traffic_pct: pct }),
+        });
+        if (!response.ok) {
+          throw new Error(`Failed to update traffic for ${id}`);
+        }
+      });

-      setVersions(prev => prev.map(v => ({
-        ...v,
-        traffic_percentage: trafficValues[v.id] ?? v.traffic_percentage,
-      })));
+      await Promise.all(updates);
+
+      // Refresh data from server
+      await fetchScrapers();
      setEditingTraffic(null);
    } catch (error) {
      console.error('Failed to update traffic:', error);
+      alert('Failed to update traffic allocation. Please try again.');
    } finally {
      setIsUpdatingTraffic(false);
    }
@@ -130,26 +169,21 @@ export default function ScrapersPage() {
  const promoteVersion = async (version: ScraperVersion) => {
    setIsProcessing(true);
    try {
-      // Mock API call
-      await new Promise(resolve => setTimeout(resolve, 500));
+      const response = await fetch(`${API_BASE}/api/admin/scrapers/${version.id}/promote?traffic_pct=80`, {
+        method: 'POST',
+      });

-      setVersions(prev => prev.map(v => {
-        if (v.variant === 'stable') {
-          return { ...v, variant: 'beta' as const, traffic_percentage: 10 };
-        }
-        if (v.id === version.id) {
-          return {
-            ...v,
-            variant: 'stable' as const,
-            traffic_percentage: 90,
-            promoted_at: new Date().toISOString(),
-          };
-        }
-        return v;
-      }));
+      if (!response.ok) {
+        const error = await response.json();
+        throw new Error(error.detail || 'Failed to promote version');
+      }
+
+      // Refresh data from server
+      await fetchScrapers();
      setConfirmAction(null);
    } catch (error) {
      console.error('Failed to promote version:', error);
+      alert(error instanceof Error ? error.message : 'Failed to promote version');
    } finally {
      setIsProcessing(false);
    }
@@ -159,25 +193,21 @@ export default function ScrapersPage() {
  const deprecateVersion = async (version: ScraperVersion) => {
    setIsProcessing(true);
    try {
-      // Mock API call
-      await new Promise(resolve => setTimeout(resolve, 500));
+      const response = await fetch(`${API_BASE}/api/admin/scrapers/${version.id}/deprecate`, {
+        method: 'POST',
+      });

-      // Redistribute traffic
-      const activeVersions = versions.filter(v => v.status === 'active' && v.id !== version.id);
-      const redistributedTraffic = version.traffic_percentage / activeVersions.length;
+      if (!response.ok) {
+        const error = await response.json();
+        throw new Error(error.detail || 'Failed to deprecate version');
+      }

-      setVersions(prev => prev.map(v => {
-        if (v.id === version.id) {
-          return { ...v, status: 'deprecated' as const, traffic_percentage: 0 };
-        }
-        if (v.status === 'active') {
-          return { ...v, traffic_percentage: v.traffic_percentage + redistributedTraffic };
-        }
-        return v;
-      }));
+      // Refresh data from server
+      await fetchScrapers();
      setConfirmAction(null);
    } catch (error) {
      console.error('Failed to deprecate version:', error);
+      alert(error instanceof Error ? error.message : 'Failed to deprecate version');
    } finally {
      setIsProcessing(false);
    }
@@ -187,29 +217,40 @@ export default function ScrapersPage() {
  const handleAddVersion = async (e: React.FormEvent) => {
    e.preventDefault();

-    const newScraperVersion: ScraperVersion = {
-      id: Date.now().toString(),
-      version: newVersion.version,
-      variant: newVersion.variant,
-      traffic_percentage: newVersion.traffic_percentage,
-      jobs_24h: 0,
-      success_rate: 0,
-      avg_duration: 0,
-      status: newVersion.traffic_percentage > 0 ? 'active' : 'inactive',
-      module_path: newVersion.module_path,
-      function_name: newVersion.function_name,
-      created_at: new Date().toISOString(),
-    };
+    try {
+      const response = await fetch(`${API_BASE}/api/admin/scrapers`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          job_type: 'google_reviews',
+          version: newVersion.version,
+          variant: newVersion.variant,
+          module_path: newVersion.module_path,
+          function_name: newVersion.function_name,
+          traffic_pct: newVersion.traffic_percentage,
+        }),
+      });

-    setVersions(prev => [...prev, newScraperVersion]);
-    setNewVersion({
-      version: '',
-      variant: 'beta',
-      module_path: '',
-      function_name: '',
-      traffic_percentage: 0,
-    });
-    setShowAddForm(false);
+      if (!response.ok) {
+        const error = await response.json();
+        throw new Error(error.detail || 'Failed to add version');
+      }
+
+      // Refresh data from server
+      await fetchScrapers();
+
+      setNewVersion({
+        version: '',
+        variant: 'beta',
+        module_path: '',
+        function_name: '',
+        traffic_percentage: 0,
+      });
+      setShowAddForm(false);
+    } catch (error) {
+      console.error('Failed to add version:', error);
+      alert(error instanceof Error ? error.message : 'Failed to add version');
+    }
  };

  // Variant badge styling
@@ -240,6 +281,41 @@ export default function ScrapersPage() {
    }
  };

+  // Loading state
+  if (isLoadingVersions) {
+    return (
+      <div className="h-full flex items-center justify-center">
+        <div className="text-center">
+          <div className="w-8 h-8 border-4 border-blue-500 border-t-transparent rounded-full animate-spin mx-auto mb-4" />
+          <p className="text-gray-600">Loading scrapers...</p>
+        </div>
+      </div>
+    );
+  }
+
+  // Error state
+  if (loadError) {
+    return (
+      <div className="h-full flex items-center justify-center">
+        <div className="text-center">
+          <div className="w-16 h-16 bg-red-100 rounded-full flex items-center justify-center mx-auto mb-4">
+            <svg className="w-8 h-8 text-red-600" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+              <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
+            </svg>
+          </div>
+          <h3 className="text-lg font-semibold text-gray-800 mb-2">Failed to load scrapers</h3>
+          <p className="text-gray-600 mb-4">{loadError}</p>
+          <button
+            onClick={() => { setIsLoadingVersions(true); fetchScrapers(); }}
+            className="px-4 py-2 bg-blue-600 text-white rounded-lg font-semibold hover:bg-blue-700 transition-colors"
+          >
+            Try Again
+          </button>
+        </div>
+      </div>
+    );
+  }
+
  return (
    <div className="h-full overflow-y-auto p-6">
      {/* Header */}