diff --git a/Dockerfile b/Dockerfile index ee8b8fc..35f4409 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,6 +55,12 @@ RUN pip install --no-cache-dir -r requirements-production.txt # Copy application code COPY modules/ ./modules/ +COPY api/ ./api/ +COPY core/ ./core/ +COPY scrapers/ ./scrapers/ +COPY services/ ./services/ +COPY utils/ ./utils/ +COPY workers/ ./workers/ COPY api_server_production.py . COPY config.yaml . diff --git a/web/app/dashboard/page.tsx b/web/app/dashboard/page.tsx index 51a69d8..5c8419d 100644 --- a/web/app/dashboard/page.tsx +++ b/web/app/dashboard/page.tsx @@ -1,18 +1,23 @@ 'use client'; import Link from 'next/link'; -import { useState, useMemo } from 'react'; +import { useState, useMemo, useEffect } from 'react'; import { useJobs } from '@/contexts/JobsContext'; import { JobStatus } from '@/components/ScraperTest'; -// Mock data for initial development - will be replaced with API data -const MOCK_CLIENTS = [ - { client_id: 'client-001', job_count: 45, success_rate: 94.2 }, - { client_id: 'client-002', job_count: 38, success_rate: 89.5 }, - { client_id: 'client-003', job_count: 27, success_rate: 96.3 }, - { client_id: 'client-004', job_count: 19, success_rate: 84.2 }, - { client_id: 'client-005', job_count: 12, success_rate: 91.7 }, -]; +// API base URL +const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000'; + +// Client stats type from API +interface ClientStats { + client_id: string; + source: string | null; + total_jobs: number; + completed: number; + failed: number; + success_rate: number; + total_reviews: number; +} function formatDuration(seconds: number): string { if (seconds < 60) return `${seconds.toFixed(1)}s`; @@ -55,6 +60,26 @@ function getErrorType(errorMessage: string | null): string { export default function DashboardPage() { const { jobs, isLoading } = useJobs(); const [currentDate] = useState(new Date()); + const [clients, setClients] = useState([]); + const [clientsLoading, setClientsLoading] = useState(true); + + // Fetch client stats from API + useEffect(() => { + async function fetchClients() { + try { + const response = await fetch(`${API_BASE}/api/dashboard/by-client?limit=5`); + if (response.ok) { + const data = await response.json(); + setClients(data); + } + } catch (error) { + console.error('Failed to fetch client stats:', error); + } finally { + setClientsLoading(false); + } + } + fetchClients(); + }, []); // Calculate stats from jobs data const stats = useMemo(() => { @@ -453,42 +478,67 @@ export default function DashboardPage() { View all -
- {MOCK_CLIENTS.map((client, index) => ( -
+
+
+ ) : clients.length === 0 ? ( +
+ -
- - {index + 1} - -
-

- {client.client_id} -

-

- {client.job_count} jobs + + +

No client data yet

+

Run jobs with requester metadata to see stats

+
+ ) : ( +
+ {clients.map((client, index) => ( +
+
+ + {index + 1} + +
+

+ {client.client_id} +

+

+ {client.total_jobs} jobs + {client.source && ` ยท ${client.source}`} +

+
+
+
+

= 90 + ? 'text-green-600' + : client.success_rate >= 80 + ? 'text-yellow-600' + : 'text-red-600' + }`} + > + {client.success_rate.toFixed(1)}%

+

success

-
-

= 90 - ? 'text-green-600' - : client.success_rate >= 80 - ? 'text-yellow-600' - : 'text-red-600' - }`} - > - {client.success_rate}% -

-

success

-
-
- ))} -
+ ))} +
+ )}
diff --git a/web/app/dashboard/scrapers/page.tsx b/web/app/dashboard/scrapers/page.tsx index 309cd58..17d5fd1 100644 --- a/web/app/dashboard/scrapers/page.tsx +++ b/web/app/dashboard/scrapers/page.tsx @@ -1,8 +1,31 @@ 'use client'; -import React, { useState, useCallback, useMemo } from 'react'; +import React, { useState, useCallback, useMemo, useEffect } from 'react'; -// Types +// API base URL +const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000'; + +// Types from API +interface ScraperStats { + total_jobs: number; + success_rate: number; + avg_duration: number; +} + +interface ApiScraperVersion { + id: string; + job_type: string; + version: string; + variant: 'stable' | 'beta' | 'canary'; + is_default: boolean; + traffic_pct: number; + module_path: string; + function_name: string | null; + deprecated_at: string | null; + stats: ScraperStats; +} + +// Internal UI type interface ScraperVersion { id: string; version: string; @@ -16,59 +39,65 @@ interface ScraperVersion { function_name: string; created_at: string; promoted_at?: string; + is_default?: boolean; } -// Mock data - replace with API calls -const mockScraperVersions: ScraperVersion[] = [ - { - id: '1', - version: '1.0.0', - variant: 'stable', - traffic_percentage: 90, - jobs_24h: 150, - success_rate: 95.2, - avg_duration: 42, - status: 'active', - module_path: 'scrapers.google_reviews', - function_name: 'scrape_reviews_v1', - created_at: '2024-01-01T00:00:00Z', - promoted_at: '2024-01-15T00:00:00Z', - }, - { - id: '2', - version: '1.1.0', - variant: 'beta', - traffic_percentage: 10, - jobs_24h: 15, - success_rate: 97.1, - avg_duration: 38, - status: 'active', - module_path: 'scrapers.google_reviews_v2', - function_name: 'scrape_reviews_v2', - created_at: '2024-01-20T00:00:00Z', - }, - { - id: '3', - version: '1.2.0-alpha', - variant: 'canary', - traffic_percentage: 0, - jobs_24h: 0, - success_rate: 0, - avg_duration: 0, - status: 'inactive', - module_path: 'scrapers.google_reviews_v3', - function_name: 'scrape_reviews_v3', - created_at: '2024-01-22T00:00:00Z', - }, -]; +// Transform API response to internal format +function transformApiScraper(api: ApiScraperVersion): ScraperVersion { + let status: 'active' | 'deprecated' | 'inactive' = 'inactive'; + if (api.deprecated_at) { + status = 'deprecated'; + } else if (api.traffic_pct > 0) { + status = 'active'; + } + + return { + id: api.id, + version: api.version, + variant: api.variant, + traffic_percentage: api.traffic_pct, + jobs_24h: api.stats.total_jobs, + success_rate: api.stats.success_rate, + avg_duration: api.stats.avg_duration, + status, + module_path: api.module_path, + function_name: api.function_name || 'scrape', + created_at: new Date().toISOString(), + is_default: api.is_default, + }; +} export default function ScrapersPage() { - const [versions, setVersions] = useState(mockScraperVersions); + const [versions, setVersions] = useState([]); + const [isLoadingVersions, setIsLoadingVersions] = useState(true); + const [loadError, setLoadError] = useState(null); const [showAddForm, setShowAddForm] = useState(false); const [editingTraffic, setEditingTraffic] = useState(null); const [trafficValues, setTrafficValues] = useState>({}); const [isUpdatingTraffic, setIsUpdatingTraffic] = useState(false); + // Fetch scrapers from API + const fetchScrapers = useCallback(async () => { + try { + setLoadError(null); + const response = await fetch(`${API_BASE}/api/admin/scrapers`); + if (!response.ok) { + throw new Error(`Failed to fetch scrapers: ${response.statusText}`); + } + const data: ApiScraperVersion[] = await response.json(); + setVersions(data.map(transformApiScraper)); + } catch (error) { + console.error('Failed to fetch scrapers:', error); + setLoadError(error instanceof Error ? error.message : 'Failed to load scrapers'); + } finally { + setIsLoadingVersions(false); + } + }, []); + + useEffect(() => { + fetchScrapers(); + }, [fetchScrapers]); + // Confirmation modal state const [confirmAction, setConfirmAction] = useState<{ type: 'promote' | 'deprecate' | 'delete'; @@ -111,16 +140,26 @@ export default function ScrapersPage() { setIsUpdatingTraffic(true); try { - // Mock API call - await new Promise(resolve => setTimeout(resolve, 500)); + // Update each scraper's traffic via API + const updates = Object.entries(trafficValues).map(async ([id, pct]) => { + const response = await fetch(`${API_BASE}/api/admin/scrapers/${id}/traffic`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ traffic_pct: pct }), + }); + if (!response.ok) { + throw new Error(`Failed to update traffic for ${id}`); + } + }); - setVersions(prev => prev.map(v => ({ - ...v, - traffic_percentage: trafficValues[v.id] ?? v.traffic_percentage, - }))); + await Promise.all(updates); + + // Refresh data from server + await fetchScrapers(); setEditingTraffic(null); } catch (error) { console.error('Failed to update traffic:', error); + alert('Failed to update traffic allocation. Please try again.'); } finally { setIsUpdatingTraffic(false); } @@ -130,26 +169,21 @@ export default function ScrapersPage() { const promoteVersion = async (version: ScraperVersion) => { setIsProcessing(true); try { - // Mock API call - await new Promise(resolve => setTimeout(resolve, 500)); + const response = await fetch(`${API_BASE}/api/admin/scrapers/${version.id}/promote?traffic_pct=80`, { + method: 'POST', + }); - setVersions(prev => prev.map(v => { - if (v.variant === 'stable') { - return { ...v, variant: 'beta' as const, traffic_percentage: 10 }; - } - if (v.id === version.id) { - return { - ...v, - variant: 'stable' as const, - traffic_percentage: 90, - promoted_at: new Date().toISOString(), - }; - } - return v; - })); + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Failed to promote version'); + } + + // Refresh data from server + await fetchScrapers(); setConfirmAction(null); } catch (error) { console.error('Failed to promote version:', error); + alert(error instanceof Error ? error.message : 'Failed to promote version'); } finally { setIsProcessing(false); } @@ -159,25 +193,21 @@ export default function ScrapersPage() { const deprecateVersion = async (version: ScraperVersion) => { setIsProcessing(true); try { - // Mock API call - await new Promise(resolve => setTimeout(resolve, 500)); + const response = await fetch(`${API_BASE}/api/admin/scrapers/${version.id}/deprecate`, { + method: 'POST', + }); - // Redistribute traffic - const activeVersions = versions.filter(v => v.status === 'active' && v.id !== version.id); - const redistributedTraffic = version.traffic_percentage / activeVersions.length; + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Failed to deprecate version'); + } - setVersions(prev => prev.map(v => { - if (v.id === version.id) { - return { ...v, status: 'deprecated' as const, traffic_percentage: 0 }; - } - if (v.status === 'active') { - return { ...v, traffic_percentage: v.traffic_percentage + redistributedTraffic }; - } - return v; - })); + // Refresh data from server + await fetchScrapers(); setConfirmAction(null); } catch (error) { console.error('Failed to deprecate version:', error); + alert(error instanceof Error ? error.message : 'Failed to deprecate version'); } finally { setIsProcessing(false); } @@ -187,29 +217,40 @@ export default function ScrapersPage() { const handleAddVersion = async (e: React.FormEvent) => { e.preventDefault(); - const newScraperVersion: ScraperVersion = { - id: Date.now().toString(), - version: newVersion.version, - variant: newVersion.variant, - traffic_percentage: newVersion.traffic_percentage, - jobs_24h: 0, - success_rate: 0, - avg_duration: 0, - status: newVersion.traffic_percentage > 0 ? 'active' : 'inactive', - module_path: newVersion.module_path, - function_name: newVersion.function_name, - created_at: new Date().toISOString(), - }; + try { + const response = await fetch(`${API_BASE}/api/admin/scrapers`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + job_type: 'google_reviews', + version: newVersion.version, + variant: newVersion.variant, + module_path: newVersion.module_path, + function_name: newVersion.function_name, + traffic_pct: newVersion.traffic_percentage, + }), + }); - setVersions(prev => [...prev, newScraperVersion]); - setNewVersion({ - version: '', - variant: 'beta', - module_path: '', - function_name: '', - traffic_percentage: 0, - }); - setShowAddForm(false); + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Failed to add version'); + } + + // Refresh data from server + await fetchScrapers(); + + setNewVersion({ + version: '', + variant: 'beta', + module_path: '', + function_name: '', + traffic_percentage: 0, + }); + setShowAddForm(false); + } catch (error) { + console.error('Failed to add version:', error); + alert(error instanceof Error ? error.message : 'Failed to add version'); + } }; // Variant badge styling @@ -240,6 +281,41 @@ export default function ScrapersPage() { } }; + // Loading state + if (isLoadingVersions) { + return ( +
+
+
+

Loading scrapers...

+
+
+ ); + } + + // Error state + if (loadError) { + return ( +
+
+
+ + + +
+

Failed to load scrapers

+

{loadError}

+ +
+
+ ); + } + return (
{/* Header */}