Align artifacts with canonical URT v5.1 specification
Fixes inconsistencies discovered during audit against urt-taxonomy/: - urt_profile ENUM: Add 'lite' and 'core' profiles (was missing) - USN format: Use canonical regex from spec (was non-compliant) - USN valence encoding: Add V0 (0) and V± (±) support - USN grammar: Add Lite (URT:L:) and Core (URT:C:) formats - Dimension codes: Fix temporal (TC/TR/TH/TF), evidence (ES/EI/EC), comparative (CR-N/CR-B/CR-W/CR-S) in decisions doc - LLM contract: Full USN regex validation pattern Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Alejandro Gutiérrez
@@ -1,17 +1,8 @@
|
||||
'use client';
|
||||
|
||||
import { useState, useEffect, useRef, useCallback } from 'react';
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import ReviewAnalytics from './ReviewAnalytics';
|
||||
|
||||
const API_BASE = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000';
|
||||
|
||||
interface ScraperType {
|
||||
job_type: string;
|
||||
version: string;
|
||||
variant: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
interface Review {
|
||||
author: string;
|
||||
rating: number;
|
||||
@@ -69,10 +60,6 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
||||
const [businessImage, setBusinessImage] = useState<string | null>(null);
|
||||
const [businessCategory, setBusinessCategory] = useState<string | null>(null);
|
||||
|
||||
// Scraper type selection
|
||||
const [availableScrapers, setAvailableScrapers] = useState<ScraperType[]>([]);
|
||||
const [selectedScraper, setSelectedScraper] = useState<ScraperType | null>(null);
|
||||
const [scrapersLoading, setScrapersLoading] = useState(true);
|
||||
const [userFingerprint, setUserFingerprint] = useState<{
|
||||
geolocation?: {lat: number, lng: number},
|
||||
userAgent?: string,
|
||||
@@ -132,48 +119,6 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
||||
collectFingerprint();
|
||||
}, []);
|
||||
|
||||
// Fetch available scraper types on mount
|
||||
const fetchScrapers = useCallback(async () => {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/api/admin/scrapers`);
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
// Transform to ScraperType format and filter to active scrapers
|
||||
const scrapers: ScraperType[] = data
|
||||
.filter((s: { deprecated_at: string | null; traffic_pct: number }) => !s.deprecated_at && s.traffic_pct > 0)
|
||||
.map((s: { job_type: string; version: string; variant: string }) => ({
|
||||
job_type: s.job_type,
|
||||
version: s.version,
|
||||
variant: s.variant,
|
||||
// Format job_type nicely: google_reviews or google-reviews -> "Google Reviews"
|
||||
label: `${s.job_type.split(/[-_]/).map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')} v${s.version}${s.variant !== 'stable' ? ` (${s.variant})` : ''}`,
|
||||
}));
|
||||
setAvailableScrapers(scrapers);
|
||||
// Auto-select first scraper (usually google-reviews stable)
|
||||
if (scrapers.length > 0 && !selectedScraper) {
|
||||
setSelectedScraper(scrapers[0]);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch scrapers:', err);
|
||||
// Fallback to default google-reviews
|
||||
const defaultScraper: ScraperType = {
|
||||
job_type: 'google-reviews',
|
||||
version: '1.0.0',
|
||||
variant: 'stable',
|
||||
label: 'Google Reviews v1.0.0',
|
||||
};
|
||||
setAvailableScrapers([defaultScraper]);
|
||||
setSelectedScraper(defaultScraper);
|
||||
} finally {
|
||||
setScrapersLoading(false);
|
||||
}
|
||||
}, [selectedScraper]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchScrapers();
|
||||
}, [fetchScrapers]);
|
||||
|
||||
const pollingIntervals = useRef<Map<string, NodeJS.Timeout>>(new Map());
|
||||
const abortControllerRef = useRef<AbortController | null>(null);
|
||||
|
||||
@@ -379,11 +324,6 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
||||
const url = `https://www.google.com/maps/search/?api=1&query=${encodeURIComponent(searchedQuery)}&hl=en`;
|
||||
|
||||
try {
|
||||
// Use selected scraper or default to google-reviews
|
||||
const jobType = selectedScraper?.job_type || 'google-reviews';
|
||||
const scraperVersion = selectedScraper?.version;
|
||||
const scraperVariant = selectedScraper?.variant;
|
||||
|
||||
const response = await fetch('/api/scrape', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -395,10 +335,8 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
||||
total_reviews_snapshot: availableReviewCount,
|
||||
geolocation: userFingerprint.geolocation,
|
||||
browser_fingerprint: userFingerprint, // Pass full fingerprint
|
||||
// Include scraper selection
|
||||
job_type: jobType,
|
||||
scraper_version: scraperVersion,
|
||||
scraper_variant: scraperVariant,
|
||||
// Google Reviews scraper (this component is specific to Google Reviews)
|
||||
job_type: 'google-reviews',
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -502,68 +440,6 @@ export default function ScraperTest({ onJobsChange, onSelectReviews }: ScraperTe
|
||||
|
||||
return (
|
||||
<div className="w-full max-w-4xl mx-auto">
|
||||
{/* Scraper Type Selection */}
|
||||
<div className="mb-4 p-4 bg-white border-2 border-gray-200 rounded-xl shadow-sm">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="w-10 h-10 bg-gradient-to-br from-blue-500 to-indigo-600 rounded-lg flex items-center justify-center">
|
||||
<svg className="w-5 h-5 text-white" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19.428 15.428a2 2 0 00-1.022-.547l-2.387-.477a6 6 0 00-3.86.517l-.318.158a6 6 0 01-3.86.517L6.05 15.21a2 2 0 00-1.806.547M8 4h8l-1 1v5.172a2 2 0 00.586 1.414l5 5c1.26 1.26.367 3.414-1.415 3.414H4.828c-1.782 0-2.674-2.154-1.414-3.414l5-5A2 2 0 009 10.172V5L8 4z" />
|
||||
</svg>
|
||||
</div>
|
||||
<div>
|
||||
<label className="block text-sm font-bold text-gray-700">Scraper Type</label>
|
||||
<p className="text-xs text-gray-500">Select the type of data to scrape</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{scrapersLoading ? (
|
||||
<div className="flex items-center gap-2 text-gray-400">
|
||||
<div className="w-4 h-4 border-2 border-gray-300 border-t-blue-500 rounded-full animate-spin"></div>
|
||||
<span className="text-sm">Loading...</span>
|
||||
</div>
|
||||
) : (
|
||||
<select
|
||||
value={selectedScraper ? `${selectedScraper.job_type}:${selectedScraper.version}:${selectedScraper.variant}` : ''}
|
||||
onChange={(e) => {
|
||||
const [jobType, version, variant] = e.target.value.split(':');
|
||||
const scraper = availableScrapers.find(
|
||||
s => s.job_type === jobType && s.version === version && s.variant === variant
|
||||
);
|
||||
if (scraper) setSelectedScraper(scraper);
|
||||
}}
|
||||
className="px-4 py-2 bg-gray-50 border-2 border-gray-300 rounded-lg text-sm font-medium text-gray-700 hover:border-blue-500 focus:border-blue-500 focus:ring-2 focus:ring-blue-100 outline-none transition-all cursor-pointer min-w-[200px]"
|
||||
>
|
||||
{availableScrapers.map((scraper) => (
|
||||
<option
|
||||
key={`${scraper.job_type}:${scraper.version}:${scraper.variant}`}
|
||||
value={`${scraper.job_type}:${scraper.version}:${scraper.variant}`}
|
||||
>
|
||||
{scraper.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Show selected scraper info */}
|
||||
{selectedScraper && (
|
||||
<div className="mt-3 pt-3 border-t border-gray-100 flex items-center gap-2 text-xs text-gray-500">
|
||||
<span className="px-2 py-0.5 bg-blue-100 text-blue-700 rounded font-medium">
|
||||
{selectedScraper.job_type.split(/[-_]/).map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')}
|
||||
</span>
|
||||
<span className="px-2 py-0.5 bg-gray-100 text-gray-600 rounded">
|
||||
v{selectedScraper.version}
|
||||
</span>
|
||||
{selectedScraper.variant !== 'stable' && (
|
||||
<span className="px-2 py-0.5 bg-yellow-100 text-yellow-700 rounded font-medium">
|
||||
{selectedScraper.variant}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Test URL Quick Select */}
|
||||
<div className="mb-4 p-3 bg-gray-50 border-2 border-gray-200 rounded-xl">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
|
||||
Reference in New Issue
Block a user