From 544e028c3f5d7440fe524c315402d54f6cf61415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Guti=C3=A9rrez?= <35082514+alezmad@users.noreply.github.com> Date: Sat, 24 Jan 2026 15:22:08 +0000 Subject: [PATCH] Phase 0: Project restructure to ReviewIQ platform architecture New structure: - scrapers/google_reviews/v1_0_0.py (was modules/scraper_clean.py) - scrapers/base.py (BaseScraper interface) - scrapers/registry.py (ScraperRegistry for version routing) - core/database.py, models.py, config.py, enums.py - utils/logger.py, crash_analyzer.py, health_checks.py, helpers.py, date_converter.py - workers/chrome_pool.py - services/webhook_service.py - api/ routes structure (empty, ready for Phase 2) - tests/ structure mirroring source All imports updated in: - api_server_production.py (7 import paths updated) - utils/health_checks.py (scraper import path) Legacy modules moved to modules/_legacy/: - data_storage.py, image_handler.py, s3_handler.py (unused) Syntax verified, frontend build passing. Co-Authored-By: Claude Opus 4.5 --- .artifacts/ReviewIQ-Architecture-v2.md | 1143 ++++++++ .artifacts/ReviewIQ-Architecture-v3.2.md | 2306 +++++++++++++++++ .artifacts/ReviewIQ-Architecture-v3.md | 1513 +++++++++++ .artifacts/ReviewIQ-v32-Decisions.md | 183 ++ .artifacts/URT-v5.1-Reference.md | 331 +++ api/__init__.py | 0 api/middleware/__init__.py | 0 api/routes/__init__.py | 0 api_server_production.py | 14 +- core/__init__.py | 0 {modules => core}/config.py | 0 {modules => core}/database.py | 13 +- core/enums.py | 14 + {modules => core}/models.py | 4 +- modules/{ => _legacy}/data_storage.py | 0 modules/{ => _legacy}/image_handler.py | 0 modules/{ => _legacy}/s3_handler.py | 0 scrapers/__init__.py | 10 + scrapers/base.py | 97 + scrapers/google_reviews/__init__.py | 21 + .../google_reviews/v1_0_0.py | 9 +- scrapers/registry.py | 138 + services/__init__.py | 0 .../webhook_service.py | 0 tests/api/__init__.py | 0 tests/integration/__init__.py | 0 tests/scrapers/__init__.py | 0 tests/scrapers/google_reviews/__init__.py | 0 tests/services/__init__.py | 0 utils/__init__.py | 0 {modules => utils}/crash_analyzer.py | 0 {modules => utils}/date_converter.py | 0 {modules => utils}/health_checks.py | 16 +- modules/utils.py => utils/helpers.py | 0 .../structured_logger.py => utils/logger.py | 0 workers/__init__.py | 0 {modules => workers}/chrome_pool.py | 0 37 files changed, 5782 insertions(+), 30 deletions(-) create mode 100644 .artifacts/ReviewIQ-Architecture-v2.md create mode 100644 .artifacts/ReviewIQ-Architecture-v3.2.md create mode 100644 .artifacts/ReviewIQ-Architecture-v3.md create mode 100644 .artifacts/ReviewIQ-v32-Decisions.md create mode 100644 .artifacts/URT-v5.1-Reference.md create mode 100644 api/__init__.py create mode 100644 api/middleware/__init__.py create mode 100644 api/routes/__init__.py create mode 100644 core/__init__.py rename {modules => core}/config.py (100%) rename {modules => core}/database.py (99%) create mode 100644 core/enums.py rename {modules => core}/models.py (97%) rename modules/{ => _legacy}/data_storage.py (100%) rename modules/{ => _legacy}/image_handler.py (100%) rename modules/{ => _legacy}/s3_handler.py (100%) create mode 100644 scrapers/__init__.py create mode 100644 scrapers/base.py create mode 100644 scrapers/google_reviews/__init__.py rename modules/scraper_clean.py => scrapers/google_reviews/v1_0_0.py (99%) create mode 100644 scrapers/registry.py create mode 100644 services/__init__.py rename modules/webhooks.py => services/webhook_service.py (100%) create mode 100644 tests/api/__init__.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/scrapers/__init__.py create mode 100644 tests/scrapers/google_reviews/__init__.py create mode 100644 tests/services/__init__.py create mode 100644 utils/__init__.py rename {modules => utils}/crash_analyzer.py (100%) rename {modules => utils}/date_converter.py (100%) rename {modules => utils}/health_checks.py (95%) rename modules/utils.py => utils/helpers.py (100%) rename modules/structured_logger.py => utils/logger.py (100%) create mode 100644 workers/__init__.py rename {modules => workers}/chrome_pool.py (100%) diff --git a/.artifacts/ReviewIQ-Architecture-v2.md b/.artifacts/ReviewIQ-Architecture-v2.md new file mode 100644 index 0000000..8a92395 --- /dev/null +++ b/.artifacts/ReviewIQ-Architecture-v2.md @@ -0,0 +1,1143 @@ +# ReviewIQ: Review Intelligence Pipeline + +**Version**: 2.0 +**Status**: Architecture Specification +**Date**: 2026-01-24 + +--- + +## Executive Summary + +ReviewIQ transforms customer reviews into actionable business intelligence through a three-stage pipeline: + +1. **Ingest** — LLM-powered URT classification with semantic embeddings +2. **Analyze** — Issue lifecycle management with sub-pattern discovery +3. **Report** — Statistically rigorous insights with trend detection + +**Design Principles**: +- **Accuracy over heuristics**: LLM classification at ingest (~$0.0002/review) +- **Taxonomy as structure**: URT provides stable, interpretable categories +- **Local ML for depth**: Sub-clustering reveals actionable patterns within categories +- **Feedback loop**: CR (Comparative Reference) signals verify resolution effectiveness + +--- + +## Part 1: System Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ REVIEWIQ PIPELINE │ +├─────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────┐ ┌──────────────────────────────────────────────────────┐ │ +│ │ │ │ INGEST LAYER │ │ +│ │ Reviews │────▶│ ┌─────────┐ ┌─────────┐ ┌─────────────────┐ │ │ +│ │ (Input) │ │ │ Embed │ │ LLM │ │ Store │ │ │ +│ │ │ │ │ Review │───▶│Classify │───▶│ (PostgreSQL) │ │ │ +│ └─────────────┘ │ └─────────┘ └─────────┘ └─────────────────┘ │ │ +│ │ │ │ │ │ +│ │ │ ~$0.00 │ ~$0.0002 │ │ +│ │ │ (local) │ per review │ │ +│ └──────┼──────────────────────────────┼─────────────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ ISSUE AGGREGATION │ │ +│ │ │ │ +│ │ V- classified reviews ───▶ Match or Create Issue ───▶ Track State │ │ +│ │ │ │ +│ │ Rules: Same URT code + entity + location + time window = same issue │ │ +│ │ States: DETECTED → ACKNOWLEDGED → IN_PROGRESS → RESOLVED → VERIFIED │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ REPORT GENERATION │ │ +│ │ │ │ +│ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────┐ │ │ +│ │ │ Aggregate │ │ Sub-Cluster │ │ Trend │ │ LLM │ │ │ +│ │ │ by URT Code │──▶│ Within Codes │──▶│ Analysis │──▶│ Narrate │ │ │ +│ │ │ (SQL) │ │ (HDBSCAN) │ │ (CR + Rate) │ │ │ │ │ +│ │ └──────────────┘ └──────────────┘ └──────────────┘ └──────────┘ │ │ +│ │ $0.00 $0.00 $0.00 ~$0.15 │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ │ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ OUTPUT │ │ +│ │ │ │ +│ │ • Executive Summary with statistically defensible claims │ │ +│ │ • Issues ranked by priority with sub-pattern breakdown │ │ +│ │ • Strengths with trend signals │ │ +│ │ • Staff performance insights │ │ +│ │ • Actionable recommendations │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Part 2: Ingest Layer + +### 2.1 Design Philosophy + +The review is the atomic unit. We do not split reviews into fragments — this preserves context and enables accurate classification. A single review may contain multiple topics; URT's multi-coding (primary + up to 2 secondary codes) handles this naturally. + +### 2.2 Dual Processing + +Each review undergoes two parallel operations: + +```python +async def ingest_review(review: dict) -> dict: + """Ingest a single review: embed + classify.""" + + text = review['text'].strip() + + # Parallel execution + embedding_task = asyncio.create_task(embed_review(text)) + classification_task = asyncio.create_task(classify_review_llm(text)) + + embedding = await embedding_task + classification = await classification_task + + return { + 'review_id': review['review_id'], + 'business_id': review['business_id'], + 'text': text, + 'embedding': embedding, + 'date': review['date'], + 'rating': review.get('rating'), + **classification, # URT codes, valence, intensity, etc. + } +``` + +### 2.3 Embedding + +Local multilingual embeddings for semantic capabilities: + +```python +from sentence_transformers import SentenceTransformer + +model = SentenceTransformer('intfloat/multilingual-e5-small') + +def embed_review(text: str) -> np.ndarray: + """Generate normalized embedding for semantic search and clustering.""" + + # e5 models perform better with instruction prefix + embedding = model.encode( + f"passage: {text}", + normalize_embeddings=True + ) + return embedding # 384 dimensions +``` + +**Why embed if we have URT codes?** +- Sub-clustering within URT codes (pattern discovery) +- Semantic quote selection (centroid-closest) +- Similarity search for emerging patterns +- Backup for low-confidence classifications + +### 2.4 LLM Classification + +Single LLM call extracts complete URT classification: + +```python +CLASSIFICATION_PROMPT = """You are a customer feedback classifier using the Universal Review Taxonomy (URT). + +Analyze the review and return JSON with: + +{ + "urt_primary": "X1.23", // Main URT subcode + "urt_secondary": ["Y2.34"], // 0-2 additional codes (different domains only) + "valence": "V-", // V+, V-, V0, V± + "intensity": "I2", // I1 (mild), I2 (moderate), I3 (strong) + "comparative": "CR-N", // CR-N (none), CR-B (better), CR-W (worse), CR-S (same) + "staff_mentions": ["Mike"], // Employee names mentioned + "quotes": { // Key phrase for each code + "X1.23": "exact phrase from review", + "Y2.34": "another phrase" + } +} + +URT DOMAINS (choose primary from most impactful): +- O (Offering): Product/service quality, function, completeness, fit +- P (People): Staff attitude, competence, responsiveness, communication +- J (Journey): Timing, ease, reliability, resolution +- E (Environment): Physical space, digital interface, ambiance, safety +- A (Access): Availability, accessibility, inclusivity, convenience +- V (Value): Price, transparency, effort, worth +- R (Relationship): Trust, dependability, recovery, loyalty + +RULES: +1. Primary = what customer is MOST affected by +2. Secondary must be DIFFERENT domains (P1.02 + P3.01 is invalid) +3. V± only when genuinely mixed (positive AND negative on different topics) +4. CR-B/W/S only for explicit self-comparison ("better than last time", "still broken") +5. quotes must be EXACT phrases from the review text + +Return valid JSON only.""" + + +async def classify_review_llm(text: str) -> dict: + """Complete URT classification via LLM.""" + + response = await llm.chat( + model="gpt-4o-mini", # ~$0.0002 per review + messages=[ + {"role": "system", "content": CLASSIFICATION_PROMPT}, + {"role": "user", "content": text} + ], + response_format={"type": "json_object"}, + temperature=0.1 # Low temperature for consistency + ) + + return json.loads(response.content) +``` + +### 2.5 Batch Processing for Efficiency + +For bulk ingestion, batch multiple reviews per LLM call: + +```python +async def classify_batch(reviews: list[dict], batch_size: int = 10) -> list[dict]: + """Process reviews in batches for ~40% cost reduction.""" + + results = [] + for i in range(0, len(reviews), batch_size): + batch = reviews[i:i+batch_size] + + prompt = BATCH_CLASSIFICATION_PROMPT + "\n\nREVIEWS:\n" + for j, review in enumerate(batch): + prompt += f"\n[{j}] {review['text']}\n---\n" + + response = await llm.chat( + model="gpt-4o-mini", + messages=[{"role": "system", "content": prompt}], + response_format={"type": "json_object"} + ) + + batch_results = json.loads(response.content)["classifications"] + results.extend(batch_results) + + return results +``` + +### 2.6 Data Model + +```sql +-- Core review storage with URT classification +CREATE TABLE reviews ( + review_id TEXT PRIMARY KEY, + business_id TEXT NOT NULL, + text TEXT NOT NULL, + embedding VECTOR(384), + date TIMESTAMP NOT NULL, + rating SMALLINT, + + -- URT Classification (from LLM) + urt_primary TEXT NOT NULL, -- 'J1.01', 'P1.02', etc. + urt_secondary TEXT[] DEFAULT '{}', -- Max 2 + valence TEXT NOT NULL, -- 'V+', 'V-', 'V0', 'V±' + intensity TEXT NOT NULL, -- 'I1', 'I2', 'I3' + comparative TEXT DEFAULT 'CR-N', -- 'CR-N', 'CR-B', 'CR-W', 'CR-S' + + -- Extracted entities + staff_mentions TEXT[] DEFAULT '{}', + quotes JSONB, -- {"code": "phrase", ...} + + -- Metadata + created_at TIMESTAMP DEFAULT NOW(), + classification_model TEXT DEFAULT 'gpt-4o-mini' +); + +-- Indexes for query patterns +CREATE INDEX idx_reviews_business_date ON reviews(business_id, date DESC); +CREATE INDEX idx_reviews_urt_primary ON reviews(business_id, urt_primary); +CREATE INDEX idx_reviews_valence ON reviews(business_id, valence, date); +CREATE INDEX idx_reviews_comparative ON reviews(comparative) WHERE comparative != 'CR-N'; +CREATE INDEX idx_reviews_embedding ON reviews USING hnsw (embedding vector_cosine_ops); +``` + +--- + +## Part 3: Issue Lifecycle Management + +Following the URT Issue Lifecycle Framework (C1), negative feedback (V-) generates trackable issues. + +### 3.1 Issue Aggregation + +Multiple reviews about the same problem aggregate into a single issue: + +```python +def aggregate_to_issue(review: dict) -> str: + """Match review to existing issue or create new one.""" + + if review['valence'] not in ('V-', 'V±'): + return None # Only negative feedback creates issues + + # Find matching open issues + matching = db.query(""" + SELECT issue_id, primary_subcode, entity, location + FROM issues + WHERE business_id = %s + AND primary_subcode = %s + AND state NOT IN ('VERIFIED', 'DECLINED') + AND created_at > NOW() - INTERVAL '30 days' + """, [review['business_id'], review['urt_primary']]) + + for issue in matching: + if is_same_issue(review, issue): + # Aggregate to existing issue + add_span_to_issue(issue['issue_id'], review) + recalculate_priority(issue['issue_id']) + return issue['issue_id'] + + # Check intensity threshold for new issue creation + if should_create_issue(review): + return create_issue(review) + + return None # Stored in buffer for future aggregation + + +def should_create_issue(review: dict) -> bool: + """Intensity-based issue creation thresholds.""" + + if review['intensity'] == 'I3': + return True # Critical = immediate issue + + # Check aggregation buffer for patterns + similar_count = count_similar_in_buffer(review, window_days=30) + + if review['intensity'] == 'I2' and similar_count >= 2: + return True # Moderate + 2 others = issue + + if review['intensity'] == 'I1' and similar_count >= 4: + return True # Mild + 4 others = issue + + return False +``` + +### 3.2 Issue State Machine + +``` + DETECTED + │ + ┌────────────┼────────────┐ + ▼ ▼ ▼ + ACKNOWLEDGED DECLINED (escalate) + │ │ + ▼ │ + IN_PROGRESS │ + │ │ + ▼ │ + RESOLVED ◀────────────────────┘ + │ + ┌─────┼─────┐ + ▼ ▼ + VERIFIED REOPENED + │ + └──▶ (back to IN_PROGRESS) +``` + +### 3.3 Priority Scoring + +```python +def calculate_priority(issue: dict) -> float: + """ + Priority combines intensity, volume, recency, and recurrence. + + P = I_weight × (1 + log(span_count)) × decay(days) × recurrence_boost × trend_modifier + """ + + INTENSITY_WEIGHTS = {'I1': 1.0, 'I2': 2.0, 'I3': 4.0} + + i_weight = INTENSITY_WEIGHTS[issue['max_intensity']] + volume_factor = 1 + math.log(issue['span_count']) + + days_old = (datetime.now() - issue['created_at']).days + decay = math.exp(-0.023 * days_old) # Half-life ~30 days + + recurrence_boost = 1.0 + 0.5 * math.log2(issue['reopen_count'] + 1) + + # Trend modifier from CR signals + if issue['recent_cr_w_count'] >= 2: + trend_modifier = 1.3 # Worsening + elif issue['recent_cr_b_count'] >= 2: + trend_modifier = 0.7 # Improving + else: + trend_modifier = 1.0 # Stable + + return i_weight * volume_factor * decay * recurrence_boost * trend_modifier +``` + +### 3.4 Resolution Verification via CR Signals + +The Comparative Reference (CR) dimension enables automatic verification: + +```python +def process_cr_signal(review: dict): + """Handle CR-B/W/S signals for issue lifecycle.""" + + if review['comparative'] == 'CR-N': + return + + # Find resolved issues with matching code + resolved_issues = db.query(""" + SELECT issue_id, state, resolved_at + FROM issues + WHERE business_id = %s + AND primary_subcode = %s + AND state IN ('RESOLVED', 'VERIFIED') + AND resolved_at > NOW() - INTERVAL '60 days' + """, [review['business_id'], review['urt_primary']]) + + for issue in resolved_issues: + if review['comparative'] == 'CR-B': + # Improvement signal → verify resolution + if issue['state'] == 'RESOLVED': + verify_issue(issue['issue_id'], review['review_id']) + + elif review['comparative'] in ('CR-S', 'CR-W'): + # Unchanged or worsening → reopen + reopen_issue(issue['issue_id'], review['review_id']) + + if review['comparative'] == 'CR-W': + escalate_issue(issue['issue_id'], reason='REGRESSION') +``` + +### 3.5 Issue Data Model + +```sql +CREATE TABLE issues ( + issue_id TEXT PRIMARY KEY, + business_id TEXT NOT NULL, + primary_subcode TEXT NOT NULL, + domain TEXT NOT NULL, + + -- State + state TEXT NOT NULL DEFAULT 'DETECTED', + priority_score FLOAT NOT NULL, + confidence_score FLOAT NOT NULL, + + -- Aggregation + review_ids TEXT[] NOT NULL, + span_count INT NOT NULL DEFAULT 1, + max_intensity TEXT NOT NULL, + + -- Ownership + owner_team TEXT, + owner_individual TEXT, + + -- Timestamps + created_at TIMESTAMP DEFAULT NOW(), + acknowledged_at TIMESTAMP, + resolved_at TIMESTAMP, + verified_at TIMESTAMP, + + -- Resolution + reopen_count INT DEFAULT 0, + resolution_code TEXT, + resolution_notes TEXT, + decline_reason TEXT, + + -- Context + entity TEXT, -- Product, staff member, feature + location TEXT, -- Physical or logical location + causal_codes TEXT[], -- CD-O, MG-T, etc. + + -- Verification + verification_window_days INT DEFAULT 60 +); + +CREATE TABLE issue_events ( + event_id SERIAL PRIMARY KEY, + issue_id TEXT REFERENCES issues(issue_id), + event_type TEXT NOT NULL, -- 'state_change', 'span_added', 'priority_update' + from_state TEXT, + to_state TEXT, + actor TEXT, + notes TEXT, + review_id TEXT, -- Triggering review if applicable + created_at TIMESTAMP DEFAULT NOW() +); +``` + +--- + +## Part 4: Report Generation + +### 4.1 Report Structure + +```python +def generate_report(business_id: str, start: date, end: date) -> dict: + """Generate comprehensive business intelligence report.""" + + # 1. Aggregate statistics by URT code + code_stats = compute_code_statistics(business_id, start, end) + + # 2. Deep analysis of top issues (sub-clustering) + top_issues = analyze_top_issues(business_id, code_stats, start, end) + + # 3. Strength analysis + strengths = analyze_strengths(business_id, code_stats, start, end) + + # 4. Trend analysis (vs prior period) + trends = compute_trends(business_id, start, end) + + # 5. Staff insights + staff = analyze_staff_mentions(business_id, start, end) + + # 6. Open issues summary + open_issues = get_open_issues(business_id) + + # 7. Build payload + payload = build_report_payload( + business_id, start, end, + top_issues, strengths, trends, staff, open_issues + ) + + # 8. LLM narration + narrative = await generate_narrative(payload) + + return { + 'payload': payload, + 'narrative': narrative, + 'generated_at': datetime.now().isoformat() + } +``` + +### 4.2 Statistics Computation + +Review-level presence with Wilson confidence intervals: + +```python +def compute_code_statistics(business_id: str, start: date, end: date) -> list[dict]: + """Aggregate statistics by URT code with confidence intervals.""" + + stats = db.query(""" + WITH review_codes AS ( + SELECT + review_id, + urt_primary as code, + valence, + intensity + FROM reviews + WHERE business_id = %s AND date BETWEEN %s AND %s + + UNION ALL + + SELECT + review_id, + unnest(urt_secondary) as code, + valence, + intensity + FROM reviews + WHERE business_id = %s AND date BETWEEN %s AND %s + AND array_length(urt_secondary, 1) > 0 + ), + code_stats AS ( + SELECT + code, + COUNT(DISTINCT review_id) as k, + COUNT(DISTINCT review_id) FILTER (WHERE valence = 'V-') as k_neg, + COUNT(DISTINCT review_id) FILTER (WHERE valence = 'V+') as k_pos, + MAX(CASE intensity + WHEN 'I3' THEN 3 WHEN 'I2' THEN 2 ELSE 1 END) as max_intensity + FROM review_codes + GROUP BY code + ) + SELECT + cs.*, + (SELECT COUNT(DISTINCT review_id) FROM reviews + WHERE business_id = %s AND date BETWEEN %s AND %s) as n + FROM code_stats cs + WHERE k >= 3 + ORDER BY k_neg DESC + """, [business_id, start, end] * 4) + + results = [] + for row in stats: + n = row['n'] + + # Wilson confidence intervals + ci_neg = wilson_ci(row['k_neg'], n) if row['k_neg'] > 0 else (0, 0) + ci_pos = wilson_ci(row['k_pos'], n) if row['k_pos'] > 0 else (0, 0) + + results.append({ + 'code': row['code'], + 'domain': row['code'][0], + 'name': URT_CODE_NAMES[row['code']], + 'k': row['k'], + 'k_neg': row['k_neg'], + 'k_pos': row['k_pos'], + 'n': n, + 'rate_neg': row['k_neg'] / n if n > 0 else 0, + 'rate_pos': row['k_pos'] / n if n > 0 else 0, + 'ci_neg': ci_neg, + 'ci_pos': ci_pos, + 'max_intensity': f"I{row['max_intensity']}", + }) + + return results + + +def wilson_ci(k: int, n: int, z: float = 1.96) -> tuple[float, float]: + """Wilson score interval for binomial proportion.""" + if n == 0: + return (0.0, 0.0) + + p = k / n + denom = 1 + z**2 / n + center = (p + z**2 / (2*n)) / denom + margin = z * math.sqrt((p*(1-p) + z**2/(4*n)) / n) / denom + + return (max(0, center - margin), min(1, center + margin)) +``` + +### 4.3 Sub-Pattern Discovery (Local ML) + +The key insight: **LLM gives categories, local ML reveals patterns within categories.** + +```python +def analyze_top_issues(business_id: str, code_stats: list, + start: date, end: date, top_k: int = 5) -> list[dict]: + """Deep analysis of top negative codes with sub-clustering.""" + + # Filter to significant negative codes + issues_to_analyze = [ + cs for cs in code_stats + if cs['k_neg'] >= 8 and cs['ci_neg'][1] - cs['ci_neg'][0] <= 0.30 + ][:top_k] + + results = [] + for code_stat in issues_to_analyze: + code = code_stat['code'] + + # Fetch all negative reviews for this code + reviews = db.query(""" + SELECT review_id, text, embedding, intensity, quotes, date + FROM reviews + WHERE business_id = %s + AND (urt_primary = %s OR %s = ANY(urt_secondary)) + AND valence IN ('V-', 'V±') + AND date BETWEEN %s AND %s + """, [business_id, code, code, start, end]) + + # Sub-cluster to find patterns + sub_patterns = discover_sub_patterns(reviews, code) + + results.append({ + 'code': code, + 'name': code_stat['name'], + 'total_reviews': code_stat['k_neg'], + 'rate': code_stat['rate_neg'], + 'ci': code_stat['ci_neg'], + 'max_intensity': code_stat['max_intensity'], + 'sub_patterns': sub_patterns, + }) + + return results + + +def discover_sub_patterns(reviews: list[dict], code: str, + min_cluster_size: int = 3) -> list[dict]: + """Cluster reviews within a URT code to find actionable sub-patterns.""" + + if len(reviews) < min_cluster_size * 2: + # Too few for meaningful clustering + return [{ + 'label': 'General', + 'count': len(reviews), + 'percentage': 1.0, + 'representative_quote': select_representative(reviews, code), + 'sharpest_quote': select_sharpest(reviews, code), + }] + + embeddings = np.array([r['embedding'] for r in reviews]) + + # HDBSCAN for small datasets, KMeans for larger + if len(reviews) < 500: + clusterer = hdbscan.HDBSCAN( + min_cluster_size=min_cluster_size, + min_samples=2, + metric='euclidean' + ) + labels = clusterer.fit_predict(embeddings) + else: + k = min(8, max(3, int(np.sqrt(len(reviews) / 5)))) + kmeans = KMeans(n_clusters=k, n_init=3) + labels = kmeans.fit_predict(embeddings) + + # Group by cluster + clusters = {} + for review, label in zip(reviews, labels): + if label == -1: # Noise + continue + if label not in clusters: + clusters[label] = [] + clusters[label].append(review) + + # Build sub-pattern descriptions + patterns = [] + for label, cluster_reviews in clusters.items(): + if len(cluster_reviews) < min_cluster_size: + continue + + cluster_embeddings = np.array([r['embedding'] for r in cluster_reviews]) + centroid = cluster_embeddings.mean(axis=0) + centroid /= np.linalg.norm(centroid) + + patterns.append({ + 'label': extract_cluster_label(cluster_reviews, code), + 'count': len(cluster_reviews), + 'percentage': len(cluster_reviews) / len(reviews), + 'representative_quote': select_representative(cluster_reviews, code, centroid), + 'sharpest_quote': select_sharpest(cluster_reviews, code), + 'avg_intensity': np.mean([ + {'I1': 1, 'I2': 2, 'I3': 3}[r['intensity']] + for r in cluster_reviews + ]), + 'centroid': centroid, # For trend matching + }) + + patterns.sort(key=lambda x: x['count'], reverse=True) + return patterns[:4] # Top 4 sub-patterns + + +def select_representative(reviews: list, code: str, + centroid: np.ndarray = None) -> str: + """Select quote closest to centroid (most representative).""" + + if centroid is None: + embeddings = np.array([r['embedding'] for r in reviews]) + centroid = embeddings.mean(axis=0) + centroid /= np.linalg.norm(centroid) + + best_review = max(reviews, key=lambda r: r['embedding'] @ centroid) + + # Return the extracted quote for this code, or truncated text + if best_review.get('quotes') and code in best_review['quotes']: + return best_review['quotes'][code] + return best_review['text'][:150] + + +def select_sharpest(reviews: list, code: str) -> str: + """Select highest intensity quote (sharpest criticism).""" + + intensity_order = {'I3': 3, 'I2': 2, 'I1': 1} + best_review = max(reviews, key=lambda r: intensity_order.get(r['intensity'], 0)) + + if best_review.get('quotes') and code in best_review['quotes']: + return best_review['quotes'][code] + return best_review['text'][:150] + + +def extract_cluster_label(reviews: list, code: str) -> str: + """Generate a concise label for the cluster.""" + + # Extract common phrases from quotes + texts = [] + for r in reviews: + if r.get('quotes') and code in r['quotes']: + texts.append(r['quotes'][code].lower()) + else: + texts.append(r['text'][:100].lower()) + + # Find distinctive 2-3 word phrases + from collections import Counter + + all_text = ' '.join(texts) + words = re.findall(r'\b[a-z]{3,}\b', all_text) + + # Bigrams, skip stopwords + stopwords = {'the', 'and', 'was', 'were', 'for', 'that', 'this', 'with', 'but', 'have', 'had'} + bigrams = [ + f"{words[i]} {words[i+1]}" + for i in range(len(words)-1) + if words[i] not in stopwords and words[i+1] not in stopwords + ] + + counts = Counter(bigrams) + if counts: + label = counts.most_common(1)[0][0] + return label.title() + + return URT_CODE_NAMES.get(code, "General") +``` + +### 4.4 Trend Analysis + +Combine rate comparison with CR signals: + +```python +def compute_trends(business_id: str, current_start: date, + current_end: date) -> dict: + """Compute trends vs prior period using rates and CR signals.""" + + period_length = (current_end - current_start).days + prior_start = current_start - timedelta(days=period_length) + prior_end = current_start + + # Current period stats + current_stats = compute_code_statistics(business_id, current_start, current_end) + current_map = {cs['code']: cs for cs in current_stats} + + # Prior period stats + prior_stats = compute_code_statistics(business_id, prior_start, prior_end) + prior_map = {cs['code']: cs for cs in prior_stats} + + # CR signals in current period + cr_signals = db.query(""" + SELECT urt_primary as code, comparative, COUNT(*) as count + FROM reviews + WHERE business_id = %s + AND date BETWEEN %s AND %s + AND comparative != 'CR-N' + GROUP BY urt_primary, comparative + """, [business_id, current_start, current_end]) + + cr_map = {} + for row in cr_signals: + if row['code'] not in cr_map: + cr_map[row['code']] = {'CR-B': 0, 'CR-W': 0, 'CR-S': 0} + cr_map[row['code']][row['comparative']] = row['count'] + + # Compute trends + trends = {} + for code, current in current_map.items(): + prior = prior_map.get(code, {'rate_neg': 0, 'rate_pos': 0, 'k_neg': 0, 'k_pos': 0}) + cr = cr_map.get(code, {'CR-B': 0, 'CR-W': 0, 'CR-S': 0}) + + # Rate-based trend (issues) + rate_trend_neg = current['rate_neg'] - prior['rate_neg'] + + # CR-enhanced trend signal + if cr['CR-W'] >= 2: + trend_signal = 'worsening' + elif cr['CR-B'] >= 2: + trend_signal = 'improving' + elif cr['CR-S'] >= 2: + trend_signal = 'persistent' + elif rate_trend_neg > 0.05: + trend_signal = 'worsening' + elif rate_trend_neg < -0.05: + trend_signal = 'improving' + else: + trend_signal = 'stable' + + trends[code] = { + 'rate_change_neg': rate_trend_neg, + 'rate_change_pos': current['rate_pos'] - prior['rate_pos'], + 'signal': trend_signal, + 'cr_better': cr['CR-B'], + 'cr_worse': cr['CR-W'], + 'cr_same': cr['CR-S'], + } + + return trends +``` + +### 4.5 Staff Analysis + +```python +def analyze_staff_mentions(business_id: str, start: date, end: date) -> dict: + """Aggregate staff performance from mentions.""" + + staff_data = db.query(""" + SELECT + unnest(staff_mentions) as staff_name, + valence, + intensity, + urt_primary, + quotes + FROM reviews + WHERE business_id = %s + AND date BETWEEN %s AND %s + AND array_length(staff_mentions, 1) > 0 + """, [business_id, start, end]) + + staff_map = {} + for row in staff_data: + name = row['staff_name'] + if name not in staff_map: + staff_map[name] = { + 'positive': [], + 'negative': [], + 'codes': Counter() + } + + if row['valence'] in ('V+',): + staff_map[name]['positive'].append(row) + elif row['valence'] in ('V-',): + staff_map[name]['negative'].append(row) + + staff_map[name]['codes'][row['urt_primary']] += 1 + + # Build summary + staff_summary = [] + for name, data in staff_map.items(): + total = len(data['positive']) + len(data['negative']) + if total < 2: + continue # Need multiple mentions + + staff_summary.append({ + 'name': name, + 'total_mentions': total, + 'positive': len(data['positive']), + 'negative': len(data['negative']), + 'sentiment_ratio': len(data['positive']) / total, + 'top_codes': data['codes'].most_common(3), + 'sample_praise': data['positive'][0]['quotes'] if data['positive'] else None, + 'sample_criticism': data['negative'][0]['quotes'] if data['negative'] else None, + }) + + staff_summary.sort(key=lambda x: x['total_mentions'], reverse=True) + + return { + 'staff': staff_summary, + 'top_performer': max(staff_summary, key=lambda x: x['sentiment_ratio']) if staff_summary else None, + 'needs_attention': [s for s in staff_summary if s['sentiment_ratio'] < 0.5], + } +``` + +### 4.6 LLM Narrative Generation + +```python +NARRATIVE_PROMPT = """You are a business intelligence analyst writing an executive summary of customer feedback. + +You MUST follow these rules: +1. Only state claims supported by the provided data +2. Include specific numbers (percentages, counts) for every claim +3. Do not invent or hallucinate any statistics +4. Be direct and actionable, not vague +5. Highlight the most impactful findings first + +The report payload follows. Write a concise executive summary (~300 words) covering: +- Top issues with their sub-patterns and severity +- Notable strengths +- Trend signals (improving/worsening/persistent) +- Staff highlights +- 2-3 prioritized recommendations + +REPORT DATA: +{payload}""" + + +async def generate_narrative(payload: dict) -> str: + """Generate executive narrative from structured payload.""" + + response = await llm.chat( + model="gpt-4o", # Use stronger model for narrative + messages=[ + {"role": "system", "content": NARRATIVE_PROMPT.format( + payload=json.dumps(payload, indent=2) + )} + ], + temperature=0.3 + ) + + return response.content +``` + +--- + +## Part 5: Report Output Example + +### 5.1 Structured Payload + +```json +{ + "business_id": "rest_12345", + "period": "2026-01-01 to 2026-01-31", + "total_reviews": 234, + + "issues": [ + { + "code": "J1.01", + "name": "Wait Time", + "total_reviews": 47, + "rate": 0.201, + "ci": [0.153, 0.258], + "max_intensity": "I3", + "trend": { + "signal": "worsening", + "cr_worse": 3, + "rate_change": 0.042 + }, + "sub_patterns": [ + { + "label": "Table Seating", + "count": 20, + "percentage": 0.426, + "representative_quote": "Waited 45 minutes for a table even with reservation", + "sharpest_quote": "HOUR wait on a Tuesday. Unacceptable." + }, + { + "label": "Food After Ordering", + "count": 15, + "percentage": 0.319, + "representative_quote": "Food took 40 minutes after we ordered", + "sharpest_quote": "Over an hour for cold pasta" + }, + { + "label": "Check Payment", + "count": 12, + "percentage": 0.255, + "representative_quote": "Had to flag someone down just to pay", + "sharpest_quote": "20 minutes for the check, ridiculous" + } + ] + } + ], + + "strengths": [ + { + "code": "O2.02", + "name": "Craftsmanship", + "total_reviews": 89, + "rate": 0.380, + "ci": [0.318, 0.446], + "trend": {"signal": "stable"}, + "representative_quote": "The pasta is clearly made fresh, incredible quality" + } + ], + + "staff": { + "top_performer": { + "name": "Maria", + "mentions": 12, + "sentiment_ratio": 0.917 + }, + "needs_attention": [ + { + "name": "Tom", + "mentions": 8, + "sentiment_ratio": 0.375, + "top_issues": ["P1.02", "P3.01"] + } + ] + }, + + "open_issues": [ + { + "issue_id": "ISSUE-2026-0142", + "code": "J1.01", + "state": "IN_PROGRESS", + "priority": 7.45, + "days_open": 12 + } + ] +} +``` + +### 5.2 Generated Narrative + +> **Executive Summary: January 2026** +> +> Analysis of 234 reviews reveals **wait times as the critical issue**, affecting 20.1% of customers (95% CI: 15.3%-25.8%) — a worsening trend with 3 explicit "worse than before" signals this month. +> +> **Wait Time Breakdown:** +> - **Seating delays (43%)**: Customers report 30-60 minute waits despite reservations. *"Waited 45 minutes for a table even with reservation."* +> - **Kitchen delays (32%)**: Food taking 40+ minutes after ordering. *"Over an hour for cold pasta."* +> - **Checkout friction (26%)**: Difficulty getting the check. *"20 minutes for the check, ridiculous."* +> +> **Strengths remain strong**: Food craftsmanship praised in 38% of reviews, stable month-over-month. *"The pasta is clearly made fresh, incredible quality."* +> +> **Staff Notes**: Maria received 12 mentions with 92% positive sentiment. Tom (8 mentions, 38% positive) shows patterns in P1.02 (Respect) and P3.01 (Attentiveness) — recommend coaching session. +> +> **Prioritized Recommendations:** +> 1. **Immediate**: Audit reservation system — seating bottleneck is primary wait issue +> 2. **This Week**: Review kitchen workflow for food delivery timing +> 3. **This Month**: Implement checkout process training (e.g., table check-in rotation) +> +> One high-priority issue (ISSUE-2026-0142) is in progress with 12 days elapsed. + +--- + +## Part 6: Cost Model + +| Stage | When | Cost | Notes | +|-------|------|------|-------| +| **Embedding** | Per review ingested | $0.00 | Local model, ~50ms/review | +| **LLM Classification** | Per review ingested | ~$0.0002 | GPT-4o-mini, batched | +| **Issue Aggregation** | Per V- review | $0.00 | SQL queries | +| **Sub-Clustering** | Per report | $0.00 | HDBSCAN/KMeans, <1s | +| **Trend Analysis** | Per report | $0.00 | SQL + computation | +| **LLM Narrative** | Per report | ~$0.15 | GPT-4o, single call | + +**Total Costs:** + +| Volume | Monthly Ingest | Reports (10/month) | Total | +|--------|---------------|-------------------|-------| +| 1K reviews | $0.20 | $1.50 | **$1.70** | +| 10K reviews | $2.00 | $1.50 | **$3.50** | +| 100K reviews | $20.00 | $1.50 | **$21.50** | + +--- + +## Part 7: Implementation Checklist + +### Phase 1: Core Pipeline +- [ ] Set up PostgreSQL with pgvector extension +- [ ] Implement embedding generation (multilingual-e5-small) +- [ ] Build LLM classification module with batching +- [ ] Create review ingestion pipeline +- [ ] Implement URT code reference data + +### Phase 2: Issue Lifecycle +- [ ] Implement issue aggregation logic +- [ ] Build state machine with transitions +- [ ] Create priority scoring function +- [ ] Add CR signal processing for verification +- [ ] Set up issue event logging + +### Phase 3: Report Generation +- [ ] Build statistics aggregation queries +- [ ] Implement sub-pattern clustering +- [ ] Add trend analysis with CR integration +- [ ] Create staff analysis module +- [ ] Build narrative generation prompt + +### Phase 4: Integration +- [ ] API endpoints for ingestion +- [ ] Report generation endpoint +- [ ] Issue management endpoints +- [ ] Dashboard queries +- [ ] Alert/notification hooks + +--- + +## Part 8: Key Innovations + +| Innovation | Benefit | +|------------|---------| +| **LLM at ingest, not report** | Accurate classification amortized across all reports | +| **URT as structure** | Stable, interpretable categories; no clustering drift | +| **Multi-coding** | Handle complex reviews without fragmentation | +| **Sub-clustering within codes** | Actionable patterns beyond category level | +| **CR for verification** | Automatic resolution validation from customer feedback | +| **Review as unit** | Preserve context; avoid embedding quality loss | +| **Issue lifecycle** | Operational tracking with statistical rigor | + +--- + +## Document Control + +| Field | Value | +|-------|-------| +| **Document** | ReviewIQ Architecture v2.0 | +| **Status** | Specification Complete | +| **Date** | 2026-01-24 | +| **Dependencies** | URT Specification v5.1, Issue Lifecycle Framework C1 | +| **Cost Target** | <$25/month at 100K reviews | +| **Accuracy Target** | >90% URT classification, >85% sub-pattern relevance | + +--- + +*End of ReviewIQ Architecture v2.0* diff --git a/.artifacts/ReviewIQ-Architecture-v3.2.md b/.artifacts/ReviewIQ-Architecture-v3.2.md new file mode 100644 index 0000000..c470b9c --- /dev/null +++ b/.artifacts/ReviewIQ-Architecture-v3.2.md @@ -0,0 +1,2306 @@ +# ReviewIQ: Review Intelligence Pipeline + +**Version**: 3.2.0 +**Status**: Architecture Specification (Reviewed) +**Date**: 2026-01-24 + +--- + +## Executive Summary + +ReviewIQ v3.2 transforms Google Reviews into actionable business intelligence through a scalable, KPI-ready pipeline. This version introduces the **span layer** — a fine-grained extraction model that identifies and classifies individual semantic units within each review, enabling richer issue routing, causal analysis, and entity-aware aggregation. + +**What's New in v3.2**: +- **Span layer**: `review_spans` table extracts individual semantic units from review text +- **URT ENUM types**: Strongly-typed classification fields with database-enforced constraints +- **Causal chain support**: `profile='full'` spans can capture cause/effect relationships +- **Entity extraction**: Named entities (staff, products, locations) linked to spans +- **Reprocessing pattern**: Soft-switch `is_active` flag for atomic span replacement +- **Deterministic issue routing**: SHA256-based issue IDs from grouping keys +- **1:1 span-to-issue mapping**: Each span belongs to exactly one issue + +**Design Principles**: +- **Google Reviews only** (for now) — but schema is source-agnostic +- **Relational over arrays** — scales, queries, joins +- **Facts-first reporting** — pre-aggregated spine for fast dashboards +- **KPI-joinable** — `(business_id, place_id, period_date, bucket_type)` as universal key +- **Tenant-scoped locations** — same place_id can exist for multiple businesses +- **Span-first classification** — spans are the atomic unit of analysis; review-level is derived + +--- + +## Part 1: System Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────────────┐ +│ REVIEWIQ v3.2 PIPELINE │ +├─────────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ │ +│ │ Google │ │ +│ │ Reviews │ │ +│ │ (API) │ │ +│ └──────┬───────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐ │ +│ │ A) SOURCE & STORAGE │ │ +│ │ │ │ +│ │ google_connector ───▶ reviews_raw (immutable JSON + metadata) │ │ +│ │ │ │ +│ └──────────────────────────────────┬──────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐ │ +│ │ B) ENRICHMENT │ │ +│ │ │ │ +│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ +│ │ │Normalize │──▶│ LLM │──▶│ Embed │──▶│ Trust │ │ │ +│ │ │ + Map │ │ Classify │ │ (local) │ │ Score │ │ │ +│ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │ │ +│ │ │ │ │ │ │ +│ │ └──────────────┴─────────────┬───────────────┘ │ │ +│ │ ▼ │ │ +│ │ ┌────────────────────────────────┐ │ │ +│ │ │ reviews_enriched │ │ │ +│ │ │ │ │ │ │ +│ │ │ ▼ │ │ │ +│ │ │ review_spans (NEW) │◀── Span Extraction │ │ +│ │ │ (per-span classification) │ │ │ +│ │ └────────────────────────────────┘ │ │ +│ │ │ │ +│ └──────────────────────────────┬──────────────────────────────────────────────┘ │ +│ │ │ +│ ┌────────────┴────────────┐ │ +│ ▼ ▼ │ +│ ┌────────────────────────────┐ ┌────────────────────────────────┐ │ +│ │ C) OPERATIONALIZATION │ │ D) ANALYTICS SPINE │ │ +│ │ │ │ │ │ +│ │ review_spans │ │ Daily/Weekly Jobs: │ │ +│ │ │ │ │ │ │ +│ │ ▼ │ │ review_spans │ │ +│ │ issue_spans (1:1 link) │ │ │ │ │ +│ │ │ │ │ ▼ │ │ +│ │ ▼ │ │ fact_timeseries │ │ +│ │ issues (update counters) │ │ (pre-aggregated metrics) │ │ +│ │ │ │ │ │ │ +│ │ ▼ │ │ Keys: │ │ +│ │ issue_events (log) │ │ • business_id │ │ +│ │ │ │ • place_id (or 'ALL') │ │ +│ └────────────────────────────┘ │ • subject_type/id │ │ +│ │ │ • period_date │ │ +│ │ │ • bucket_type │ │ +│ │ └────────────────────────────────┘ │ +│ └────────────┬────────────┘ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐ │ +│ │ E) REPORTING │ │ +│ │ │ │ +│ │ fact_timeseries ──┬──▶ Statistics & Trends │ │ +│ │ │ │ │ +│ │ issues + spans ───┼──▶ Issue Rankings & Drill-Down │ │ +│ │ │ │ │ +│ │ embeddings ───────┼──▶ Sub-Pattern Clustering │ │ +│ │ │ │ │ +│ │ competitors ──────┴──▶ Benchmark Comparisons │ │ +│ │ │ │ │ +│ │ ▼ │ │ +│ │ LLM Narrative Generation │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Part 2: Data Model (SQL DDL) + +### 2.0 Required Extensions + +```sql +-- btree_gist: Enables GiST index on btree-compatible types (for exclusion constraints) +CREATE EXTENSION IF NOT EXISTS btree_gist; + +-- pgcrypto: Provides cryptographic functions (for SHA256-based ID generation) +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +-- pgvector: Vector similarity search (for embeddings) +CREATE EXTENSION IF NOT EXISTS vector; +``` + +### 2.1 ENUM Types + +```sql +-- URT classification enums (strongly-typed, database-enforced) +CREATE TYPE urt_valence AS ENUM ('V+', 'V-', 'V0', 'V±'); +CREATE TYPE urt_intensity AS ENUM ('I1', 'I2', 'I3'); +CREATE TYPE urt_specificity AS ENUM ('S1', 'S2', 'S3'); +CREATE TYPE urt_actionability AS ENUM ('A1', 'A2', 'A3'); +CREATE TYPE urt_temporal AS ENUM ('TC', 'TR', 'TH', 'TF'); +CREATE TYPE urt_evidence AS ENUM ('ES', 'EI', 'EC'); +CREATE TYPE urt_comparative AS ENUM ('CR-N', 'CR-B', 'CR-W', 'CR-S'); +CREATE TYPE urt_profile AS ENUM ('standard', 'full'); +CREATE TYPE urt_confidence AS ENUM ('high', 'medium', 'low'); +CREATE TYPE urt_relation AS ENUM ('cause_of', 'effect_of', 'contrast', 'resolution'); +CREATE TYPE urt_entity_type AS ENUM ('location', 'staff', 'product', 'process', 'time', 'other'); +``` + +### 2.2 Dimension Tables + +```sql +-- Business locations (multi-tenant: same place_id can exist for multiple businesses) +-- Includes both owned locations and tracked competitor locations +CREATE TABLE locations ( + business_id TEXT NOT NULL, -- Internal business identifier + place_id TEXT NOT NULL, -- Google Place ID + location_type TEXT NOT NULL DEFAULT 'owned' + CHECK (location_type IN ('owned', 'competitor')), + display_name TEXT NOT NULL, + address TEXT, + city TEXT, + state TEXT, + country TEXT, + timezone TEXT, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + + PRIMARY KEY (business_id, place_id) +); + +CREATE INDEX idx_locations_place ON locations(place_id); +CREATE INDEX idx_locations_owned ON locations(business_id) + WHERE location_type = 'owned'; + +-- URT code reference +CREATE TABLE urt_codes ( + code TEXT PRIMARY KEY, -- 'J1.01', 'P1.02', etc. + domain CHAR(1) NOT NULL, -- O, P, J, E, A, V, R + category TEXT NOT NULL, + subcategory TEXT NOT NULL, + display_name TEXT NOT NULL, + description TEXT, + keywords TEXT[] -- For search/matching +); + +-- Competitor mapping (separate from locations - no fake business_ids) +CREATE TABLE competitors ( + id SERIAL PRIMARY KEY, + business_id TEXT NOT NULL, -- Your business + competitor_place_id TEXT NOT NULL, -- Competitor's Google Place ID + competitor_name TEXT NOT NULL, + relationship TEXT DEFAULT 'direct', -- 'direct', 'indirect', 'aspirational' + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT NOW(), + + UNIQUE(business_id, competitor_place_id) +); + +CREATE INDEX idx_competitors_business ON competitors(business_id); +``` + +### 2.3 Reviews Tables (Raw + Enriched) + +```sql +-- Immutable raw review storage (audit + reprocessing) +CREATE TABLE reviews_raw ( + id SERIAL PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'google', + review_id TEXT NOT NULL, -- Google review ID + place_id TEXT NOT NULL, -- Google Place ID + + -- Raw payload + raw_payload JSONB NOT NULL, -- Complete API response + review_text TEXT, -- Extracted for indexing + rating SMALLINT, + review_time TIMESTAMP, + reviewer_name TEXT, + reviewer_id TEXT, + + -- Versioning (Google reviews can be edited) + review_version INT DEFAULT 1, + + -- Ingestion metadata + pulled_at TIMESTAMP DEFAULT NOW(), + + UNIQUE(source, review_id, review_version) +); + +CREATE INDEX idx_reviews_raw_place ON reviews_raw(place_id, review_time DESC); +CREATE INDEX idx_reviews_raw_lookup ON reviews_raw(source, review_id); + +-- Enriched review with LLM classification + embeddings (versioned) +-- Supports edited reviews: each version is a separate row +CREATE TABLE reviews_enriched ( + -- Versioned primary key (handles edited reviews) + source TEXT NOT NULL DEFAULT 'google', + review_id TEXT NOT NULL, -- Matches reviews_raw.review_id + review_version INT NOT NULL DEFAULT 1, + is_latest BOOLEAN NOT NULL DEFAULT TRUE, + + -- Link to raw (specific version) + raw_id INT NOT NULL REFERENCES reviews_raw(id), + + -- Identity + business_id TEXT NOT NULL, + place_id TEXT NOT NULL, + + -- Core content + text TEXT NOT NULL, + text_normalized TEXT, -- Cleaned for processing + rating SMALLINT, + review_time TIMESTAMP NOT NULL, + language TEXT, + + -- URT Classification (from LLM) — review-level summary, derived from spans in v3.2 + urt_primary TEXT NOT NULL, -- 'J1.01', 'P1.02', etc. + urt_secondary TEXT[] DEFAULT '{}', -- Max 2, different domains + valence TEXT NOT NULL, -- 'V+', 'V-', 'V0', 'V±' + intensity TEXT NOT NULL, -- 'I1', 'I2', 'I3' + comparative TEXT DEFAULT 'CR-N', -- 'CR-N', 'CR-B', 'CR-W', 'CR-S' + + -- Extracted entities (summary from spans) + staff_mentions TEXT[] DEFAULT '{}', + quotes JSONB, -- {"code": "phrase", ...} + + -- Embedding + embedding VECTOR(384), + + -- Quality control + trust_score FLOAT DEFAULT 1.0, -- 0.0 to 1.0 + dedup_group_id TEXT, -- Tenant-scoped: format "{business_id}:{hash}" + is_suspicious BOOLEAN DEFAULT FALSE, + + -- Processing metadata + classification_model TEXT, + classification_confidence JSONB, -- Per-field confidence scores + processed_at TIMESTAMP DEFAULT NOW(), + model_version TEXT, + + -- KPI-ready hooks (nullable, computed later) + kpi_impact_estimate FLOAT, + kpi_last_computed_at TIMESTAMP, + + PRIMARY KEY (source, review_id, review_version) +); + +-- Indexes for common query patterns +CREATE INDEX idx_enriched_latest ON reviews_enriched(source, review_id) + WHERE is_latest = TRUE; +CREATE INDEX idx_enriched_business_date ON reviews_enriched(business_id, review_time DESC) + WHERE is_latest = TRUE; +CREATE INDEX idx_enriched_place_date ON reviews_enriched(place_id, review_time DESC) + WHERE is_latest = TRUE; +CREATE INDEX idx_enriched_urt_primary ON reviews_enriched(business_id, urt_primary) + WHERE is_latest = TRUE; +CREATE INDEX idx_enriched_valence ON reviews_enriched(business_id, valence, review_time) + WHERE is_latest = TRUE; +CREATE INDEX idx_enriched_comparative ON reviews_enriched(comparative) + WHERE comparative != 'CR-N' AND is_latest = TRUE; +CREATE INDEX idx_enriched_trust ON reviews_enriched(trust_score) + WHERE trust_score < 0.5 AND is_latest = TRUE; +CREATE INDEX idx_enriched_embedding ON reviews_enriched + USING hnsw (embedding vector_cosine_ops); + +-- FK to locations (tenant-scoped) +ALTER TABLE reviews_enriched + ADD CONSTRAINT fk_enriched_location + FOREIGN KEY (business_id, place_id) REFERENCES locations(business_id, place_id); + +-- Enforce tenant-scoped dedup format +ALTER TABLE reviews_enriched + ADD CONSTRAINT chk_dedup_scoped + CHECK (dedup_group_id IS NULL OR dedup_group_id LIKE business_id || ':%'); +``` + +### 2.4 Span Layer (NEW in v3.2) + +The span layer extracts individual semantic units from review text. Each span represents a single classifiable statement with its own URT code, valence, intensity, and optional entity reference. + +```sql +-- Review spans: fine-grained semantic units within reviews +CREATE TABLE review_spans ( + span_id TEXT PRIMARY KEY, -- Deterministic ID (see §9.5) + + -- Parent review reference + business_id TEXT NOT NULL, + place_id TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'google', + review_id TEXT NOT NULL, + review_version INT NOT NULL, + + -- Span position (within review text) + span_index INT NOT NULL CHECK (span_index >= 0), + span_text TEXT NOT NULL, + span_start INT NOT NULL CHECK (span_start >= 0), + span_end INT NOT NULL, + + -- Profile level (standard vs full classification) + profile urt_profile NOT NULL DEFAULT 'standard', + + -- URT Classification (strongly-typed) + urt_primary TEXT NOT NULL, -- Tier-3 code: 'J1.01', 'P2.03', etc. + urt_secondary TEXT[] NOT NULL DEFAULT '{}', + valence urt_valence NOT NULL, + intensity urt_intensity NOT NULL, + comparative urt_comparative NOT NULL DEFAULT 'CR-N', + + -- Extended classification (standard profile) + specificity urt_specificity NOT NULL DEFAULT 'S2', + actionability urt_actionability NOT NULL DEFAULT 'A2', + temporal urt_temporal NOT NULL DEFAULT 'TC', + evidence urt_evidence NOT NULL DEFAULT 'ES', + + -- Causal relations (full profile only) + relation_type urt_relation, + related_span_id TEXT REFERENCES review_spans(span_id), + causal_chain JSONB, -- Full profile: structured cause/effect + + -- Entity extraction + entity TEXT, -- Raw entity mention + entity_type urt_entity_type, + entity_normalized TEXT, -- Normalized form for grouping + + -- Span state + is_primary BOOLEAN NOT NULL DEFAULT FALSE, -- Primary span for this review + is_active BOOLEAN NOT NULL DEFAULT TRUE, -- Soft-delete for reprocessing + review_time TIMESTAMP NOT NULL, -- Denormalized from parent review + + -- Processing metadata + confidence urt_confidence DEFAULT 'medium', + usn TEXT, -- URT Semantic Notation string + model_version TEXT, + ingest_batch_id TEXT, -- For atomic reprocessing + created_at TIMESTAMP DEFAULT NOW(), + + -- Uniqueness within review + UNIQUE (source, review_id, review_version, span_index) +); + +-- Constraints for review_spans +ALTER TABLE review_spans + ADD CONSTRAINT chk_span_end + CHECK (span_end > span_start); + +ALTER TABLE review_spans + ADD CONSTRAINT chk_primary_tier3 + CHECK (urt_primary ~ '^[OPJEAVR][1-4]\.[0-9]{2}$'); + +ALTER TABLE review_spans + ADD CONSTRAINT chk_secondary_max2 + CHECK (cardinality(urt_secondary) <= 2); + +-- Validate each element in urt_secondary matches tier-3 pattern +ALTER TABLE review_spans + ADD CONSTRAINT chk_secondary_tier3 + CHECK ( + urt_secondary = '{}' OR + (SELECT bool_and(elem ~ '^[OPJEAVR][1-4]\.[0-9]{2}$') FROM unnest(urt_secondary) AS elem) + ); + +-- causal_chain only allowed for full profile +ALTER TABLE review_spans + ADD CONSTRAINT chk_full_only_fields + CHECK ( + profile = 'full' OR causal_chain IS NULL + ); + +-- No self-referential relations +ALTER TABLE review_spans + ADD CONSTRAINT chk_no_self_relation + CHECK (related_span_id IS NULL OR related_span_id != span_id); + +-- USN format validation based on profile +-- Standard: V[+-0±]:I[123]:CODE (e.g., "V-:I2:J1.01") +-- Full: V[+-0±]:I[123]:CODE:S[123]:A[123]:T[CRHF]:E[SIC] (e.g., "V-:I3:J1.01:S2:A2:TC:ES") +ALTER TABLE review_spans + ADD CONSTRAINT chk_usn_format + CHECK ( + usn IS NULL OR + (profile = 'standard' AND usn ~ '^V[+\-0±]:I[123]:[OPJEAVR][1-4]\.[0-9]{2}$') OR + (profile = 'full' AND usn ~ '^V[+\-0±]:I[123]:[OPJEAVR][1-4]\.[0-9]{2}:S[123]:A[123]:T[CRHF]:E[SIC]$') + ); + +-- Foreign keys for review_spans +ALTER TABLE review_spans + ADD CONSTRAINT fk_spans_review + FOREIGN KEY (source, review_id, review_version) + REFERENCES reviews_enriched(source, review_id, review_version) + ON DELETE CASCADE; + +ALTER TABLE review_spans + ADD CONSTRAINT fk_spans_location + FOREIGN KEY (business_id, place_id) + REFERENCES locations(business_id, place_id); + +ALTER TABLE review_spans + ADD CONSTRAINT fk_spans_urt_primary + FOREIGN KEY (urt_primary) + REFERENCES urt_codes(code); + +-- Indexes for review_spans +CREATE UNIQUE INDEX uq_spans_active_order + ON review_spans(source, review_id, review_version, span_index) + WHERE is_active = TRUE; + +CREATE UNIQUE INDEX uq_spans_one_primary_active + ON review_spans(source, review_id, review_version) + WHERE is_active = TRUE AND is_primary = TRUE; + +CREATE INDEX idx_spans_review + ON review_spans(source, review_id, review_version) + WHERE is_active = TRUE; + +CREATE INDEX idx_spans_business_time + ON review_spans(business_id, review_time DESC) + WHERE is_active = TRUE; + +CREATE INDEX idx_spans_issue_routing + ON review_spans(business_id, place_id, urt_primary, entity_normalized) + WHERE is_active = TRUE AND valence IN ('V-', 'V±'); + +CREATE INDEX idx_spans_entity + ON review_spans(business_id, entity_normalized) + WHERE entity_normalized IS NOT NULL AND is_active = TRUE; + +CREATE INDEX idx_spans_batch + ON review_spans(ingest_batch_id) + WHERE ingest_batch_id IS NOT NULL; + +-- Exclusion constraint: no overlapping spans within same active review version +CREATE INDEX ex_spans_no_overlap + ON review_spans + USING gist ( + source, + review_id, + review_version, + int4range(span_start, span_end) WITH && + ) + WHERE is_active = TRUE; + +-- Note: The above index enables checking for overlaps but does not enforce exclusion. +-- For strict enforcement, use: +ALTER TABLE review_spans + ADD CONSTRAINT ex_spans_no_overlap_constraint + EXCLUDE USING gist ( + source WITH =, + review_id WITH =, + review_version WITH =, + int4range(span_start, span_end) WITH && + ) + WHERE (is_active = TRUE); +``` + +### 2.5 Issue Tables (Relational, Span-Based) + +**v3.2 Issue Key**: `(business_id, place_id, urt_primary, entity_normalized)` — entity matching is now active. `entity_normalized` defaults to NULL in v3.2; distinct entities create distinct issues in v3.3+. + +```sql +-- Issues (aggregated problems) +CREATE TABLE issues ( + issue_id TEXT PRIMARY KEY, -- Deterministic SHA256-based ID + + -- Grouping keys (v3.2: code + place + entity) + business_id TEXT NOT NULL, + place_id TEXT NOT NULL, + primary_subcode TEXT NOT NULL, -- URT code + domain CHAR(1) NOT NULL, + + -- State machine + state TEXT NOT NULL DEFAULT 'DETECTED', + priority_score FLOAT NOT NULL, + confidence_score FLOAT NOT NULL, + + -- Aggregated metrics (updated via triggers/jobs) + span_count INT NOT NULL DEFAULT 1, + max_intensity TEXT NOT NULL, + avg_trust_score FLOAT DEFAULT 1.0, + + -- CR counters (rolling 30-day window) + cr_better_count INT DEFAULT 0, + cr_worse_count INT DEFAULT 0, + cr_same_count INT DEFAULT 0, + + -- Star drag proxy (avg rating when this issue present vs absent) + star_drag_estimate FLOAT, + + -- Ownership + owner_team TEXT, + owner_individual TEXT, + + -- Timestamps + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + acknowledged_at TIMESTAMP, + resolved_at TIMESTAMP, + verified_at TIMESTAMP, + + -- Resolution + reopen_count INT DEFAULT 0, + resolution_code TEXT, + resolution_notes TEXT, + decline_reason TEXT, + + -- Context (v3.2: entity extraction active) + entity TEXT, -- Product, staff member, feature + entity_normalized TEXT, -- Normalized for grouping (defaults NULL in v3.2) + + -- KPI-ready hooks (nullable) + kpi_impact_estimate FLOAT, + kpi_impact_confidence FLOAT, + kpi_last_computed_at TIMESTAMP +); + +CREATE INDEX idx_issues_business ON issues(business_id, state, priority_score DESC); +CREATE INDEX idx_issues_place ON issues(place_id, state); +CREATE INDEX idx_issues_code ON issues(business_id, primary_subcode); +CREATE INDEX idx_issues_open ON issues(business_id) + WHERE state NOT IN ('VERIFIED', 'DECLINED'); +CREATE INDEX idx_issues_entity ON issues(business_id, entity_normalized) + WHERE entity_normalized IS NOT NULL; + +-- FK to locations (tenant-scoped) +ALTER TABLE issues + ADD CONSTRAINT fk_issues_location + FOREIGN KEY (business_id, place_id) REFERENCES locations(business_id, place_id); + +-- Issue spans: 1:1 link from span to issue (each span belongs to exactly one issue) +CREATE TABLE issue_spans ( + id SERIAL PRIMARY KEY, + issue_id TEXT NOT NULL REFERENCES issues(issue_id) ON DELETE CASCADE, + + -- Span reference (unique constraint enforces 1:1) + span_id TEXT NOT NULL REFERENCES review_spans(span_id) ON DELETE CASCADE, + + -- Denormalized for queries (copied from span) + source TEXT NOT NULL DEFAULT 'google', + review_id TEXT NOT NULL, + review_version INT NOT NULL, + + -- Classification snapshot + is_primary_match BOOLEAN DEFAULT TRUE, -- Primary vs secondary code match + intensity TEXT NOT NULL, -- Copied from span for fast queries + review_time TIMESTAMP NOT NULL, -- Denormalized for timeline queries + weight FLOAT DEFAULT 1.0, -- For weighted aggregation + + created_at TIMESTAMP DEFAULT NOW(), + + -- One span → exactly one issue (1:1 mapping) + CONSTRAINT uq_issue_spans_span UNIQUE (span_id) +); + +CREATE INDEX idx_issue_spans_issue ON issue_spans(issue_id); +CREATE INDEX idx_issue_spans_review ON issue_spans(source, review_id, review_version); +CREATE INDEX idx_issue_spans_time ON issue_spans(issue_id, review_time DESC); + +-- Issue events (audit log) +CREATE TABLE issue_events ( + event_id SERIAL PRIMARY KEY, + issue_id TEXT NOT NULL REFERENCES issues(issue_id), + + event_type TEXT NOT NULL, -- 'state_change', 'span_added', 'priority_update' + from_state TEXT, + to_state TEXT, + + actor TEXT, -- User or 'system' + notes TEXT, + + -- Triggering span/review reference + span_id TEXT, + source TEXT DEFAULT 'google', + review_id TEXT, + review_version INT, + + metadata JSONB, -- Additional context + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_events_issue ON issue_events(issue_id, created_at DESC); +CREATE INDEX idx_events_span ON issue_events(span_id) + WHERE span_id IS NOT NULL; +CREATE INDEX idx_events_review ON issue_events(source, review_id, review_version) + WHERE review_id IS NOT NULL; +``` + +### 2.6 Unified Analytics Spine + +**Design Decision**: Sentinel value conventions (do not normalize): +- `place_id = 'ALL'` — spatial rollup (all locations) +- `subject_id = 'all'` — semantic rollup (all subjects within type) + +Case matters: `'ALL'` ≠ `'all'`. This avoids NULL handling while keeping the schema simple. + +```sql +-- Fact table: pre-aggregated time-series metrics +CREATE TABLE fact_timeseries ( + id SERIAL PRIMARY KEY, + + -- Universal join keys (KPI-ready) + business_id TEXT NOT NULL, + place_id TEXT NOT NULL, -- 'ALL' = all locations rollup + period_date DATE NOT NULL, + bucket_type TEXT NOT NULL, -- 'day', 'week', 'month' + + -- Subject (what we're measuring) + subject_type TEXT NOT NULL, -- 'urt_code', 'domain', 'overall', 'issue' + subject_id TEXT NOT NULL, -- Code, domain letter, issue_id, or 'all' + + -- Volume metrics + review_count INT NOT NULL DEFAULT 0, + span_count INT NOT NULL DEFAULT 0, -- v3.2: span-level counting + negative_count INT NOT NULL DEFAULT 0, + positive_count INT NOT NULL DEFAULT 0, + neutral_count INT NOT NULL DEFAULT 0, + mixed_count INT NOT NULL DEFAULT 0, + + -- Strength metrics (intensity-weighted) + strength_score FLOAT NOT NULL DEFAULT 0, + negative_strength FLOAT NOT NULL DEFAULT 0, + positive_strength FLOAT NOT NULL DEFAULT 0, + + -- Rating metrics + avg_rating FLOAT, + rating_count INT DEFAULT 0, + + -- Intensity distribution + i1_count INT DEFAULT 0, + i2_count INT DEFAULT 0, + i3_count INT DEFAULT 0, + + -- CR signals + cr_better INT DEFAULT 0, + cr_worse INT DEFAULT 0, + cr_same INT DEFAULT 0, + + -- Trust-weighted variants (v3.2: now populated) + trust_weighted_strength FLOAT, + trust_weighted_negative FLOAT, + + -- Metadata + computed_at TIMESTAMP DEFAULT NOW(), + + UNIQUE(business_id, place_id, period_date, bucket_type, subject_type, subject_id) +); + +-- Validate 'ALL' sentinel +ALTER TABLE fact_timeseries + ADD CONSTRAINT chk_place_id_format + CHECK (place_id = 'ALL' OR place_id ~ '^[a-zA-Z0-9_-]+$'); + +-- Optimized indexes for reporting queries +CREATE INDEX idx_facts_lookup ON fact_timeseries( + business_id, place_id, subject_type, subject_id, period_date DESC +); +CREATE INDEX idx_facts_period ON fact_timeseries( + business_id, period_date, bucket_type +); +CREATE INDEX idx_facts_code ON fact_timeseries(subject_type, subject_id) + WHERE subject_type = 'urt_code'; +CREATE INDEX idx_facts_all_locations ON fact_timeseries(business_id, period_date) + WHERE place_id = 'ALL'; +CREATE INDEX idx_facts_issue ON fact_timeseries(subject_id) + WHERE subject_type = 'issue'; +``` + +**v3.2 Fact Population Scope**: +| subject_type | Populated | Notes | +|--------------|-----------|-------| +| `overall` | Mandatory | Business-wide + per-location | +| `urt_code` | Mandatory | Per URT code (from spans) | +| `domain` | Derived | Rollup from urt_code at query time | +| `issue` | Recommended | Per-issue timelines | + +**v3.2 Rollup Rules**: +- `place_id='ALL'` includes **owned locations only** (not competitors) +- Competitor facts live at their `competitor_place_id`, never in `'ALL'` rollup +- Competitor comparison queries explicitly join on `competitor_place_id` +- Span-level metrics (`span_count`, intensity distribution) are now primary + +**v3.2 Trust Score Usage**: +- `trust_score` applied to **issue priority scoring** and **filtering** +- `trust_weighted_strength` / `trust_weighted_negative` now **populated** in v3.2 +- Formula: `SUM(trust_score * intensity_weight)` per fact row + +### 2.7 Sub-Patterns (Persistent Clustering Results) + +```sql +-- Stored sub-pattern clustering results +CREATE TABLE subpatterns ( + id SERIAL PRIMARY KEY, + + -- Parent + subject_type TEXT NOT NULL, -- 'urt_code', 'issue' + subject_id TEXT NOT NULL, + business_id TEXT NOT NULL, + place_id TEXT, -- NULL = all locations + + -- Period + period_start DATE NOT NULL, + period_end DATE NOT NULL, + + -- Cluster info + cluster_id INT NOT NULL, + label TEXT NOT NULL, + + -- Metrics + review_count INT NOT NULL, + span_count INT NOT NULL, -- v3.2: span-level + percentage FLOAT NOT NULL, + avg_intensity FLOAT, + + -- Representative content + representative_span_id TEXT, -- v3.2: span reference + representative_quote TEXT, + sharpest_span_id TEXT, + sharpest_quote TEXT, + + -- Embedding (for trend matching) + centroid VECTOR(384), + + -- Metadata + computed_at TIMESTAMP DEFAULT NOW(), + + UNIQUE(subject_type, subject_id, business_id, place_id, period_start, period_end, cluster_id) +); + +CREATE INDEX idx_subpatterns_lookup ON subpatterns( + subject_type, subject_id, business_id, period_end DESC +); +``` + +--- + +## Part 3: Triggers and Functions + +### 3.1 Span Validation Triggers + +```sql +-- Trigger 1: Validate span_end <= length(review text) +CREATE OR REPLACE FUNCTION trg_review_spans_validate_bounds() +RETURNS TRIGGER AS $$ +DECLARE + review_text_length INT; +BEGIN + SELECT length(text) INTO review_text_length + FROM reviews_enriched + WHERE source = NEW.source + AND review_id = NEW.review_id + AND review_version = NEW.review_version; + + IF review_text_length IS NULL THEN + RAISE EXCEPTION 'Parent review not found: (%, %, %)', + NEW.source, NEW.review_id, NEW.review_version; + END IF; + + IF NEW.span_end > review_text_length THEN + RAISE EXCEPTION 'span_end (%) exceeds review text length (%)', + NEW.span_end, review_text_length; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_review_spans_validate_bounds + BEFORE INSERT OR UPDATE ON review_spans + FOR EACH ROW + EXECUTE FUNCTION trg_review_spans_validate_bounds(); + +-- Trigger 2: Validate span_text matches parent substring (conditional) +-- Enabled via session setting: SET reviewiq.validate_span_text = 'on'; +CREATE OR REPLACE FUNCTION trg_review_spans_validate_text() +RETURNS TRIGGER AS $$ +DECLARE + review_text TEXT; + expected_text TEXT; + validate_enabled TEXT; +BEGIN + -- Check if validation is enabled via session setting + BEGIN + validate_enabled := current_setting('reviewiq.validate_span_text', true); + EXCEPTION WHEN OTHERS THEN + validate_enabled := 'off'; + END; + + IF validate_enabled != 'on' THEN + RETURN NEW; + END IF; + + SELECT text INTO review_text + FROM reviews_enriched + WHERE source = NEW.source + AND review_id = NEW.review_id + AND review_version = NEW.review_version; + + expected_text := substring(review_text FROM NEW.span_start + 1 FOR NEW.span_end - NEW.span_start); + + IF NEW.span_text != expected_text THEN + RAISE EXCEPTION 'span_text mismatch: expected "%" but got "%"', + left(expected_text, 50), left(NEW.span_text, 50); + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_review_spans_validate_text + BEFORE INSERT OR UPDATE ON review_spans + FOR EACH ROW + EXECUTE FUNCTION trg_review_spans_validate_text(); + +-- Trigger 3: Validate causal_chain JSONB structure +CREATE OR REPLACE FUNCTION trg_review_spans_validate_causal_chain() +RETURNS TRIGGER AS $$ +BEGIN + IF NEW.causal_chain IS NOT NULL THEN + -- Validate structure using helper function + IF NOT urt_validate_causal_chain(NEW.causal_chain) THEN + RAISE EXCEPTION 'Invalid causal_chain structure'; + END IF; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_review_spans_validate_causal_chain + BEFORE INSERT OR UPDATE ON review_spans + FOR EACH ROW + WHEN (NEW.causal_chain IS NOT NULL) + EXECUTE FUNCTION trg_review_spans_validate_causal_chain(); +``` + +### 3.2 Causal Chain Validation Function + +```sql +-- Validate causal chain structure, codes, and ordering +CREATE OR REPLACE FUNCTION urt_validate_causal_chain(chain JSONB) +RETURNS BOOLEAN AS $$ +DECLARE + link JSONB; + link_code TEXT; + link_role TEXT; + link_order INT; + prev_order INT := -1; + valid_roles TEXT[] := ARRAY['cause', 'effect', 'context', 'outcome']; +BEGIN + -- Must be an array + IF jsonb_typeof(chain) != 'array' THEN + RETURN FALSE; + END IF; + + -- Empty array is valid + IF jsonb_array_length(chain) = 0 THEN + RETURN TRUE; + END IF; + + -- Validate each link + FOR link IN SELECT * FROM jsonb_array_elements(chain) + LOOP + -- Required fields + IF NOT (link ? 'code' AND link ? 'role' AND link ? 'order') THEN + RETURN FALSE; + END IF; + + link_code := link->>'code'; + link_role := link->>'role'; + link_order := (link->>'order')::INT; + + -- Validate code format (tier-3) + IF link_code !~ '^[OPJEAVR][1-4]\.[0-9]{2}$' THEN + RETURN FALSE; + END IF; + + -- Validate role + IF NOT (link_role = ANY(valid_roles)) THEN + RETURN FALSE; + END IF; + + -- Validate order is increasing + IF link_order <= prev_order THEN + RETURN FALSE; + END IF; + prev_order := link_order; + END LOOP; + + RETURN TRUE; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +``` + +### 3.3 Span Relation Validation + +```sql +-- Validate related_span_id references span from same review +CREATE OR REPLACE FUNCTION validate_review_relations( + p_source TEXT, + p_review_id TEXT, + p_review_version INT +) +RETURNS BOOLEAN AS $$ +DECLARE + invalid_count INT; +BEGIN + SELECT COUNT(*) INTO invalid_count + FROM review_spans s + WHERE s.source = p_source + AND s.review_id = p_review_id + AND s.review_version = p_review_version + AND s.related_span_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM review_spans r + WHERE r.span_id = s.related_span_id + AND r.source = s.source + AND r.review_id = s.review_id + AND r.review_version = s.review_version + ); + + RETURN invalid_count = 0; +END; +$$ LANGUAGE plpgsql; +``` + +### 3.4 Active Span Set Validation + +```sql +-- Validate exactly one active span set per review version +CREATE OR REPLACE FUNCTION validate_active_spans( + p_source TEXT, + p_review_id TEXT, + p_review_version INT +) +RETURNS BOOLEAN AS $$ +DECLARE + active_count INT; + primary_count INT; +BEGIN + -- Count active spans + SELECT COUNT(*), COUNT(*) FILTER (WHERE is_primary) + INTO active_count, primary_count + FROM review_spans + WHERE source = p_source + AND review_id = p_review_id + AND review_version = p_review_version + AND is_active = TRUE; + + -- Must have at least one active span + IF active_count = 0 THEN + RETURN FALSE; + END IF; + + -- Must have exactly one primary span + IF primary_count != 1 THEN + RETURN FALSE; + END IF; + + RETURN TRUE; +END; +$$ LANGUAGE plpgsql; +``` + +### 3.5 Primary Span Selection + +```sql +-- Deterministic primary span selection: I3 > I2 > I1, V- > V± > V0 > V+, then span_index +CREATE OR REPLACE FUNCTION set_primary_span( + p_source TEXT, + p_review_id TEXT, + p_review_version INT +) +RETURNS TEXT AS $$ +DECLARE + selected_span_id TEXT; +BEGIN + -- Clear existing primary + UPDATE review_spans + SET is_primary = FALSE + WHERE source = p_source + AND review_id = p_review_id + AND review_version = p_review_version + AND is_active = TRUE + AND is_primary = TRUE; + + -- Select new primary using deterministic ordering + SELECT span_id INTO selected_span_id + FROM review_spans + WHERE source = p_source + AND review_id = p_review_id + AND review_version = p_review_version + AND is_active = TRUE + ORDER BY + -- Intensity: I3 > I2 > I1 + CASE intensity + WHEN 'I3' THEN 1 + WHEN 'I2' THEN 2 + WHEN 'I1' THEN 3 + END, + -- Valence: V- > V± > V0 > V+ + CASE valence + WHEN 'V-' THEN 1 + WHEN 'V±' THEN 2 + WHEN 'V0' THEN 3 + WHEN 'V+' THEN 4 + END, + -- Tiebreaker: first span + span_index + LIMIT 1; + + -- Set as primary + IF selected_span_id IS NOT NULL THEN + UPDATE review_spans + SET is_primary = TRUE + WHERE span_id = selected_span_id; + END IF; + + RETURN selected_span_id; +END; +$$ LANGUAGE plpgsql; +``` + +### 3.6 Deterministic Issue ID Generation + +```sql +-- Generate deterministic issue_id from grouping key using SHA256 +CREATE OR REPLACE FUNCTION generate_issue_id( + p_business_id TEXT, + p_place_id TEXT, + p_urt_primary TEXT, + p_entity_normalized TEXT DEFAULT NULL +) +RETURNS TEXT AS $$ +DECLARE + grouping_key TEXT; + hash_bytes BYTEA; +BEGIN + -- Build grouping key (entity_normalized defaults to empty string if NULL) + grouping_key := p_business_id || '|' || p_place_id || '|' || p_urt_primary || '|' || COALESCE(p_entity_normalized, ''); + + -- Generate SHA256 hash + hash_bytes := digest(grouping_key, 'sha256'); + + -- Return first 16 chars of hex encoding (64 bits of entropy) + RETURN 'ISS-' || left(encode(hash_bytes, 'hex'), 16); +END; +$$ LANGUAGE plpgsql IMMUTABLE; +``` + +--- + +## Part 4: Ingest Layer + +### 4.1 Google Connector + +```python +async def pull_reviews(place_id: str, since: datetime = None) -> list[dict]: + """Fetch new/updated reviews from Google Places API.""" + + reviews = await google_places_client.get_reviews(place_id, since=since) + + for review in reviews: + await store_raw_review(place_id, review) + + return reviews + + +async def store_raw_review(place_id: str, review: dict) -> int: + """Store immutable raw review payload.""" + + existing = await db.query_one(""" + SELECT id, review_version FROM reviews_raw + WHERE source = 'google' AND review_id = %s + ORDER BY review_version DESC LIMIT 1 + """, [review['review_id']]) + + version = 1 + if existing: + if content_changed(existing, review): + version = existing['review_version'] + 1 + else: + return existing['id'] + + return await db.insert(""" + INSERT INTO reviews_raw ( + source, review_id, place_id, raw_payload, + review_text, rating, review_time, reviewer_name, reviewer_id, + review_version, pulled_at + ) VALUES ( + 'google', %s, %s, %s, + %s, %s, %s, %s, %s, + %s, NOW() + ) + RETURNING id + """, [ + review['review_id'], place_id, json.dumps(review), + review.get('text'), review.get('rating'), + review.get('time'), review.get('author_name'), review.get('author_id'), + version + ]) +``` + +### 4.2 Enrichment Pipeline + +```python +async def enrich_review(raw_id: int, business_id: str) -> dict: + """ + Full enrichment: normalize → classify → embed → trust score → extract spans. + + Args: + raw_id: ID from reviews_raw + business_id: Tenant context (passed from ingest job, not looked up) + """ + + raw = await db.query_one( + "SELECT * FROM reviews_raw WHERE id = %s", [raw_id] + ) + + # 1. Normalize + text = normalize_text(raw['review_text']) + + # 2. Validate place_id exists under this tenant (owned or competitor) + location = await db.query_one( + "SELECT display_name, location_type FROM locations WHERE business_id = %s AND place_id = %s", + [business_id, raw['place_id']] + ) + if not location: + raise ValueError(f"place_id {raw['place_id']} not registered for business {business_id}") + + # 3. Parallel: LLM classify (with span extraction) + embed + classify_task = asyncio.create_task(classify_review_with_spans(text)) + embed_task = asyncio.create_task(embed_review(text)) + + classification = await classify_task + embedding = await embed_task + + # 4. Trust score + trust_score = compute_trust_score(raw, text, classification) + + # 5. Dedup check + dedup_group_id = await find_dedup_group(embedding, raw['place_id']) + + # 6. Mark previous versions as not-latest + await db.execute(""" + UPDATE reviews_enriched + SET is_latest = FALSE + WHERE source = 'google' AND review_id = %s AND is_latest = TRUE + """, [raw['review_id']]) + + # 7. Store enriched (versioned) + enriched = { + 'source': 'google', + 'review_id': raw['review_id'], + 'review_version': raw['review_version'], + 'is_latest': True, + 'raw_id': raw_id, + 'business_id': business_id, + 'place_id': raw['place_id'], + 'text': raw['review_text'], + 'text_normalized': text, + 'rating': raw['rating'], + 'review_time': raw['review_time'], + 'language': detect_language(text), + 'embedding': embedding, + 'trust_score': trust_score, + 'dedup_group_id': dedup_group_id, + # Review-level classification derived from primary span + 'urt_primary': classification['spans'][0]['urt_primary'] if classification['spans'] else 'O1.01', + 'valence': classification['review_valence'], + 'intensity': classification['review_intensity'], + **classification.get('review_meta', {}), + } + + await upsert_enriched_review(enriched) + + # 8. Extract and store spans (v3.2) + batch_id = f"batch-{raw['review_id']}-{raw['review_version']}-{int(time.time())}" + await store_review_spans( + enriched, + classification['spans'], + batch_id + ) + + return enriched +``` + +### 4.3 LLM Classification with Span Extraction + +```python +CLASSIFICATION_PROMPT = """You are a customer feedback classifier using the Universal Review Taxonomy (URT). + +Analyze the review and extract SPANS (individual semantic units). Each span is a phrase or sentence expressing one classifiable idea. + +Return JSON: + +{ + "spans": [ + { + "text": "exact phrase from review", + "start": 0, + "end": 25, + "urt_primary": "X1.23", + "urt_secondary": [], + "valence": "V-", + "intensity": "I2", + "comparative": "CR-N", + "specificity": "S2", + "actionability": "A2", + "temporal": "TC", + "evidence": "ES", + "entity": "Mike", + "entity_type": "staff", + "confidence": "high" + } + ], + "review_valence": "V-", + "review_intensity": "I2", + "review_meta": { + "staff_mentions": ["Mike"], + "comparative": "CR-N" + } +} + +URT DOMAINS: +- O (Offering): Product/service quality, function, completeness +- P (People): Staff attitude, competence, responsiveness +- J (Journey): Timing, ease, reliability, resolution +- E (Environment): Physical space, digital interface, ambiance +- A (Access): Availability, accessibility, convenience +- V (Value): Price, transparency, worth +- R (Relationship): Trust, dependability, loyalty + +SPAN RULES: +1. Each span = one classifiable semantic unit +2. Spans must not overlap +3. text must be EXACT substring from review +4. start/end are character offsets (0-indexed) +5. First span with highest intensity + negative valence becomes primary + +INTENSITY: +- I1: Mild observation, passing mention +- I2: Moderate emphasis, clear statement +- I3: Strong emotion, repeated emphasis, dealbreaker + +SPECIFICITY: +- S1: Vague ("it was bad") +- S2: Specific ("the wait was 30 minutes") +- S3: Precise ("waited 32 minutes on Tuesday at 6pm") + +ACTIONABILITY: +- A1: No clear action ("didn't like it") +- A2: Implied action ("too slow") +- A3: Explicit action ("need more cashiers during rush hour") + +TEMPORAL: +- TC: Current/recent experience +- TR: Recurring pattern +- TH: Historical comparison +- TF: Future expectation + +EVIDENCE: +- ES: Subjective opinion +- EI: Indirect evidence +- EC: Concrete/verifiable + +Return valid JSON only.""" + + +async def classify_review_with_spans(text: str) -> dict: + """LLM-powered URT classification with span extraction.""" + + response = await llm.chat( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": CLASSIFICATION_PROMPT}, + {"role": "user", "content": text} + ], + response_format={"type": "json_object"}, + temperature=0.1 + ) + + result = json.loads(response.content) + result['classification_model'] = 'gpt-4o-mini' + + return result +``` + +### 4.4 Span Storage + +```python +async def store_review_spans( + enriched: dict, + spans: list[dict], + batch_id: str +) -> list[str]: + """ + Store extracted spans with soft-switch pattern. + Returns list of span_ids. + """ + + span_ids = [] + + for idx, span in enumerate(spans): + # Generate deterministic span_id + span_id = generate_span_id( + enriched['source'], + enriched['review_id'], + enriched['review_version'], + idx + ) + + # Build USN string + usn = build_usn(span, profile='standard') + + await db.execute(""" + INSERT INTO review_spans ( + span_id, business_id, place_id, source, review_id, review_version, + span_index, span_text, span_start, span_end, + profile, urt_primary, urt_secondary, valence, intensity, comparative, + specificity, actionability, temporal, evidence, + entity, entity_type, entity_normalized, + is_primary, is_active, review_time, + confidence, usn, model_version, ingest_batch_id + ) VALUES ( + %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, + %s, %s, %s, + %s, %s, %s, + %s, %s, %s, %s + ) + """, [ + span_id, enriched['business_id'], enriched['place_id'], + enriched['source'], enriched['review_id'], enriched['review_version'], + idx, span['text'], span['start'], span['end'], + 'standard', span['urt_primary'], span.get('urt_secondary', []), + span['valence'], span['intensity'], span.get('comparative', 'CR-N'), + span.get('specificity', 'S2'), span.get('actionability', 'A2'), + span.get('temporal', 'TC'), span.get('evidence', 'ES'), + span.get('entity'), span.get('entity_type'), + normalize_entity(span.get('entity')), + False, # is_primary set later + False, # is_active=FALSE until validated + enriched['review_time'], + span.get('confidence', 'medium'), usn, 'gpt-4o-mini', batch_id + ]) + + span_ids.append(span_id) + + # Set primary span + await set_primary_span_for_batch( + enriched['source'], + enriched['review_id'], + enriched['review_version'], + batch_id + ) + + # Atomic activation (soft-switch) + await activate_span_batch( + enriched['source'], + enriched['review_id'], + enriched['review_version'], + batch_id + ) + + return span_ids + + +def generate_span_id(source: str, review_id: str, version: int, index: int) -> str: + """Generate deterministic span ID.""" + key = f"{source}|{review_id}|{version}|{index}" + hash_bytes = hashlib.sha256(key.encode()).digest() + return f"SPN-{hash_bytes[:8].hex()}" + + +def build_usn(span: dict, profile: str = 'standard') -> str: + """Build URT Semantic Notation string.""" + base = f"V{span['valence'][-1]}:I{span['intensity'][-1]}:{span['urt_primary']}" + if profile == 'full': + base += f":S{span.get('specificity', 'S2')[-1]}" + base += f":A{span.get('actionability', 'A2')[-1]}" + base += f":T{span.get('temporal', 'TC')[-1]}" + base += f":E{span.get('evidence', 'ES')[-1]}" + return base +``` + +### 4.5 Reprocessing Pattern + +The soft-switch pattern enables atomic span replacement without downtime: + +```python +async def reprocess_review_spans( + source: str, + review_id: str, + review_version: int +) -> str: + """ + Reprocess spans for a review using soft-switch pattern. + Returns new batch_id. + """ + + # 1. Fetch review + review = await db.query_one(""" + SELECT * FROM reviews_enriched + WHERE source = %s AND review_id = %s AND review_version = %s + """, [source, review_id, review_version]) + + # 2. Re-classify with LLM + classification = await classify_review_with_spans(review['text']) + + # 3. Generate new batch ID + batch_id = f"reprocess-{review_id}-{review_version}-{int(time.time())}" + + # 4. INSERT new spans with is_active=FALSE + for idx, span in enumerate(classification['spans']): + span_id = generate_span_id(source, review_id, review_version, idx) + # ... insert with is_active=FALSE, ingest_batch_id=batch_id + + # 5. Validate new spans + if not await validate_span_set(source, review_id, review_version, batch_id): + # Rollback: delete invalid batch + await db.execute(""" + DELETE FROM review_spans + WHERE ingest_batch_id = %s + """, [batch_id]) + raise ValueError("New span set failed validation") + + # 6. Set primary span for new batch + await set_primary_span_for_batch(source, review_id, review_version, batch_id) + + # 7. Atomic switch + async with db.transaction(): + # Deactivate old spans + await db.execute(""" + UPDATE review_spans + SET is_active = FALSE + WHERE source = %s AND review_id = %s AND review_version = %s + AND is_active = TRUE + AND ingest_batch_id != %s + """, [source, review_id, review_version, batch_id]) + + # Activate new spans + await db.execute(""" + UPDATE review_spans + SET is_active = TRUE + WHERE ingest_batch_id = %s + """, [batch_id]) + + return batch_id + + +async def activate_span_batch( + source: str, + review_id: str, + review_version: int, + batch_id: str +): + """Atomically switch from old spans to new batch.""" + + async with db.transaction(): + # Deactivate existing active spans + await db.execute(""" + UPDATE review_spans + SET is_active = FALSE + WHERE source = %s AND review_id = %s AND review_version = %s + AND is_active = TRUE + AND ingest_batch_id != %s + """, [source, review_id, review_version, batch_id]) + + # Activate new batch + await db.execute(""" + UPDATE review_spans + SET is_active = TRUE + WHERE ingest_batch_id = %s + """, [batch_id]) +``` + +### 4.6 Trust Score Computation + +```python +def compute_trust_score(raw: dict, text: str, classification: dict) -> float: + """ + Compute trust score (0.0 to 1.0) based on review quality signals. + Low trust = likely spam, fake, or low-quality. + """ + score = 1.0 + + # Length penalty + word_count = len(text.split()) + if word_count < 5: + score *= 0.5 + elif word_count > 500: + score *= 0.8 + + # Rating/sentiment mismatch + rating = raw.get('rating') + valence = classification.get('review_valence') + if rating and valence: + if rating >= 4 and valence == 'V-': + score *= 0.7 + elif rating <= 2 and valence == 'V+': + score *= 0.7 + + # Generic text patterns + if is_generic_review(text): + score *= 0.6 + + # Span confidence + spans = classification.get('spans', []) + if spans: + low_conf_count = sum(1 for s in spans if s.get('confidence') == 'low') + if low_conf_count > len(spans) / 2: + score *= 0.9 + + return max(0.1, min(1.0, score)) +``` + +--- + +## Part 5: Issue Lifecycle Management + +### 5.1 Issue Routing (Span-Based) + +**v3.2 Issue Key**: `(business_id, place_id, urt_primary, entity_normalized)` + +```python +async def route_span_to_issue(span: dict) -> Optional[str]: + """ + Route a span to an existing or new issue. + Returns issue_id or None if span doesn't warrant an issue. + """ + + # Only negative/mixed spans create issues + if span['valence'] not in ('V-', 'V±'): + return None + + # Generate deterministic issue_id from grouping key + issue_id = await db.query_one(""" + SELECT generate_issue_id(%s, %s, %s, %s) as issue_id + """, [ + span['business_id'], + span['place_id'], + span['urt_primary'], + span.get('entity_normalized') # NULL in v3.2 + ]) + issue_id = issue_id['issue_id'] + + # Check if issue exists + existing = await db.query_one(""" + SELECT issue_id, state, span_count + FROM issues + WHERE issue_id = %s + """, [issue_id]) + + if existing: + # Add span to existing issue + await add_span_to_issue(existing['issue_id'], span) + return existing['issue_id'] + + # Create new issue + if should_create_issue(span): + return await create_issue_from_span(span, issue_id) + + return None + + +async def create_issue_from_span(span: dict, issue_id: str) -> str: + """Create a new issue from a span.""" + + domain = span['urt_primary'][0] + + await db.execute(""" + INSERT INTO issues ( + issue_id, business_id, place_id, primary_subcode, domain, + state, priority_score, confidence_score, + span_count, max_intensity, entity, entity_normalized + ) VALUES ( + %s, %s, %s, %s, %s, + 'DETECTED', %s, %s, + 1, %s, %s, %s + ) + """, [ + issue_id, span['business_id'], span['place_id'], + span['urt_primary'], domain, + compute_initial_priority(span), + confidence_to_score(span.get('confidence', 'medium')), + span['intensity'], + span.get('entity'), span.get('entity_normalized') + ]) + + # Link span to issue + await db.execute(""" + INSERT INTO issue_spans ( + issue_id, span_id, source, review_id, review_version, + is_primary_match, intensity, review_time + ) VALUES ( + %s, %s, %s, %s, %s, + TRUE, %s, %s + ) + """, [ + issue_id, span['span_id'], span['source'], + span['review_id'], span['review_version'], + span['intensity'], span['review_time'] + ]) + + await log_issue_event(issue_id, 'created', span_id=span['span_id']) + + return issue_id + + +async def add_span_to_issue(issue_id: str, span: dict): + """Add span to existing issue and update counters.""" + + # Insert span link (1:1 mapping enforced by UNIQUE constraint) + await db.execute(""" + INSERT INTO issue_spans ( + issue_id, span_id, source, review_id, review_version, + is_primary_match, intensity, review_time + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (span_id) DO NOTHING + """, [ + issue_id, span['span_id'], span['source'], + span['review_id'], span['review_version'], + True, span['intensity'], span['review_time'] + ]) + + # Update issue counters + await db.execute(""" + UPDATE issues SET + span_count = (SELECT COUNT(*) FROM issue_spans WHERE issue_id = %s), + max_intensity = ( + SELECT CASE MAX(CASE intensity + WHEN 'I3' THEN 3 WHEN 'I2' THEN 2 ELSE 1 END) + WHEN 3 THEN 'I3' WHEN 2 THEN 'I2' ELSE 'I1' END + FROM issue_spans WHERE issue_id = %s + ), + updated_at = NOW() + WHERE issue_id = %s + """, [issue_id, issue_id, issue_id]) + + await recalculate_priority(issue_id) + await log_issue_event( + issue_id, 'span_added', + span_id=span['span_id'], + source=span['source'], + review_id=span['review_id'], + review_version=span['review_version'] + ) +``` + +### 5.2 Priority Scoring (Trust-Weighted) + +```python +INTENSITY_WEIGHTS = {'I1': 1.0, 'I2': 2.0, 'I3': 4.0} + +async def recalculate_priority(issue_id: str): + """ + Priority = intensity × volume × decay × recurrence × trend × trust + """ + + issue = await db.query_one(""" + SELECT + i.*, + (SELECT AVG(re.trust_score) + FROM issue_spans s + JOIN review_spans rs ON s.span_id = rs.span_id + JOIN reviews_enriched re ON (rs.source, rs.review_id, rs.review_version) + = (re.source, re.review_id, re.review_version) + WHERE s.issue_id = i.issue_id) as avg_trust + FROM issues i + WHERE i.issue_id = %s + """, [issue_id]) + + intensity_num = {'I1': 1, 'I2': 2, 'I3': 3}.get(issue['max_intensity'], 1) + i_weight = INTENSITY_WEIGHTS.get(f"I{intensity_num}", 1.0) + + volume_factor = 1 + math.log(max(1, issue['span_count'])) + + days_old = (datetime.now() - issue['created_at']).days + decay = math.exp(-0.023 * days_old) + + recurrence_boost = 1.0 + 0.5 * math.log2(issue['reopen_count'] + 1) + + if issue['cr_worse_count'] >= 2: + trend_modifier = 1.3 + elif issue['cr_better_count'] >= 2: + trend_modifier = 0.7 + else: + trend_modifier = 1.0 + + trust_factor = issue['avg_trust'] or 1.0 + + priority = ( + i_weight * volume_factor * decay * + recurrence_boost * trend_modifier * trust_factor + ) + + await db.execute(""" + UPDATE issues SET + priority_score = %s, + avg_trust_score = %s, + updated_at = NOW() + WHERE issue_id = %s + """, [priority, issue['avg_trust'], issue_id]) +``` + +### 5.3 Issue Span Drill-Down + +```python +async def get_issue_spans(issue_id: str, + sort_by: str = 'date', + limit: int = 50, + offset: int = 0) -> list[dict]: + """Fetch all spans for an issue with full details.""" + + order_clause = { + 'date': 's.review_time DESC', + 'intensity': "CASE s.intensity WHEN 'I3' THEN 1 WHEN 'I2' THEN 2 ELSE 3 END", + 'trust': 're.trust_score DESC', + }.get(sort_by, 's.review_time DESC') + + return await db.query(f""" + SELECT + rs.span_id, + rs.span_text, + rs.span_start, + rs.span_end, + rs.urt_primary, + rs.valence, + rs.intensity, + rs.specificity, + rs.actionability, + rs.entity, + rs.entity_type, + rs.usn, + s.review_time, + s.is_primary_match, + re.review_id, + re.review_version, + re.text as review_text, + re.rating, + re.trust_score, + l.display_name as location_name + FROM issue_spans s + JOIN review_spans rs ON s.span_id = rs.span_id + JOIN reviews_enriched re ON (rs.source, rs.review_id, rs.review_version) + = (re.source, re.review_id, re.review_version) + JOIN locations l ON (rs.business_id, rs.place_id) = (l.business_id, l.place_id) + WHERE s.issue_id = %s + AND rs.is_active = TRUE + ORDER BY {order_clause} + LIMIT %s OFFSET %s + """, [issue_id, limit, offset]) +``` + +### 5.4 Strength Score + +``` +Strength Score = Σ (intensity_weight) + +Where: + I1 (mild) → weight = 1 + I2 (moderate) → weight = 2 + I3 (strong) → weight = 4 + +One I3 span = 4 I1 spans = 2 I2 spans +``` + +--- + +## Part 6: Analytics Spine (Fact Population) + +### 6.1 Daily Fact Aggregation Job + +```python +async def populate_facts(business_id: str, date: date, bucket_type: str = 'day'): + """ + Aggregate spans into fact_timeseries. Run daily. + + v3.2 populates: + - subject_type='overall', subject_id='all' (per location + 'ALL') + - subject_type='urt_code', subject_id= (per location + 'ALL') + - subject_type='issue', subject_id= (per issue) + """ + + if bucket_type == 'day': + period_start = date + period_end = date + timedelta(days=1) + elif bucket_type == 'week': + period_start = date - timedelta(days=date.weekday()) + period_end = period_start + timedelta(days=7) + elif bucket_type == 'month': + period_start = date.replace(day=1) + next_month = period_start.replace(day=28) + timedelta(days=4) + period_end = next_month.replace(day=1) + + # Get owned locations (competitors excluded from 'ALL' rollup) + owned_locations = await db.query( + "SELECT place_id FROM locations WHERE business_id = %s AND is_active = TRUE AND location_type = 'owned'", + [business_id] + ) + owned_place_ids = [loc['place_id'] for loc in owned_locations] + + # Per-location facts (owned) + for loc in owned_locations: + await populate_location_facts_from_spans( + business_id, loc['place_id'], period_start, period_end, bucket_type + ) + + # All-locations rollup (owned only — place_id='ALL') + await populate_all_locations_facts_from_spans( + business_id, owned_place_ids, period_start, period_end, bucket_type + ) + + # Issue facts + await populate_issue_facts(business_id, period_start, period_end, bucket_type) + + +async def populate_location_facts_from_spans( + business_id: str, + place_id: str, + period_start: date, + period_end: date, + bucket_type: str +): + """Populate facts for a single location from spans.""" + + # Aggregate by URT code from spans + code_stats = await db.query(""" + SELECT + rs.urt_primary as code, + COUNT(DISTINCT re.review_id) as review_count, + COUNT(*) as span_count, + SUM(CASE WHEN rs.valence = 'V-' THEN 1 ELSE 0 END) as negative_count, + SUM(CASE WHEN rs.valence = 'V+' THEN 1 ELSE 0 END) as positive_count, + SUM(CASE WHEN rs.valence = 'V0' THEN 1 ELSE 0 END) as neutral_count, + SUM(CASE WHEN rs.valence = 'V±' THEN 1 ELSE 0 END) as mixed_count, + SUM(CASE rs.intensity::text + WHEN 'I1' THEN 1 WHEN 'I2' THEN 2 WHEN 'I3' THEN 4 ELSE 0 + END) as strength_score, + SUM(CASE WHEN rs.valence = 'V-' THEN + CASE rs.intensity::text WHEN 'I1' THEN 1 WHEN 'I2' THEN 2 WHEN 'I3' THEN 4 ELSE 0 END + ELSE 0 END) as negative_strength, + SUM(CASE WHEN rs.valence = 'V+' THEN + CASE rs.intensity::text WHEN 'I1' THEN 1 WHEN 'I2' THEN 2 WHEN 'I3' THEN 4 ELSE 0 END + ELSE 0 END) as positive_strength, + SUM(CASE WHEN rs.intensity::text = 'I1' THEN 1 ELSE 0 END) as i1_count, + SUM(CASE WHEN rs.intensity::text = 'I2' THEN 1 ELSE 0 END) as i2_count, + SUM(CASE WHEN rs.intensity::text = 'I3' THEN 1 ELSE 0 END) as i3_count, + SUM(CASE WHEN rs.comparative::text = 'CR-B' THEN 1 ELSE 0 END) as cr_better, + SUM(CASE WHEN rs.comparative::text = 'CR-W' THEN 1 ELSE 0 END) as cr_worse, + SUM(CASE WHEN rs.comparative::text = 'CR-S' THEN 1 ELSE 0 END) as cr_same, + -- Trust-weighted metrics (v3.2) + SUM(re.trust_score * CASE rs.intensity::text + WHEN 'I1' THEN 1 WHEN 'I2' THEN 2 WHEN 'I3' THEN 4 ELSE 0 + END) as trust_weighted_strength, + SUM(CASE WHEN rs.valence = 'V-' THEN + re.trust_score * CASE rs.intensity::text WHEN 'I1' THEN 1 WHEN 'I2' THEN 2 WHEN 'I3' THEN 4 ELSE 0 END + ELSE 0 END) as trust_weighted_negative, + AVG(re.rating) as avg_rating, + COUNT(re.rating) as rating_count + FROM review_spans rs + JOIN reviews_enriched re ON (rs.source, rs.review_id, rs.review_version) + = (re.source, re.review_id, re.review_version) + WHERE rs.business_id = %s + AND rs.place_id = %s + AND rs.review_time >= %s AND rs.review_time < %s + AND rs.is_active = TRUE + AND re.is_latest = TRUE + GROUP BY rs.urt_primary + """, [business_id, place_id, period_start, period_end]) + + for stat in code_stats: + await upsert_fact( + business_id=business_id, + place_id=place_id, + period_date=period_start, + bucket_type=bucket_type, + subject_type='urt_code', + subject_id=stat['code'], + metrics=stat + ) + + # Aggregate overall + overall = await db.query_one(""" + SELECT + COUNT(DISTINCT re.review_id) as review_count, + COUNT(*) as span_count, + SUM(CASE WHEN rs.valence = 'V-' THEN 1 ELSE 0 END) as negative_count, + SUM(CASE WHEN rs.valence = 'V+' THEN 1 ELSE 0 END) as positive_count, + AVG(re.rating) as avg_rating + FROM review_spans rs + JOIN reviews_enriched re ON (rs.source, rs.review_id, rs.review_version) + = (re.source, re.review_id, re.review_version) + WHERE rs.business_id = %s + AND rs.place_id = %s + AND rs.review_time >= %s AND rs.review_time < %s + AND rs.is_active = TRUE + AND re.is_latest = TRUE + """, [business_id, place_id, period_start, period_end]) + + await upsert_fact( + business_id=business_id, + place_id=place_id, + period_date=period_start, + bucket_type=bucket_type, + subject_type='overall', + subject_id='all', + metrics=overall + ) +``` + +### 6.2 Timeline Query (For Charts) + +```python +async def get_timeline(business_id: str, + place_id: Optional[str], + subject_type: str, + subject_id: str, + start: date, + end: date, + bucket_type: str = 'week') -> list[dict]: + """ + Query pre-aggregated facts for line charts. + + Args: + place_id: Specific place_id, or None for 'ALL' locations rollup + """ + + # Use 'ALL' sentinel for all-locations queries + effective_place_id = place_id if place_id else 'ALL' + + return await db.query(""" + SELECT + period_date, + review_count, + span_count, + negative_count, + positive_count, + strength_score, + negative_strength, + avg_rating, + cr_better, + cr_worse, + cr_same, + trust_weighted_strength, + trust_weighted_negative + FROM fact_timeseries + WHERE business_id = %s + AND place_id = %s + AND subject_type = %s + AND subject_id = %s + AND bucket_type = %s + AND period_date BETWEEN %s AND %s + ORDER BY period_date + """, [business_id, effective_place_id, subject_type, subject_id, bucket_type, start, end]) +``` + +--- + +## Part 7: Competitor Analysis + +### 7.1 Competitor Setup (Clean Model) + +Competitors are tracked in both `competitors` (relationship metadata) and `locations` (with `location_type='competitor'`). This preserves FK integrity and enables consistent joins for display names/timezones. + +**Competitor Review Storage Rule**: Competitor reviews are stored with the **customer's business_id** and the **competitor's place_id**: + +``` +reviews_enriched.business_id = +reviews_enriched.place_id = +``` + +The `locations.location_type` column distinguishes ownership: +- `'owned'` — customer's own locations +- `'competitor'` — tracked competitor locations + +```python +async def setup_competitor(business_id: str, competitor_place_id: str, + competitor_name: str, relationship: str = 'direct'): + """Register a competitor for tracking.""" + + # 1. Add to locations with location_type='competitor' + await db.execute(""" + INSERT INTO locations (business_id, place_id, location_type, display_name) + VALUES (%s, %s, 'competitor', %s) + ON CONFLICT (business_id, place_id) DO UPDATE SET + display_name = EXCLUDED.display_name + """, [business_id, competitor_place_id, competitor_name]) + + # 2. Track relationship metadata in competitors table + await db.execute(""" + INSERT INTO competitors (business_id, competitor_place_id, competitor_name, relationship) + VALUES (%s, %s, %s, %s) + ON CONFLICT (business_id, competitor_place_id) DO UPDATE SET + competitor_name = EXCLUDED.competitor_name, + relationship = EXCLUDED.relationship + """, [business_id, competitor_place_id, competitor_name, relationship]) +``` + +### 7.2 Competitor Comparison + +```python +async def get_competitor_comparison(business_id: str, code: str, + start: date, end: date) -> dict: + """Compare your URT metrics against competitors.""" + + # Your metrics (from 'ALL' rollup) + your_metrics = await db.query_one(""" + SELECT + SUM(negative_strength) as negative_strength, + SUM(span_count) as span_count, + AVG(avg_rating) as avg_rating, + SUM(trust_weighted_negative) as trust_weighted_negative + FROM fact_timeseries + WHERE business_id = %s + AND place_id = 'ALL' + AND subject_type = 'urt_code' + AND subject_id = %s + AND period_date BETWEEN %s AND %s + """, [business_id, code, start, end]) + + # Competitor metrics + competitors = await db.query(""" + SELECT competitor_place_id, competitor_name + FROM competitors WHERE business_id = %s AND is_active = TRUE + """, [business_id]) + + comparison = { + 'your_business': your_metrics or {}, + 'competitors': [] + } + + for comp in competitors: + comp_metrics = await db.query_one(""" + SELECT + SUM(negative_strength) as negative_strength, + SUM(span_count) as span_count, + AVG(avg_rating) as avg_rating + FROM fact_timeseries + WHERE business_id = %s + AND place_id = %s + AND subject_type = 'urt_code' + AND subject_id = %s + AND period_date BETWEEN %s AND %s + """, [business_id, comp['competitor_place_id'], code, start, end]) + + comparison['competitors'].append({ + 'name': comp['competitor_name'], + 'place_id': comp['competitor_place_id'], + **(comp_metrics or {}) + }) + + return comparison +``` + +--- + +## Part 8: Report Generation (Facts-First) + +```python +async def generate_report(business_id: str, place_id: Optional[str], + start: date, end: date) -> dict: + """Generate report from pre-aggregated facts.""" + + effective_place_id = place_id if place_id else 'ALL' + + # 1. Top issues from facts (fast) + top_issues = await get_top_issues_from_facts(business_id, effective_place_id, start, end) + + # 2. Strengths from facts + strengths = await get_strengths_from_facts(business_id, effective_place_id, start, end) + + # 3. Sub-patterns with span references + for issue in top_issues[:5]: + patterns = await discover_and_store_subpatterns( + business_id, effective_place_id, issue['code'], start, end + ) + issue['sub_patterns'] = patterns + + # 4. Trends from facts + trends = await compute_trends_from_facts(business_id, effective_place_id, start, end) + + # 5. Entity analysis (v3.2) + entities = await analyze_entities(business_id, effective_place_id, start, end) + + # 6. Competitor benchmarks + competitors = await get_competitor_benchmarks(business_id, start, end) + + payload = { + 'business_id': business_id, + 'place_id': place_id, + 'period': f"{start} to {end}", + 'issues': top_issues, + 'strengths': strengths, + 'trends': trends, + 'entities': entities, + 'competitors': competitors, + } + + narrative = await generate_narrative(payload) + + return { + 'payload': payload, + 'narrative': narrative, + 'generated_at': datetime.now().isoformat() + } + + +async def analyze_entities(business_id: str, place_id: str, + start: date, end: date) -> list[dict]: + """Analyze entity mentions from spans.""" + + return await db.query(""" + SELECT + rs.entity_normalized, + rs.entity_type::text, + COUNT(*) as mention_count, + SUM(CASE WHEN rs.valence = 'V-' THEN 1 ELSE 0 END) as negative_count, + SUM(CASE WHEN rs.valence = 'V+' THEN 1 ELSE 0 END) as positive_count, + AVG(CASE rs.intensity::text + WHEN 'I1' THEN 1 WHEN 'I2' THEN 2 WHEN 'I3' THEN 3 + END) as avg_intensity, + array_agg(DISTINCT rs.urt_primary) as codes + FROM review_spans rs + WHERE rs.business_id = %s + AND (rs.place_id = %s OR %s = 'ALL') + AND rs.review_time >= %s AND rs.review_time < %s + AND rs.entity_normalized IS NOT NULL + AND rs.is_active = TRUE + GROUP BY rs.entity_normalized, rs.entity_type + ORDER BY mention_count DESC + LIMIT 20 + """, [business_id, place_id, place_id, start, end]) +``` + +--- + +## Part 9: KPI-Ready Hooks + +### 9.1 Future KPI Integration (Interface Only) + +```sql +-- Future: KPI fact table with same grain +CREATE TABLE fact_kpi_timeseries ( + id SERIAL PRIMARY KEY, + + -- Same join keys as fact_timeseries + business_id TEXT NOT NULL, + place_id TEXT NOT NULL, -- 'ALL' for rollups + period_date DATE NOT NULL, + bucket_type TEXT NOT NULL, + + -- KPI metrics + revenue DECIMAL(12,2), + transactions INT, + cancellations INT, + refunds DECIMAL(12,2), + support_tickets INT, + + computed_at TIMESTAMP DEFAULT NOW(), + + UNIQUE(business_id, place_id, period_date, bucket_type) +); + +-- Join reviews and KPIs: +SELECT + r.period_date, + r.negative_strength, + r.trust_weighted_negative, + k.revenue, + k.cancellations +FROM fact_timeseries r +JOIN fact_kpi_timeseries k USING (business_id, place_id, period_date, bucket_type) +WHERE r.subject_type = 'overall' AND r.subject_id = 'all'; +``` + +--- + +## Part 10: Cost Model + +| Stage | When | Cost | Notes | +|-------|------|------|-------| +| **Raw Storage** | Per review | $0.00 | ~1KB per review | +| **Embedding** | Per review | $0.00 | Local model, ~50ms | +| **LLM Classification + Spans** | Per review | ~$0.0003 | GPT-4o-mini (larger prompt) | +| **Fact Aggregation** | Daily job | $0.00 | SQL, <1 minute | +| **Sub-Clustering** | Per report | $0.00 | HDBSCAN, <1s | +| **LLM Narrative** | Per report | ~$0.15 | GPT-4o | + +**Total Costs:** + +| Volume | Monthly Ingest | Reports (10/mo) | Total | +|--------|---------------|-----------------|-------| +| 1K reviews | $0.30 | $1.50 | **$1.80** | +| 10K reviews | $3.00 | $1.50 | **$4.50** | +| 100K reviews | $30.00 | $1.50 | **$31.50** | + +--- + +## Part 11: Key Innovations + +| Innovation | Benefit | +|------------|---------| +| **Span layer** | Fine-grained classification at semantic unit level | +| **1:1 span-to-issue** | Clean data model, no ambiguity in routing | +| **Deterministic issue IDs** | SHA256-based, reproducible from grouping key | +| **Soft-switch reprocessing** | Atomic span replacement without downtime | +| **URT ENUM types** | Database-enforced classification constraints | +| **Causal chain support** | Full profile enables cause/effect analysis | +| **Entity extraction** | Named entity routing for targeted issues | +| **Trust-weighted facts** | Spam resistance with weighted aggregation | +| **USN notation** | Compact semantic notation for spans | + +--- + +## Document Control + +| Field | Value | +|-------|-------| +| **Document** | ReviewIQ Architecture v3.2.0 | +| **Status** | Specification Complete | +| **Date** | 2026-01-24 | +| **Dependencies** | URT Specification v5.1, Issue Lifecycle Framework C1 | +| **Source** | Google Reviews only | +| **Cost Target** | <$35/month at 100K reviews | + +### Changelog + +| Version | Changes | +|---------|---------| +| v3.0 | Issue lifecycle, strength scores, timeline charts | +| v3.1 | Relational refactor: issue_spans, fact_timeseries, raw/enriched split, multi-location, competitors, trust scoring | +| v3.1.1 | Versioned enriched PK, tenant-scoped locations, 'ALL' sentinel, competitor cleanup | +| v3.1.2 | Versioned issue_spans FK, competitor business_id rule, trust-weighted facts deferred, location_type flag | +| v3.2.0 | **Span layer**: review_spans table, URT ENUM types, causal chain support, entity extraction, reprocessing pattern, deterministic issue IDs, 1:1 span mapping | + +### New in v3.2.0 + +| Feature | Description | +|---------|-------------| +| **review_spans table** | Fine-grained semantic unit extraction from reviews | +| **URT ENUM types** | 12 strongly-typed enums for classification fields | +| **Required extensions** | btree_gist for exclusion constraints, pgcrypto for SHA256 | +| **Span constraints** | chk_span_end, chk_primary_tier3, chk_secondary_max2, chk_secondary_tier3, chk_full_only_fields, chk_no_self_relation, chk_usn_format | +| **Span indexes** | Active ordering, one-primary enforcement, non-overlap exclusion, issue routing | +| **Validation triggers** | Bounds validation, text matching, causal chain structure | +| **Helper functions** | urt_validate_causal_chain, validate_review_relations, validate_active_spans, set_primary_span, generate_issue_id | +| **Reprocessing pattern** | Soft-switch with is_active flag and ingest_batch_id | +| **issue_spans rewrite** | 1:1 span-to-issue mapping with UNIQUE(span_id) | +| **Trust-weighted facts** | trust_weighted_strength, trust_weighted_negative now populated | +| **span_count in facts** | Span-level counting alongside review_count | + +### Deferred to v3.3+ + +| Feature | Reason | +|---------|--------| +| Distinct entity issues | `entity_normalized` defaults to NULL in v3.2; v3.3 creates separate issues per entity | +| Journey step inference | Needs better grounding data | +| Intent signals extraction | Needs action playbooks | +| Stability score tracking | Premature for current version | +| Span embeddings | Per-span vectors for sub-clustering | + +--- + +*End of ReviewIQ Architecture v3.2.0* diff --git a/.artifacts/ReviewIQ-Architecture-v3.md b/.artifacts/ReviewIQ-Architecture-v3.md new file mode 100644 index 0000000..5dff12f --- /dev/null +++ b/.artifacts/ReviewIQ-Architecture-v3.md @@ -0,0 +1,1513 @@ +# ReviewIQ: Review Intelligence Pipeline + +**Version**: 3.0 +**Status**: Architecture Specification +**Date**: 2026-01-24 + +--- + +## Executive Summary + +ReviewIQ transforms customer reviews into actionable business intelligence through a three-stage pipeline: + +1. **Ingest** — LLM-powered URT classification with semantic embeddings +2. **Analyze** — Issue lifecycle management with sub-pattern discovery +3. **Report** — Statistically rigorous insights with trend detection + +**Design Principles**: +- **Accuracy over heuristics**: LLM classification at ingest (~$0.0002/review) +- **Taxonomy as structure**: URT provides stable, interpretable categories +- **Local ML for depth**: Sub-clustering reveals actionable patterns within categories +- **Feedback loop**: CR (Comparative Reference) signals verify resolution effectiveness + +--- + +## Part 1: System Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ REVIEWIQ PIPELINE │ +├─────────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────┐ ┌──────────────────────────────────────────────────────┐ │ +│ │ │ │ INGEST LAYER │ │ +│ │ Reviews │────▶│ ┌─────────┐ ┌─────────┐ ┌─────────────────┐ │ │ +│ │ (Input) │ │ │ Embed │ │ LLM │ │ Store │ │ │ +│ │ │ │ │ Review │───▶│Classify │───▶│ (PostgreSQL) │ │ │ +│ └─────────────┘ │ └─────────┘ └─────────┘ └─────────────────┘ │ │ +│ │ │ │ │ │ +│ │ │ ~$0.00 │ ~$0.0002 │ │ +│ │ │ (local) │ per review │ │ +│ └──────┼──────────────────────────────┼─────────────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ ISSUE AGGREGATION │ │ +│ │ │ │ +│ │ V- classified reviews ───▶ Match or Create Issue ───▶ Track State │ │ +│ │ │ │ +│ │ Rules: Same URT code + entity + location + time window = same issue │ │ +│ │ States: DETECTED → ACKNOWLEDGED → IN_PROGRESS → RESOLVED → VERIFIED │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ REPORT GENERATION │ │ +│ │ │ │ +│ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────┐ │ │ +│ │ │ Aggregate │ │ Sub-Cluster │ │ Trend │ │ LLM │ │ │ +│ │ │ by URT Code │──▶│ Within Codes │──▶│ Analysis │──▶│ Narrate │ │ │ +│ │ │ (SQL) │ │ (HDBSCAN) │ │ (CR + Rate) │ │ │ │ │ +│ │ └──────────────┘ └──────────────┘ └──────────────┘ └──────────┘ │ │ +│ │ $0.00 $0.00 $0.00 ~$0.15 │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ │ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────────────────┐ │ +│ │ OUTPUT │ │ +│ │ │ │ +│ │ • Executive Summary with statistically defensible claims │ │ +│ │ • Issues ranked by priority with sub-pattern breakdown │ │ +│ │ • Strengths with trend signals │ │ +│ │ • Staff performance insights │ │ +│ │ • Actionable recommendations │ │ +│ │ │ │ +│ └──────────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Part 2: Ingest Layer + +### 2.1 Design Philosophy + +The review is the atomic unit. We do not split reviews into fragments — this preserves context and enables accurate classification. A single review may contain multiple topics; URT's multi-coding (primary + up to 2 secondary codes) handles this naturally. + +### 2.2 Dual Processing + +Each review undergoes two parallel operations: + +```python +async def ingest_review(review: dict) -> dict: + """Ingest a single review: embed + classify.""" + + text = review['text'].strip() + + # Parallel execution + embedding_task = asyncio.create_task(embed_review(text)) + classification_task = asyncio.create_task(classify_review_llm(text)) + + embedding = await embedding_task + classification = await classification_task + + return { + 'review_id': review['review_id'], + 'business_id': review['business_id'], + 'text': text, + 'embedding': embedding, + 'date': review['date'], + 'rating': review.get('rating'), + **classification, # URT codes, valence, intensity, etc. + } +``` + +### 2.3 Embedding + +Local multilingual embeddings for semantic capabilities: + +```python +from sentence_transformers import SentenceTransformer + +model = SentenceTransformer('intfloat/multilingual-e5-small') + +def embed_review(text: str) -> np.ndarray: + """Generate normalized embedding for semantic search and clustering.""" + + # e5 models perform better with instruction prefix + embedding = model.encode( + f"passage: {text}", + normalize_embeddings=True + ) + return embedding # 384 dimensions +``` + +**Why embed if we have URT codes?** +- Sub-clustering within URT codes (pattern discovery) +- Semantic quote selection (centroid-closest) +- Similarity search for emerging patterns +- Backup for low-confidence classifications + +### 2.4 LLM Classification + +Single LLM call extracts complete URT classification: + +```python +CLASSIFICATION_PROMPT = """You are a customer feedback classifier using the Universal Review Taxonomy (URT). + +Analyze the review and return JSON with: + +{ + "urt_primary": "X1.23", // Main URT subcode + "urt_secondary": ["Y2.34"], // 0-2 additional codes (different domains only) + "valence": "V-", // V+, V-, V0, V± + "intensity": "I2", // I1 (mild), I2 (moderate), I3 (strong) + "comparative": "CR-N", // CR-N (none), CR-B (better), CR-W (worse), CR-S (same) + "staff_mentions": ["Mike"], // Employee names mentioned + "quotes": { // Key phrase for each code + "X1.23": "exact phrase from review", + "Y2.34": "another phrase" + } +} + +URT DOMAINS (choose primary from most impactful): +- O (Offering): Product/service quality, function, completeness, fit +- P (People): Staff attitude, competence, responsiveness, communication +- J (Journey): Timing, ease, reliability, resolution +- E (Environment): Physical space, digital interface, ambiance, safety +- A (Access): Availability, accessibility, inclusivity, convenience +- V (Value): Price, transparency, effort, worth +- R (Relationship): Trust, dependability, recovery, loyalty + +RULES: +1. Primary = what customer is MOST affected by +2. Secondary must be DIFFERENT domains (P1.02 + P3.01 is invalid) +3. V± only when genuinely mixed (positive AND negative on different topics) +4. CR-B/W/S only for explicit self-comparison ("better than last time", "still broken") +5. quotes must be EXACT phrases from the review text + +Return valid JSON only.""" + + +async def classify_review_llm(text: str) -> dict: + """Complete URT classification via LLM.""" + + response = await llm.chat( + model="gpt-4o-mini", # ~$0.0002 per review + messages=[ + {"role": "system", "content": CLASSIFICATION_PROMPT}, + {"role": "user", "content": text} + ], + response_format={"type": "json_object"}, + temperature=0.1 # Low temperature for consistency + ) + + return json.loads(response.content) +``` + +### 2.5 Batch Processing for Efficiency + +For bulk ingestion, batch multiple reviews per LLM call: + +```python +async def classify_batch(reviews: list[dict], batch_size: int = 10) -> list[dict]: + """Process reviews in batches for ~40% cost reduction.""" + + results = [] + for i in range(0, len(reviews), batch_size): + batch = reviews[i:i+batch_size] + + prompt = BATCH_CLASSIFICATION_PROMPT + "\n\nREVIEWS:\n" + for j, review in enumerate(batch): + prompt += f"\n[{j}] {review['text']}\n---\n" + + response = await llm.chat( + model="gpt-4o-mini", + messages=[{"role": "system", "content": prompt}], + response_format={"type": "json_object"} + ) + + batch_results = json.loads(response.content)["classifications"] + results.extend(batch_results) + + return results +``` + +### 2.6 Data Model + +```sql +-- Core review storage with URT classification +CREATE TABLE reviews ( + review_id TEXT PRIMARY KEY, + business_id TEXT NOT NULL, + text TEXT NOT NULL, + embedding VECTOR(384), + date TIMESTAMP NOT NULL, + rating SMALLINT, + + -- URT Classification (from LLM) + urt_primary TEXT NOT NULL, -- 'J1.01', 'P1.02', etc. + urt_secondary TEXT[] DEFAULT '{}', -- Max 2 + valence TEXT NOT NULL, -- 'V+', 'V-', 'V0', 'V±' + intensity TEXT NOT NULL, -- 'I1', 'I2', 'I3' + comparative TEXT DEFAULT 'CR-N', -- 'CR-N', 'CR-B', 'CR-W', 'CR-S' + + -- Extracted entities + staff_mentions TEXT[] DEFAULT '{}', + quotes JSONB, -- {"code": "phrase", ...} + + -- Metadata + created_at TIMESTAMP DEFAULT NOW(), + classification_model TEXT DEFAULT 'gpt-4o-mini' +); + +-- Indexes for query patterns +CREATE INDEX idx_reviews_business_date ON reviews(business_id, date DESC); +CREATE INDEX idx_reviews_urt_primary ON reviews(business_id, urt_primary); +CREATE INDEX idx_reviews_valence ON reviews(business_id, valence, date); +CREATE INDEX idx_reviews_comparative ON reviews(comparative) WHERE comparative != 'CR-N'; +CREATE INDEX idx_reviews_embedding ON reviews USING hnsw (embedding vector_cosine_ops); +``` + +--- + +## Part 3: Issue Lifecycle Management + +Following the URT Issue Lifecycle Framework (C1), negative feedback (V-) generates trackable issues. + +### 3.1 Issue Aggregation + +Multiple reviews about the same problem aggregate into a single issue: + +```python +def aggregate_to_issue(review: dict) -> str: + """Match review to existing issue or create new one.""" + + if review['valence'] not in ('V-', 'V±'): + return None # Only negative feedback creates issues + + # Find matching open issues + matching = db.query(""" + SELECT issue_id, primary_subcode, entity, location + FROM issues + WHERE business_id = %s + AND primary_subcode = %s + AND state NOT IN ('VERIFIED', 'DECLINED') + AND created_at > NOW() - INTERVAL '30 days' + """, [review['business_id'], review['urt_primary']]) + + for issue in matching: + if is_same_issue(review, issue): + # Aggregate to existing issue + add_span_to_issue(issue['issue_id'], review) + recalculate_priority(issue['issue_id']) + return issue['issue_id'] + + # Check intensity threshold for new issue creation + if should_create_issue(review): + return create_issue(review) + + return None # Stored in buffer for future aggregation + + +def should_create_issue(review: dict) -> bool: + """Intensity-based issue creation thresholds.""" + + if review['intensity'] == 'I3': + return True # Critical = immediate issue + + # Check aggregation buffer for patterns + similar_count = count_similar_in_buffer(review, window_days=30) + + if review['intensity'] == 'I2' and similar_count >= 2: + return True # Moderate + 2 others = issue + + if review['intensity'] == 'I1' and similar_count >= 4: + return True # Mild + 4 others = issue + + return False +``` + +### 3.2 Issue State Machine + +``` + DETECTED + │ + ┌────────────┼────────────┐ + ▼ ▼ ▼ + ACKNOWLEDGED DECLINED (escalate) + │ │ + ▼ │ + IN_PROGRESS │ + │ │ + ▼ │ + RESOLVED ◀────────────────────┘ + │ + ┌─────┼─────┐ + ▼ ▼ + VERIFIED REOPENED + │ + └──▶ (back to IN_PROGRESS) +``` + +### 3.3 Priority Scoring + +```python +def calculate_priority(issue: dict) -> float: + """ + Priority combines intensity, volume, recency, and recurrence. + + P = I_weight × (1 + log(span_count)) × decay(days) × recurrence_boost × trend_modifier + """ + + INTENSITY_WEIGHTS = {'I1': 1.0, 'I2': 2.0, 'I3': 4.0} + + i_weight = INTENSITY_WEIGHTS[issue['max_intensity']] + volume_factor = 1 + math.log(issue['span_count']) + + days_old = (datetime.now() - issue['created_at']).days + decay = math.exp(-0.023 * days_old) # Half-life ~30 days + + recurrence_boost = 1.0 + 0.5 * math.log2(issue['reopen_count'] + 1) + + # Trend modifier from CR signals + if issue['recent_cr_w_count'] >= 2: + trend_modifier = 1.3 # Worsening + elif issue['recent_cr_b_count'] >= 2: + trend_modifier = 0.7 # Improving + else: + trend_modifier = 1.0 # Stable + + return i_weight * volume_factor * decay * recurrence_boost * trend_modifier +``` + +### 3.4 Resolution Verification via CR Signals + +The Comparative Reference (CR) dimension enables automatic verification: + +```python +def process_cr_signal(review: dict): + """Handle CR-B/W/S signals for issue lifecycle.""" + + if review['comparative'] == 'CR-N': + return + + # Find resolved issues with matching code + resolved_issues = db.query(""" + SELECT issue_id, state, resolved_at + FROM issues + WHERE business_id = %s + AND primary_subcode = %s + AND state IN ('RESOLVED', 'VERIFIED') + AND resolved_at > NOW() - INTERVAL '60 days' + """, [review['business_id'], review['urt_primary']]) + + for issue in resolved_issues: + if review['comparative'] == 'CR-B': + # Improvement signal → verify resolution + if issue['state'] == 'RESOLVED': + verify_issue(issue['issue_id'], review['review_id']) + + elif review['comparative'] in ('CR-S', 'CR-W'): + # Unchanged or worsening → reopen + reopen_issue(issue['issue_id'], review['review_id']) + + if review['comparative'] == 'CR-W': + escalate_issue(issue['issue_id'], reason='REGRESSION') +``` + +### 3.5 Issue Data Model + +```sql +CREATE TABLE issues ( + issue_id TEXT PRIMARY KEY, + business_id TEXT NOT NULL, + primary_subcode TEXT NOT NULL, + domain TEXT NOT NULL, + + -- State + state TEXT NOT NULL DEFAULT 'DETECTED', + priority_score FLOAT NOT NULL, + confidence_score FLOAT NOT NULL, + + -- Aggregation + review_ids TEXT[] NOT NULL, + span_count INT NOT NULL DEFAULT 1, + max_intensity TEXT NOT NULL, + + -- Ownership + owner_team TEXT, + owner_individual TEXT, + + -- Timestamps + created_at TIMESTAMP DEFAULT NOW(), + acknowledged_at TIMESTAMP, + resolved_at TIMESTAMP, + verified_at TIMESTAMP, + + -- Resolution + reopen_count INT DEFAULT 0, + resolution_code TEXT, + resolution_notes TEXT, + decline_reason TEXT, + + -- Context + entity TEXT, -- Product, staff member, feature + location TEXT, -- Physical or logical location + causal_codes TEXT[], -- CD-O, MG-T, etc. + + -- Verification + verification_window_days INT DEFAULT 60 +); + +CREATE TABLE issue_events ( + event_id SERIAL PRIMARY KEY, + issue_id TEXT REFERENCES issues(issue_id), + event_type TEXT NOT NULL, -- 'state_change', 'span_added', 'priority_update' + from_state TEXT, + to_state TEXT, + actor TEXT, + notes TEXT, + review_id TEXT, -- Triggering review if applicable + created_at TIMESTAMP DEFAULT NOW() +); + +-- Time-series aggregation for impact charts +CREATE TABLE issue_timeseries ( + id SERIAL PRIMARY KEY, + business_id TEXT NOT NULL, + code TEXT NOT NULL, -- URT code (e.g., 'J1.01') + period DATE NOT NULL, -- Bucket date (day/week/month) + bucket_type TEXT NOT NULL, -- 'day', 'week', 'month' + + -- Counts + review_count INT NOT NULL DEFAULT 0, + negative_count INT NOT NULL DEFAULT 0, + positive_count INT NOT NULL DEFAULT 0, + + -- Strength metrics + strength_score FLOAT NOT NULL DEFAULT 0, -- Weighted by intensity + avg_intensity FLOAT, + max_intensity TEXT, + + -- CR signals in period + cr_better INT DEFAULT 0, + cr_worse INT DEFAULT 0, + cr_same INT DEFAULT 0, + + UNIQUE(business_id, code, period, bucket_type) +); + +CREATE INDEX idx_timeseries_lookup ON issue_timeseries(business_id, code, period); +``` + +### 3.6 Issue Review Drill-Down + +Retrieve all reviews belonging to a specific issue for detailed inspection: + +```python +def get_issue_reviews(issue_id: str, + sort_by: str = 'date', + limit: int = 50) -> list[dict]: + """Fetch all reviews aggregated into an issue.""" + + issue = db.query_one(""" + SELECT issue_id, review_ids, primary_subcode, business_id + FROM issues WHERE issue_id = %s + """, [issue_id]) + + if not issue: + return [] + + order_clause = { + 'date': 'date DESC', + 'intensity': "CASE intensity WHEN 'I3' THEN 1 WHEN 'I2' THEN 2 ELSE 3 END", + 'relevance': 'date DESC' # Could enhance with embedding similarity + }.get(sort_by, 'date DESC') + + reviews = db.query(f""" + SELECT + review_id, + text, + date, + rating, + valence, + intensity, + comparative, + quotes->>%s as quote, + staff_mentions + FROM reviews + WHERE review_id = ANY(%s) + ORDER BY {order_clause} + LIMIT %s + """, [issue['primary_subcode'], issue['review_ids'], limit]) + + return [{ + **r, + 'intensity_weight': {'I1': 1, 'I2': 2, 'I3': 4}[r['intensity']] + } for r in reviews] +``` + +### 3.7 Strength Score Aggregation + +A unified metric combining volume and intensity for impact measurement: + +``` +Strength Score = Σ (intensity_weight × review_count) + +Where: + I1 (mild) → weight = 1 + I2 (moderate) → weight = 2 + I3 (strong) → weight = 4 +``` + +```python +INTENSITY_WEIGHTS = {'I1': 1, 'I2': 2, 'I3': 4} + +def compute_strength_score(reviews: list[dict]) -> float: + """ + Aggregate strength from multiple reviews. + + A single I3 review (weight=4) has same impact as: + - 4 I1 reviews, or + - 2 I2 reviews + + This captures that one "furious" customer signals more + than four "mildly annoyed" customers. + """ + return sum(INTENSITY_WEIGHTS.get(r['intensity'], 1) for r in reviews) + + +def compute_strength_by_code(business_id: str, code: str, + start: date, end: date) -> dict: + """Compute strength metrics for a URT code.""" + + reviews = db.query(""" + SELECT intensity, valence, date + FROM reviews + WHERE business_id = %s + AND (urt_primary = %s OR %s = ANY(urt_secondary)) + AND date BETWEEN %s AND %s + """, [business_id, code, code, start, end]) + + neg_reviews = [r for r in reviews if r['valence'] in ('V-', 'V±')] + pos_reviews = [r for r in reviews if r['valence'] == 'V+'] + + return { + 'code': code, + 'total_count': len(reviews), + 'negative_count': len(neg_reviews), + 'positive_count': len(pos_reviews), + 'negative_strength': compute_strength_score(neg_reviews), + 'positive_strength': compute_strength_score(pos_reviews), + 'avg_intensity': np.mean([ + INTENSITY_WEIGHTS[r['intensity']] for r in reviews + ]) if reviews else 0, + 'max_intensity': max( + (r['intensity'] for r in reviews), + key=lambda i: INTENSITY_WEIGHTS.get(i, 0), + default='I1' + ) + } +``` + +### 3.8 Impact Timeline (Time-Series Aggregation) + +Generate data for line charts showing issue/strength evolution over time: + +```python +def build_impact_timeline(business_id: str, code: str, + start: date, end: date, + bucket: str = 'week') -> list[dict]: + """ + Time-series strength aggregation for charts. + + Returns data points for plotting: + - X-axis: time periods + - Y-axis: strength score (or count, rate) + """ + + timeline = db.query(""" + SELECT + date_trunc(%s, date)::date as period, + COUNT(*) as review_count, + COUNT(*) FILTER (WHERE valence IN ('V-', 'V±')) as negative_count, + COUNT(*) FILTER (WHERE valence = 'V+') as positive_count, + SUM(CASE intensity + WHEN 'I3' THEN 4 + WHEN 'I2' THEN 2 + ELSE 1 + END) as strength_score, + SUM(CASE intensity + WHEN 'I3' THEN 4 + WHEN 'I2' THEN 2 + ELSE 1 + END) FILTER (WHERE valence IN ('V-', 'V±')) as negative_strength, + AVG(CASE intensity + WHEN 'I3' THEN 3 + WHEN 'I2' THEN 2 + ELSE 1 + END) as avg_intensity, + MAX(CASE intensity + WHEN 'I3' THEN 3 + WHEN 'I2' THEN 2 + ELSE 1 + END) as max_intensity_num, + COUNT(*) FILTER (WHERE comparative = 'CR-B') as cr_better, + COUNT(*) FILTER (WHERE comparative = 'CR-W') as cr_worse, + COUNT(*) FILTER (WHERE comparative = 'CR-S') as cr_same + FROM reviews + WHERE business_id = %s + AND (urt_primary = %s OR %s = ANY(urt_secondary)) + AND date BETWEEN %s AND %s + GROUP BY 1 + ORDER BY 1 + """, [bucket, business_id, code, code, start, end]) + + # Fill gaps for continuous chart + return fill_timeline_gaps(timeline, start, end, bucket) + + +def fill_timeline_gaps(data: list[dict], start: date, end: date, + bucket: str) -> list[dict]: + """Ensure continuous timeline with zero-fill for missing periods.""" + + from pandas import date_range + + freq = {'day': 'D', 'week': 'W-MON', 'month': 'MS'}[bucket] + all_periods = date_range(start, end, freq=freq) + + data_map = {row['period']: row for row in data} + + result = [] + for period in all_periods: + period_date = period.date() + if period_date in data_map: + result.append(data_map[period_date]) + else: + result.append({ + 'period': period_date, + 'review_count': 0, + 'negative_count': 0, + 'positive_count': 0, + 'strength_score': 0, + 'negative_strength': 0, + 'avg_intensity': None, + 'max_intensity_num': None, + 'cr_better': 0, + 'cr_worse': 0, + 'cr_same': 0 + }) + + return result + + +def get_issue_impact_chart_data(issue_id: str, + months_back: int = 6) -> dict: + """ + Generate chart-ready data for a specific issue. + + Returns structure suitable for Recharts/Chart.js: + { + "issue": {...}, + "timeline": [ + {"period": "2026-01-06", "strength": 12, "count": 5, ...}, + {"period": "2026-01-13", "strength": 8, "count": 3, ...}, + ... + ], + "summary": { + "total_strength": 156, + "peak_period": "2026-01-06", + "trend": "improving" + } + } + """ + + issue = db.query_one(""" + SELECT issue_id, primary_subcode, business_id, created_at + FROM issues WHERE issue_id = %s + """, [issue_id]) + + end = date.today() + start = end - timedelta(days=months_back * 30) + + timeline = build_impact_timeline( + issue['business_id'], + issue['primary_subcode'], + start, end, + bucket='week' + ) + + # Compute summary stats + total_strength = sum(t['negative_strength'] or 0 for t in timeline) + peak = max(timeline, key=lambda t: t['negative_strength'] or 0) + + # Trend: compare last 4 weeks vs prior 4 weeks + recent = timeline[-4:] if len(timeline) >= 4 else timeline + prior = timeline[-8:-4] if len(timeline) >= 8 else [] + + recent_avg = np.mean([t['negative_strength'] or 0 for t in recent]) + prior_avg = np.mean([t['negative_strength'] or 0 for t in prior]) if prior else recent_avg + + if recent_avg < prior_avg * 0.7: + trend = 'improving' + elif recent_avg > prior_avg * 1.3: + trend = 'worsening' + else: + trend = 'stable' + + return { + 'issue': { + 'issue_id': issue['issue_id'], + 'code': issue['primary_subcode'], + 'name': URT_CODE_NAMES.get(issue['primary_subcode'], issue['primary_subcode']) + }, + 'timeline': [ + { + 'period': t['period'].isoformat(), + 'strength': t['negative_strength'] or 0, + 'count': t['negative_count'], + 'avg_intensity': round(t['avg_intensity'], 2) if t['avg_intensity'] else None, + 'cr_signals': { + 'better': t['cr_better'], + 'worse': t['cr_worse'], + 'same': t['cr_same'] + } + } + for t in timeline + ], + 'summary': { + 'total_strength': total_strength, + 'peak_period': peak['period'].isoformat(), + 'peak_strength': peak['negative_strength'] or 0, + 'trend': trend + } + } +``` + +### 3.9 Timeline Data Model (Chart-Ready) + +```typescript +// TypeScript interface for frontend consumption + +interface IssueTimelinePoint { + period: string; // ISO date "2026-01-06" + strength: number; // Weighted strength score + count: number; // Raw review count + avg_intensity: number | null; + cr_signals: { + better: number; + worse: number; + same: number; + }; +} + +interface IssueImpactChart { + issue: { + issue_id: string; + code: string; // "J1.01" + name: string; // "Wait Time" + }; + timeline: IssueTimelinePoint[]; + summary: { + total_strength: number; + peak_period: string; + peak_strength: number; + trend: 'improving' | 'worsening' | 'stable'; + }; +} + +// Example Recharts usage: +// +// +// +// +``` + +--- + +## Part 4: Report Generation + +### 4.1 Report Structure + +```python +def generate_report(business_id: str, start: date, end: date) -> dict: + """Generate comprehensive business intelligence report.""" + + # 1. Aggregate statistics by URT code + code_stats = compute_code_statistics(business_id, start, end) + + # 2. Deep analysis of top issues (sub-clustering) + top_issues = analyze_top_issues(business_id, code_stats, start, end) + + # 3. Strength analysis + strengths = analyze_strengths(business_id, code_stats, start, end) + + # 4. Trend analysis (vs prior period) + trends = compute_trends(business_id, start, end) + + # 5. Staff insights + staff = analyze_staff_mentions(business_id, start, end) + + # 6. Open issues summary + open_issues = get_open_issues(business_id) + + # 7. Build payload + payload = build_report_payload( + business_id, start, end, + top_issues, strengths, trends, staff, open_issues + ) + + # 8. LLM narration + narrative = await generate_narrative(payload) + + return { + 'payload': payload, + 'narrative': narrative, + 'generated_at': datetime.now().isoformat() + } +``` + +### 4.2 Statistics Computation + +Review-level presence with Wilson confidence intervals: + +```python +def compute_code_statistics(business_id: str, start: date, end: date) -> list[dict]: + """Aggregate statistics by URT code with confidence intervals.""" + + stats = db.query(""" + WITH review_codes AS ( + SELECT + review_id, + urt_primary as code, + valence, + intensity + FROM reviews + WHERE business_id = %s AND date BETWEEN %s AND %s + + UNION ALL + + SELECT + review_id, + unnest(urt_secondary) as code, + valence, + intensity + FROM reviews + WHERE business_id = %s AND date BETWEEN %s AND %s + AND array_length(urt_secondary, 1) > 0 + ), + code_stats AS ( + SELECT + code, + COUNT(DISTINCT review_id) as k, + COUNT(DISTINCT review_id) FILTER (WHERE valence = 'V-') as k_neg, + COUNT(DISTINCT review_id) FILTER (WHERE valence = 'V+') as k_pos, + MAX(CASE intensity + WHEN 'I3' THEN 3 WHEN 'I2' THEN 2 ELSE 1 END) as max_intensity + FROM review_codes + GROUP BY code + ) + SELECT + cs.*, + (SELECT COUNT(DISTINCT review_id) FROM reviews + WHERE business_id = %s AND date BETWEEN %s AND %s) as n + FROM code_stats cs + WHERE k >= 3 + ORDER BY k_neg DESC + """, [business_id, start, end] * 4) + + results = [] + for row in stats: + n = row['n'] + + # Wilson confidence intervals + ci_neg = wilson_ci(row['k_neg'], n) if row['k_neg'] > 0 else (0, 0) + ci_pos = wilson_ci(row['k_pos'], n) if row['k_pos'] > 0 else (0, 0) + + results.append({ + 'code': row['code'], + 'domain': row['code'][0], + 'name': URT_CODE_NAMES[row['code']], + 'k': row['k'], + 'k_neg': row['k_neg'], + 'k_pos': row['k_pos'], + 'n': n, + 'rate_neg': row['k_neg'] / n if n > 0 else 0, + 'rate_pos': row['k_pos'] / n if n > 0 else 0, + 'ci_neg': ci_neg, + 'ci_pos': ci_pos, + 'max_intensity': f"I{row['max_intensity']}", + }) + + return results + + +def wilson_ci(k: int, n: int, z: float = 1.96) -> tuple[float, float]: + """Wilson score interval for binomial proportion.""" + if n == 0: + return (0.0, 0.0) + + p = k / n + denom = 1 + z**2 / n + center = (p + z**2 / (2*n)) / denom + margin = z * math.sqrt((p*(1-p) + z**2/(4*n)) / n) / denom + + return (max(0, center - margin), min(1, center + margin)) +``` + +### 4.3 Sub-Pattern Discovery (Local ML) + +The key insight: **LLM gives categories, local ML reveals patterns within categories.** + +```python +def analyze_top_issues(business_id: str, code_stats: list, + start: date, end: date, top_k: int = 5) -> list[dict]: + """Deep analysis of top negative codes with sub-clustering.""" + + # Filter to significant negative codes + issues_to_analyze = [ + cs for cs in code_stats + if cs['k_neg'] >= 8 and cs['ci_neg'][1] - cs['ci_neg'][0] <= 0.30 + ][:top_k] + + results = [] + for code_stat in issues_to_analyze: + code = code_stat['code'] + + # Fetch all negative reviews for this code + reviews = db.query(""" + SELECT review_id, text, embedding, intensity, quotes, date + FROM reviews + WHERE business_id = %s + AND (urt_primary = %s OR %s = ANY(urt_secondary)) + AND valence IN ('V-', 'V±') + AND date BETWEEN %s AND %s + """, [business_id, code, code, start, end]) + + # Sub-cluster to find patterns + sub_patterns = discover_sub_patterns(reviews, code) + + results.append({ + 'code': code, + 'name': code_stat['name'], + 'total_reviews': code_stat['k_neg'], + 'rate': code_stat['rate_neg'], + 'ci': code_stat['ci_neg'], + 'max_intensity': code_stat['max_intensity'], + 'sub_patterns': sub_patterns, + }) + + return results + + +def discover_sub_patterns(reviews: list[dict], code: str, + min_cluster_size: int = 3) -> list[dict]: + """Cluster reviews within a URT code to find actionable sub-patterns.""" + + if len(reviews) < min_cluster_size * 2: + # Too few for meaningful clustering + return [{ + 'label': 'General', + 'count': len(reviews), + 'percentage': 1.0, + 'representative_quote': select_representative(reviews, code), + 'sharpest_quote': select_sharpest(reviews, code), + }] + + embeddings = np.array([r['embedding'] for r in reviews]) + + # HDBSCAN for small datasets, KMeans for larger + if len(reviews) < 500: + clusterer = hdbscan.HDBSCAN( + min_cluster_size=min_cluster_size, + min_samples=2, + metric='euclidean' + ) + labels = clusterer.fit_predict(embeddings) + else: + k = min(8, max(3, int(np.sqrt(len(reviews) / 5)))) + kmeans = KMeans(n_clusters=k, n_init=3) + labels = kmeans.fit_predict(embeddings) + + # Group by cluster + clusters = {} + for review, label in zip(reviews, labels): + if label == -1: # Noise + continue + if label not in clusters: + clusters[label] = [] + clusters[label].append(review) + + # Build sub-pattern descriptions + patterns = [] + for label, cluster_reviews in clusters.items(): + if len(cluster_reviews) < min_cluster_size: + continue + + cluster_embeddings = np.array([r['embedding'] for r in cluster_reviews]) + centroid = cluster_embeddings.mean(axis=0) + centroid /= np.linalg.norm(centroid) + + patterns.append({ + 'label': extract_cluster_label(cluster_reviews, code), + 'count': len(cluster_reviews), + 'percentage': len(cluster_reviews) / len(reviews), + 'representative_quote': select_representative(cluster_reviews, code, centroid), + 'sharpest_quote': select_sharpest(cluster_reviews, code), + 'avg_intensity': np.mean([ + {'I1': 1, 'I2': 2, 'I3': 3}[r['intensity']] + for r in cluster_reviews + ]), + 'centroid': centroid, # For trend matching + }) + + patterns.sort(key=lambda x: x['count'], reverse=True) + return patterns[:4] # Top 4 sub-patterns + + +def select_representative(reviews: list, code: str, + centroid: np.ndarray = None) -> str: + """Select quote closest to centroid (most representative).""" + + if centroid is None: + embeddings = np.array([r['embedding'] for r in reviews]) + centroid = embeddings.mean(axis=0) + centroid /= np.linalg.norm(centroid) + + best_review = max(reviews, key=lambda r: r['embedding'] @ centroid) + + # Return the extracted quote for this code, or truncated text + if best_review.get('quotes') and code in best_review['quotes']: + return best_review['quotes'][code] + return best_review['text'][:150] + + +def select_sharpest(reviews: list, code: str) -> str: + """Select highest intensity quote (sharpest criticism).""" + + intensity_order = {'I3': 3, 'I2': 2, 'I1': 1} + best_review = max(reviews, key=lambda r: intensity_order.get(r['intensity'], 0)) + + if best_review.get('quotes') and code in best_review['quotes']: + return best_review['quotes'][code] + return best_review['text'][:150] + + +def extract_cluster_label(reviews: list, code: str) -> str: + """Generate a concise label for the cluster.""" + + # Extract common phrases from quotes + texts = [] + for r in reviews: + if r.get('quotes') and code in r['quotes']: + texts.append(r['quotes'][code].lower()) + else: + texts.append(r['text'][:100].lower()) + + # Find distinctive 2-3 word phrases + from collections import Counter + + all_text = ' '.join(texts) + words = re.findall(r'\b[a-z]{3,}\b', all_text) + + # Bigrams, skip stopwords + stopwords = {'the', 'and', 'was', 'were', 'for', 'that', 'this', 'with', 'but', 'have', 'had'} + bigrams = [ + f"{words[i]} {words[i+1]}" + for i in range(len(words)-1) + if words[i] not in stopwords and words[i+1] not in stopwords + ] + + counts = Counter(bigrams) + if counts: + label = counts.most_common(1)[0][0] + return label.title() + + return URT_CODE_NAMES.get(code, "General") +``` + +### 4.4 Trend Analysis + +Combine rate comparison with CR signals: + +```python +def compute_trends(business_id: str, current_start: date, + current_end: date) -> dict: + """Compute trends vs prior period using rates and CR signals.""" + + period_length = (current_end - current_start).days + prior_start = current_start - timedelta(days=period_length) + prior_end = current_start + + # Current period stats + current_stats = compute_code_statistics(business_id, current_start, current_end) + current_map = {cs['code']: cs for cs in current_stats} + + # Prior period stats + prior_stats = compute_code_statistics(business_id, prior_start, prior_end) + prior_map = {cs['code']: cs for cs in prior_stats} + + # CR signals in current period + cr_signals = db.query(""" + SELECT urt_primary as code, comparative, COUNT(*) as count + FROM reviews + WHERE business_id = %s + AND date BETWEEN %s AND %s + AND comparative != 'CR-N' + GROUP BY urt_primary, comparative + """, [business_id, current_start, current_end]) + + cr_map = {} + for row in cr_signals: + if row['code'] not in cr_map: + cr_map[row['code']] = {'CR-B': 0, 'CR-W': 0, 'CR-S': 0} + cr_map[row['code']][row['comparative']] = row['count'] + + # Compute trends + trends = {} + for code, current in current_map.items(): + prior = prior_map.get(code, {'rate_neg': 0, 'rate_pos': 0, 'k_neg': 0, 'k_pos': 0}) + cr = cr_map.get(code, {'CR-B': 0, 'CR-W': 0, 'CR-S': 0}) + + # Rate-based trend (issues) + rate_trend_neg = current['rate_neg'] - prior['rate_neg'] + + # CR-enhanced trend signal + if cr['CR-W'] >= 2: + trend_signal = 'worsening' + elif cr['CR-B'] >= 2: + trend_signal = 'improving' + elif cr['CR-S'] >= 2: + trend_signal = 'persistent' + elif rate_trend_neg > 0.05: + trend_signal = 'worsening' + elif rate_trend_neg < -0.05: + trend_signal = 'improving' + else: + trend_signal = 'stable' + + trends[code] = { + 'rate_change_neg': rate_trend_neg, + 'rate_change_pos': current['rate_pos'] - prior['rate_pos'], + 'signal': trend_signal, + 'cr_better': cr['CR-B'], + 'cr_worse': cr['CR-W'], + 'cr_same': cr['CR-S'], + } + + return trends +``` + +### 4.5 Staff Analysis + +```python +def analyze_staff_mentions(business_id: str, start: date, end: date) -> dict: + """Aggregate staff performance from mentions.""" + + staff_data = db.query(""" + SELECT + unnest(staff_mentions) as staff_name, + valence, + intensity, + urt_primary, + quotes + FROM reviews + WHERE business_id = %s + AND date BETWEEN %s AND %s + AND array_length(staff_mentions, 1) > 0 + """, [business_id, start, end]) + + staff_map = {} + for row in staff_data: + name = row['staff_name'] + if name not in staff_map: + staff_map[name] = { + 'positive': [], + 'negative': [], + 'codes': Counter() + } + + if row['valence'] in ('V+',): + staff_map[name]['positive'].append(row) + elif row['valence'] in ('V-',): + staff_map[name]['negative'].append(row) + + staff_map[name]['codes'][row['urt_primary']] += 1 + + # Build summary + staff_summary = [] + for name, data in staff_map.items(): + total = len(data['positive']) + len(data['negative']) + if total < 2: + continue # Need multiple mentions + + staff_summary.append({ + 'name': name, + 'total_mentions': total, + 'positive': len(data['positive']), + 'negative': len(data['negative']), + 'sentiment_ratio': len(data['positive']) / total, + 'top_codes': data['codes'].most_common(3), + 'sample_praise': data['positive'][0]['quotes'] if data['positive'] else None, + 'sample_criticism': data['negative'][0]['quotes'] if data['negative'] else None, + }) + + staff_summary.sort(key=lambda x: x['total_mentions'], reverse=True) + + return { + 'staff': staff_summary, + 'top_performer': max(staff_summary, key=lambda x: x['sentiment_ratio']) if staff_summary else None, + 'needs_attention': [s for s in staff_summary if s['sentiment_ratio'] < 0.5], + } +``` + +### 4.6 LLM Narrative Generation + +```python +NARRATIVE_PROMPT = """You are a business intelligence analyst writing an executive summary of customer feedback. + +You MUST follow these rules: +1. Only state claims supported by the provided data +2. Include specific numbers (percentages, counts) for every claim +3. Do not invent or hallucinate any statistics +4. Be direct and actionable, not vague +5. Highlight the most impactful findings first + +The report payload follows. Write a concise executive summary (~300 words) covering: +- Top issues with their sub-patterns and severity +- Notable strengths +- Trend signals (improving/worsening/persistent) +- Staff highlights +- 2-3 prioritized recommendations + +REPORT DATA: +{payload}""" + + +async def generate_narrative(payload: dict) -> str: + """Generate executive narrative from structured payload.""" + + response = await llm.chat( + model="gpt-4o", # Use stronger model for narrative + messages=[ + {"role": "system", "content": NARRATIVE_PROMPT.format( + payload=json.dumps(payload, indent=2) + )} + ], + temperature=0.3 + ) + + return response.content +``` + +--- + +## Part 5: Report Output Example + +### 5.1 Structured Payload + +```json +{ + "business_id": "rest_12345", + "period": "2026-01-01 to 2026-01-31", + "total_reviews": 234, + + "issues": [ + { + "code": "J1.01", + "name": "Wait Time", + "total_reviews": 47, + "rate": 0.201, + "ci": [0.153, 0.258], + "max_intensity": "I3", + "trend": { + "signal": "worsening", + "cr_worse": 3, + "rate_change": 0.042 + }, + "sub_patterns": [ + { + "label": "Table Seating", + "count": 20, + "percentage": 0.426, + "representative_quote": "Waited 45 minutes for a table even with reservation", + "sharpest_quote": "HOUR wait on a Tuesday. Unacceptable." + }, + { + "label": "Food After Ordering", + "count": 15, + "percentage": 0.319, + "representative_quote": "Food took 40 minutes after we ordered", + "sharpest_quote": "Over an hour for cold pasta" + }, + { + "label": "Check Payment", + "count": 12, + "percentage": 0.255, + "representative_quote": "Had to flag someone down just to pay", + "sharpest_quote": "20 minutes for the check, ridiculous" + } + ] + } + ], + + "strengths": [ + { + "code": "O2.02", + "name": "Craftsmanship", + "total_reviews": 89, + "rate": 0.380, + "ci": [0.318, 0.446], + "trend": {"signal": "stable"}, + "representative_quote": "The pasta is clearly made fresh, incredible quality" + } + ], + + "staff": { + "top_performer": { + "name": "Maria", + "mentions": 12, + "sentiment_ratio": 0.917 + }, + "needs_attention": [ + { + "name": "Tom", + "mentions": 8, + "sentiment_ratio": 0.375, + "top_issues": ["P1.02", "P3.01"] + } + ] + }, + + "open_issues": [ + { + "issue_id": "ISSUE-2026-0142", + "code": "J1.01", + "state": "IN_PROGRESS", + "priority": 7.45, + "days_open": 12 + } + ] +} +``` + +### 5.2 Generated Narrative + +> **Executive Summary: January 2026** +> +> Analysis of 234 reviews reveals **wait times as the critical issue**, affecting 20.1% of customers (95% CI: 15.3%-25.8%) — a worsening trend with 3 explicit "worse than before" signals this month. +> +> **Wait Time Breakdown:** +> - **Seating delays (43%)**: Customers report 30-60 minute waits despite reservations. *"Waited 45 minutes for a table even with reservation."* +> - **Kitchen delays (32%)**: Food taking 40+ minutes after ordering. *"Over an hour for cold pasta."* +> - **Checkout friction (26%)**: Difficulty getting the check. *"20 minutes for the check, ridiculous."* +> +> **Strengths remain strong**: Food craftsmanship praised in 38% of reviews, stable month-over-month. *"The pasta is clearly made fresh, incredible quality."* +> +> **Staff Notes**: Maria received 12 mentions with 92% positive sentiment. Tom (8 mentions, 38% positive) shows patterns in P1.02 (Respect) and P3.01 (Attentiveness) — recommend coaching session. +> +> **Prioritized Recommendations:** +> 1. **Immediate**: Audit reservation system — seating bottleneck is primary wait issue +> 2. **This Week**: Review kitchen workflow for food delivery timing +> 3. **This Month**: Implement checkout process training (e.g., table check-in rotation) +> +> One high-priority issue (ISSUE-2026-0142) is in progress with 12 days elapsed. + +--- + +## Part 6: Cost Model + +| Stage | When | Cost | Notes | +|-------|------|------|-------| +| **Embedding** | Per review ingested | $0.00 | Local model, ~50ms/review | +| **LLM Classification** | Per review ingested | ~$0.0002 | GPT-4o-mini, batched | +| **Issue Aggregation** | Per V- review | $0.00 | SQL queries | +| **Sub-Clustering** | Per report | $0.00 | HDBSCAN/KMeans, <1s | +| **Trend Analysis** | Per report | $0.00 | SQL + computation | +| **LLM Narrative** | Per report | ~$0.15 | GPT-4o, single call | + +**Total Costs:** + +| Volume | Monthly Ingest | Reports (10/month) | Total | +|--------|---------------|-------------------|-------| +| 1K reviews | $0.20 | $1.50 | **$1.70** | +| 10K reviews | $2.00 | $1.50 | **$3.50** | +| 100K reviews | $20.00 | $1.50 | **$21.50** | + +--- + +## Part 7: Implementation Checklist + +### Phase 1: Core Pipeline +- [ ] Set up PostgreSQL with pgvector extension +- [ ] Implement embedding generation (multilingual-e5-small) +- [ ] Build LLM classification module with batching +- [ ] Create review ingestion pipeline +- [ ] Implement URT code reference data + +### Phase 2: Issue Lifecycle +- [ ] Implement issue aggregation logic +- [ ] Build state machine with transitions +- [ ] Create priority scoring function +- [ ] Add CR signal processing for verification +- [ ] Set up issue event logging +- [ ] Implement strength score aggregation +- [ ] Build issue review drill-down query +- [ ] Create impact timeline aggregation +- [ ] Set up issue_timeseries table and population + +### Phase 3: Report Generation +- [ ] Build statistics aggregation queries +- [ ] Implement sub-pattern clustering +- [ ] Add trend analysis with CR integration +- [ ] Create staff analysis module +- [ ] Build narrative generation prompt + +### Phase 4: Integration +- [ ] API endpoints for ingestion +- [ ] Report generation endpoint +- [ ] Issue management endpoints +- [ ] Issue timeline chart endpoint +- [ ] Issue review table endpoint +- [ ] Dashboard queries +- [ ] Alert/notification hooks + +--- + +## Part 8: Key Innovations + +| Innovation | Benefit | +|------------|---------| +| **LLM at ingest, not report** | Accurate classification amortized across all reports | +| **URT as structure** | Stable, interpretable categories; no clustering drift | +| **Multi-coding** | Handle complex reviews without fragmentation | +| **Sub-clustering within codes** | Actionable patterns beyond category level | +| **CR for verification** | Automatic resolution validation from customer feedback | +| **Review as unit** | Preserve context; avoid embedding quality loss | +| **Issue lifecycle** | Operational tracking with statistical rigor | +| **Strength score** | Unified impact metric: volume × intensity | +| **Impact timeline** | Time-series visualization for trend analysis | +| **Issue drill-down** | Full review table for any aggregated issue | + +--- + +## Document Control + +| Field | Value | +|-------|-------| +| **Document** | ReviewIQ Architecture v3.0 | +| **Status** | Specification Complete | +| **Date** | 2026-01-24 | +| **Dependencies** | URT Specification v5.1, Issue Lifecycle Framework C1 | +| **Cost Target** | <$25/month at 100K reviews | +| **Accuracy Target** | >90% URT classification, >85% sub-pattern relevance | + +### Changelog v3.0 + +| Addition | Description | +|----------|-------------| +| **3.6 Issue Review Drill-Down** | Query to fetch all reviews for a specific issue | +| **3.7 Strength Score Aggregation** | Unified metric: count × intensity weight | +| **3.8 Impact Timeline** | Time-series aggregation for line charts | +| **3.9 Timeline Data Model** | TypeScript interface for frontend charts | +| **issue_timeseries table** | Persistent time-bucketed aggregation | + +--- + +*End of ReviewIQ Architecture v3.0* diff --git a/.artifacts/ReviewIQ-v32-Decisions.md b/.artifacts/ReviewIQ-v32-Decisions.md new file mode 100644 index 0000000..28e0a38 --- /dev/null +++ b/.artifacts/ReviewIQ-v32-Decisions.md @@ -0,0 +1,183 @@ +# ReviewIQ v3.2 Design Decisions + +> Fast context-recovery document — all key decisions without the full spec. + +--- + +## 1. Markpoint + +``` +ID: reviewiq-v32-span-layer-2026-01-24-001 +Status: v3.2 span layer complete +Based on: v3.1.2 (commit f998277) +``` + +--- + +## 2. Core Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Span granularity | Clause/topic-level | Preserves multi-domain signal | +| span_id format | ULID (TEXT) | Survives re-segmentation | +| Span offsets | Required (NOT NULL) | Deterministic reconstruction | +| Offsets reference | reviews_enriched.text | Not text_normalized | +| Span → Issue mapping | One-to-one (UNIQUE span_id) | Atomic unit per issue | +| Primary span enforcement | Partial unique index | Exactly one per review version | +| Primary selection | I3>I2>I1, V->V±>V0>V+, span_index | Deterministic, stable | +| Reprocessing strategy | Soft-switch with is_active | No transient empty states | +| Span overlap | GiST exclusion constraint | Non-overlapping ranges enforced | +| Secondary codes | Array with cardinality ≤ 2 | Could normalize to link table later | +| Causal chain storage | JSONB | Flexibility, normalize later if needed | +| relation_type vs causal_chain | Separate concerns | relation = within-review, causal = root cause | +| Dimension columns | Postgres ENUMs | Type safety, storage efficiency | +| Trust score floor | 0.2 (GREATEST clamp) | Prevent multiplicative collapse | +| Issue routing key | (business_id, place_id, urt_primary, entity_normalized) | Deterministic, entity-aware | +| Issue ID generation | SHA256 via pgcrypto | Deterministic, collision-resistant | +| Text validation trigger | Conditional via session setting | Performance: skip in bulk loads | +| Relation validation | Application-level post-insert | Handles insertion order | + +--- + +## 3. Extensions Required + +| Extension | Purpose | +|-----------|---------| +| `btree_gist` | Exclusion constraint for non-overlapping spans | +| `pgcrypto` | SHA256-based issue ID generation | + +--- + +## 4. New Tables + +| Table | Purpose | +|-------|---------| +| `review_spans` | Span-level URT classification | +| `review_span_secondary_codes` | (Optional) Normalized secondary codes | + +--- + +## 5. Modified Tables + +| Table | Changes | +|-------|---------| +| `issue_spans` | Added `span_id` FK (NOT NULL), removed direct review FK as canonical | + +--- + +## 6. New ENUM Types + +**Valence & Intensity:** +- `urt_valence` — V-, V±, V0, V+ +- `urt_intensity` — I1, I2, I3 + +**Specificity & Actionability:** +- `urt_specificity` — S1, S2, S3 +- `urt_actionability` — A1, A2, A3 + +**Context & Evidence:** +- `urt_temporal` — T1, T2, T3 +- `urt_evidence` — E1, E2, E3 +- `urt_comparative` — CR1, CR2, CR3 + +**Classification:** +- `urt_profile` — factual, emotional, comparative, etc. +- `urt_confidence` — low, medium, high +- `urt_relation` — elaborates, contrasts, causes, etc. +- `urt_entity_type` — person, product, location, etc. + +--- + +## 7. Key Functions + +| Function | Purpose | +|----------|---------| +| `urt_validate_causal_chain()` | Validates causal JSONB structure | +| `validate_review_relations()` | Ensures related_span_id same-parent | +| `validate_active_spans()` | Ensures valid active span set | +| `set_primary_span()` | Deterministic primary selection | +| `generate_issue_id()` | SHA256-based issue ID | + +--- + +## 8. Key Triggers + +| Trigger | Purpose | +|---------|---------| +| `review_spans_validate_bounds` | span_end ≤ text length | +| `review_spans_validate_text` | span_text matches substring | +| `review_spans_validate_causal_chain` | causal_chain JSONB valid | + +--- + +## 9. USN Format + +``` +Standard: URT:S:{codes}:{V}{I}:{S}{A}{T}.{E}.{CR} +Full: URT:F:{codes}:{V}{I}:{S}{A}{T}.{E}.{CR}:{causal} +``` + +**Examples:** +- `URT:S:SVC.SPD:V-I3:S3A3T2.E2.CR1` — Specific service speed complaint +- `URT:F:PRD.QUA:V+I2:S2A1T1.E3.CR2:staff→training` — Product quality praise with causal chain + +--- + +## 10. Span Boundary Rules + +1. **Split on contrasting conjunctions** — "but", "however", "although" +2. **Split on topic/target change** — Different entity or aspect +3. **Split on valence change** — Positive → Negative or vice versa +4. **Split on domain change** — SVC → PRD → AMB +5. **Keep cause→effect together** — Causal chain stays in one span + +--- + +## 11. Deferred to v3.3+ + +| Item | Reason | +|------|--------| +| Entity extraction implementation | Requires NER pipeline | +| Trust-weighted fact aggregation | Needs more span data | +| Secondary domain enforcement | App-level validation sufficient | +| Span-based fact counting | Currently review-based, optimize later | + +--- + +## 12. Open Questions Resolved + +| Question | Resolution | +|----------|------------| +| Span → Issue cardinality? | **One-to-one** (not many-to-many) | +| Offsets nullable for LLM-inferred? | **No** — required, NOT NULL | +| Reprocessing strategy? | **Soft-switch** with is_active flag | +| TEXT vs ENUM for dimensions? | **ENUMs** — committed to Postgres | + +--- + +## Quick Reference + +### Primary Span Selection Algorithm + +``` +ORDER BY: + 1. intensity DESC (I3 > I2 > I1) + 2. valence ASC (V- > V± > V0 > V+) + 3. span_index ASC (first wins ties) +``` + +### Issue Routing Key + +```sql +(business_id, place_id, urt_primary, entity_normalized) +``` + +### Trust Score Calculation + +```sql +GREATEST(0.2, base_trust * modifiers) -- Floor prevents collapse +``` + +--- + +*Last updated: 2026-01-24* diff --git a/.artifacts/URT-v5.1-Reference.md b/.artifacts/URT-v5.1-Reference.md new file mode 100644 index 0000000..77138aa --- /dev/null +++ b/.artifacts/URT-v5.1-Reference.md @@ -0,0 +1,331 @@ +# Universal Review Taxonomy (URT) v5.1 Reference + +## Overview + +The Universal Review Taxonomy (URT) is a classification system for customer feedback. It provides a structured approach to categorizing, annotating, and analyzing review content across any industry. + +### Key Characteristics + +- **Three Profiles**: Core, Standard, Full (increasing detail) +- **Seven Domains**: Covering all aspects of customer experience +- **Tier-3 Canonical Codes**: Format `X#.##` (e.g., J1.02, P2.15) +- **Dimensional Annotation**: Valence, intensity, specificity, and more +- **Causal Analysis**: Root cause chains (Full profile) + +--- + +## Domain Codes + +URT organizes feedback into seven domains, each identified by a single letter. + +| Domain | Letter | Description | +|--------|--------|-------------| +| Offering | O | Product/service quality | +| Price | P | Value, pricing, promotions | +| Journey | J | Customer experience, timing, process | +| Environment | E | Physical/digital space | +| Attitude | A | Staff behavior, service attitude | +| Voice | V | Brand, communication, marketing | +| Relationship | R | Loyalty, trust, long-term relationship | + +### Tier-3 Code Format + +``` +Pattern: [OPJEAVR][1-4]\.[0-9]{2} +``` + +Examples: +- `J1.02` - Journey domain, category 1, subcategory 02 +- `P2.15` - Price domain, category 2, subcategory 15 +- `A3.01` - Attitude domain, category 3, subcategory 01 + +--- + +## Dimension Codes + +### Valence + +Indicates the sentiment direction of the feedback. + +| Code | Meaning | +|------|---------| +| V+ | Positive | +| V- | Negative | +| V0 | Neutral | +| V± | Mixed | + +### Intensity + +Indicates the strength of the expressed sentiment. + +| Code | Meaning | +|------|---------| +| I1 | Low intensity | +| I2 | Moderate intensity | +| I3 | High intensity | + +### Specificity (Standard+) + +Indicates how detailed the feedback is. + +| Code | Meaning | +|------|---------| +| S1 | Low - vague, general | +| S2 | Medium - some detail | +| S3 | High - specific, precise | + +### Actionability (Standard+) + +Indicates whether clear actions can be derived from the feedback. + +| Code | Meaning | +|------|---------| +| A1 | None - no clear action | +| A2 | Unclear - possible actions | +| A3 | Clear - specific actionable | + +### Temporal (Standard+) + +Indicates the time frame referenced in the feedback. + +| Code | Meaning | Markers | +|------|---------|---------| +| TC | Current - this visit | "today", "this time", "yesterday" | +| TR | Recent - last few visits | "lately", "recently", "again" | +| TH | Historical - long-standing | "for years", "always", "historically" | +| TF | Future - expectations | "I won't come back", "next time" | + +**Default**: TC when no temporal language exists. + +### Evidence (Standard+) + +Indicates how the information was obtained from the text. + +| Code | Meaning | Example | +|------|---------|---------| +| ES | Stated - explicit in text | "Waited 45 minutes" | +| EI | Inferred - logically entailed | "Took 3 weeks to reply" → slow response | +| EC | Contextual - depends on context | "That happened again" | + +**Default**: ES. Use EI/EC only when needed. + +### Comparative + +Indicates whether the feedback compares to alternatives. + +| Code | Meaning | +|------|---------| +| CR-N | No comparison | +| CR-B | Better than alternatives | +| CR-W | Worse than alternatives | +| CR-S | Same as alternatives | + +--- + +## USN (URT String Notation) + +USN is a compact string encoding for URT annotations. + +### Grammar + +``` +Standard: URT:S:{codes}:{V}{I}:{S}{A}{T}.{E}.{CR} +Full: URT:F:{codes}:{V}{I}:{S}{A}{T}.{E}.{CR}:{causal} +``` + +### Encoding Rules + +**Valence**: +- `+` for V+ +- `-` for V- + +**Intensity**: +- `1` for I1 +- `2` for I2 +- `3` for I3 + +### Examples + +**Standard Profile**: +``` +URT:S:J1.03:-2:22TC.ES.N +``` +Decoded: +- Profile: Standard +- Code: J1.03 +- Valence: V- (negative) +- Intensity: I2 +- Specificity: S2 +- Actionability: A2 +- Temporal: TC +- Evidence: ES +- Comparative: CR-N + +**Full Profile with Causal Chain**: +``` +URT:F:J1.01+A1.04:-3:23TR.EI.S:CD.O,MG.O +``` +Decoded: +- Profile: Full +- Codes: J1.01, A1.04 +- Valence: V- (negative) +- Intensity: I3 +- Specificity: S2 +- Actionability: A3 +- Temporal: TR +- Evidence: EI +- Comparative: CR-S +- Causal: CD.O (Conditions-Operational), MG.O (Management-Oversight) + +--- + +## Causal Chain (Full Profile Only) + +The causal chain identifies root causes across three layers, ordered from immediate to systemic. + +### Layers + +| Layer | Codes | Scope | +|-------|-------|-------| +| conditions | CD-S, CD-T, CD-E, CD-F, CD-O | Staff State, Team Dynamics, Equipment, Facility, Operational | +| management | MG-P, MG-T, MG-O, MG-R, MG-C | Planning, Training, Oversight, Resources, Communication | +| systemic | SY-R, SY-P, SY-C, SY-S, SY-H, SY-X | Resource Decisions, Policy, Culture, Standards, Human Capital, External | + +### Code Reference + +**Conditions Layer**: +- `CD-S` - Staff State +- `CD-T` - Team Dynamics +- `CD-E` - Equipment +- `CD-F` - Facility +- `CD-O` - Operational + +**Management Layer**: +- `MG-P` - Planning +- `MG-T` - Training +- `MG-O` - Oversight +- `MG-R` - Resources +- `MG-C` - Communication + +**Systemic Layer**: +- `SY-R` - Resource Decisions +- `SY-P` - Policy +- `SY-C` - Culture +- `SY-S` - Standards +- `SY-H` - Human Capital +- `SY-X` - External + +### JSONB Schema + +```json +[ + {"layer": "conditions", "code": "CD-O", "evidence": "ES"}, + {"layer": "management", "code": "MG-P", "evidence": "EI"} +] +``` + +### Constraints + +- Maximum 3 entries (one per layer) +- Only include when text explicitly supports it +- Order: conditions → management → systemic + +--- + +## Span Boundary Detection Rules + +Spans are detected at the clause/topic level, not sentence level. + +### Split Rules (in priority order) + +1. **Split on contrasting conjunctions**: but, however, although, despite, yet +2. **Split when subject/target changes** (topic shift) +3. **Split when valence changes** (positive ↔ negative) +4. **Split when domain changes** (O/P/J/E/A/V/R) +5. **Keep together** for cause→effect within same feedback unit + +### Guidelines + +- **Maximum**: ~3 spans per sentence +- **Validation**: If 4+ spans detected, re-check for over-splitting + +### Example + +**Input**: +> "The food was great but the service was slow and the bathroom was dirty." + +**Output**: 3 spans +1. "The food was great" (Offering, positive) +2. "the service was slow" (Journey/Attitude, negative) +3. "the bathroom was dirty" (Environment, negative) + +**Reasoning**: Topic shift + domain shift at each boundary. + +--- + +## Primary Span Selection + +When a review contains multiple spans, select the primary span using these criteria in order: + +### Selection Priority + +1. **Highest intensity** (I3 > I2 > I1) +2. **Tie-break**: Negative over positive (V- > V± > V0 > V+) +3. **Tie-break**: Earliest span_index + +### Example + +Given spans: +- Span 0: I2, V+ +- Span 1: I3, V+ +- Span 2: I3, V- + +**Primary**: Span 2 (highest intensity I3, negative valence wins tie-break) + +--- + +## Secondary Codes Rules + +Secondary codes capture additional topics mentioned in a span. + +### Constraints + +- **Maximum**: 2 secondary codes +- **Format**: Must be Tier-3 (X#.##) +- **Recommendation**: Should be different domain from primary + +### Example + +Primary: `J1.03` (Journey) +Secondary: `A2.01`, `E1.05` (Attitude, Environment) + +--- + +## Quick Reference Card + +### Profiles + +| Profile | Dimensions | Causal Chain | +|---------|------------|--------------| +| Core | V, I | No | +| Standard | V, I, S, A, T, E, CR | No | +| Full | V, I, S, A, T, E, CR | Yes | + +### USN Quick Format + +``` +URT:{S|F}:{tier3_codes}:{valence}{intensity}:{SAT}.{E}.{CR}[:{causal}] +``` + +### Domain Letters + +``` +O P J E A V R +│ │ │ │ │ │ └─ Relationship +│ │ │ │ │ └─── Voice +│ │ │ │ └───── Attitude +│ │ │ └─────── Environment +│ │ └───────── Journey +│ └─────────── Price +└───────────── Offering +``` diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/middleware/__init__.py b/api/middleware/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/routes/__init__.py b/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api_server_production.py b/api_server_production.py index f7b0733..4e3ec92 100644 --- a/api_server_production.py +++ b/api_server_production.py @@ -20,13 +20,13 @@ from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel, HttpUrl, Field from fastapi.responses import JSONResponse, StreamingResponse -from modules.database import DatabaseManager, JobStatus -from modules.webhooks import WebhookDispatcher, WebhookManager -from modules.health_checks import HealthCheckSystem -from modules.scraper_clean import fast_scrape_reviews, LogCapture, get_business_card_info # Clean scraper -from modules.crash_analyzer import analyze_crash, summarize_crash_patterns, apply_auto_fix -from modules.structured_logger import StructuredLogger, LogEntry -from modules.chrome_pool import ( +from core.database import DatabaseManager, JobStatus +from services.webhook_service import WebhookDispatcher, WebhookManager +from utils.health_checks import HealthCheckSystem +from scrapers.google_reviews.v1_0_0 import fast_scrape_reviews, LogCapture, get_business_card_info # Clean scraper +from utils.crash_analyzer import analyze_crash, summarize_crash_patterns, apply_auto_fix +from utils.logger import StructuredLogger, LogEntry +from workers.chrome_pool import ( start_worker_pools, stop_worker_pools, get_validation_worker, diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/config.py b/core/config.py similarity index 100% rename from modules/config.py rename to core/config.py diff --git a/modules/database.py b/core/database.py similarity index 99% rename from modules/database.py rename to core/database.py index f288e43..aaeb842 100644 --- a/modules/database.py +++ b/core/database.py @@ -8,22 +8,13 @@ import json from datetime import datetime from typing import Optional, List, Dict, Any from uuid import UUID, uuid4 -from enum import Enum import logging +from core.enums import JobStatus + log = logging.getLogger(__name__) -class JobStatus(str, Enum): - """Job status enumeration""" - PENDING = "pending" - RUNNING = "running" - COMPLETED = "completed" - FAILED = "failed" - CANCELLED = "cancelled" - PARTIAL = "partial" # Job crashed but has partial reviews saved - - class DatabaseManager: """PostgreSQL database manager with connection pooling""" diff --git a/core/enums.py b/core/enums.py new file mode 100644 index 0000000..2f0dfcb --- /dev/null +++ b/core/enums.py @@ -0,0 +1,14 @@ +""" +Enumerations for the ReviewIQ project. +""" +from enum import Enum + + +class JobStatus(str, Enum): + """Job status enumeration""" + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + PARTIAL = "partial" # Job crashed but has partial reviews saved diff --git a/modules/models.py b/core/models.py similarity index 97% rename from modules/models.py rename to core/models.py index b30c1ec..c1b3d32 100644 --- a/modules/models.py +++ b/core/models.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from selenium.webdriver.remote.webelement import WebElement -from modules.utils import (try_find, first_text, first_attr, safe_int, detect_lang, parse_date_to_iso) +from utils.helpers import (try_find, first_text, first_attr, safe_int, detect_lang, parse_date_to_iso) @dataclass @@ -27,7 +27,7 @@ class RawReview: owner_date: str = "" owner_text: str = "" review_date: str = "" # ISO format date - + # Translation fields translations: dict = field(default_factory=dict) # Store translations by language code diff --git a/modules/data_storage.py b/modules/_legacy/data_storage.py similarity index 100% rename from modules/data_storage.py rename to modules/_legacy/data_storage.py diff --git a/modules/image_handler.py b/modules/_legacy/image_handler.py similarity index 100% rename from modules/image_handler.py rename to modules/_legacy/image_handler.py diff --git a/modules/s3_handler.py b/modules/_legacy/s3_handler.py similarity index 100% rename from modules/s3_handler.py rename to modules/_legacy/s3_handler.py diff --git a/scrapers/__init__.py b/scrapers/__init__.py new file mode 100644 index 0000000..cfd8412 --- /dev/null +++ b/scrapers/__init__.py @@ -0,0 +1,10 @@ +""" +Scrapers Package + +This package contains all scraper implementations for the ReviewIQ system. +""" + +from scrapers.base import BaseScraper +from scrapers.registry import ScraperRegistry, registry + +__all__ = ["BaseScraper", "ScraperRegistry", "registry"] diff --git a/scrapers/base.py b/scrapers/base.py new file mode 100644 index 0000000..e2df076 --- /dev/null +++ b/scrapers/base.py @@ -0,0 +1,97 @@ +""" +Base Scraper Interface + +This module defines the abstract base class that all scrapers must implement. +It ensures consistent interface across different scraper implementations. +""" + +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Optional + + +class BaseScraper(ABC): + """ + Abstract base class for all scrapers in the ReviewIQ system. + + All concrete scraper implementations must inherit from this class + and implement the required abstract methods. + """ + + @abstractmethod + def scrape( + self, + driver: Any, + url: str, + max_reviews: int = 5000, + timeout_no_new: int = 15, + flush_callback: Optional[Callable[[List[Dict]], None]] = None, + flush_batch_size: int = 500, + progress_callback: Optional[Callable[[int, Optional[int]], None]] = None, + validation_only: bool = False + ) -> Dict[str, Any]: + """ + Scrape reviews from the given URL. + + Args: + driver: WebDriver instance (e.g., Selenium WebDriver) + url: The URL to scrape reviews from + max_reviews: Maximum number of reviews to collect + timeout_no_new: Seconds to wait with no new reviews before stopping + flush_callback: Optional callback called with reviews batches for streaming + flush_batch_size: Number of reviews before triggering flush_callback + progress_callback: Optional callback(current_count, total_count) for progress + validation_only: If True, return early after extracting metadata only + + Returns: + Dictionary containing: + - reviews: List of review dictionaries + - total: Total number of reviews collected + - error: Error message if any, None otherwise + - Additional scraper-specific metadata + """ + pass + + @abstractmethod + def validate_url(self, url: str) -> bool: + """ + Validate if the given URL is supported by this scraper. + + Args: + url: The URL to validate + + Returns: + True if the URL is valid for this scraper, False otherwise + """ + pass + + @abstractmethod + def get_business_info(self, driver: Any, url: str) -> Dict[str, Any]: + """ + Extract business information from the URL without scraping reviews. + + Args: + driver: WebDriver instance + url: The URL to extract info from + + Returns: + Dictionary containing business metadata (name, rating, address, etc.) + """ + pass + + @property + @abstractmethod + def name(self) -> str: + """Return the human-readable name of this scraper.""" + pass + + @property + @abstractmethod + def version(self) -> str: + """Return the version string of this scraper.""" + pass + + @property + @abstractmethod + def supported_domains(self) -> List[str]: + """Return list of domains this scraper supports.""" + pass diff --git a/scrapers/google_reviews/__init__.py b/scrapers/google_reviews/__init__.py new file mode 100644 index 0000000..218455d --- /dev/null +++ b/scrapers/google_reviews/__init__.py @@ -0,0 +1,21 @@ +""" +Google Reviews Scraper Package + +This package contains the Google Reviews scraper implementations. +""" + +from scrapers.google_reviews.v1_0_0 import ( + scrape_reviews, + fast_scrape_reviews, + get_business_card_info, + extract_about_info, + LogCapture, +) + +__all__ = [ + "scrape_reviews", + "fast_scrape_reviews", + "get_business_card_info", + "extract_about_info", + "LogCapture", +] diff --git a/modules/scraper_clean.py b/scrapers/google_reviews/v1_0_0.py similarity index 99% rename from modules/scraper_clean.py rename to scrapers/google_reviews/v1_0_0.py index a8048a2..7595f70 100644 --- a/modules/scraper_clean.py +++ b/scrapers/google_reviews/v1_0_0.py @@ -1,7 +1,12 @@ """ -Clean Google Maps Reviews Scraper +Google Reviews Scraper v1.0.0 + +This module provides the core Google Maps reviews scraping functionality. - Simple down scrolling - DOM scraping + API interception + +Version: 1.0.0 +Migrated from: modules/scraper_clean.py """ import re @@ -12,7 +17,7 @@ from datetime import datetime from typing import List, Optional from selenium.webdriver.common.by import By -from modules.structured_logger import StructuredLogger +from utils.logger import StructuredLogger def get_chrome_memory(driver) -> Optional[int]: """Get Chrome memory usage in MB using CDP.""" diff --git a/scrapers/registry.py b/scrapers/registry.py new file mode 100644 index 0000000..213d88c --- /dev/null +++ b/scrapers/registry.py @@ -0,0 +1,138 @@ +""" +Scraper Registry + +This module provides a registry for managing and discovering scrapers. +It allows dynamic registration and lookup of scraper implementations. +""" + +from typing import Dict, List, Optional, Type + +from scrapers.base import BaseScraper + + +class ScraperRegistry: + """ + Registry for managing scraper implementations. + + The registry allows: + - Registering scrapers by name and version + - Looking up scrapers by domain or name + - Listing all available scrapers + + Usage: + registry = ScraperRegistry() + registry.register(GoogleReviewsScraper) + scraper = registry.get_scraper_for_url("https://google.com/maps/place/...") + """ + + _instance: Optional["ScraperRegistry"] = None + _scrapers: Dict[str, Type[BaseScraper]] + + def __new__(cls) -> "ScraperRegistry": + """Singleton pattern to ensure one global registry.""" + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._scrapers = {} + cls._instance._domain_map = {} + return cls._instance + + def register(self, scraper_class: Type[BaseScraper], name: Optional[str] = None) -> None: + """ + Register a scraper class with the registry. + + Args: + scraper_class: The scraper class to register (must inherit from BaseScraper) + name: Optional name override, defaults to scraper_class.name property + """ + # Create a temporary instance to get properties + # Note: In production, we might want scraper_class to have class-level properties + instance = scraper_class.__new__(scraper_class) + + scraper_name = name or instance.name + scraper_version = instance.version + key = f"{scraper_name}:{scraper_version}" + + self._scrapers[key] = scraper_class + + # Map domains to this scraper + for domain in instance.supported_domains: + if domain not in self._domain_map: + self._domain_map[domain] = [] + self._domain_map[domain].append(key) + + def get_scraper(self, name: str, version: Optional[str] = None) -> Optional[Type[BaseScraper]]: + """ + Get a scraper class by name and optional version. + + Args: + name: The scraper name + version: Optional version string. If not provided, returns the latest. + + Returns: + The scraper class, or None if not found + """ + if version: + key = f"{name}:{version}" + return self._scrapers.get(key) + + # Find latest version for this name + matching = [k for k in self._scrapers.keys() if k.startswith(f"{name}:")] + if not matching: + return None + + # Sort by version and return latest + matching.sort(reverse=True) + return self._scrapers.get(matching[0]) + + def get_scraper_for_url(self, url: str) -> Optional[Type[BaseScraper]]: + """ + Find a suitable scraper for the given URL. + + Args: + url: The URL to find a scraper for + + Returns: + The scraper class that can handle this URL, or None if no match + """ + from urllib.parse import urlparse + + parsed = urlparse(url) + domain = parsed.netloc.lower() + + # Remove www. prefix for matching + if domain.startswith("www."): + domain = domain[4:] + + scraper_keys = self._domain_map.get(domain, []) + if not scraper_keys: + return None + + # Return the latest version + scraper_keys.sort(reverse=True) + return self._scrapers.get(scraper_keys[0]) + + def list_scrapers(self) -> List[Dict[str, str]]: + """ + List all registered scrapers. + + Returns: + List of dictionaries with scraper info (name, version, domains) + """ + result = [] + for key, scraper_class in self._scrapers.items(): + instance = scraper_class.__new__(scraper_class) + result.append({ + "name": instance.name, + "version": instance.version, + "domains": instance.supported_domains + }) + return result + + def clear(self) -> None: + """Clear all registered scrapers. Useful for testing.""" + self._scrapers.clear() + self._domain_map.clear() + + +# Global registry instance +registry = ScraperRegistry() diff --git a/services/__init__.py b/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/webhooks.py b/services/webhook_service.py similarity index 100% rename from modules/webhooks.py rename to services/webhook_service.py diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scrapers/__init__.py b/tests/scrapers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scrapers/google_reviews/__init__.py b/tests/scrapers/google_reviews/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/__init__.py b/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/crash_analyzer.py b/utils/crash_analyzer.py similarity index 100% rename from modules/crash_analyzer.py rename to utils/crash_analyzer.py diff --git a/modules/date_converter.py b/utils/date_converter.py similarity index 100% rename from modules/date_converter.py rename to utils/date_converter.py diff --git a/modules/health_checks.py b/utils/health_checks.py similarity index 95% rename from modules/health_checks.py rename to utils/health_checks.py index 2210559..86f5b76 100644 --- a/modules/health_checks.py +++ b/utils/health_checks.py @@ -67,7 +67,7 @@ class CanaryMonitor: # Alert if multiple consecutive failures if self.consecutive_failures >= 3: await self.send_alert( - f"🚨 CRITICAL: Scraper canary failed {self.consecutive_failures} times in a row! " + f"CRITICAL: Scraper canary failed {self.consecutive_failures} times in a row! " f"Last error: {str(e)[:200]}" ) @@ -90,7 +90,7 @@ class CanaryMonitor: - Scrape time is reasonable - Data structure is valid """ - from modules.scraper_clean import fast_scrape_reviews + from scrapers.google_reviews.v1_0_0 import fast_scrape_reviews log.info(f"Running canary scrape test on {self.test_url[:60]}...") self.last_run = datetime.now() @@ -121,7 +121,7 @@ class CanaryMonitor: if all_passed: # Success! log.info( - f"✅ Canary test PASSED: {result['count']} reviews in {result['time']:.1f}s" + f"Canary test PASSED: {result['count']} reviews in {result['time']:.1f}s" ) self.consecutive_failures = 0 self.last_success = datetime.now() @@ -144,7 +144,7 @@ class CanaryMonitor: # Validation failed failed_checks = [k for k, v in checks.items() if not v] log.error( - f"❌ Canary test FAILED: validation failed on {failed_checks}" + f"Canary test FAILED: validation failed on {failed_checks}" ) self.consecutive_failures += 1 self.last_result = { @@ -167,12 +167,12 @@ class CanaryMonitor: # Alert on failure if self.consecutive_failures >= 3: await self.send_alert( - f"🚨 CRITICAL: Canary validation failed {self.consecutive_failures} times! " + f"CRITICAL: Canary validation failed {self.consecutive_failures} times! " f"Failed checks: {failed_checks}" ) except asyncio.TimeoutError: - log.error("❌ Canary test TIMEOUT (>60s)") + log.error("Canary test TIMEOUT (>60s)") self.consecutive_failures += 1 self.last_result = { "status": "timeout", @@ -186,11 +186,11 @@ class CanaryMonitor: if self.consecutive_failures >= 3: await self.send_alert( - f"🚨 CRITICAL: Canary timeout {self.consecutive_failures} times!" + f"CRITICAL: Canary timeout {self.consecutive_failures} times!" ) except Exception as e: - log.error(f"❌ Canary test ERROR: {e}") + log.error(f"Canary test ERROR: {e}") self.consecutive_failures += 1 self.last_result = { "status": "error", diff --git a/modules/utils.py b/utils/helpers.py similarity index 100% rename from modules/utils.py rename to utils/helpers.py diff --git a/modules/structured_logger.py b/utils/logger.py similarity index 100% rename from modules/structured_logger.py rename to utils/logger.py diff --git a/workers/__init__.py b/workers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/chrome_pool.py b/workers/chrome_pool.py similarity index 100% rename from modules/chrome_pool.py rename to workers/chrome_pool.py