Initial commit - WhyRating Engine (Google Reviews Scraper)

2026-02-02 18:19:00 +00:00
parent 0543a08242
commit 2206ddeff2
136 changed files with 51138 additions and 855 deletions
--- a/packages/reviewiq-pipeline/scripts/fix_l1_configs.py
+++ b/packages/reviewiq-pipeline/scripts/fix_l1_configs.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""
+Fix L1 configs based on validation results.
+
+Applies fixes discovered during validation:
+1. Enable primitives that were disabled but appearing frequently
+2. Remove weights for primitives with zero appearances
+3. Add weights for high-frequency unweighted primitives
+"""
+
+import json
+from pathlib import Path
+
+CONFIGS_DIR = Path(__file__).parent.parent / "data" / "primitive_configs" / "l1"
+
+# Fixes based on validation results
+# Format: { sector: { "enable": [primitives], "disable": [primitives], "add_weight": {prim: weight}, "remove_weight": [prims] } }
+FIXES = {
+    "ENTERTAINMENT": {
+        "enable": ["CRAFT", "CONSISTENCY", "COMMUNICATION", "FRICTION"],
+        "disable": [],
+        "add_weight": {},
+        "remove_weight": ["CONDITION"],  # 0 appearances despite 1.4x weight
+    },
+    "FOOD_DINING": {
+        "enable": ["PRICE_LEVEL", "ACCESSIBILITY", "PRICE_TRANSPARENCY", "FRICTION", "EFFECTIVENESS"],
+        "disable": [],
+        "add_weight": {},
+        "remove_weight": [],
+    },
+    "AUTOMOTIVE": {
+        "enable": ["CRAFT", "CONSISTENCY", "PRICE_LEVEL", "AMBIANCE"],
+        "disable": [],
+        "add_weight": {},
+        "remove_weight": [],
+    },
+    "HEALTHCARE": {
+        "enable": ["CRAFT", "PRICE_LEVEL", "AMBIANCE"],
+        "disable": [],
+        "add_weight": {},
+        "remove_weight": [],
+    },
+    "RETAIL_SHOPPING": {
+        "enable": ["CRAFT", "PRICE_LEVEL", "AMBIANCE"],
+        "disable": [],
+        "add_weight": {},
+        "remove_weight": [],
+    },
+    "HOSPITALITY_TRAVEL": {
+        "enable": ["CRAFT", "CONSISTENCY", "PRICE_LEVEL"],
+        "disable": [],
+        "add_weight": {},
+        "remove_weight": [],
+    },
+    "PERSONAL_SERVICES": {
+        "enable": ["PRICE_LEVEL", "SPEED", "FRICTION"],
+        "disable": [],
+        "add_weight": {},
+        "remove_weight": [],
+    },
+}
+
+
+def fix_config(sector_code: str, fixes: dict) -> dict:
+    """Apply fixes to a sector config."""
+    config_path = CONFIGS_DIR / f"{sector_code.lower()}_config.json"
+
+    if not config_path.exists():
+        print(f"  ⚠️  Config not found: {config_path}")
+        return None
+
+    with open(config_path) as f:
+        config = json.load(f)
+
+    enabled = set(config.get("enabled", []))
+    disabled = set(config.get("disabled", []))
+    weights = config.get("weights", {})
+
+    changes = []
+
+    # Apply enables (move from disabled to enabled)
+    for prim in fixes.get("enable", []):
+        if prim in disabled:
+            disabled.remove(prim)
+            enabled.add(prim)
+            changes.append(f"✓ Enabled {prim}")
+        elif prim not in enabled:
+            enabled.add(prim)
+            changes.append(f"✓ Added {prim} to enabled")
+
+    # Apply disables (move from enabled to disabled)
+    for prim in fixes.get("disable", []):
+        if prim in enabled:
+            enabled.remove(prim)
+            disabled.add(prim)
+            changes.append(f"✗ Disabled {prim}")
+
+    # Add weights
+    for prim, weight in fixes.get("add_weight", {}).items():
+        if prim not in weights:
+            weights[prim] = weight
+            changes.append(f"⚖️ Added weight {prim}: {weight}x")
+
+    # Remove weights
+    for prim in fixes.get("remove_weight", []):
+        if prim in weights:
+            del weights[prim]
+            changes.append(f"⚖️ Removed weight for {prim}")
+
+    # Update config
+    config["enabled"] = sorted(enabled)
+    config["disabled"] = sorted(disabled)
+    config["weights"] = dict(sorted(weights.items()))
+    config["config_version"] = "1.1"  # Bump version
+
+    # Save
+    with open(config_path, "w") as f:
+        json.dump(config, f, indent=2)
+        f.write("\n")
+
+    return changes
+
+
+def main():
+    print("=" * 60)
+    print("L1 CONFIG FIXER - Applying validation-based fixes")
+    print("=" * 60)
+
+    total_changes = 0
+
+    for sector, fixes in FIXES.items():
+        print(f"\n📁 {sector}")
+        changes = fix_config(sector, fixes)
+        if changes:
+            for change in changes:
+                print(f"   {change}")
+            total_changes += len(changes)
+        else:
+            print("   No changes applied")
+
+    print(f"\n{'=' * 60}")
+    print(f"Total changes applied: {total_changes}")
+    print("Config version bumped to 1.1")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()