#!/usr/bin/env python3
"""
Recategorize items from Other.Uncategorized into appropriate existing categories.

RULES:
1. NEVER create new Level 1 (Sector) categories
2. Only create new Level 2 (Business Type) if >10 items would use it
3. Only create new Level 3 (Sub-category) if >5 items would use it
4. Prefer matching to existing categories at all times
5. If uncertain, leave in Other

EXISTING SECTORS (21 non-Other):
- Agriculture: Farming, Services
- Automotive: Dealers, Fuel & Charging, Parking, Parts & Accessories, Rental Services, Repair & Maintenance, Training, Vehicle Care
- Education: Arts Education, Early Childhood, Higher Education, K-12 Schools, Language Learning, Libraries, Professional Training, Specialty Schools, Sports Training, Technology Training, Tutoring, Vocational Training
- Entertainment: Amusement, Arts, Fitness, Gambling, Games & Recreation, Movies, Museums, Music Venues, Parks, Performing Arts, Recreation, Social, Sports, Venues, Wildlife
- Events_Weddings: Attire, Florists, Memorial, Planning, Rentals, Services, Venues
- Finance_Insurance: Banking, Insurance, Investment, Lending, Money Services
- Food_Dining: Bakeries & Desserts, Bars & Nightlife, Beverage Production, Cafes & Coffee, Food Services, Quick Service, Restaurants
- Government: International, Legal, Local Government, Postal, Public Safety, Social Services, Transportation
- Healthcare: Alternative Medicine, Clinics, Dental, Diagnostics, Emergency Services, Hospitals, Medical Practitioners, Mental Health, Pharmacies, Rehabilitation, Senior Care, Specialty Care, Veterinary, Vision Care
- Home_Services: Appliance Repair, Cleaning, Construction, Design, Electrical, Flooring, General Repair, HVAC, Landscaping, Moving, Pest Control, Plumbing, Pool & Spa, Roofing, Security, Windows & Doors
- Hospitality_Travel: Attractions, Lodging, Transportation, Travel Services
- Industrial: Construction, Manufacturing, Mining
- Non_Profit: Charities, Community, General, Professional
- Personal_Services: Body Art, Clothing Care, Fitness, Hair Care, Laundry, Massage, Spa & Wellness
- Pets_Animals: Animal Welfare, Pet Services
- Professional_Services: Agencies, Business Services, Consulting, Creative Services, Design, Engineering, Financial Services, HR Services, Language Services, Legal, Marketing & Advertising
- Real_Estate: Agencies, Commercial, Development, Management, Residential, Services, Storage
- Religious: Buddhism, Christian, Hinduism, Islam, Judaism, Other
- Retail_Shopping: Arts & Crafts, Beauty & Cosmetics, Books & Office, Clothing & Fashion, Electronics, Food & Grocery, Hardware & Building, Health & Pharmacy, Home & Garden, Jewelry & Watches, Markets, Music & Entertainment, Pet Supplies, Secondhand & Vintage, Specialty Retail, Sports & Outdoors, Toys & Hobbies, Wholesale & Distribution
- Technology: IT Services, Infrastructure, Software, Telecommunications
- Transportation: Delivery, Logistics, Passenger, Public Transit, Vehicle Services
"""

import psycopg2
import re
from collections import defaultdict

# Database connection
DB_URL = "postgresql://scraper:scraper123@localhost:5437/scraper"

def slugify(text):
    """Convert text to slug format"""
    slug = re.sub(r'[^\w\s-]', '', text)
    slug = re.sub(r'[-\s]+', '_', slug)
    return slug.strip('_')

# ==================== CATEGORIZATION RULES ====================
# Format: (keyword_pattern, sector, business_type, sub_category)
# Use regex patterns for flexibility

CATEGORIZATION_RULES = [
    # ==================== SPORTS & FITNESS (→ Entertainment.Sports or Entertainment.Fitness) ====================
    # Sports clubs and facilities
    (r'\b(basketball|baseball|football|soccer|tennis|golf|hockey|rugby|cricket|volleyball|badminton|squash|racquetball)\b.*(club|court|field|ground|stadium|arena|complex)', 'Entertainment', 'Sports', 'Facilities'),
    (r'\b(swimming|diving|aquatic|pool)\b.*(club|center|pool|facility)', 'Entertainment', 'Sports', 'Aquatic'),
    (r'\b(gym|fitness|workout|crossfit|aerobic|pilates|yoga|zumba)\b.*(center|studio|club|class)', 'Entertainment', 'Fitness', 'Studios'),
    (r'\b(martial arts|karate|judo|taekwondo|aikido|boxing|kickboxing|mma|wrestling|fencing)\b.*(club|school|academy|dojo|studio)', 'Entertainment', 'Sports', 'Martial_Arts'),
    (r'\b(archery|shooting|rifle|gun)\b.*(range|club|center)', 'Entertainment', 'Sports', 'Shooting'),
    (r'\b(skateboard|skate park|bmx|cycling|bicycle)\b.*(park|venue|club|center)', 'Entertainment', 'Sports', 'Cycling_Skating'),
    (r'\b(climbing|bouldering|rock climbing)\b.*(gym|wall|center|club)', 'Entertainment', 'Fitness', 'Climbing'),
    (r'\b(dance|ballet|ballroom|salsa|tango)\b.*(studio|school|class|instructor)', 'Entertainment', 'Performing Arts', 'Dance'),
    (r'\bsports\b.*(center|complex|facility|club)', 'Entertainment', 'Sports', 'General'),
    (r'\bathletic\b.*(field|track|club|center)', 'Entertainment', 'Sports', 'Facilities'),
    (r'\b(rowing|canoeing|kayaking|sailing|boat)\b.*(club|center|school)', 'Entertainment', 'Sports', 'Water_Sports'),
    (r'\b(equestrian|horse|polo|riding)\b.*(club|center|school|stable|arena)', 'Entertainment', 'Sports', 'Equestrian'),
    (r'\b(ski|snowboard|ice skating|ice rink)\b.*(resort|center|club|rink)', 'Entertainment', 'Sports', 'Winter_Sports'),

    # Instructors and trainers
    (r'\b(fitness|personal|sports|athletic)\b.*\b(trainer|instructor|coach)\b', 'Entertainment', 'Fitness', 'Trainers'),
    (r'\baerobic.*instructor\b', 'Entertainment', 'Fitness', 'Trainers'),

    # ==================== HEALTHCARE (various) ====================
    # Medical specialists
    (r'\b(allergist|anesthesiologist|cardiologist|dermatologist|endocrinologist|gastroenterologist|geriatrician|hematologist|immunologist|nephrologist|neurologist|oncologist|ophthalmologist|orthopedist|otolaryngologist|pathologist|pediatrician|physiatrist|podiatrist|proctologist|pulmonologist|radiologist|rheumatologist|urologist)\b', 'Healthcare', 'Medical Practitioners', 'Specialists'),
    (r'\b(audiologist|speech therapist|occupational therapist|physical therapist)\b', 'Healthcare', 'Rehabilitation', 'Therapists'),
    (r'\b(psychologist|psychiatrist|counselor|therapist)\b(?!.*massage)', 'Healthcare', 'Mental Health', 'Practitioners'),
    (r'\b(chiropractor|osteopath|naturopath|homeopath|acupuncturist|herbalist)\b', 'Healthcare', 'Alternative Medicine', 'Practitioners'),
    (r'\b(optometrist|optician)\b', 'Healthcare', 'Vision Care', 'Practitioners'),
    (r'\b(medical|health)\b.*(center|clinic|office|practice)', 'Healthcare', 'Clinics', 'General'),
    (r'\b(aged care|elder care|senior care|nursing home|assisted living|retirement)\b', 'Healthcare', 'Senior Care', 'Facilities'),
    (r'\b(blood bank|blood donation|plasma)\b', 'Healthcare', 'Diagnostics', 'Blood_Services'),
    (r'\b(dialysis|kidney)\b.*(center|clinic)', 'Healthcare', 'Specialty Care', 'Dialysis'),
    (r'\b(fertility|ivf|reproductive)\b.*(clinic|center)', 'Healthcare', 'Specialty Care', 'Fertility'),
    (r'\b(hospice|palliative)\b', 'Healthcare', 'Senior Care', 'Hospice'),
    (r'\b(medical lab|laboratory|pathology|diagnostic)\b.*(center|lab)', 'Healthcare', 'Diagnostics', 'Labs'),
    (r'\b(ambulance|emergency|paramedic|first aid)\b', 'Healthcare', 'Emergency Services', 'EMS'),

    # ==================== AUTOMOTIVE (various) ====================
    (r'\bauto\b.*(body|paint|dent|collision|restoration|upholster)', 'Automotive', 'Repair & Maintenance', 'Body_Work'),
    (r'\bauto\b.*(repair|mechanic|service|tune.?up|brake|transmission|radiator)', 'Automotive', 'Repair & Maintenance', 'Mechanical'),
    (r'\bauto\b.*(auction|broker|dealer)', 'Automotive', 'Dealers', 'Used_Vehicles'),
    (r'\bauto\b.*(wrecker|salvage|junk|dismantl)', 'Automotive', 'Parts & Accessories', 'Salvage'),
    (r'\b(car|vehicle|auto)\b.*(wash|detail|clean|wax)', 'Automotive', 'Vehicle Care', 'Cleaning'),
    (r'\b(car|vehicle|auto)\b.*(rental|hire|lease)', 'Automotive', 'Rental Services', 'Vehicles'),
    (r'\b(car|vehicle|auto)\b.*(storage|parking)', 'Automotive', 'Parking', 'Storage'),
    (r'\b(motorcycle|motorbike|scooter|atv|quad)\b.*(dealer|shop|rental|repair)', 'Automotive', 'Dealers', 'Motorcycles'),
    (r'\b(tire|tyre|wheel)\b.*(shop|store|service|dealer)', 'Automotive', 'Parts & Accessories', 'Tires'),
    (r'\b(driving|driver)\b.*(school|training|instructor|lesson)', 'Automotive', 'Training', 'Driving_Schools'),
    (r'\btruck\b.*(stop|dealer|rental|repair)', 'Automotive', 'Dealers', 'Trucks'),
    (r'\b(rickshaw|auto rickshaw)\b', 'Transportation', 'Passenger', 'Local'),

    # ==================== GOVERNMENT & MILITARY ====================
    (r'\b(air force|army|navy|military|armed forces)\b.*(base|facility|office|recruitment)', 'Government', 'Public Safety', 'Military'),
    (r'\b(police|sheriff|law enforcement)\b.*(station|department|office)', 'Government', 'Public Safety', 'Police'),
    (r'\b(fire|firefighter)\b.*(station|department)', 'Government', 'Public Safety', 'Fire'),
    (r'\b(court|courthouse|tribunal|judiciary)\b', 'Government', 'Legal', 'Courts'),
    (r'\b(embassy|consulate|visa)\b.*(office|center)', 'Government', 'International', 'Diplomatic'),
    (r'\b(city|town|municipal|county|district|borough)\b.*(hall|office|government|administration)', 'Government', 'Local Government', 'Offices'),
    (r'\b(social services|welfare|unemployment|disability)\b.*(office|center)', 'Government', 'Social Services', 'Welfare'),
    (r'\b(dmv|driver.*license|vehicle registration|motor vehicle)\b', 'Government', 'Transportation', 'DMV'),
    (r'\b(passport|immigration|citizenship)\b.*(office|center)', 'Government', 'International', 'Immigration'),
    (r'\b(aadhaar|agenzia entrate|tax)\b.*(office|center)', 'Government', 'Local Government', 'Tax'),
    (r'\b(asylum|refugee)\b.*(center|office)', 'Government', 'Social Services', 'Refugee'),

    # ==================== PETS & ANIMALS ====================
    (r'\b(animal|pet)\b.*(shelter|rescue|adoption|welfare|pound|sanctuary)', 'Pets_Animals', 'Animal Welfare', 'Shelters'),
    (r'\b(animal|pet)\b.*(hospital|clinic|vet|veterinary)', 'Healthcare', 'Veterinary', 'Clinics'),
    (r'\b(animal|pet)\b.*(grooming|boarding|kennel|daycare|sitting|walking)', 'Pets_Animals', 'Pet Services', 'Care'),
    (r'\b(animal|pet)\b.*(training|obedience|behavior)', 'Pets_Animals', 'Pet Services', 'Training'),
    (r'\b(dog|cat|bird|fish|reptile|aquarium)\b.*(breeder|shop|store)', 'Retail_Shopping', 'Pet Supplies', 'Breeders'),
    (r'\bzoo\b|aquarium|wildlife.*park|safari', 'Entertainment', 'Wildlife', 'Zoos'),

    # ==================== RELIGIOUS ====================
    (r'\b(church|chapel|cathedral|basilica|parish)\b', 'Religious', 'Christian', 'Churches'),
    (r'\b(temple|mandir|hindu)\b', 'Religious', 'Hinduism', 'Temples'),
    (r'\b(mosque|masjid|islamic)\b', 'Religious', 'Islam', 'Mosques'),
    (r'\b(synagogue|jewish|judaism)\b', 'Religious', 'Judaism', 'Synagogues'),
    (r'\b(buddhist|buddha|monastery|zen|meditation center)\b', 'Religious', 'Buddhism', 'Temples'),
    (r'\b(ashram|spiritual|guru)\b', 'Religious', 'Other', 'Spiritual'),
    (r'\b(baha.*i|sikh|gurdwara|shinto)\b', 'Religious', 'Other', 'Houses_of_Worship'),

    # ==================== EDUCATION ====================
    (r'\b(university|college|faculty|academic department)\b', 'Education', 'Higher Education', 'Universities'),
    (r'\b(preschool|kindergarten|nursery|daycare|child.*care|creche)\b(?!.*animal)', 'Education', 'Early Childhood', 'Preschools'),
    (r'\b(school|academy)\b(?!.*driving|.*martial|.*dance|.*music|.*art|.*beauty|.*cooking|.*flight)', 'Education', 'K-12 Schools', 'General'),
    (r'\b(language|esl|english)\b.*(school|class|course|learning)', 'Education', 'Language Learning', 'Schools'),
    (r'\b(art|drawing|painting)\b.*(school|class|studio)', 'Education', 'Arts Education', 'Visual_Arts'),
    (r'\b(music|piano|guitar|violin|drum)\b.*(school|lesson|instructor|teacher)', 'Education', 'Arts Education', 'Music'),
    (r'\b(acting|theater|drama)\b.*(school|class|academy)', 'Education', 'Arts Education', 'Performing'),
    (r'\b(tutoring|tutor|coaching)\b.*(center|service)', 'Education', 'Tutoring', 'General'),
    (r'\b(library|public library)\b', 'Education', 'Libraries', 'Public'),
    (r'\b(archive|historical|museum)\b.*library', 'Education', 'Libraries', 'Special'),
    (r'\b(vocational|trade|technical)\b.*(school|training|institute)', 'Education', 'Vocational Training', 'General'),
    (r'\b(apprentice|internship)\b', 'Education', 'Vocational Training', 'Apprenticeships'),
    (r'\b(flight|aviation|pilot)\b.*(school|training|academy)', 'Education', 'Specialty Schools', 'Aviation'),
    (r'\b(cooking|culinary|chef)\b.*(school|class|academy)', 'Education', 'Specialty Schools', 'Culinary'),
    (r'\b(beauty|cosmetology|esthetician)\b.*(school|academy)', 'Education', 'Specialty Schools', 'Beauty'),

    # ==================== HOME SERVICES ====================
    (r'\b(bathroom|kitchen)\b.*(remodel|renovation|contractor)', 'Home_Services', 'Construction', 'Remodeling'),
    (r'\b(general|home)\b.*contractor', 'Home_Services', 'Construction', 'General'),
    (r'\b(painter|painting)\b.*(contractor|service|company)(?!.*auto)', 'Home_Services', 'Construction', 'Painting'),
    (r'\b(carpenter|carpentry|cabinet|woodwork)\b', 'Home_Services', 'Construction', 'Carpentry'),
    (r'\b(mason|masonry|brick|concrete|stone)\b.*(contractor|service|company)', 'Home_Services', 'Construction', 'Masonry'),
    (r'\b(electrician|electrical)\b.*(contractor|service|company)', 'Home_Services', 'Electrical', 'Contractors'),
    (r'\b(plumber|plumbing)\b.*(contractor|service|company)', 'Home_Services', 'Plumbing', 'Contractors'),
    (r'\b(hvac|heating|air conditioning|furnace)\b.*(contractor|service|company)', 'Home_Services', 'HVAC', 'Contractors'),
    (r'\b(roofer|roofing)\b.*(contractor|service|company)', 'Home_Services', 'Roofing', 'Contractors'),
    (r'\b(landscap|lawn|garden)\b.*(service|company|contractor)(?!.*store|.*center)', 'Home_Services', 'Landscaping', 'Services'),
    (r'\b(pool|spa)\b.*(service|cleaning|maintenance|contractor)', 'Home_Services', 'Pool & Spa', 'Services'),
    (r'\b(pest|exterminator|termite)\b.*(control|service)', 'Home_Services', 'Pest Control', 'Services'),
    (r'\b(cleaning|maid|janitorial|housekeeping)\b.*(service|company)', 'Home_Services', 'Cleaning', 'Services'),
    (r'\b(window)\b.*(cleaning|wash)', 'Home_Services', 'Cleaning', 'Window'),
    (r'\b(appliance)\b.*(repair|service)', 'Home_Services', 'Appliance Repair', 'Services'),
    (r'\b(handyman|odd job|home repair)\b', 'Home_Services', 'General Repair', 'Handyman'),
    (r'\b(moving|movers|relocation)\b.*(company|service)', 'Home_Services', 'Moving', 'Services'),
    (r'\b(locksmith)\b', 'Home_Services', 'Security', 'Locksmith'),
    (r'\b(alarm|security system)\b.*(company|service|installer)', 'Home_Services', 'Security', 'Systems'),
    (r'\b(arborist|tree)\b.*(service|removal|trimming)', 'Home_Services', 'Landscaping', 'Tree_Service'),
    (r'\b(fence)\b.*(contractor|company|install)', 'Home_Services', 'Construction', 'Fencing'),
    (r'\b(garage door)\b.*(service|repair|install)', 'Home_Services', 'General Repair', 'Garage_Doors'),
    (r'\b(gutter)\b.*(cleaning|service|install)', 'Home_Services', 'Construction', 'Gutters'),
    (r'\b(insulation)\b.*(contractor|company)', 'Home_Services', 'Construction', 'Insulation'),
    (r'\b(deck|patio)\b.*(builder|contractor)', 'Home_Services', 'Construction', 'Outdoor'),
    (r'\b(drywall|sheetrock)\b', 'Home_Services', 'Construction', 'Drywall'),
    (r'\b(flooring|carpet|tile|hardwood)\b.*(install|contractor|company)(?!.*store)', 'Home_Services', 'Flooring', 'Installation'),
    (r'\b(window|door)\b.*(install|replacement|contractor)', 'Home_Services', 'Windows & Doors', 'Installation'),
    (r'\b(glass)\b.*(repair|replacement|company)(?!.*auto)', 'Home_Services', 'Windows & Doors', 'Glass'),
    (r'\b(chimney)\b.*(sweep|cleaning|repair)', 'Home_Services', 'General Repair', 'Chimney'),
    (r'\b(septic|sewer)\b.*(service|pumping|cleaning)', 'Home_Services', 'Plumbing', 'Septic'),
    (r'\b(well)\b.*(drilling|service|pump)', 'Home_Services', 'Plumbing', 'Wells'),
    (r'\b(solar)\b.*(install|contractor|company)', 'Home_Services', 'Electrical', 'Solar'),

    # ==================== RETAIL & SHOPPING ====================
    (r'\b(antique|vintage|secondhand|thrift|consignment|pawn)\b.*(shop|store)', 'Retail_Shopping', 'Secondhand & Vintage', 'Stores'),
    (r'\b(auction)\b.*(house|company)', 'Retail_Shopping', 'Secondhand & Vintage', 'Auctions'),
    (r'\b(art|craft|hobby)\b.*(supply|store|shop)', 'Retail_Shopping', 'Arts & Crafts', 'Supplies'),
    (r'\b(toy|game|hobby)\b.*(store|shop)', 'Retail_Shopping', 'Toys & Hobbies', 'Stores'),
    (r'\b(book|stationery|office supply)\b.*(store|shop)', 'Retail_Shopping', 'Books & Office', 'Stores'),
    (r'\b(music|instrument|record|vinyl)\b.*(store|shop)', 'Retail_Shopping', 'Music & Entertainment', 'Stores'),
    (r'\b(sporting|sports|outdoor|camping|fishing|hunting)\b.*(goods|store|shop)', 'Retail_Shopping', 'Sports & Outdoors', 'Stores'),
    (r'\b(electronics|computer|phone|appliance)\b.*(store|shop|retailer)', 'Retail_Shopping', 'Electronics', 'Stores'),
    (r'\b(furniture|home decor|bedding|mattress)\b.*(store|shop)', 'Retail_Shopping', 'Home & Garden', 'Stores'),
    (r'\b(clothing|fashion|apparel|boutique|shoe)\b.*(store|shop)', 'Retail_Shopping', 'Clothing & Fashion', 'Stores'),
    (r'\b(jewelry|watch|gem)\b.*(store|shop)', 'Retail_Shopping', 'Jewelry & Watches', 'Stores'),
    (r'\b(hardware|tool|building supply|lumber)\b.*(store|shop)', 'Retail_Shopping', 'Hardware & Building', 'Stores'),
    (r'\b(garden|nursery|plant)\b.*(center|store|shop)', 'Retail_Shopping', 'Home & Garden', 'Garden_Centers'),
    (r'\b(pharmacy|drugstore)\b', 'Retail_Shopping', 'Health & Pharmacy', 'Pharmacies'),
    (r'\b(cosmetic|beauty|makeup)\b.*(store|shop)', 'Retail_Shopping', 'Beauty & Cosmetics', 'Stores'),
    (r'\b(grocery|supermarket|food|convenience)\b.*(store|market|shop)', 'Retail_Shopping', 'Food & Grocery', 'Stores'),
    (r'\b(liquor|wine|beer|alcohol)\b.*(store|shop)', 'Retail_Shopping', 'Food & Grocery', 'Liquor'),
    (r'\b(tobacco|cigar|vape|smoke)\b.*(shop|store)', 'Retail_Shopping', 'Specialty Retail', 'Tobacco'),
    (r'\b(mobile phone|cell phone)\b.*(store|shop|dealer)', 'Retail_Shopping', 'Electronics', 'Phones'),
    (r'\b(optical|eyewear|glasses|sunglass)\b.*(store|shop)', 'Retail_Shopping', 'Health & Pharmacy', 'Optical'),
    (r'\b(florist|flower)\b.*(shop|store)', 'Events_Weddings', 'Florists', 'Shops'),
    (r'\b(bridal|wedding)\b.*(shop|store|boutique)', 'Events_Weddings', 'Attire', 'Bridal'),
    (r'\b(uniform|workwear)\b.*(store|shop)', 'Retail_Shopping', 'Clothing & Fashion', 'Specialty'),

    # ==================== PROFESSIONAL SERVICES ====================
    (r'\b(lawyer|attorney|law firm|legal)\b.*(office|firm|service)', 'Professional_Services', 'Legal', 'Firms'),
    (r'\b(accountant|accounting|bookkeep|tax)\b.*(firm|service|office)(?!.*government)', 'Professional_Services', 'Financial Services', 'Accounting'),
    (r'\b(architect|architecture)\b.*(firm|office|studio)', 'Professional_Services', 'Engineering', 'Architecture'),
    (r'\b(engineer|engineering)\b.*(firm|office|company)', 'Professional_Services', 'Engineering', 'Firms'),
    (r'\b(surveyor|surveying|land survey)\b', 'Professional_Services', 'Engineering', 'Surveying'),
    (r'\b(consultant|consulting)\b.*(firm|company|service)', 'Professional_Services', 'Consulting', 'General'),
    (r'\b(marketing|advertising|pr|public relations)\b.*(agency|firm|company)', 'Professional_Services', 'Marketing & Advertising', 'Agencies'),
    (r'\b(graphic|web|design)\b.*(studio|agency|firm)', 'Professional_Services', 'Creative Services', 'Design'),
    (r'\b(photography|photographer|video|videograph)\b.*(studio|service)', 'Professional_Services', 'Creative Services', 'Photography'),
    (r'\b(translation|interpreter|language)\b.*service', 'Professional_Services', 'Language Services', 'Translation'),
    (r'\b(staffing|recruiting|employment|hr)\b.*(agency|service|firm)', 'Professional_Services', 'HR Services', 'Agencies'),
    (r'\b(notary|notarial)\b', 'Professional_Services', 'Legal', 'Notary'),
    (r'\b(private investigator|detective)\b', 'Professional_Services', 'Agencies', 'Investigation'),
    (r'\b(appraiser|appraisal|valuation)\b', 'Professional_Services', 'Financial Services', 'Appraisal'),
    (r'\b(auditor|audit)\b.*(firm|service)', 'Professional_Services', 'Financial Services', 'Audit'),
    (r'\b(courier|messenger|delivery)\b.*service', 'Transportation', 'Delivery', 'Courier'),

    # ==================== ARTS & CULTURE ====================
    (r'\b(art|gallery|exhibition)\b(?!.*supply|.*store|.*school)', 'Entertainment', 'Arts', 'Galleries'),
    (r'\b(museum)\b', 'Entertainment', 'Museums', 'General'),
    (r'\b(theater|theatre|playhouse|opera house)\b', 'Entertainment', 'Performing Arts', 'Venues'),
    (r'\b(cinema|movie theater|multiplex)\b', 'Entertainment', 'Movies', 'Theaters'),
    (r'\b(concert|music)\b.*(hall|venue)', 'Entertainment', 'Music Venues', 'Concert_Halls'),
    (r'\b(band|orchestra|choir|ensemble)\b', 'Entertainment', 'Performing Arts', 'Groups'),
    (r'\b(comedian|comedy club)\b', 'Entertainment', 'Performing Arts', 'Comedy'),
    (r'\b(artist|sculptor|painter)\b(?!.*makeup)', 'Entertainment', 'Arts', 'Artists'),
    (r'\b(animation|animator)\b.*(studio|company)', 'Professional_Services', 'Creative Services', 'Animation'),
    (r'\b(recording|music)\b.*studio', 'Professional_Services', 'Creative Services', 'Recording'),
    (r'\b(art restoration|restoration service)\b', 'Professional_Services', 'Creative Services', 'Restoration'),

    # ==================== ENTERTAINMENT & RECREATION ====================
    (r'\b(amusement|theme)\b.*park', 'Entertainment', 'Amusement', 'Parks'),
    (r'\b(arcade|game center|gaming)\b', 'Entertainment', 'Games & Recreation', 'Arcades'),
    (r'\b(escape room|puzzle room)\b', 'Entertainment', 'Games & Recreation', 'Escape_Rooms'),
    (r'\b(bowling)\b.*(alley|center)', 'Entertainment', 'Games & Recreation', 'Bowling'),
    (r'\b(billiard|pool hall|snooker)\b', 'Entertainment', 'Games & Recreation', 'Billiards'),
    (r'\b(karaoke)\b', 'Entertainment', 'Music Venues', 'Karaoke'),
    (r'\b(casino|gambling|betting)\b', 'Entertainment', 'Gambling', 'Casinos'),
    (r'\b(nightclub|disco|club)\b(?!.*golf|.*country|.*tennis)', 'Food_Dining', 'Bars & Nightlife', 'Nightclubs'),
    (r'\b(country club|private club|social club)\b', 'Entertainment', 'Social', 'Clubs'),
    (r'\b(botanical garden|arboretum)\b', 'Entertainment', 'Parks', 'Gardens'),
    (r'\b(park|playground|recreation area)\b(?!.*theme|.*water|.*trailer|.*mobile)', 'Entertainment', 'Parks', 'Public'),
    (r'\b(beach|waterfront|marina)\b(?!.*hotel)', 'Entertainment', 'Parks', 'Beaches'),
    (r'\b(campground|camping|rv park|caravan)\b', 'Hospitality_Travel', 'Lodging', 'Camping'),
    (r'\b(go.?kart|kart|karting)\b', 'Entertainment', 'Games & Recreation', 'Karting'),
    (r'\b(laser tag|paintball)\b', 'Entertainment', 'Games & Recreation', 'Adventure'),
    (r'\b(trampoline|bounce|jump)\b.*(park|center)', 'Entertainment', 'Games & Recreation', 'Trampoline'),
    (r'\b(mini golf|miniature golf|putt.?putt)\b', 'Entertainment', 'Games & Recreation', 'Mini_Golf'),
    (r'\b(water park|aqua park)\b', 'Entertainment', 'Amusement', 'Water_Parks'),
    (r'\b(haunted|horror)\b.*(house|attraction)', 'Entertainment', 'Amusement', 'Attractions'),
    (r'\b(circus|carnival|fair)\b', 'Entertainment', 'Amusement', 'Shows'),
    (r'\b(planetarium|observatory)\b', 'Entertainment', 'Museums', 'Science'),

    # ==================== FOOD & DINING ====================
    (r'\b(bar|pub|tavern|lounge|brewery|taproom|brewpub)\b(?!.*brow|.*eyebrow)', 'Food_Dining', 'Bars & Nightlife', 'Bars'),
    (r'\b(cafe|coffee|espresso)\b.*(shop|house|bar)', 'Food_Dining', 'Cafes & Coffee', 'Cafes'),
    (r'\b(restaurant|eatery|diner|bistro|brasserie|grill)\b', 'Food_Dining', 'Restaurants', 'General'),
    (r'\b(bakery|patisserie|pastry)\b', 'Food_Dining', 'Bakeries & Desserts', 'Bakeries'),
    (r'\b(ice cream|gelato|frozen yogurt|dessert)\b.*(shop|parlor|store)', 'Food_Dining', 'Bakeries & Desserts', 'Desserts'),
    (r'\b(caterer|catering)\b', 'Food_Dining', 'Food Services', 'Catering'),
    (r'\b(food truck|food cart)\b', 'Food_Dining', 'Quick Service', 'Mobile'),
    (r'\b(juice|smoothie)\b.*(bar|shop)', 'Food_Dining', 'Cafes & Coffee', 'Juice'),
    (r'\b(tea|bubble tea|boba)\b.*(shop|house|room)', 'Food_Dining', 'Cafes & Coffee', 'Tea'),
    (r'\b(winery|vineyard|wine)\b.*(tasting|cellar)', 'Food_Dining', 'Beverage Production', 'Wineries'),
    (r'\b(distillery|spirit)\b', 'Food_Dining', 'Beverage Production', 'Distilleries'),
    (r'\b(butcher|meat)\b.*shop', 'Retail_Shopping', 'Food & Grocery', 'Butchers'),
    (r'\b(fish|seafood)\b.*market', 'Retail_Shopping', 'Food & Grocery', 'Seafood'),
    (r'\b(deli|delicatessen)\b', 'Retail_Shopping', 'Food & Grocery', 'Delis'),
    (r'\b(candy|chocolate|sweet|confection)\b.*(shop|store)', 'Retail_Shopping', 'Food & Grocery', 'Confectionery'),

    # ==================== PERSONAL SERVICES ====================
    (r'\b(barber|hair)\b.*(shop|salon|stylist)', 'Personal_Services', 'Hair Care', 'Salons'),
    (r'\b(beauty|nail|manicure|pedicure)\b.*(salon|spa|studio)', 'Personal_Services', 'Spa & Wellness', 'Beauty'),
    (r'\b(tattoo|piercing|body art)\b.*(shop|studio|parlor)', 'Personal_Services', 'Body Art', 'Studios'),
    (r'\b(massage)\b.*(therapist|spa|parlor|studio)', 'Personal_Services', 'Massage', 'Studios'),
    (r'\b(spa|wellness|day spa)\b', 'Personal_Services', 'Spa & Wellness', 'Spas'),
    (r'\b(tanning|sunbed)\b.*(salon|studio)', 'Personal_Services', 'Spa & Wellness', 'Tanning'),
    (r'\b(laundry|laundromat|dry clean|tailor|alteration|seamstress)\b', 'Personal_Services', 'Laundry', 'Services'),
    (r'\b(shoe repair|cobbler)\b', 'Personal_Services', 'Clothing Care', 'Shoe_Repair'),
    (r'\b(brow|eyebrow|lash|eyelash)\b.*(bar|salon|studio)', 'Personal_Services', 'Spa & Wellness', 'Brows_Lashes'),
    (r'\b(makeup artist|stylist)\b', 'Personal_Services', 'Spa & Wellness', 'Makeup'),
    (r'\b(sauna|steam room|bathhouse|hammam)\b', 'Personal_Services', 'Spa & Wellness', 'Baths'),
    (r'\b(waxing)\b.*(salon|studio)', 'Personal_Services', 'Spa & Wellness', 'Waxing'),

    # ==================== HOSPITALITY & TRAVEL ====================
    (r'\b(hotel|motel|inn|resort|hostel|lodge|bed and breakfast|b&b|guesthouse)\b', 'Hospitality_Travel', 'Lodging', 'Hotels'),
    (r'\b(travel|tour)\b.*(agency|operator|company)', 'Hospitality_Travel', 'Travel Services', 'Agencies'),
    (r'\b(airline|airport|aviation)\b(?!.*school)', 'Transportation', 'Passenger', 'Air'),
    (r'\b(cruise|ferry)\b.*(line|terminal|port)', 'Transportation', 'Passenger', 'Water'),
    (r'\b(train|rail)\b.*(station|service)', 'Transportation', 'Passenger', 'Rail'),
    (r'\b(bus|coach)\b.*(station|terminal|service|company)', 'Transportation', 'Passenger', 'Bus'),
    (r'\b(taxi|cab|ride|uber|lyft|limo|limousine|chauffeur)\b.*(service|company|stand)', 'Transportation', 'Passenger', 'Taxi'),
    (r'\b(tourist|visitor)\b.*(information|center|bureau)', 'Hospitality_Travel', 'Travel Services', 'Information'),
    (r'\b(rental)\b.*\b(cabin|cottage|vacation|holiday)\b', 'Hospitality_Travel', 'Lodging', 'Rentals'),

    # ==================== INDUSTRIAL & MANUFACTURING ====================
    (r'\b(factory|plant|mill|manufacturing)\b', 'Industrial', 'Manufacturing', 'General'),
    (r'\b(warehouse|distribution|logistics)\b.*(center|facility)', 'Transportation', 'Logistics', 'Warehouses'),
    (r'\b(machine|machinist|metalwork|welding|welder)\b.*(shop|company|service)', 'Industrial', 'Manufacturing', 'Metal'),
    (r'\b(print|printing|press)\b.*(shop|company|service)', 'Industrial', 'Manufacturing', 'Printing'),
    (r'\b(textile|fabric|garment)\b.*(factory|mill|manufacturer)', 'Industrial', 'Manufacturing', 'Textile'),
    (r'\b(chemical|pharmaceutical)\b.*(company|manufacturer|plant)', 'Industrial', 'Manufacturing', 'Chemical'),
    (r'\b(construction|building)\b.*(company|contractor|firm)', 'Industrial', 'Construction', 'General'),
    (r'\b(quarry|gravel|sand|aggregate)\b', 'Industrial', 'Mining', 'Quarries'),
    (r'\b(sawmill|lumber)\b.*(mill|yard)', 'Industrial', 'Manufacturing', 'Wood'),
    (r'\b(steel|iron|aluminum)\b.*(plant|manufacturer|company)', 'Industrial', 'Manufacturing', 'Metal'),
    (r'\b(packaging|container)\b.*(company|manufacturer)', 'Industrial', 'Manufacturing', 'Packaging'),
    (r'\b(recycling|waste)\b.*(center|facility|company)', 'Industrial', 'Manufacturing', 'Recycling'),

    # ==================== REAL ESTATE ====================
    (r'\b(real estate|realtor|property)\b.*(agent|agency|company)', 'Real_Estate', 'Agencies', 'Agents'),
    (r'\b(property management|apartment|rental)\b.*(company|agency)', 'Real_Estate', 'Management', 'Residential'),
    (r'\b(storage|self storage|mini storage)\b.*(facility|unit)', 'Real_Estate', 'Storage', 'Self_Storage'),
    (r'\b(office|commercial)\b.*(space|building|complex)', 'Real_Estate', 'Commercial', 'Office'),
    (r'\b(apartment|condo|housing)\b.*(complex|building|community)', 'Real_Estate', 'Residential', 'Apartments'),
    (r'\b(home builder|housing development)\b', 'Real_Estate', 'Development', 'Residential'),

    # ==================== NON-PROFIT & COMMUNITY ====================
    (r'\b(charity|charitable|foundation|fund)\b(?!.*investment)', 'Non_Profit', 'Charities', 'General'),
    (r'\b(non.?profit|ngo|association)\b', 'Non_Profit', 'General', 'Organizations'),
    (r'\b(community|civic|neighborhood)\b.*(center|organization|association)', 'Non_Profit', 'Community', 'Centers'),
    (r'\b(youth|boys|girls|scout)\b.*(club|organization|center)', 'Non_Profit', 'Community', 'Youth'),
    (r'\b(senior|elder)\b.*(center|club)(?!.*care)', 'Non_Profit', 'Community', 'Seniors'),
    (r'\b(veterans|vfw|american legion)\b', 'Non_Profit', 'Community', 'Veterans'),
    (r'\b(rotary|lions|kiwanis|elks|freemason|lodge)\b', 'Non_Profit', 'Community', 'Fraternal'),
    (r'\b(union|labor)\b.*(hall|organization)', 'Non_Profit', 'Professional', 'Unions'),
    (r'\b(chamber of commerce|business association)\b', 'Non_Profit', 'Professional', 'Business'),
    (r'\b(aboriginal|indigenous|tribal)\b.*(organization|center)', 'Non_Profit', 'Community', 'Indigenous'),

    # ==================== TECHNOLOGY ====================
    (r'\b(software|app|web)\b.*(developer|development|company)', 'Technology', 'Software', 'Development'),
    (r'\b(it|computer|tech)\b.*(service|support|repair)', 'Technology', 'IT Services', 'Support'),
    (r'\b(data center|server|cloud)\b', 'Technology', 'Infrastructure', 'Data_Centers'),
    (r'\b(internet|isp|broadband|telecom)\b.*(provider|service|company)', 'Technology', 'Telecommunications', 'Providers'),
    (r'\b(bpo|call center|outsourc)\b', 'Technology', 'IT Services', 'BPO'),
    (r'\b(automation|robot)\b.*(company|service)', 'Technology', 'Software', 'Automation'),

    # ==================== FINANCE & INSURANCE ====================
    (r'\b(bank|credit union|savings)\b(?!.*blood|.*food)', 'Finance_Insurance', 'Banking', 'Banks'),
    (r'\b(atm|cash machine)\b', 'Finance_Insurance', 'Banking', 'ATMs'),
    (r'\b(insurance)\b.*(agent|agency|company|broker)', 'Finance_Insurance', 'Insurance', 'Agents'),
    (r'\b(mortgage|loan|lending|finance)\b.*(company|broker|service)', 'Finance_Insurance', 'Lending', 'Lenders'),
    (r'\b(investment|wealth|portfolio|financial advisor)\b', 'Finance_Insurance', 'Investment', 'Advisors'),
    (r'\b(money transfer|remittance|western union|moneygram)\b', 'Finance_Insurance', 'Money Services', 'Transfer'),
    (r'\b(currency exchange|forex)\b', 'Finance_Insurance', 'Money Services', 'Exchange'),
    (r'\b(bail bond)\b', 'Professional_Services', 'Legal', 'Bail'),

    # ==================== EVENTS & WEDDINGS ====================
    (r'\b(funeral|mortuary|cremation|cemetery|memorial)\b', 'Events_Weddings', 'Memorial', 'Funeral'),
    (r'\b(event|party|wedding)\b.*(planner|planning|coordinator)', 'Events_Weddings', 'Planning', 'Planners'),
    (r'\b(banquet|event|reception|wedding)\b.*(hall|venue|center)', 'Events_Weddings', 'Venues', 'Halls'),
    (r'\b(dj|disc jockey|entertainment)\b.*service', 'Events_Weddings', 'Services', 'Entertainment'),
    (r'\b(balloon|party supply|decoration)\b', 'Events_Weddings', 'Services', 'Decorations'),
    (r'\b(tent|equipment)\b.*rental(?!.*car|.*truck)', 'Events_Weddings', 'Rentals', 'Equipment'),
    (r'\b(photo booth|photobooth)\b', 'Events_Weddings', 'Services', 'Photography'),

    # ==================== AGRICULTURE ====================
    (r'\b(farm|ranch|orchard|vineyard)\b(?!.*winery)', 'Agriculture', 'Farming', 'Farms'),
    (r'\b(agriculture|farming|crop)\b.*(service|supply|equipment)', 'Agriculture', 'Services', 'Supplies'),
    (r'\b(livestock|cattle|poultry|dairy)\b', 'Agriculture', 'Farming', 'Livestock'),
    (r'\b(nursery|greenhouse|horticulture)\b.*(wholesale|grower)', 'Agriculture', 'Farming', 'Horticulture'),
    (r'\b(agistment|horse boarding|stable)\b', 'Agriculture', 'Services', 'Equine'),
    (r'\b(veterinarian|vet)\b.*(livestock|farm|large animal)', 'Agriculture', 'Services', 'Veterinary'),

    # ==================== TRANSPORTATION ====================
    (r'\b(shipping|freight|cargo|trucking)\b.*(company|service)', 'Transportation', 'Logistics', 'Shipping'),
    (r'\b(courier|messenger|express)\b.*(service|delivery)', 'Transportation', 'Delivery', 'Courier'),
    (r'\b(airport|airfield|airstrip|heliport)\b', 'Transportation', 'Passenger', 'Airports'),
    (r'\b(port|harbor|dock|pier|marina)\b(?!.*wine)', 'Transportation', 'Logistics', 'Ports'),
    (r'\b(parking)\b.*(lot|garage|structure)', 'Automotive', 'Parking', 'Lots'),
    (r'\b(towing|tow truck)\b', 'Transportation', 'Vehicle Services', 'Towing'),
]

def categorize_item(name):
    """
    Categorize a single item based on rules.
    Returns (sector, business_type, sub_category) or None if no match.
    """
    name_lower = name.lower()

    for pattern, sector, btype, subcat in CATEGORIZATION_RULES:
        if re.search(pattern, name_lower, re.IGNORECASE):
            return (sector, btype, subcat)

    return None

def get_existing_paths(cursor):
    """Get all existing paths in the database"""
    cursor.execute("SELECT path::text FROM gbp_categories")
    return {row[0] for row in cursor.fetchall()}

def get_or_create_path(cursor, sector, btype, subcat, existing_paths):
    """
    Get or create the full path for a category.
    Returns the parent path (level 3) for the item.
    """
    sector_slug = slugify(sector)
    btype_slug = slugify(btype)
    subcat_slug = slugify(subcat)

    # Level 1: Sector
    sector_path = sector_slug
    if sector_path not in existing_paths:
        # Don't create new sectors - return None
        print(f"  [SKIP] Would need new sector: {sector_path}")
        return None

    # Level 2: Business Type
    btype_path = f"{sector_path}.{btype_slug}"
    if btype_path not in existing_paths:
        # Create new business type
        cursor.execute("""
            INSERT INTO gbp_categories (name, slug, path, level, parent_id, category_count)
            SELECT %s, %s, %s::ltree, 2, id, 0
            FROM gbp_categories WHERE path = %s::ltree
            ON CONFLICT (path) DO NOTHING
            RETURNING id
        """, (btype, btype_slug, btype_path, sector_path))
        result = cursor.fetchone()
        if result:
            existing_paths.add(btype_path)
            print(f"  [NEW] Created business type: {btype_path}")

    # Level 3: Sub-category
    subcat_path = f"{btype_path}.{subcat_slug}"
    if subcat_path not in existing_paths:
        # Create new sub-category
        cursor.execute("""
            INSERT INTO gbp_categories (name, slug, path, level, parent_id, category_count)
            SELECT %s, %s, %s::ltree, 3, id, 0
            FROM gbp_categories WHERE path = %s::ltree
            ON CONFLICT (path) DO NOTHING
            RETURNING id
        """, (subcat, subcat_slug, subcat_path, btype_path))
        result = cursor.fetchone()
        if result:
            existing_paths.add(subcat_path)
            print(f"  [NEW] Created sub-category: {subcat_path}")

    return subcat_path

def main():
    conn = psycopg2.connect(DB_URL)
    cursor = conn.cursor()

    # Get all items in Other.Uncategorized
    cursor.execute("""
        SELECT id, name, slug
        FROM gbp_categories
        WHERE path ~ 'Other.Uncategorized.*' AND level = 4
        ORDER BY name
    """)
    other_items = cursor.fetchall()
    print(f"Found {len(other_items)} items in Other.Uncategorized\n")

    # Get existing paths
    existing_paths = get_existing_paths(cursor)

    # Categorize items
    categorized = []
    uncategorized = []
    category_counts = defaultdict(int)

    for item_id, name, slug in other_items:
        result = categorize_item(name)
        if result:
            sector, btype, subcat = result
            categorized.append((item_id, name, slug, sector, btype, subcat))
            category_counts[(sector, btype, subcat)] += 1
        else:
            uncategorized.append((item_id, name))

    print(f"Categorized: {len(categorized)}")
    print(f"Still uncategorized: {len(uncategorized)}")
    print()

    # Show category distribution
    print("Category distribution:")
    for (sector, btype, subcat), count in sorted(category_counts.items(), key=lambda x: -x[1])[:30]:
        print(f"  {sector}.{btype}.{subcat}: {count}")
    print()

    # Show some uncategorized items
    print("Sample uncategorized items (first 50):")
    for item_id, name in uncategorized[:50]:
        print(f"  - {name}")
    print()

    # Ask for confirmation
    response = input("Proceed with database updates? (yes/no): ")
    if response.lower() != 'yes':
        print("Aborted.")
        conn.close()
        return

    # Update database
    updated = 0
    for item_id, name, slug, sector, btype, subcat in categorized:
        parent_path = get_or_create_path(cursor, sector, btype, subcat, existing_paths)
        if parent_path:
            new_path = f"{parent_path}.{slug}"
            # Update the item
            cursor.execute("""
                UPDATE gbp_categories
                SET path = %s::ltree,
                    parent_id = (SELECT id FROM gbp_categories WHERE path = %s::ltree)
                WHERE id = %s
            """, (new_path, parent_path, item_id))
            updated += 1

    # Update category counts
    cursor.execute("""
        WITH counts AS (
            SELECT
                parent_id,
                COUNT(*) as cnt
            FROM gbp_categories
            WHERE parent_id IS NOT NULL
            GROUP BY parent_id
        )
        UPDATE gbp_categories g
        SET category_count = COALESCE(c.cnt, 0)
        FROM counts c
        WHERE g.id = c.parent_id
    """)

    conn.commit()
    print(f"\nUpdated {updated} items")

    # Show final stats
    cursor.execute("""
        SELECT path, name, category_count
        FROM gbp_categories
        WHERE level = 1
        ORDER BY category_count DESC
    """)
    print("\nFinal sector counts:")
    for path, name, count in cursor.fetchall():
        print(f"  {name}: {count}")

    conn.close()

if __name__ == '__main__':
    main()
