#!/usr/bin/env python3
"""Scores a company-fields receipt run: which lead-scoring fields each provider
returned for each domain, and how far the providers' headcounts disagree.
Reads the masked receipts written by run-company-fields-fixture.sh; writes
src/data/company-fields/<date>.json. Deterministic; no LLM.
Usage: scripts/score-company-fields.py 2026-09-17
"""
import glob, json, os, re, sys

DATE = sys.argv[1]
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SRC = os.path.join(ROOT, 'evidence', 'receipts', DATE)
OUT = os.path.join(ROOT, 'src', 'data', 'company-fields')


def get(d, path):
    for k in path.split('.'):
        if not isinstance(d, dict):
            return None
        d = d.get(k)
    return d


def filled(v):
    return v not in (None, '', [], {}) and v != 0


# Where each field lives in each provider's raw payload. None = the provider has no such field.
FIELDS = {
    'employees_count': {'thecompaniesapi': 'about.totalEmployeesExact', 'hunter': 'data.metrics.employeesCount', 'apollo': 'organization.estimated_num_employees', 'findymail': None, 'pdl': 'employee_count'},
    'employees_band': {'thecompaniesapi': 'about.totalEmployees', 'hunter': 'data.metrics.employees', 'apollo': None, 'findymail': 'company_size', 'pdl': 'size'},
    'industry': {'thecompaniesapi': 'about.industry', 'hunter': 'data.category.industry', 'apollo': 'organization.industry', 'findymail': 'industry', 'pdl': 'industry'},
    'industry_code': {'thecompaniesapi': None, 'hunter': 'data.category.naicsCode', 'apollo': 'organization.naics_codes', 'findymail': None, 'pdl': 'naics'},
    'country': {'thecompaniesapi': 'locations.headquarters.country.name', 'hunter': 'data.geo.country', 'apollo': 'organization.country', 'findymail': 'country', 'pdl': 'location.country'},
    'founded': {'thecompaniesapi': 'about.yearFounded', 'hunter': 'data.foundedYear', 'apollo': 'organization.founded_year', 'findymail': None, 'pdl': 'founded'},
    'revenue': {'thecompaniesapi': 'finances.revenue', 'hunter': 'data.metrics.estimatedAnnualRevenue', 'apollo': 'organization.annual_revenue', 'findymail': None, 'pdl': 'inferred_revenue'},
    'funding_total': {'thecompaniesapi': None, 'hunter': 'data.metrics.raised', 'apollo': 'organization.total_funding', 'findymail': None, 'pdl': 'total_funding_raised'},
    'tech_stack': {'thecompaniesapi': 'technologies.active', 'hunter': 'data.tech', 'apollo': 'organization.technology_names', 'findymail': None, 'pdl': None},
    'headcount_growth': {'thecompaniesapi': None, 'hunter': None, 'apollo': 'organization.organization_headcount_twelve_month_growth', 'findymail': None, 'pdl': None},
    'linkedin_url': {'thecompaniesapi': 'socials.linkedin.url', 'hunter': 'data.linkedin.handle', 'apollo': 'organization.linkedin_url', 'findymail': 'linkedin_url', 'pdl': 'linkedin_url'},
}


def band_low(v):
    """Lower bound of a band like '1K-5K', '1001-5000', 'over-10k', '10001+'."""
    if v is None:
        return None
    s = str(v).lower().replace(',', '')
    m = re.match(r'(?:over-)?(\d+)(k?)', s)
    if not m:
        return None
    n = int(m.group(1)) * (1000 if m.group(2) else 1)
    return n


def bucket(n):  # the size buckets a lead-scoring rule typically uses
    if n is None:
        return None
    return '1-200' if n <= 200 else '201-1,000' if n <= 1000 else '1,001-5,000' if n <= 5000 else '5,001+'


groups = {}
for f in sorted(glob.glob(os.path.join(SRC, 'company-fields-*.json'))):
    label = os.path.basename(f)[:-5]
    route, cid = re.match(r'company-fields-(\w+)-(c\d+)$', label).groups()
    d = json.load(open(f))
    t = d.get('_treg') or {}
    groups.setdefault(route, []).append({'id': cid, 'served_by': (t.get('served_by') or '').split('.')[0], 'hit': t.get('outcome') == 'hit', 'usd': (t.get('charged_micro') or 0) / 1e6, 'raw': d.get('raw') or {}, 'tried': [(x.get('endpoint_id'), x.get('outcome'), x.get('status')) for x in t.get('tried', [])]})

domains = {c['id']: c['domain'] for c in json.load(open(os.path.join(ROOT, 'evidence', 'fixtures', 'enrichment-fixture-v1.json')))['companies']}
out = {'date': DATE, 'routes': {}, 'headcount': {}, 'excluded': {}}
for route, rows in groups.items():
    served = {r['served_by'] for r in rows}
    pinned_ok = route == 'routed' or served == {route}
    if not pinned_ok:  # the pinned provider did not answer (e.g. rate-limited) and the call fell through
        out['excluded'][route] = {'n': len(rows), 'served_by': sorted(served), 'first_try': sorted({x for r in rows for x in [r['tried'][0]] if r['tried']})}
        continue
    prov = rows[0]['served_by'] if route == 'routed' else route
    fill = {}
    for field, paths in FIELDS.items():
        p = paths.get(prov)
        fill[field] = None if p is None else sum(1 for r in rows if filled(get(r['raw'], p)))
    hits = sum(r['hit'] for r in rows)
    usd = sum(r['usd'] for r in rows)
    out['routes'][route] = {'provider': prov, 'n': len(rows), 'hits': hits, 'usd_total': round(usd, 6), 'usd_per_hit': round(usd / hits, 6) if hits else None, 'fill': fill}
    for r in rows:
        cnt = get(r['raw'], FIELDS['employees_count'][prov]) if FIELDS['employees_count'][prov] else None
        band = get(r['raw'], FIELDS['employees_band'][prov]) if FIELDS['employees_band'][prov] else None
        out['headcount'].setdefault(r['id'], {'domain': domains[r['id']]})[prov] = {'count': cnt, 'band': band, 'bucket': bucket(cnt if cnt else band_low(band))}

# Bucket agreement per domain across every provider that returned a headcount
agree = 0
for cid, h in out['headcount'].items():
    b = {k: v['bucket'] for k, v in h.items() if k != 'domain' and v['bucket']}
    h['buckets_distinct'] = len(set(b.values()))
    agree += h['buckets_distinct'] == 1
out['headcount_bucket_agreement'] = {'domains': len(out['headcount']), 'all_same_bucket': agree}

# A provider whose exact count falls outside its own band
def band_range(v):
    nums = [int(n) * (1000 if k else 1) for n, k in re.findall(r'(\d+)(k?)', str(v).lower().replace(',', ''))]
    if not nums:
        return None
    return (nums[0], nums[1] if len(nums) > 1 else float('inf'))


out['band_count_mismatch'] = []
for cid, h in out['headcount'].items():
    for k, v in h.items():
        rng = isinstance(v, dict) and v['count'] and v['band'] and band_range(v['band'])
        if rng and not (rng[0] - 1 <= v['count'] <= rng[1]):
            out['band_count_mismatch'].append({'id': cid, 'domain': h['domain'], 'provider': k, 'count': v['count'], 'band': v['band']})

# Country and founded year: domains where the providers that answered do not all agree
CC = {'us': 'united states', 'ca': 'canada', 'au': 'australia', 'il': 'israel'}
disagree = {'country': [], 'founded': []}
for field in disagree:
    per = {}
    for route, rows in groups.items():
        if route not in out['routes']:
            continue
        prov = out['routes'][route]['provider']
        for r in rows:
            v = get(r['raw'], FIELDS[field][prov]) if FIELDS[field][prov] else None
            if filled(v):
                v = str(v).lower()
                per.setdefault(r['id'], {})[prov] = CC.get(v, v)
    for cid, vals in per.items():
        if len(set(vals.values())) > 1:
            disagree[field].append({'id': cid, 'domain': domains[cid], 'values': vals})
out['disagreements'] = disagree

# How old the record is, where the provider says: oldest and newest stamp across the run
STAMPS = {'thecompaniesapi': 'meta.syncedAt', 'hunter': 'data.indexedAt', 'pdl': 'dataset_version'}
out['record_stamps'] = {}
for route, info in out['routes'].items():
    path = STAMPS.get(info['provider'])
    vals = sorted(str(v)[:10] for r in groups[route] for v in [get(r['raw'], path) if path else None] if v)
    if vals:
        out['record_stamps'][info['provider']] = {'field': path, 'n': len(vals), 'min': vals[0], 'max': vals[-1]}

os.makedirs(OUT, exist_ok=True)
json.dump(out, open(os.path.join(OUT, f'{DATE}.json'), 'w'), indent=2)
print(json.dumps({k: v for k, v in out.items() if k != 'headcount'}, indent=1))
for cid, h in out['headcount'].items():
    print(cid, h['domain'], h['buckets_distinct'], {k: (v['count'], v['band'], v['bucket']) for k, v in h.items() if isinstance(v, dict)})