#!/usr/bin/env python3
"""Google Maps -> website -> email -> verify, one category in one city per query.

Runs each step through the treg CLI (https://treg.to/docs) and stores every
response with personal email local parts masked (j***@domain) before it is
written. Unmasked addresses live only in memory, long enough to be verified.

Steps per place that lists a website:
  1. places   serper.google.serp.maps {q}                      (one call per query)
  2. site     treg.web.extract on the homepage and /contact    (emails printed on the site)
  3. domain   hunter.companies.emails ?domain&limit=10         (emails Hunter has seen for the domain)
  4. owner    treg.people.search {company_domain, title}       (a named owner or principal)
  5. find     treg.people.email.find {full_name, domain}       (that person's work email)
  6. verify   treg.people.email.verify on every unique address from steps 2, 3 and 5

Usage: scripts/run-maps-email-chain.py <out-dir> "dentist in Austin, TX" ["plumber in Denver, CO" ...]
Writes <out-dir>/raw/*.json (masked, gitignored) and <out-dir>/_calls.jsonl.
"""
import json, re, subprocess, sys, time
from pathlib import Path
from urllib.parse import urlparse

OUT = Path(sys.argv[1]); QUERIES = sys.argv[2:]
RAW = OUT / "raw"; RAW.mkdir(parents=True, exist_ok=True)
EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
# role inboxes: the local part names a function, not a person (classified before masking)
GENERIC = re.compile(r"^(info|contact|office|hello|hi|admin|frontdesk|front\.?desk|appointments?|appts?|reception|service|services|sales|support|team|mail|email|scheduling|schedule|billing|accounts?|dispatch|customerservice|help|inquir(y|ies)|enquir(y|ies)|smile|smiles|care|patients?|newpatients?|hr|jobs|careers|marketing|webmaster|noreply|no-reply|booking|bookings|ask|general|staff|clinic|dental|dentist|plumbing|orders?)[0-9]*$")
OWNER_TITLES = "owner,founder,president,principal,dds,dentist,ceo"

def mask(s):
    return re.sub(r"([A-Za-z0-9._%+-])[A-Za-z0-9._%+-]*@([A-Za-z0-9.-]+\.[A-Za-z]{2,})", r"\1***@\2", s)

def call(label, endpoint, body, method="POST", headers=()):
    cmd = ["treg", "call", endpoint, "--method", method, "--data", json.dumps(body)]
    for h in headers: cmd += ["--header", h]
    t0 = time.time()
    p = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
    ms = int((time.time() - t0) * 1000)
    text = p.stdout or p.stderr
    (RAW / f"{label}.json").write_text(mask(text))
    try: data = json.loads(text)
    except Exception: data = {"_unparsed": text[:500]}
    t = data.get("_treg", {}) if isinstance(data, dict) else {}
    with open(OUT / "_calls.jsonl", "a") as f:
        f.write(json.dumps({"label": label, "endpoint": endpoint, "request": mask(json.dumps(body)), "exit": p.returncode,
                            "latency_ms": ms, "served_by": t.get("served_by"), "charged_micro": t.get("charged_micro"),
                            "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}) + "\n")
    print(label, endpoint, f"{ms}ms", "exit", p.returncode, t.get("served_by"), file=sys.stderr)
    return data

def out(d): return d.get("output", d) if isinstance(d, dict) else {}

def domain_of(url):
    h = urlparse(url if "//" in url else "https://" + url).netloc.lower()
    return h[4:] if h.startswith("www.") else h

places = []
for qi, q in enumerate(QUERIES):
    d = call(f"places-q{qi}", "serper.google.serp.maps", {"q": q, "hl": "en"})
    for i, p in enumerate(out(d).get("places", [])):
        places.append({"id": f"q{qi}-p{i+1:02d}", "query": q, "name": p.get("title"), "website": p.get("website"),
                       "category": p.get("type"), "rating_count": p.get("ratingCount")})

results = []
DONE = OUT / "places"; DONE.mkdir(exist_ok=True)
for p in places:
    if (DONE / f"{p['id']}.json").exists():  # resume: a finished place is not called again
        results.append(json.loads((DONE / f"{p['id']}.json").read_text())); continue
    r = dict(p); r["site_emails"] = []; r["hunter_emails"] = []; r["owner_found"] = False; r["found_email"] = None
    found = {}  # unmasked address -> set of sources (memory only)
    if not p["website"]:
        (DONE / f"{p['id']}.json").write_text(json.dumps(r)); results.append(r); continue
    dom = domain_of(p["website"]); r["domain"] = dom
    home = p["website"]
    for tag, url in (("home", home), ("contact", f"https://{urlparse(home).netloc}/contact")):
        d = call(f"site-{tag}-{p['id']}", "treg.web.extract", {"url": url}, headers=["X-Treg-Route-Max-Cost: 0.01"])
        for e in set(EMAIL.findall(json.dumps(out(d)))):
            e = e.lower().rstrip(".")
            if e.endswith((".png", ".jpg", ".webp", ".svg", ".gif")) or "sentry" in e or "wixpress" in e: continue
            found.setdefault(e, set()).add("site")
    d = call(f"hunter-{p['id']}", "hunter.companies.emails", {"domain": dom, "limit": 10}, method="GET")
    hd = out(d).get("data", {}) if isinstance(out(d), dict) else {}
    r["hunter_accept_all"] = hd.get("accept_all")
    for e in hd.get("emails", []) or []:
        v = (e.get("value") or "").lower()
        if v: found.setdefault(v, set()).add("hunter:" + (e.get("type") or "?"))
    d = call(f"owner-{p['id']}", "treg.people.search", {"company_domain": dom, "title": OWNER_TITLES})
    people = out(d).get("people") or out(d).get("results") or [] if isinstance(out(d), dict) else []
    same = [x for x in people if isinstance(x, dict) and dom in json.dumps(x).lower()]
    r["owner_rows"] = len(people); r["owner_rows_on_domain"] = len(same)
    if same:
        x = same[0]
        name = x.get("full_name") or " ".join(filter(None, [x.get("first_name"), x.get("last_name")]))
        r["owner_found"] = bool(name); r["owner_title"] = x.get("title")
        if name:
            d = call(f"find-{p['id']}", "treg.people.email.find", {"full_name": name, "domain": dom},
                     headers=["X-Treg-Route-Max-Cost: 0.05"])
            o = out(d); e = (o.get("email") if isinstance(o, dict) else None) or ""
            if e:
                found.setdefault(e.lower(), set()).add("find"); r["found_email"] = mask(e.lower())
    r["emails"] = []
    for e, src in sorted(found.items()):
        local, _, edom = e.partition("@")
        d = call(f"verify-{p['id']}-{len(r['emails'])}", "treg.people.email.verify", {"email": e},
                 headers=["X-Treg-Route-Max-Cost: 0.01"])
        o = out(d) if isinstance(out(d), dict) else {}
        r["emails"].append({"email": mask(e), "on_domain": edom == dom, "sources": sorted(src),
                            "kind": "generic" if GENERIC.match(local) else "named",
                            "status": o.get("status") or o.get("result") or o.get("verdict"),
                            "catch_all": o.get("catch_all") if "catch_all" in o else o.get("accept_all"),
                            "role": o.get("role") if "role" in o else o.get("is_role"),
                            "served_by": d.get("_treg", {}).get("served_by") if isinstance(d, dict) else None})
    (DONE / f"{p['id']}.json").write_text(json.dumps(r))
    results.append(r)

(OUT / "chain.json").write_text(json.dumps({"queries": QUERIES, "places": results}, indent=1))
print(f"wrote {len(results)} places", file=sys.stderr)