#!/usr/bin/env python3
"""
MercadoLibre Certificate Transparency + DNS recon orchestrator.

Sources:
  - ctlogs.dev    (/v1/domain, /v1/subdomains, /v1/cert for full SANs)
  - certspotter   (/v1/issuances with expand=dns_names)
  - crt.sh        (best-effort, retries)
  - Censys v1     : decommissioned - skipped (needs v2 API key)
  - crt.cloudflare.com : DNS dead - skipped

Outputs:
  - certs/<source>/*.json  : raw cert data per domain
  - certs/all_certs.json   : merged, deduplicated
  - subdomains/subdomains_merged.txt
  - dns/dns_resolution.txt : domain -> IPs
  - REPORT.md
"""

import json
import os
import subprocess
import sys
import time
import urllib.parse
from collections import defaultdict
from datetime import datetime, timezone

BASE = os.path.expanduser("~/Desktop/mercadolibre_recon")
INPUT_FILE = os.path.expanduser("~/Desktop/mercadolibre.txt")
CERTS_DIR = os.path.join(BASE, "certs")
SUBS_DIR = os.path.join(BASE, "subdomains")
DNS_DIR = os.path.join(BASE, "dns")

for d in (CERTS_DIR, SUBS_DIR, DNS_DIR):
    os.makedirs(d, exist_ok=True)

CTLOGS_BASE = "https://api.ctlogs.dev/v1"
CERTSPOTTER_BASE = "https://api.certspotter.com/v1"

# ---- util ----------------------------------------------------------------

def sh(cmd, timeout=60):
    """Run shell command, return (rc, stdout, stderr)."""
    try:
        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
        return r.returncode, r.stdout, r.stderr
    except subprocess.TimeoutExpired:
        return 124, "", "timeout"


def curl_json(url, headers=None, timeout=45):
    """GET a URL with curl and parse JSON. Returns (status_code, parsed_or_text)."""
    hdr = " ".join(f"-H '{h}'" for h in (headers or []))
    cmd = f"curl -sS --max-time {timeout} {hdr} '{url}' -w '\\n%{{http_code}}'"
    rc, out, err = sh(cmd, timeout=timeout + 5)
    if not out:
        return 0, None
    if out.strip().isdigit():
        return int(out.strip()), None
    # last line is the status code
    lines = out.strip().rsplit("\n", 1)
    body = lines[0]
    code = int(lines[1]) if len(lines) > 1 and lines[1].isdigit() else 0
    try:
        return code, json.loads(body)
    except Exception:
        return code, body


def rate_limit_sleep():
    time.sleep(1.5)  # be polite to free endpoints


# ---- domain list ----------------------------------------------------------

with open(INPUT_FILE) as f:
    INPUT_DOMAINS = [ln.strip() for ln in f if ln.strip()]

# registrable (base) domains for extra expansion
BASES = set()
for d in INPUT_DOMAINS:
    parts = d.rsplit(".", 3)
    # handle co.uk-like TLDs (not present here, but safe)
    two_level = {"com.ar", "com.br", "com.mx", "com.co"}
    if len(parts) >= 3 and ".".join(parts[-2:]) in two_level:
        BASES.add(".".join(parts[-3:]))
    else:
        BASES.add(".".join(parts[-2:]))
BASES = sorted(BASES)

print(f"[+] {len(INPUT_DOMAINS)} input domains")
print(f"[+] base (registrable) domains: {BASES}")

# ---- 1. ctlogs.dev sweep ---------------------------------------------------

ctlogs_certs = []          # list of CertRow dicts
ctlogs_certs_by_domain = defaultdict(list)
ctlogs_hosts = set()       # hostnames discovered
ctlogs_rate = {"limit": "?", "remaining": "?"}

# cache: if ctlogs_dev.json exists from a previous run, reuse it and skip API calls
ctlogs_cache_rows = None
_ctlogs_cache_file = os.path.join(CERTS_DIR, "ctlogs_dev.json")
if os.path.exists(_ctlogs_cache_file) and os.path.getsize(_ctlogs_cache_file) > 100:
    try:
        with open(_ctlogs_cache_file) as _f:
            _ctlogs_cached = json.load(_f)
        ctlogs_certs = _ctlogs_cached.get("certs", [])
        ctlogs_hosts = set(_ctlogs_cached.get("hosts", []))
        for _c in ctlogs_certs:
            _m = _c.get("match", "")
            if _m:
                ctlogs_hosts.add(_m.lower())
            ctlogs_certs_by_domain[_c.get("_source_domain", "")].append(_c)
        ctlogs_cache_rows = ctlogs_certs
        print(f"  [cache] ctlogs.dev: loaded {len(ctlogs_certs)} certs, {len(ctlogs_hosts)} hosts - skipping API")
    except Exception as _e:
        print(f"  [warn] ctlogs cache unreadable ({_e}) - will re-query")


def ctlogs_get(url, save_as=None):
    """GET ctlogs.dev endpoint with rate-limit headers captured."""
    cmd = f"curl -sS --max-time 40 -D /tmp/ctlogs_hdr.txt '{url}'"
    rc, out, err = sh(cmd, timeout=50)
    hdrs = {}
    if os.path.exists("/tmp/ctlogs_hdr.txt"):
        for ln in open("/tmp/ctlogs_hdr.txt"):
            if ":" in ln:
                k, v = ln.split(":", 1)
                hdrs[k.strip().lower()] = v.strip()
    ctlogs_rate["limit"] = hdrs.get("x-hourly-limit", "?")
    ctlogs_rate["remaining"] = hdrs.get("x-hourly-remaining", "?")
    if save_as and out:
        with open(save_as, "w") as f:
            f.write(out)
    try:
        return json.loads(out) if out else {}
    except Exception:
        return {}


def ctlogs_paginate(path, max_pages=3):
    """Paginate a ctlogs.dev list endpoint."""
    all_rows = []
    after = ""
    for _ in range(max_pages):
        url = f"{CTLOGS_BASE}{path}"
        if after:
            url += f"?after={after}"
        data = ctlogs_get(url)
        rows = data.get("rows", [])
        all_rows.extend(rows)
        after = data.get("next_cursor", "")
        if not after:
            break
        rate_limit_sleep()
    return all_rows


print("\n=== [1] ctlogs.dev: /v1/domain for each input domain ===")
for i, d in enumerate([] if ctlogs_cache_rows is not None else INPUT_DOMAINS, 1):
    path = f"/domain/{urllib.parse.quote(d)}"
    rows = ctlogs_paginate(path, max_pages=2)
    for r in rows:
        r["_source_domain"] = d
        ctlogs_certs.append(r)
        ctlogs_certs_by_domain[d].append(r)
    print(f"  [{i:2d}/{len(INPUT_DOMAINS)}] {d:45s} -> {len(rows):3d} certs (rate remaining: {ctlogs_rate['remaining']})")
    rate_limit_sleep()

print("\n=== [2] ctlogs.dev: /v1/subdomains for base domains (find more) ===")
for b in ([] if ctlogs_cache_rows is not None else BASES):
    path = f"/subdomains/{urllib.parse.quote(b)}"
    rows = ctlogs_paginate(path, max_pages=5)
    new_hosts = set()
    for r in rows:
        m = r.get("match", "")
        if m:
            new_hosts.add(m.lower())
            r["_source_base"] = b
            ctlogs_certs.append(r)
    ctlogs_hosts |= new_hosts
    print(f"  base {b:25s} -> {len(rows):4d} certs, {len(new_hosts):4d} distinct hostnames (rate remaining: {ctlogs_rate['remaining']})")
    rate_limit_sleep()

# dedupe certs by id
seen = set()
ctlogs_certs_dedup = []
for c in ctlogs_certs:
    k = c.get("id", "")
    if k and k not in seen:
        seen.add(k)
        ctlogs_certs_dedup.append(c)
ctlogs_certs = ctlogs_certs_dedup
print(f"  total unique ctlogs.dev certs: {len(ctlogs_certs)}")

with open(os.path.join(CERTS_DIR, "ctlogs_dev.json"), "w") as f:
    json.dump({"fetched_at": datetime.now(timezone.utc).isoformat(),
               "rate_limit": ctlogs_rate,
               "certs": ctlogs_certs,
               "hosts": sorted(ctlogs_hosts)}, f, indent=2)

# ---- 2. certspotter --------------------------------------------------------

print("\n=== [3] certspotter: issuances with full dns_names ===")
certspotter_certs = []
certspotter_hosts = set()
_cs_cache = os.path.join(CERTS_DIR, "certspotter.json")
_cs_from_cache = False
if os.path.exists(_cs_cache) and os.path.getsize(_cs_cache) > 100:
    try:
        with open(_cs_cache) as _f:
            _cached = json.load(_f)
        certspotter_certs = _cached.get("issuances", [])
        certspotter_hosts = set(_cached.get("hosts", []))
        _cs_from_cache = True
        print(f"  [cache] certspotter: loaded {len(certspotter_certs)} issuances - skipping API")
    except Exception as _e:
        print(f"  [warn] certspotter cache unreadable ({_e}) - will re-query")
for i, b in enumerate([] if _cs_from_cache else BASES, 1):
    url = f"{CERTSPOTTER_BASE}/issuances?domain={urllib.parse.quote(b)}&include_subdomains=true&expand=dns_names"
    code, data = curl_json(url)
    if code == 200 and isinstance(data, list):
        certspotter_certs.extend(data)
        for c in data:
            for n in c.get("dns_names", []):
                certspotter_hosts.add(n.lower())
        print(f"  [{i}/{len(BASES)}] {b:25s} -> {len(data):4d} issuances (http {code})")
    else:
        body = data if not isinstance(data, list) else ""
        print(f"  [{i}/{len(BASES)}] {b:25s} -> http {code} {str(body)[:120]}")
    rate_limit_sleep()

with open(os.path.join(CERTS_DIR, "certspotter.json"), "w") as f:
    json.dump({"fetched_at": datetime.now(timezone.utc).isoformat(),
               "issuances": certspotter_certs,
               "hosts": sorted(certspotter_hosts)}, f, indent=2)
print(f"  total certspotter issuances: {len(certspotter_certs)}, distinct hosts: {len(certspotter_hosts)}")

# ---- 3. crt.sh (best-effort with retries) ----------------------------------

print("\n=== [4] crt.sh: JSON (best effort) ===")
crtsh_certs = []
crtsh_hosts = set()
_crt_cache = os.path.join(CERTS_DIR, "crt_sh.json")
_crt_from_cache = False
if os.path.exists(_crt_cache) and os.path.getsize(_crt_cache) > 100:
    try:
        with open(_crt_cache) as _f:
            _cached = json.load(_f)
        crtsh_certs = _cached.get("entries", [])
        crtsh_hosts = set(_cached.get("hosts", []))
        _crt_from_cache = True
        print(f"  [cache] crt.sh: loaded {len(crtsh_certs)} entries - skipping API")
    except Exception as _e:
        print(f"  [warn] crt.sh cache unreadable ({_e}) - will re-query")
for i, b in enumerate([] if _crt_from_cache else BASES, 1):
    url = f"https://crt.sh/?q=%25.{urllib.parse.quote(b)}&output=json&exclude=expired"
    ok = False
    for attempt in range(3):
        code, data = curl_json(url, headers=["User-Agent: Mozilla/5.0 recon"], timeout=90)
        if code == 200 and isinstance(data, list):
            crtsh_certs.extend(data)
            for c in data:
                nv = c.get("name_value", "") or ""
                for n in nv.split("\n"):
                    n = n.strip().lstrip("*").lstrip(".")
                    if n:
                        crtsh_hosts.add(n.lower())
            print(f"  [{i}/{len(BASES)}] {b:25s} -> {len(data):4d} log entries (http {code}, attempt {attempt+1})")
            ok = True
            break
        print(f"  [{i}/{len(BASES)}] {b:25s} -> http {code}, retry {attempt+1}/3...")
        time.sleep(10)
    if not ok:
        print(f"  [{i}/{len(BASES)}] {b:25s} -> giving up")
    time.sleep(2)

with open(os.path.join(CERTS_DIR, "crt_sh.json"), "w") as f:
    json.dump({"fetched_at": datetime.now(timezone.utc).isoformat(),
               "entries": crtsh_certs,
               "hosts": sorted(crtsh_hosts)}, f, indent=2)
print(f"  total crt.sh entries: {len(crtsh_certs)}, distinct hosts: {len(crtsh_hosts)}")

# ---- 4. merge hosts --------------------------------------------------------

all_hosts = ctlogs_hosts | certspotter_hosts | crtsh_hosts | set(h.lower() for h in INPUT_DOMAINS)
# strip wildcards and keep only *.mercadolibre.com, *.mercadolivre.com, *.mercadolibre.com.ar + bases
valid_suffixes = tuple("." + b for b in BASES) + tuple(BASES)
merged = sorted(h for h in all_hosts if h.endswith(valid_suffixes) and h not in ("", "*") and not h.startswith("*"))

# related Meli-family hosts seen in cert SANs (outside the input zones)
RELATED_ZONES = (
    "meli.com", "mercadopago.com", "mlstatic.com", "mercadoshops.com",
    "mercadolibre.com.mx", "mercadopago.com.mx", "mercadopago.com.br",
    "mercadolibre.com.co", "mercadopago.com.co", "mercadolibre.cl",
    "mercadopago.com.ar", "mercadolibre.com.pe", "mercadolibre.com.uy",
    "mercadolibre.com.ve", "mercadolibre.com.ec", "mercadolibre.com.bo",
    "mercadolibre.com.gt", "mercadolibre.com.do", "mercadolibre.com.py",
    "mercadolibre.com.pa", "mercadolibre.com.cr", "mercadolibre.com.sv",
    "mercadolibre.com.hn", "mercadolibre.com.ni", "mercadolibre.com.pt",
    "mercadopago.com.py", "mercadopago.cl", "mercadopago.com.uy",
    "mercadopago.com.pe", "mercadopago.com.br", "meli.cloud", "meli.io",
    "melicloud.com", "mercadolibre.com.ar", "mercadolivre.com",
)
related = sorted(h for h in all_hosts
                 if not h.endswith(valid_suffixes) and h not in ("", "*")
                 and not h.startswith("*") and "." in h
                 and any(h == z or h.endswith("." + z) for z in RELATED_ZONES))
with open(os.path.join(SUBS_DIR, "related_domains.txt"), "w") as f:
    f.write(chr(10).join(related) + chr(10))
print(f"[+] related Meli-family hosts: {len(related)}")

with open(os.path.join(SUBS_DIR, "subdomains_merged.txt"), "w") as f:
    f.write("\n".join(merged) + "\n")
print(f"\n[+] merged unique subdomains: {len(merged)}")

# ---- 5. wildcard certs -----------------------------------------------------

wildcards = set()
for c in ctlogs_certs:
    v = c.get("match", "")
    if v.startswith("*."):
        wildcards.add(v.lower())
for c in certspotter_certs:
    for n in c.get("dns_names", []):
        if n.startswith("*."):
            wildcards.add(n.lower())
with open(os.path.join(SUBS_DIR, "wildcards_found.txt"), "w") as f:
    f.write("\n".join(sorted(wildcards)) + "\n")
print(f"[+] wildcard names found: {len(wildcards)}")

# ---- 6. DNS resolution -----------------------------------------------------

print(f"\n=== [5] DNS A/AAAA resolution for {len(merged)} hosts ===")
dns_results = {}

def resolve(host):
    rc, out, _ = sh(f"dig +short +time=3 +tries=1 A {host}", timeout=12)
    a = [l.strip() for l in out.strip().split("\n") if l.strip() and not l.strip().endswith(".")]
    rc, out, _ = sh(f"dig +short +time=3 +tries=1 AAAA {host}", timeout=12)
    aaaa = [l.strip() for l in out.strip().split("\n") if l.strip() and ":" in l]
    rc, out, _ = sh(f"dig +short +time=3 +tries=1 CNAME {host}", timeout=12)
    cname = [l.strip() for l in out.strip().split("\n") if l.strip()]
    return {"A": a, "AAAA": aaaa, "CNAME": cname}

# resolve input domains first, then new ones (cap for time)
todo = list(INPUT_DOMAINS) + [h for h in merged if h not in INPUT_DOMAINS][:400] + [h for h in related if h not in INPUT_DOMAINS][:120]
for i, h in enumerate(todo, 1):
    dns_results[h] = resolve(h)
    if i % 25 == 0:
        print(f"  resolved {i}/{len(todo)}")
    time.sleep(0.1)

with open(os.path.join(DNS_DIR, "dns_resolution.json"), "w") as f:
    json.dump(dns_results, f, indent=2)

# human-readable txt
with open(os.path.join(DNS_DIR, "dns_resolution.txt"), "w") as f:
    for h, r in sorted(dns_results.items()):
        ips = r["A"] + r["AAAA"]
        if ips:
            f.write(f"{h}\t{', '.join(ips)}\n")

resolved_with_ip = [h for h, r in dns_results.items() if r["A"] or r["AAAA"]]
print(f"  hosts with A/AAAA records: {len(resolved_with_ip)} / {len(todo)}")

# unique IP set
all_ips = set()
for h, r in dns_results.items():
    all_ips.update(r["A"])
    all_ips.update(r["AAAA"])
with open(os.path.join(DNS_DIR, "unique_ips.txt"), "w") as f:
    f.write("\n".join(sorted(all_ips)) + "\n")
print(f"  unique IPs: {len(all_ips)}")

# ---- 7. REPORT.md ----------------------------------------------------------

def fmt_date(s):
    return (s or "")[:10]

report = []
report.append("# MercadoLibre Certificate & Subdomain Recon Report")
report.append(f"\nGenerated: {datetime.now(timezone.utc).isoformat()}")
report.append(f"\nInput domains: {len(INPUT_DOMAINS)} (see `input_domains.txt`)")
report.append(f"Base domains: {', '.join(BASES)}")

report.append("\n## 1. Sources used")
report.append("| Source | Status | Result |")
report.append("|---|---|---|")
report.append(f"| ctlogs.dev | OK | {len(ctlogs_certs)} unique certs, {len(ctlogs_hosts)} hosts |")
report.append(f"| certspotter.com | OK | {len(certspotter_certs)} issuances, {len(certspotter_hosts)} hosts |")
report.append(f"| crt.sh | {'OK' if crtsh_certs else 'degraded'} | {len(crtsh_certs)} entries |")
report.append("| search.censys.io | skipped | v1 search decommissioned; needs v2 API key |")
report.append("| crt.cloudflare.com | failed | DNS does not resolve (host unreachable) |")
report.append("| dns.he.net | manual | web-only, no public JSON API |")

report.append("\n## 2. Summary numbers")
report.append(f"- Unique subdomains discovered: **{len(merged)}**")
report.append(f"- Wildcard cert names: **{len(wildcards)}**")
report.append(f"- Hosts resolving to IPs: **{len(resolved_with_ip)}**")
report.append(f"- Unique IPs: **{len(all_ips)}**")

report.append("\n## 3. Discovered subdomains (merged from CT sources)")
report.append("\n```")
report.extend(merged)
report.append("```")

report.append("\n## 4. Wildcards seen in certificates")
report.append("\n```")
report.extend(sorted(wildcards))
report.append("```")

report.append("\n## 5. DNS resolution (host -> IP)")
report.append("\n```")
for h, r in sorted(dns_results.items()):
    if r["A"] or r["AAAA"]:
        report.append(f"{h:60s} {', '.join(r['A'] + r['AAAA'])}")
report.append("```")

report.append("## Related Meli-family domains (from cert SANs, outside input zones)")
report.append("")
report.extend(related)
report.append("")

# issuer stats
issuer_count = defaultdict(int)
for c in ctlogs_certs:
    issuer_count[c.get("issuer", "?")] += 1
report.append("\n## 6. Issuer distribution (ctlogs.dev)")
report.append("\n```")
for k, v in sorted(issuer_count.items(), key=lambda x: -x[1]):
    report.append(f"{v:5d}  {k}")
report.append("```")

with open(os.path.join(BASE, "REPORT.md"), "w") as f:
    f.write("\n".join(report) + "\n")

print(f"\n[+] Report written to {os.path.join(BASE, 'REPORT.md')}")
print(f"[+] Raw data in {BASE}/certs/, {BASE}/subdomains/, {BASE}/dns/")