#!/usr/bin/env python3
"""Derive a publishable bundle from the local HAR Forensic Atlas.

The local report reconstructs a private DeepSeek conversation. Turns 3-16 map
named family, investor, immigration, and defense-contracting relationships
belonging to Aman and to third parties who did not consent to publication.

This script strips that material at the DATA layer, not the presentation layer.
The local report's "safe mode" toggle is only a CSS class over text that ships
in full inside data/report.json; anyone could read it with view-source. Nothing
here relies on client-side hiding.

What is removed:
  * prompt / assistant text for every turn classified sensitivity == "high"
  * search queries and result cards for those turns (one turn's generated
    queries pair a third party's full name with an immigration category)
  * phase labels, which name the private topics
  * search_result_domains, which leaks the immigration-lawyer domain set
  * the uploaded iMessage screenshot and every image blob derived from it
  * every .sse blob (raw assistant token streams from the private turns)
  * any remaining blob whose bytes match the identifier scan

What is kept (the actual forensic contribution):
  * all protocol, network, coverage, token/PoW, claim-audit and findings data
  * per-turn structural telemetry: timing, SSE frame counts, token usage,
    search_enabled vs search_triggered, response status, message ids
  * blob ledger entries for withheld blobs, as hash + size + reason, so the
    integrity chain stays auditable without shipping the bytes

Run from har_analysis/:  python3 sanitize_public.py
Output:                  public/
"""

from __future__ import annotations

import json
import re
import shutil
from pathlib import Path

HERE = Path(__file__).resolve().parent
SRC_DATA = HERE / "data"
SRC_BLOBS = HERE / "blobs"
OUT = HERE / "public"
OUT_DATA = OUT / "data"
OUT_BLOBS = OUT / "blobs"

# Identifier denylist. Deliberately broad: a false positive costs one withheld
# blob, a false negative publishes someone's name.
#
# The terms live in denylist.txt, NOT here. This script ships with the public
# build for auditability, and a denylist inlined in an published file would
# republish exactly the names it exists to remove. denylist.txt stays local.
DENYLIST_PATH = HERE / "denylist.txt"


def load_denylist() -> re.Pattern[str]:
    if not DENYLIST_PATH.exists():
        raise SystemExit(
            f"missing {DENYLIST_PATH.name}: the redaction terms file is required "
            "and is intentionally not published with this script"
        )
    terms = [
        line.strip()
        for line in DENYLIST_PATH.read_text(encoding="utf-8").splitlines()
        if line.strip() and not line.startswith("#")
    ]
    if not terms:
        raise SystemExit("denylist.txt is empty; refusing to build an unfiltered bundle")
    return re.compile("|".join(terms), re.IGNORECASE)


IDENTIFIERS = load_denylist()

# Above this many identifier hits in one field, the text is about people
# rather than incidentally mentioning one; drop it instead of masking.
MASK_LIMIT = 2

REDACTION_NOTE = (
    "Withheld from the public build: this turn's text described private "
    "personal relationships, including third parties who did not consent."
)

# Turn fields that carry private content and are dropped for high-sensitivity
# turns. Everything not listed here is structural telemetry and is preserved.
PRIVATE_TURN_FIELDS = ("prompt", "assistant", "searches")


def load(name: str):
    return json.loads((SRC_DATA / name).read_text(encoding="utf-8"))


def scrub_turn(turn: dict) -> dict:
    """Keep structure, drop private content for high-sensitivity turns."""
    out = dict(turn)

    # Phase labels name the private topics ("Family, investor, and
    # research-network graph"). Replace with a neutral, structural label.
    if turn.get("sensitivity") == "high":
        out["phase"] = "Withheld from public build"
        for field in PRIVATE_TURN_FIELDS:
            if field in out:
                out[field] = [] if field == "searches" else None
        out["redacted"] = True
        out["redaction_note"] = REDACTION_NOTE
        # Counts survive so length/telemetry analysis still works.
        out["search_query_count"] = 0
        out["search_result_count"] = 0
        return out

    # Low-sensitivity turns are the search-mechanics probes and carry the
    # report's technical payload. They hit the denylist only where the model
    # invents an example search query naming the account holder, inside an
    # explanation of tool-call format. Mask the identifier in place rather than
    # discarding the explanation; if a turn is dense with hits, drop it whole.
    out["redacted"] = False
    for field in ("prompt", "assistant"):
        value = out.get(field)
        if not isinstance(value, str):
            continue
        hits = IDENTIFIERS.findall(value)
        if not hits:
            continue
        if len(hits) > MASK_LIMIT:
            out[field] = None
            out["redacted"] = True
            out["redaction_note"] = REDACTION_NOTE
        else:
            out[field] = IDENTIFIERS.sub("[name redacted]", value)
            out["masked"] = True
    blob = json.dumps(out.get("searches"), ensure_ascii=False)
    if IDENTIFIERS.search(blob):
        out["searches"] = []
        out["search_query_count"] = 0
        out["search_result_count"] = 0
        out["redacted"] = True
        out["redaction_note"] = REDACTION_NOTE
    return out


# Favicon and asset URLs echo the domains the private searches hit. The
# site-icon path leaks the private topic even though no prompt text is
# involved. Domain terms come from the same local denylist, plus the
# site-icons path prefix that carries them.
TOPIC_LEAKING_URL = re.compile(
    "site-icons|labusinessjournal|" + IDENTIFIERS.pattern,
    re.IGNORECASE,
)


def scrub_url_bearing(node):
    """Recursively neutralize URLs that leak the private search topic."""
    if isinstance(node, dict):
        out = {}
        for key, value in node.items():
            if key in {"url", "path", "source_url"} and isinstance(value, str) and TOPIC_LEAKING_URL.search(value):
                out[key] = "[withheld: topic-revealing asset URL]"
            else:
                out[key] = scrub_url_bearing(value)
        return out
    if isinstance(node, list):
        return [scrub_url_bearing(v) for v in node]
    return node


def scrub_report(report: dict) -> dict:
    out = dict(report)

    out["meta"] = dict(report["meta"])
    out["meta"]["privacy"] = (
        "Public build. Private conversation content removed at the data layer; "
        "protocol, network, and coverage analysis retained in full."
    )
    out["meta"]["build"] = "public-redacted"

    session = dict(report["session"])
    session["turns"] = [scrub_turn(t) for t in report["session"]["turns"]]
    # Leaks the immigration-lawyer / personal-research domain set.
    session["search_result_domains"] = {}
    session["search_result_domains_withheld"] = True
    kept = [t for t in session["turns"] if not t.get("redacted")]
    session["search_query_count"] = sum(t.get("search_query_count", 0) for t in kept)
    session["search_result_count"] = sum(t.get("search_result_count", 0) for t in kept)
    out["session"] = session

    # The uploaded screenshot and its service-side derivatives are withheld,
    # but the OCR-vs-vision finding is a genuine technical result. Keep the
    # measurements that support it; drop the image bytes and any description
    # of what the picture showed.
    uploads = report.get("files", {}).get("uploads") or []
    images = report.get("files", {}).get("image_blobs") or []
    analysis = None
    if uploads:
        upload = uploads[0]
        analysis = {
            "reported_size": upload.get("reported_size"),
            "width": upload.get("width"),
            "height": upload.get("height"),
            "token_usage": upload.get("token_usage"),
            "model_kind": upload.get("model_kind"),
            "audit_result": upload.get("audit_result"),
            "exact_request_blob_recoverable": upload.get("exact_request_blob_recoverable"),
            "derivative_hashes": [
                {"sha256": i.get("sha256"), "size": i.get("size"), "mime": i.get("mime")}
                for i in images
                if "files.deepseeksvc.com" in (i.get("source_url") or "")
            ],
        }
    out["files"] = {
        "uploads": [],
        "image_blobs": [],
        "upload_analysis": analysis,
        "withheld": True,
        "withheld_note": (
            "One user-uploaded image and its derivatives are withheld: the "
            "screenshot shows a private third-party message thread. Hashes and "
            "parser measurements are retained so the OCR finding stays checkable."
        ),
    }
    return out


def blob_is_publishable(path: Path) -> tuple[bool, str]:
    if path.suffix == ".sse":
        return False, "raw assistant token stream from a private turn"
    if path.suffix in {".webp", ".png"} and path.stat().st_size > 1000:
        return False, "image derivative of a private uploaded screenshot"
    try:
        text = path.read_bytes().decode("utf-8", "ignore")
    except OSError:
        return False, "unreadable"
    if IDENTIFIERS.search(text):
        return False, "contains personal identifiers"
    return True, ""


def main() -> None:
    if OUT.exists():
        shutil.rmtree(OUT)
    OUT_DATA.mkdir(parents=True)
    OUT_BLOBS.mkdir(parents=True)

    raw_manifest = load("blob-manifest.json")

    # Blobs fetched from topic-revealing URLs (site favicons for the domains
    # the private searches hit) are withheld even though their bytes contain
    # no text: the asset identifies the private subject matter on its own.
    leaking_paths = {
        blob.get("path")
        for blob in raw_manifest.get("blobs", [])
        if TOPIC_LEAKING_URL.search(json.dumps(blob.get("references", ""), ensure_ascii=False))
    }

    report = scrub_url_bearing(scrub_report(load("report.json")))
    manifest = scrub_url_bearing(raw_manifest)

    # Blobs: copy the publishable ones, keep withheld ones in the ledger as
    # hash + size so the integrity chain is still auditable.
    kept_paths: set[str] = set()
    withheld = 0
    for blob_path in sorted(SRC_BLOBS.iterdir()):
        if not blob_path.is_file():
            continue
        ok, _reason = blob_is_publishable(blob_path)
        if f"blobs/{blob_path.name}" in leaking_paths:
            ok = False
        if ok:
            shutil.copy2(blob_path, OUT_BLOBS / blob_path.name)
            kept_paths.add(f"blobs/{blob_path.name}")
        else:
            withheld += 1

    out_blobs = []
    for blob in manifest.get("blobs", []):
        entry = dict(blob)
        if entry.get("path") not in kept_paths:
            entry.pop("path", None)
            entry["withheld"] = True
            entry["withheld_reason"] = "contains private session material"
        out_blobs.append(entry)
    manifest = dict(manifest)
    manifest["blobs"] = out_blobs
    manifest["build"] = "public-redacted"

    source_manifest = load("source-manifest.json")
    source_manifest = dict(source_manifest)
    # The HAR itself is the private master and is never published.
    source_manifest["files"] = [
        f for f in source_manifest.get("files", []) if not f["path"].endswith(".DS_Store")
    ]
    source_manifest["note"] = (
        "Hashes describe the local evidence bundle. The .har master and the "
        "reconstructed conversation are not published."
    )

    (OUT_DATA / "report.json").write_text(
        json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
    )
    (OUT_DATA / "report-data.js").write_text(
        "window.__HAR_REPORT__ = "
        + json.dumps(report, ensure_ascii=False).replace("</", "<\\/")
        + ";\n",
        encoding="utf-8",
    )
    (OUT_DATA / "blob-manifest.json").write_text(
        json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
    )
    (OUT_DATA / "blob-data.js").write_text(
        "window.__BLOB_MANIFEST__ = "
        + json.dumps(manifest, ensure_ascii=False).replace("</", "<\\/")
        + ";\n",
        encoding="utf-8",
    )
    (OUT_DATA / "source-manifest.json").write_text(
        json.dumps(source_manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
    )

    # Front-end: the public build has its own index.html/app.js/styles.css
    # (no safe-mode toggle, withheld-content panels, no links to local
    # evidence). They live in .public-src/ so re-running this script does not
    # clobber them. Analyzer + redaction sources ship for auditability.
    src = HERE / ".public-src"
    for name in ("index.html", "app.js", "styles.css"):
        shutil.copy2(src / name, OUT / name)
    for name in ("analyze_har.py", "sanitize_public.py"):
        shutil.copy2(HERE / name, OUT / name)

    redacted = sum(1 for t in report["session"]["turns"] if t.get("redacted"))
    print(
        json.dumps(
            {
                "turns_total": len(report["session"]["turns"]),
                "turns_redacted": redacted,
                "blobs_published": len(kept_paths),
                "blobs_withheld": withheld,
            },
            indent=2,
        )
    )


if __name__ == "__main__":
    main()