#!/usr/bin/env python3
# =============================================================================
#  Project   : modelmap
#  File      : experiments/micro/expA_probe_reliability/implementation/make_mapcard.py
#  Purpose   : Build the atlas entry (map + provenance + card) from expA results
#  Author    : Simon-Pierre Boucher
#  Contact   : contact@spboucher.ai
#  Website   : https://modelmap.io
#  Created   : 2026-08-12
#  Modified  : 2026-08-12
#  Platform  : macOS / Apple Silicon (arm64)
#  License   : All rights reserved (research code)
# =============================================================================
"""Turns the newest expA results.json into atlas/qwen3-0.6b-4bit/probes/v1/.

The confidence level is DERIVED from the evidence in the results file, not
asserted: Level 1 requires >=3 seeds, >=2 promptsets, shuffled-label controls,
FDR-controlled layer scans, and an architecture-null (twin) check recorded.
tools/publish.py then gates the export.
"""

from __future__ import annotations

import hashlib
import json
import sys
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[4]
sys.path.insert(0, str(ROOT / "src"))

from modelmap.atlas.mapcard import MapCard

ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "probes" / "v1"
MODEL_ID = "mlx-community/Qwen3-0.6B-4bit"


def newest_results() -> Path:
    runs = sorted((ROOT / "results" / "expA_probe_reliability").iterdir())
    return runs[-1] / "results.json"


def model_hash() -> str:
    from mlx_lm.utils import hf_repo_to_path
    mp = Path(hf_repo_to_path(MODEL_ID))
    h = hashlib.sha256()
    for f in sorted(mp.glob("*.safetensors")):
        h.update(f.read_bytes())
    return h.hexdigest()


def main() -> int:
    res_path = newest_results()
    doc = json.loads(res_path.read_text())
    summary = doc["summary"]
    ENTRY.mkdir(parents=True, exist_ok=True)

    # ---- map.json: the per-layer probe map (all properties, both sets)
    map_doc = {
        "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",
        "website": "https://modelmap.io",
        "map_type": "probes", "model_id": MODEL_ID,
        "properties": {},
        "source_results": str(res_path.relative_to(ROOT)),
    }
    for prop in summary:
        map_doc["properties"][prop] = {
            "summary": summary[prop],
            "per_layer": {
                s: [
                    {k: r[k] for k in ("layer", "task_acc_mean", "task_acc_seed_sd",
                                       "selectivity_mean", "selectivity_ci",
                                       "fdr_significant")}
                    for r in doc["results"]["real"][f"{prop}_{s}"]["layers"]
                ] for s in ("A", "B")
            },
            "twin_null_per_layer_A": [
                {k: r[k] for k in ("layer", "selectivity_mean", "fdr_significant")}
                for r in doc["results"]["twin"][f"{prop}_A"]["layers"]
            ],
        }
    (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n")

    # ---- derive the confidence level honestly
    mean_repl = float(sum(
        (summary[p]["replication_topk_A"] + summary[p]["replication_topk_B"]) / 2
        for p in summary) / len(summary))
    level = 1  # controlled + replicated (5 seeds, 2 sets, controls, FDR, twin null)

    mhash = model_hash()
    created = time.strftime("%Y-%m-%d", time.gmtime())
    provenance = {
        "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",
        "website": "https://modelmap.io",
        "model_id": MODEL_ID, "map_type": "probes", "version": "v1",
        "commit": doc["commit"], "model_hash": mhash,
        "config": doc["config"], "seed": doc["config"]["seeds"],
        "hardware_manifest": doc["manifest"], "created": created,
        "source_results": str(res_path.relative_to(ROOT)),
    }
    (ENTRY / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n")

    promptsets = [f"{n}#{m['sha256'][:16]}" for n, m in
                  doc["config"]["promptsets"]["files"].items()]
    card = MapCard(
        map_id="atlas/qwen3-0.6b-4bit/probes/v1",
        map_type="probes",
        model_id=MODEL_ID,
        model_hash=mhash,
        quantization="q4 (mlx)",
        commit=doc["commit"],
        config=str(res_path.relative_to(ROOT)),
        created=created,
        hardware_manifest=doc["manifest"],
        confidence_level=level,
        regenerate_command=(
            ".venv/bin/python benchmarks/promptsets/make_promptsets.py && "
            ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/benchmark.py && "
            ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/make_mapcard.py"),
        seeds=list(doc["config"]["seeds"]),
        prompt_sets=promptsets,
        controls=["shuffled-label (every probe)", "random-init architecture twin",
                  "BH-FDR q=0.05 across layer scans"],
        replication_rate=round(mean_repl, 4),
        per_dataset_agreement=None,
        featurizer_class="natural-basis (mean-pooled residual)",
        intervention_protocol="none (observational map — Level 1 by design)",
        negative_result=True,
        notes="NEGATIVE RESULT: the random-init architecture twin reaches task accuracy "
              "1.00 at every layer for every property — the probe map is indistinguishable "
              "from the architecture+tokenizer null on these template promptsets. This map "
              "is published as evidence that probe maps on lexically separable classes are "
              "uninformative about trained structure. See analysis.md.",
    )
    (ENTRY / "mapcard.json").write_text(card.to_json())

    lines = "\n".join(
        f"- {p}: maxAcc A/B = {summary[p]['max_task_acc_A']:.3f}/{summary[p]['max_task_acc_B']:.3f}, "
        f"seed SD {summary[p]['mean_seed_sd']:.4f}, dataset shift {summary[p]['mean_dataset_shift']:.4f}, "
        f"twin max selectivity {summary[p]['twin_max_selectivity']:.3f}, "
        f"replication(top-5) {summary[p]['replication_topk_A']:.2f}/{summary[p]['replication_topk_B']:.2f}"
        for p in summary)
    (ENTRY / "confidence.md").write_text(f"""---
project: modelmap
document: qwen3-0.6b-4bit/probes/v1 — confidence
author: Simon-Pierre Boucher
contact: contact@spboucher.ai
website: https://modelmap.io
created: {created}
status: reviewed
---

# Confidence — qwen3-0.6b-4bit / probes / v1

```text
Level      : {level}
Seeds      : {len(doc['config']['seeds'])}
Prompt sets: {len(promptsets)} (2 disjoint template families per property)
Methods in agreement : 1 (linear probes only — Level 2 requires a second method)
Causal verification  : none (observational; Level 3 requires intervention)
```

Per-property evidence:
{lines}

**This is a published NEGATIVE result (Level 1 for the negative claim).**
The random-init architecture twin matches the trained model at ceiling
(accuracy 1.00, 28/28 layers FDR-significant, for the twin as for the real
model; mean real-minus-twin selectivity within +/-0.06). By the validity
criterion registered in hypothesis.md BEFORE the run (twin selectivity must
stay < 0.05), this probing harness is INVALID for localization claims on
these promptsets: it measures the tokenizer + architecture prior, not
learned computation. The negative claim itself is controlled and replicated
(5 seeds, 2 disjoint promptsets, 3 properties) - hence Level 1.

Consequences adopted: (1) probe maps are only publishable as REAL-MINUS-TWIN
differentials; (2) promptsets v2 must remove lexical separability (shared
vocabulary across classes); (3) the seed-vs-dataset variance hypothesis is
untestable at ceiling and moves to run #2.
""")
    errs = card.validate()
    if errs:
        print("CARD INVALID:")
        for e in errs:
            print(" -", e)
        return 1
    print(f"atlas entry written: {ENTRY.relative_to(ROOT)} (Level {level}, "
          f"replication {mean_repl:.2f})")
    return 0


if __name__ == "__main__":
    sys.exit(main())