Code / src/modelmap/stats/replication.py
src/modelmap/stats/replication.py
83 lines
# =============================================================================
# Project : modelmap
# File : src/modelmap/stats/replication.py
# Purpose : Bootstrap CIs, BH-FDR control, selectivity, replication metrics
# Author : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# Website : https://modelmap.io
# Created : 2026-08-12
# Modified : 2026-08-12
# Platform : macOS / Apple Silicon (arm64)
# License : All rights reserved (research code)
# =============================================================================
"""Statistical core mandated by charter §8.2 and the Dead-Salmons doctrine
(notes §4.8): bootstrap confidence intervals, Benjamini-Hochberg FDR control
for unit scans, probe selectivity, and map-replication metrics.
"""
from __future__ import annotations
import numpy as np
def bootstrap_ci(
values: np.ndarray,
stat=np.mean,
n_boot: int = 10_000,
alpha: float = 0.05,
rng: np.random.Generator | None = None,
) -> tuple[float, float, float]:
"""Percentile bootstrap CI. Returns (point, lo, hi)."""
values = np.asarray(values, dtype=np.float64)
if values.size == 0:
raise ValueError("bootstrap_ci: empty sample")
rng = rng or np.random.default_rng(0)
idx = rng.integers(0, values.size, size=(n_boot, values.size))
boots = stat(values[idx], axis=1)
return (
float(stat(values)),
float(np.quantile(boots, alpha / 2)),
float(np.quantile(boots, 1 - alpha / 2)),
)
def bh_fdr(p_values: np.ndarray, q: float = 0.05) -> np.ndarray:
"""Benjamini-Hochberg: boolean mask of discoveries at FDR level q."""
p = np.asarray(p_values, dtype=np.float64)
m = p.size
if m == 0:
return np.zeros(0, dtype=bool)
order = np.argsort(p)
ranked = p[order]
thresh = q * (np.arange(1, m + 1) / m)
passing = np.nonzero(ranked <= thresh)[0]
out = np.zeros(m, dtype=bool)
if passing.size:
out[order[: passing.max() + 1]] = True
return out
def selectivity(task_accuracy: float, control_accuracy: float) -> float:
"""Hewitt & Liang selectivity: task acc minus shuffled-label control acc."""
return float(task_accuracy - control_accuracy)
def jaccard(a: set, b: set) -> float:
"""Overlap of 'important unit' sets — the replication metric of expD."""
if not a and not b:
return 1.0
return len(a & b) / len(a | b)
def replication_rate(unit_sets: list[set]) -> tuple[float, float, float]:
"""Mean pairwise Jaccard across seeds/datasets, with bootstrap CI.
This is THE first-class atlas metric (charter §8.3): does the map
reproduce under resampling?
"""
n = len(unit_sets)
if n < 2:
raise ValueError("replication_rate needs >=2 replicates")
pairs = [jaccard(unit_sets[i], unit_sets[j]) for i in range(n) for j in range(i + 1, n)]
return bootstrap_ci(np.array(pairs))