|
| 1 | +"""API surface consistency between ``statspai.__all__`` and ``registry``. |
| 2 | +
|
| 3 | +Two contracts that an agent-native library cannot afford to drift on: |
| 4 | +
|
| 5 | +1. Every name in ``statspai.__all__`` must resolve on the top-level module. |
| 6 | + (Otherwise ``from statspai import *`` raises and IDEs lie.) |
| 7 | +
|
| 8 | +2. Every registered function must resolve as ``statspai.<name>`` *or* via the |
| 9 | + submodule path documented in its registry ``example`` field. (Otherwise |
| 10 | + ``sp.help(f)`` shows a function the user cannot then call.) |
| 11 | +
|
| 12 | +A frozen baseline records the *current* asymmetry between ``__all__`` and the |
| 13 | +registry so the test passes today but fails on new drift. To accept a |
| 14 | +deliberate change, update the constants below in the same commit. |
| 15 | +""" |
| 16 | + |
| 17 | +from __future__ import annotations |
| 18 | + |
| 19 | +import importlib |
| 20 | + |
| 21 | +import pytest |
| 22 | + |
| 23 | +import statspai |
| 24 | +import statspai as sp |
| 25 | + |
| 26 | + |
| 27 | +# --------------------------------------------------------------------------- |
| 28 | +# Frozen baseline. Update deliberately when intentionally adding / removing |
| 29 | +# public surface; never silently. Both sets are alphabetised for easy diffs. |
| 30 | +# --------------------------------------------------------------------------- |
| 31 | + |
| 32 | +# Names in ``__all__`` that are *not* registry functions. These are legitimate |
| 33 | +# module / constant re-exports — the registry only tracks callable estimators. |
| 34 | +ALL_NOT_REGISTERED_BASELINE = frozenset({ |
| 35 | + "JOURNAL_PRESETS", |
| 36 | + "PAPER_TABLE_TEMPLATES", |
| 37 | + "STABILITY_TIERS", |
| 38 | + "VALIDATION_STATUSES", |
| 39 | + "epi", |
| 40 | + "exceptions", |
| 41 | + "longitudinal", |
| 42 | + "mendelian", |
| 43 | + "question", |
| 44 | + "tte", |
| 45 | +}) |
| 46 | + |
| 47 | +# Names registered in the registry but *not* in ``__all__``. Most are real |
| 48 | +# estimators that ship today on ``sp.<name>`` but were never added to the |
| 49 | +# star-import list. Reducing this set is the goal; growing it is a regression. |
| 50 | +REGISTERED_NOT_IN_ALL_BASELINE = frozenset({ |
| 51 | + "anthropic_client", |
| 52 | + "assimilative_causal", |
| 53 | + "bayes_dml", |
| 54 | + "bcf_factor_exposure", |
| 55 | + "bcf_longitudinal", |
| 56 | + "bcf_ordinal", |
| 57 | + "causal_bandit", |
| 58 | + "causal_kalman", |
| 59 | + "causal_mas", |
| 60 | + "causal_policy_forest", |
| 61 | + "conformal_continuous", |
| 62 | + "conformal_interference", |
| 63 | + "counterfactual_fairness", |
| 64 | + "counterfactual_policy_optimization", |
| 65 | + "demographic_parity", |
| 66 | + "dl_propensity_score", |
| 67 | + "echo_client", |
| 68 | + "equalized_odds", |
| 69 | + "evidence_without_injustice", |
| 70 | + "fairness_audit", |
| 71 | + "harvest_did", |
| 72 | + "heterogeneity_of_effect", |
| 73 | + "inward_outward_spillover", |
| 74 | + "long_term_from_short", |
| 75 | + "llm_causal_assess", |
| 76 | + "mr_bma", |
| 77 | + "mr_mediation", |
| 78 | + "mr_multivariable", |
| 79 | + "network_hte", |
| 80 | + "openai_client", |
| 81 | + "orthogonal_to_bias", |
| 82 | + "overlap_weighted_did", |
| 83 | + "pairwise_causal_benchmark", |
| 84 | + "paper", |
| 85 | + "particle_filter", |
| 86 | + "proximal_surrogate_index", |
| 87 | + "rwd_rct_concordance", |
| 88 | + "sharp_ope_unobserved", |
| 89 | + "shift_share_political", |
| 90 | + "shift_share_political_panel", |
| 91 | + "structural_mdp", |
| 92 | + "surrogate_index", |
| 93 | + "synth_experimental_design", |
| 94 | + "synthesise_evidence", |
| 95 | +}) |
| 96 | + |
| 97 | +# Registered functions that live on a submodule rather than the top-level |
| 98 | +# module. The registry ``example`` already documents the correct path, so this |
| 99 | +# is not a defect — just the contract the test must allow. |
| 100 | +SUBMODULE_ONLY_BASELINE = frozenset({ |
| 101 | + "anthropic_client", |
| 102 | + "echo_client", |
| 103 | + "openai_client", |
| 104 | + "particle_filter", |
| 105 | +}) |
| 106 | + |
| 107 | + |
| 108 | +# --------------------------------------------------------------------------- |
| 109 | +# Contract 1: ``__all__`` resolves |
| 110 | +# --------------------------------------------------------------------------- |
| 111 | + |
| 112 | +def test_all_names_in_dunder_all_resolve_on_module(): |
| 113 | + """Every name listed in ``statspai.__all__`` must be accessible. |
| 114 | +
|
| 115 | + A name in ``__all__`` that does not resolve means ``from statspai import *`` |
| 116 | + raises ``ImportError`` at top-level and breaks every notebook that uses |
| 117 | + the star-import idiom. |
| 118 | + """ |
| 119 | + missing = [n for n in statspai.__all__ if not hasattr(statspai, n)] |
| 120 | + assert missing == [], ( |
| 121 | + f"{len(missing)} name(s) in __all__ do not resolve: {missing!r}" |
| 122 | + ) |
| 123 | + |
| 124 | + |
| 125 | +# --------------------------------------------------------------------------- |
| 126 | +# Contract 2: registry → callable |
| 127 | +# --------------------------------------------------------------------------- |
| 128 | + |
| 129 | +def _registry_example_module(example: str | None) -> str | None: |
| 130 | + """Pull the documented submodule path out of an example like |
| 131 | + ``sp.causal_llm.anthropic_client(...)`` → ``causal_llm``. |
| 132 | + """ |
| 133 | + if not example or not isinstance(example, str): |
| 134 | + return None |
| 135 | + head = example.strip().split("(", 1)[0] |
| 136 | + parts = head.split(".") |
| 137 | + # Expect form ``sp.<sub>.<name>(...)`` or ``statspai.<sub>.<name>(...)``. |
| 138 | + if len(parts) >= 3 and parts[0] in {"sp", "statspai"}: |
| 139 | + return parts[1] |
| 140 | + return None |
| 141 | + |
| 142 | + |
| 143 | +def test_every_registered_function_is_callable_via_documented_path(): |
| 144 | + """Each registered function resolves on ``statspai`` directly, or on the |
| 145 | + submodule that its registry ``example`` documents. |
| 146 | + """ |
| 147 | + failures = [] |
| 148 | + for name in sp.list_functions(): |
| 149 | + if hasattr(statspai, name): |
| 150 | + continue # accessible at top-level — fine |
| 151 | + spec = sp.describe_function(name) or {} |
| 152 | + submod = _registry_example_module(spec.get("example")) |
| 153 | + if submod is None: |
| 154 | + failures.append( |
| 155 | + f"{name}: no example documenting submodule, " |
| 156 | + "and not resolvable as statspai.<name>" |
| 157 | + ) |
| 158 | + continue |
| 159 | + try: |
| 160 | + mod = importlib.import_module(f"statspai.{submod}") |
| 161 | + except ImportError as exc: |
| 162 | + failures.append(f"{name}: documented submodule fails to import ({exc})") |
| 163 | + continue |
| 164 | + if not hasattr(mod, name): |
| 165 | + failures.append( |
| 166 | + f"{name}: not on statspai and not on statspai.{submod}" |
| 167 | + ) |
| 168 | + assert failures == [], ( |
| 169 | + f"{len(failures)} registered function(s) are unreachable:\n " |
| 170 | + + "\n ".join(failures) |
| 171 | + ) |
| 172 | + |
| 173 | + |
| 174 | +# --------------------------------------------------------------------------- |
| 175 | +# Contract 3: drift against frozen baselines |
| 176 | +# --------------------------------------------------------------------------- |
| 177 | + |
| 178 | +@pytest.fixture(scope="module") |
| 179 | +def asymmetry(): |
| 180 | + all_set = frozenset(getattr(statspai, "__all__", [])) |
| 181 | + reg_set = frozenset(sp.list_functions()) |
| 182 | + return { |
| 183 | + "all_not_registered": all_set - reg_set, |
| 184 | + "registered_not_in_all": reg_set - all_set, |
| 185 | + } |
| 186 | + |
| 187 | + |
| 188 | +def test_all_not_registered_matches_baseline(asymmetry): |
| 189 | + """``__all__`` entries that are not registry functions are tightly |
| 190 | + enumerated. Adding a new non-function name to ``__all__`` is allowed only |
| 191 | + by updating ``ALL_NOT_REGISTERED_BASELINE`` in the same commit. |
| 192 | + """ |
| 193 | + current = asymmetry["all_not_registered"] |
| 194 | + extra = current - ALL_NOT_REGISTERED_BASELINE |
| 195 | + removed = ALL_NOT_REGISTERED_BASELINE - current |
| 196 | + assert not extra, ( |
| 197 | + "New non-function entry in __all__ — register it or update the " |
| 198 | + f"baseline: {sorted(extra)}" |
| 199 | + ) |
| 200 | + assert not removed, ( |
| 201 | + "An expected non-function entry disappeared from __all__ — " |
| 202 | + f"update the baseline if intentional: {sorted(removed)}" |
| 203 | + ) |
| 204 | + |
| 205 | + |
| 206 | +def test_registered_not_in_all_does_not_grow(asymmetry): |
| 207 | + """Functions registered but missing from ``__all__`` should shrink, never |
| 208 | + grow. Net new entries here are silent agent-discoverability regressions. |
| 209 | + """ |
| 210 | + current = asymmetry["registered_not_in_all"] |
| 211 | + new = current - REGISTERED_NOT_IN_ALL_BASELINE |
| 212 | + assert not new, ( |
| 213 | + "New registered function(s) missing from __all__ " |
| 214 | + f"(add them or update baseline): {sorted(new)}" |
| 215 | + ) |
| 216 | + |
| 217 | + |
| 218 | +def test_submodule_only_baseline_still_holds(): |
| 219 | + """Functions that live only on a submodule (documented via their example) |
| 220 | + are tightly enumerated; net new entries silently shrink the top-level |
| 221 | + surface and should be deliberate. |
| 222 | + """ |
| 223 | + actual = set() |
| 224 | + for name in sp.list_functions(): |
| 225 | + if not hasattr(statspai, name): |
| 226 | + actual.add(name) |
| 227 | + new = actual - SUBMODULE_ONLY_BASELINE |
| 228 | + assert not new, ( |
| 229 | + "Function(s) silently dropped from top-level statspai (only " |
| 230 | + f"reachable via submodule): {sorted(new)}" |
| 231 | + ) |
0 commit comments