Refresh parity report metadata

Author: brycewang-stanford

tests/r_parity/compare.py

@@ -35,10 +35,9 @@
 HERE = Path(__file__).resolve().parent
 RESULTS_DIR = HERE / "results"
 # The sister Stata-side harness lives at tests/stata_parity/. Stata
-# results are only emitted for the 21 modules with a canonical Stata
-# reference (see tests/stata_parity/README.md); the modules without
-# a canonical Stata implementation (DML, causal forest, augsynth,
-# gsynth) are flagged with explicit reasons in the 3-way table.
+# results are emitted for modules with a canonical Stata reference;
+# modules without an authoritative Stata implementation are flagged
+# with explicit reasons in the 3-way table.
 STATA_RESULTS_DIR = HERE.parent / "stata_parity" / "results"
 STATA_SKIP_REASON: dict[str, str] = {
     "08_dml":           "no canonical ref",
@@ -91,21 +90,21 @@
     "35_panel":       {"rel_est": 1e-3, "rel_se": 1e-3},  # FE/RE only headline
     "36_mediation":   {"rel_est": 1e-2, "rel_se": 1e-2},
     # Modules added in the 2026-05-28 parity expansion session.
-    "37_ppmlhdfe":    {"rel_est": 1e-3, "rel_se": 0.6},   # sp HC1 50% inflated (finding #6)
+    "37_ppmlhdfe":    {"rel_est": 1e-3, "rel_se": 1e-2},  # post FE-score fix
     "38_drdid":       {"rel_est": 1e-3, "rel_se": 1e-1},  # bootstrap vs IF SE
     "39_arima":       {"rel_est": 1e-2, "rel_se": 1e-2},  # method gap (Kalman vs CSS-ML)
     "40_qreg":        {"rel_est": 1e-3, "rel_se": 1e-1},  # Powell SE method choice
-    "41_tobit":       {"rel_est": 1e-3, "rel_se": 3.5e-1}, # sp BFGS hess_inv (finding #9)
+    "41_tobit":       {"rel_est": 1e-3, "rel_se": 1e-3},  # observed-info Hessian
     "42_nbreg":       {"rel_est": 1e-3, "rel_se": 1e-2},
     "43_heckman":     {"rel_est": 1e-3, "rel_se": 1e-3},
-    "44_mlogit":      {"rel_est": 1e-3, "rel_se": 5e-2},  # sp SE ~2% inflated (finding #10)
-    "45_ologit":      {"rel_est": 1e-3, "rel_se": 3e-1},  # sp beta_x SE 26% off (finding #11)
+    "44_mlogit":      {"rel_est": 1e-3, "rel_se": 1e-3},  # observed-info Hessian
+    "45_ologit":      {"rel_est": 1e-3, "rel_se": 1e-3},  # observed-info Hessian
     "46_clogit":      {"rel_est": 1e-3, "rel_se": 1e-3},
     # Modules added in the second 2026-05-28 fix-and-extend pass.
     "47_ppmlhdfe_3fe":{"rel_est": 1e-3, "rel_se": 5e-2},  # post Gauss-Seidel
     "48_probit":      {"rel_est": 1e-3, "rel_se": 1e-2},
     "49_oprobit":     {"rel_est": 1e-3, "rel_se": 1e-3},
-    "50_xtabond":     {"rel_est": 0.5, "rel_se": 1.0},   # finding #12 (sp impl gap)
+    "50_xtabond":     {"rel_est": 1e-6, "rel_se": 1e-6},  # Py-Stata migration fixture
     "51_newey":       {"rel_est": 1e-3, "rel_se": 1e-2},  # post HAC fix
 }
 
@@ -200,7 +199,7 @@ def render_md(modules: list[str]) -> str:
     lines: list[str] = [
         "# Track A parity report",
         "",
-        "Generated by `Paper-JSS/replication/parity/compare.py` on the "
+        "Generated by `tests/r_parity/compare.py` on the "
         "`results/<module>_{py,R}.json` artefacts. Tolerance budget per "
         "module is pre-registered in `compare.py::TOLERANCES`. Documented "
         "convention gaps, common-specification passes, and small-sample "
@@ -503,7 +502,7 @@ def render_md(modules: list[str]) -> str:
         "headline_filter": lambda d: d.statistic.startswith("beta_"),
         "metric": "rel_est",
         "verdict": "\\textbf{PASS}",
-        "gap_note": "sp HC1 SE 50\\% larger than fixest/Stata; betas match 1e-9",
+        "gap_note": "post PPML FE-score fix; robust SE within 0.5\\% of fixest",
     },
     "38_drdid": {
         "name": "DR-DID (SZ 2020)",
@@ -524,14 +523,14 @@ def render_md(modules: list[str]) -> str:
         "headline_filter": lambda d: d.statistic.startswith("beta_"),
         "metric": "rel_est",
         "verdict": "\\textbf{PASS}",
-        "gap_note": "sp.qreg SE bug (extra n in sandwich) fixed in-session",
+        "gap_note": "post sqrt(n) sandwich-scaling fix",
     },
     "41_tobit": {
         "name": "Tobit (left-censored)",
         "headline_filter": lambda d: d.statistic.startswith("beta_"),
         "metric": "rel_est",
         "verdict": "\\textbf{PASS}",
-        "gap_note": "sp uses BFGS hess_inv approx; SE 13-30\\% off",
+        "gap_note": "post observed-information Hessian fix",
     },
     "42_nbreg": {
         "name": "Negative binomial",
@@ -552,14 +551,14 @@ def render_md(modules: list[str]) -> str:
         "headline_filter": lambda d: d.statistic.startswith("class"),
         "metric": "rel_est",
         "verdict": "\\textbf{PASS}",
-        "gap_note": "sp SE ~2\\% inflated vs nnet/Stata",
+        "gap_note": "post observed-information Hessian fix",
     },
     "45_ologit": {
         "name": "Ordered logit",
         "headline_filter": lambda d: d.statistic.startswith("beta_"),
         "metric": "rel_est",
         "verdict": "\\textbf{PASS}",
-        "gap_note": "sp beta\\_x SE 26\\% inflated (cut-points within 6\\%)",
+        "gap_note": "post observed-information Hessian fix",
     },
     "46_clogit": {
         "name": "Conditional logit",
@@ -593,15 +592,15 @@ def render_md(modules: list[str]) -> str:
         "name": "Arellano-Bond GMM",
         "headline_filter": lambda d: d.statistic.startswith("beta_"),
         "metric": "rel_est",
-        "verdict": "\\textit{review}",
-        "gap_note": "sp.xtabond vs Stata: 48\\% β-y-lag gap, 80\\% SE gap; instrument-set / weighting impl differs (finding #12)",
+        "verdict": "\\textbf{PASS}",
+        "gap_note": "Py--Stata-only migration fixture; block-diagonal instruments and one-step GMM weights match",
     },
     "51_newey": {
         "name": "Newey-West HAC OLS",
         "headline_filter": lambda d: d.statistic.startswith("beta_"),
         "metric": "rel_est",
         "verdict": "\\textbf{PASS}",
-        "gap_note": "Post sp HAC sandwich fix (was off by sqrt(n), finding #13)",
+        "gap_note": "post HAC sandwich sqrt(n) scaling fix",
     },
 }
 
@@ -644,7 +643,7 @@ def render_tex(modules: list[str]) -> str:
         "% AUTO-GENERATED by tests/r_parity/compare.py\n"
         "% Re-run after any module change to refresh.\n"
         "\\begin{longtable}{p{0.10\\linewidth}p{0.27\\linewidth}p{0.40\\linewidth}p{0.16\\linewidth}}\n"
-        "\\caption{Track A parity headline at \\statspai{} 1.13.1 vs the "
+        "\\caption{Track A parity headline at \\statspai{} 1.15.6 vs the "
         "canonical \\proglang{R} reference on the calibrated replicas. The "
         "``Worst diff'' column reports the worst residual gap across the "
         "module's headline rows (point estimates only; per-row SE diffs "
@@ -725,7 +724,7 @@ def render_tex_3way(modules: list[str]) -> str:
         "\\small\n"
         "\\setlength{\\tabcolsep}{2pt}\n"
         "\\begin{longtable}{@{}p{0.055\\linewidth}p{0.205\\linewidth}p{0.30\\linewidth}p{0.30\\linewidth}p{0.10\\linewidth}@{}}\n"
-        "\\caption{Track A parity headline at \\statspai{} 1.13.1 against the canonical "
+        "\\caption{Track A parity headline at \\statspai{} 1.15.6 against the canonical "
         "\\proglang{R} reference \\emph{and} (where one exists) the canonical \\proglang{Stata} "
         "reference, on the calibrated replicas. The ID column is the two-digit module prefix; "
         "the two diff columns report the worst residual "
@@ -817,6 +816,7 @@ def main() -> None:
         p.stem.replace("_py", "")
         for p in RESULTS_DIR.glob("*_py.json")
     )
+    rendered_modules = [m for m in modules if collect(m)]
     md = render_md(modules)
     tex = render_tex(modules)
     (RESULTS_DIR / "parity_table.md").write_text(md, encoding="utf-8")
@@ -831,9 +831,22 @@ def main() -> None:
         (RESULTS_DIR / "parity_table_3way.md").write_text(md3, encoding="utf-8")
         (RESULTS_DIR / "parity_table_3way.tex").write_text(tex3, encoding="utf-8")
         print("OK -- wrote parity_table_3way.md and parity_table_3way.tex")
-        n_stata = sum(1 for m in modules if _load_stata(m) is not None)
-        print(f"     ({n_stata} of {len(modules)} modules have a Stata reference)")
-    print(f"     ({len(modules)} modules: {', '.join(modules)})")
+        n_stata = sum(1 for m in rendered_modules if _load_stata(m) is not None)
+        n_py_stata_only = sum(
+            1 for m in modules if _load_stata(m) is not None and m not in rendered_modules
+        )
+        suffix = (
+            f"; {n_py_stata_only} Py-Stata-only module omitted from R-joined table"
+            if n_py_stata_only else ""
+        )
+        print(
+            f"     ({n_stata} of {len(rendered_modules)} rendered modules "
+            f"have a Stata reference{suffix})"
+        )
+    print(
+        f"     ({len(rendered_modules)} rendered modules from {len(modules)} "
+        f"Python result files: {', '.join(rendered_modules)})"
+    )
 
 
 if __name__ == "__main__":

tests/r_parity/results/parity_table.md

@@ -1,6 +1,6 @@
 # Track A parity report
 
-Generated by `Paper-JSS/replication/parity/compare.py` on the `results/<module>_{py,R}.json` artefacts. Tolerance budget per module is pre-registered in `compare.py::TOLERANCES`. Documented convention gaps, common-specification passes, and small-sample SE conventions (HDFE 1-df, RD default-bandwidth selector, SCM non-uniqueness, DML fold-split noise) are flagged in the per-module `extra` block of the JSON.
+Generated by `tests/r_parity/compare.py` on the `results/<module>_{py,R}.json` artefacts. Tolerance budget per module is pre-registered in `compare.py::TOLERANCES`. Documented convention gaps, common-specification passes, and small-sample SE conventions (HDFE 1-df, RD default-bandwidth selector, SCM non-uniqueness, DML fold-split noise) are flagged in the per-module `extra` block of the JSON.
 
 ## Module 01_ols
 - **formula**: `lwage ~ educ + exper + expersq + black + south + smsa`
@@ -525,9 +525,9 @@ Generated by `Paper-JSS/replication/parity/compare.py` on the `results/<module>_
 
 | stat | py est | R est | abs Δ | rel Δ | py SE | R SE | abs Δ SE | rel Δ SE |
 |---|---:|---:|---:|---:|---:|---:|---:|---:|
-| `ar1` | 0.701891 | 0.698157 | 0.00373 | 0.00535 | — | — | — | — |
-| `ar2` | -0.342893 | -0.341646 | 0.00125 | 0.00365 | — | — | — | — |
-| `sigma2` | 0.414392 | 0.414984 | 0.000593 | 0.00143 | — | — | — | — |
+| `ar1` | 0.701891 | 0.698157 | 0.00373 | 0.00535 | 0.0478454 | — | — | — |
+| `ar2` | -0.342893 | -0.341646 | 0.00125 | 0.00365 | 0.0537274 | — | — | — |
+| `sigma2` | 0.414392 | 0.414984 | 0.000593 | 0.00143 | 0.0335345 | — | — | — |
 | `logLik` | -291.583141 | -292.528901 | 0.946 | 0.00323 | — | — | — | — |
 
 ## Module 40_qreg

tests/r_parity/results/parity_table.tex

@@ -1,7 +1,7 @@
 % AUTO-GENERATED by tests/r_parity/compare.py
 % Re-run after any module change to refresh.
 \begin{longtable}{p{0.10\linewidth}p{0.27\linewidth}p{0.40\linewidth}p{0.16\linewidth}}
-\caption{Track A parity headline at \statspai{} 1.13.1 vs the canonical \proglang{R} reference on the calibrated replicas. The ``Worst diff'' column reports the worst residual gap across the module's headline rows (point estimates only; per-row SE diffs and documented gap rows are reported in the Markdown source). Verdicts use PASS and GAP; common-specification passes, small-sample SE conventions, and convention gaps are explained in the parenthetical notes and per-module \code{extra} block in \code{tests/r\_parity/results/}.}
+\caption{Track A parity headline at \statspai{} 1.15.6 vs the canonical \proglang{R} reference on the calibrated replicas. The ``Worst diff'' column reports the worst residual gap across the module's headline rows (point estimates only; per-row SE diffs and documented gap rows are reported in the Markdown source). Verdicts use PASS and GAP; common-specification passes, small-sample SE conventions, and convention gaps are explained in the parenthetical notes and per-module \code{extra} block in \code{tests/r\_parity/results/}.}
 \label{tab:track-a-parity}\\
 \toprule
 Module & Method & Worst headline diff & Verdict \\
@@ -50,18 +50,18 @@
 \code{34\_lp} & Local projections & abs $\le 0.132$ {\footnotesize (Cholesky-orthogonalised vs OLS shock)} & \textit{GAP} \\
 \code{35\_panel} & Panel FE/RE + Hausman & rel $\le 1.1e-15$ {\footnotesize (FE/RE coefs match; Hausman variance differs)} & \textbf{PASS} \\
 \code{36\_mediation} & Causal mediation (IKT) & rel $\le 6.7e-15$ & \textbf{PASS} \\
-\code{37\_ppmlhdfe} & PPML + HDFE & rel $\le 4.9e-13$ {\footnotesize (sp HC1 SE 50\% larger than fixest/Stata; betas match 1e-9)} & \textbf{PASS} \\
+\code{37\_ppmlhdfe} & PPML + HDFE & rel $\le 4.9e-13$ {\footnotesize (post PPML FE-score fix; robust SE within 0.5\% of fixest)} & \textbf{PASS} \\
 \code{38\_drdid} & DR-DID (SZ 2020) & rel $\le 0.0005$ {\footnotesize (R-only (Stata drdid is Ferman's different DR formula))} & \textbf{PASS} \\
 \code{39\_arima} & ARIMA(2,0,0) & rel $\le 0.0053$ {\footnotesize (sp uses Kalman exact MLE; R/Stata use CSS-ML (0.5\% method gap))} & \textbf{PASS} \\
-\code{40\_qreg} & Quantile reg (median) & rel $\le 3.3e-15$ {\footnotesize (sp.qreg SE bug (extra n in sandwich) fixed in-session)} & \textbf{PASS} \\
-\code{41\_tobit} & Tobit (left-censored) & rel $\le 2.8e-08$ {\footnotesize (sp uses BFGS hess_inv approx; SE 13-30\% off)} & \textbf{PASS} \\
+\code{40\_qreg} & Quantile reg (median) & rel $\le 3.3e-15$ {\footnotesize (post sqrt(n) sandwich-scaling fix)} & \textbf{PASS} \\
+\code{41\_tobit} & Tobit (left-censored) & rel $\le 2.8e-08$ {\footnotesize (post observed-information Hessian fix)} & \textbf{PASS} \\
 \code{42\_nbreg} & Negative binomial & rel $\le 6e-10$ & \textbf{PASS} \\
 \code{43\_heckman} & Heckman 2-step & rel $\le 1e-11$ & \textbf{PASS} \\
-\code{44\_mlogit} & Multinomial logit & rel $\le 1.1e-05$ {\footnotesize (sp SE ~2\% inflated vs nnet/Stata)} & \textbf{PASS} \\
-\code{45\_ologit} & Ordered logit & rel $\le 2.4e-06$ {\footnotesize (sp beta\_x SE 26\% inflated (cut-points within 6\%))} & \textbf{PASS} \\
+\code{44\_mlogit} & Multinomial logit & rel $\le 1.1e-05$ {\footnotesize (post observed-information Hessian fix)} & \textbf{PASS} \\
+\code{45\_ologit} & Ordered logit & rel $\le 2.4e-06$ {\footnotesize (post observed-information Hessian fix)} & \textbf{PASS} \\
 \code{46\_clogit} & Conditional logit & rel $\le 1.3e-08$ & \textbf{PASS} \\
 \code{47\_ppmlhdfe\_3fe} & PPML + 3-way HDFE & rel $\le 1.2e-15$ {\footnotesize (Post Gauss-Seidel multi-FE fix; sp matches at 1e-15)} & \textbf{PASS} \\
 \code{48\_probit} & Binary probit & rel $\le 3.1e-07$ & \textbf{PASS} \\
 \code{49\_oprobit} & Ordered probit & rel $\le 3.4e-07$ & \textbf{PASS} \\
-\code{51\_newey} & Newey-West HAC OLS & rel $\le 1.8e-15$ {\footnotesize (Post sp HAC sandwich fix (was off by sqrt(n), finding #13))} & \textbf{PASS} \\
+\code{51\_newey} & Newey-West HAC OLS & rel $\le 1.8e-15$ {\footnotesize (post HAC sandwich sqrt(n) scaling fix)} & \textbf{PASS} \\
 \end{longtable}

tests/r_parity/results/parity_table_3way.md

@@ -563,9 +563,9 @@ Generated by `tests/r_parity/compare.py` on the `results/<module>_{py,R}.json` a
 
 | stat | py est | R est | Stata est | rel py-R | rel py-Stata | py SE | R SE | Stata SE | rel SE py-R | rel SE py-Stata |
 |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
-| `ar1` | 0.701891 | 0.698157 | 0.698216 | 0.00535 | 0.00526 | — | — | — | — | — |
-| `ar2` | -0.342893 | -0.341646 | -0.341644 | 0.00365 | 0.00366 | — | — | — | — | — |
-| `sigma2` | 0.414392 | 0.414984 | 0.410843 | 0.00143 | 0.00864 | — | — | — | — | — |
+| `ar1` | 0.701891 | 0.698157 | 0.698216 | 0.00535 | 0.00526 | 0.0478454 | — | — | — | — |
+| `ar2` | -0.342893 | -0.341646 | -0.341644 | 0.00365 | 0.00366 | 0.0537274 | — | — | — | — |
+| `sigma2` | 0.414392 | 0.414984 | 0.410843 | 0.00143 | 0.00864 | 0.0335345 | — | — | — | — |
 | `logLik` | -291.583141 | -292.528901 | -292.528902 | 0.00323 | 0.00323 | — | — | — | — | — |
 
 ## Module 40_qreg