Skip to content

Commit f40b258

Browse files
feat(scripts): schema_quality.py for JSS Table 9 numbers
New scripts/schema_quality.py iterates the live sp.list_functions() registry and reports: - Function-level coverage: total functions, those with non-empty descriptions, those with at least one typed parameter, and those with curated agent metadata cards (assumptions / pre_conditions / failure_modes / limitations / minimum_n at least one non-empty). - Parameter-level quality: total parameters across all schemas plus the number with non-empty descriptions, explicit default values, and enumerated choices. The script regenerates the numbers reported in Table 9 of the JSS manuscript (Paper-JSS/manuscript/sections/07-agent-eval.tex) and is referenced from the table caption. Re-running it after a release verifies that the table has not silently drifted from the registry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 39d6ced commit f40b258

1 file changed

Lines changed: 94 additions & 0 deletions

File tree

scripts/schema_quality.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
"""Compute the schema-quality numbers reported in Table 9 of the JSS draft.
2+
3+
Run::
4+
5+
python scripts/schema_quality.py
6+
7+
Outputs the function-level coverage and parameter-level quality
8+
statistics over the live ``sp.list_functions()`` registry.
9+
10+
The numbers feed
11+
``Paper-JSS/manuscript/sections/07-agent-eval.tex``,
12+
Table~\\ref{tab:schema-coverage}. Re-running this script after a
13+
release verifies that the table has not silently drifted from the
14+
implementation.
15+
"""
16+
from __future__ import annotations
17+
18+
import statspai as sp
19+
20+
21+
CURATED_KEYS = (
22+
"assumptions",
23+
"pre_conditions",
24+
"failure_modes",
25+
"limitations",
26+
"minimum_n",
27+
)
28+
29+
30+
def main() -> None:
31+
names = sp.list_functions()
32+
n_total = len(names)
33+
34+
n_with_desc = 0
35+
n_with_params = 0
36+
n_curated_card = 0
37+
n_param_total = 0
38+
n_param_desc = 0
39+
n_param_default = 0
40+
n_param_enum = 0
41+
42+
for name in names:
43+
try:
44+
spec = sp.describe_function(name)
45+
except Exception:
46+
spec = None
47+
desc = (spec or {}).get("description", "") if isinstance(spec, dict) else ""
48+
if isinstance(desc, str) and len(desc.strip()) > 5:
49+
n_with_desc += 1
50+
51+
sch = sp.function_schema(name)
52+
if isinstance(sch, dict):
53+
props = (sch.get("parameters", {}) or {}).get("properties", {}) or {}
54+
if props:
55+
n_with_params += 1
56+
n_param_total += len(props)
57+
for p in props.values():
58+
if not isinstance(p, dict):
59+
continue
60+
pdesc = p.get("description")
61+
if isinstance(pdesc, str) and pdesc.strip():
62+
n_param_desc += 1
63+
if "default" in p:
64+
n_param_default += 1
65+
if "enum" in p:
66+
n_param_enum += 1
67+
68+
try:
69+
card = sp.agent_card(name)
70+
except Exception:
71+
card = None
72+
if isinstance(card, dict) and any(card.get(k) for k in CURATED_KEYS):
73+
n_curated_card += 1
74+
75+
def pct(num: int, denom: int) -> str:
76+
return f"{100 * num / denom:.1f}%" if denom else "--"
77+
78+
print(f"Public surface (registered functions) : {n_total}")
79+
print(f" with non-empty description : {n_with_desc}")
80+
print(f" with at least one typed parameter : {n_with_params}")
81+
print(f" with curated agent_card : {n_curated_card} "
82+
f"(at least one of {CURATED_KEYS})")
83+
print()
84+
print(f"Total parameters across all schemas : {n_param_total}")
85+
print(f" with non-empty description : {n_param_desc} "
86+
f"({pct(n_param_desc, n_param_total)})")
87+
print(f" with explicit default value : {n_param_default} "
88+
f"({pct(n_param_default, n_param_total)})")
89+
print(f" with enumerated choices (enum) : {n_param_enum} "
90+
f"({pct(n_param_enum, n_param_total)})")
91+
92+
93+
if __name__ == "__main__":
94+
main()

0 commit comments

Comments
 (0)