docs(pid): track A shipped — CLAUDE.md, README, changelog with the live COM smoke record
scripts/smoke_pid_com.py runs the whole track against the live AutoCAD in a new document. ComBackend.connect() is lazy by design, so the script reads reachability from system_status()["connected"] rather than from connect() as the plan sketched, and closes its document without saving unless --keep is passed. On this machine (AutoCAD 2026, "25.1s (LMS Tech)") it exits 0: 5 nodes, 4 edges, 0 dangling, confidence_min 1.0, 0 critique issues, 2 XDATA chunks — identical to the headless engine on the same spec. The first attempt exited 2 on a stale win32com gen_py cache (a package folder holding only __pycache__); an environment fault, no backend change needed. A/B against v1.5.1 published as ab-v1.5.1-vs-v1.6.0-dev.json: 26/29 -> 29/29, three miss -> pass, zero regressed. check_doc_numbers.py now names the report and the two row labels in one place (AB_REPORT, AB_*_LABEL), and README.md / benchmarks/README.md carry the new rows; the v1.5.0 and v1.4.0 tables stay as history. README: P&ID row, the showcase and catalogue images, the token table re-measured (161 / 44,930 by default, 152 / 41,420 with TOOL_PACKS=core, lean 50 / 14,007), every stale count reconciled (154 -> 166 registered, 149 -> 161 advertised, corpus 158/672 -> 162/731 by the same count, 19 -> 20 groups, 6 -> 8 resources, six-task matrix with pid_roundtrip), a "Known limitations of the P&ID track" subsection, and the snapshot line at this machine's 2423 collected tests (CI has not run on this tree; noted in the changelog with the pre-existing 1369 vs 1363 drift). CLAUDE.md: section 17, sections 5 and 7 counts, TOOL_PACKS, the two 1.6 contracts and the reader rules. Spec status -> implemented. The plan and spec markdown were reformatted by ruff (fenced Python only) because `ruff format --check .` was already red on them at HEAD and the gate requires the whole tree clean. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
This commit is contained in:
co-authored by
Claude Opus 5
parent
8d9c444b76
commit
2be5e4180d
@@ -27,6 +27,12 @@ from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
#: The published A/B report the README and benchmarks/README quote. Bump this,
|
||||
#: the two row labels below and the prose together, or the gate reports drift.
|
||||
AB_REPORT = "benchmarks/results/published/ab-v1.5.1-vs-v1.6.0-dev.json"
|
||||
AB_BASELINE_LABEL = r"v1\.5\.1"
|
||||
AB_CURRENT_LABEL = r"v1\.6\.0-dev"
|
||||
|
||||
|
||||
def _read(relative: str) -> str:
|
||||
return (ROOT / relative).read_text(encoding="utf-8")
|
||||
@@ -66,7 +72,7 @@ def derived() -> dict[str, object]:
|
||||
"""Every number a document is allowed to quote, computed from the tree."""
|
||||
sys.path.insert(0, str(ROOT))
|
||||
inventory = _json("docs/tool-inventory.json")["totals"]
|
||||
ab = _json("benchmarks/results/published/ab-v1.5.0-vs-v1.5.1.json")["summary"]
|
||||
ab = _json(AB_REPORT)["summary"]
|
||||
perf = {
|
||||
w["workload_id"]: w["wall_ms"]
|
||||
for w in _json("benchmarks/results/published/perf-ezdxf.json")["workloads"]
|
||||
@@ -81,6 +87,7 @@ def derived() -> dict[str, object]:
|
||||
"resources": inventory["resources"],
|
||||
"prompts": inventory["prompts"],
|
||||
"groups": inventory["groups"],
|
||||
"ab_baseline_ref": ab["baseline_ref"],
|
||||
"ab_total": ab["total"],
|
||||
"ab_old_pass": ab["old_pass"],
|
||||
"ab_old_pct": ab["old_pct"],
|
||||
@@ -112,19 +119,29 @@ CHECKS: list[tuple[str, str, str, str]] = [
|
||||
"prompts",
|
||||
),
|
||||
("README.md", "A/B suite size", r"(\d+) deterministic headless checks", "ab_total"),
|
||||
("README.md", "A/B baseline pass", r"v1\.5\.0 \*\(baseline\)\* \| (\d+) / \d+", "ab_old_pass"),
|
||||
("README.md", "A/B suite denominator", r"v1\.5\.0 \*\(baseline\)\* \| \d+ / (\d+)", "ab_total"),
|
||||
(
|
||||
"README.md",
|
||||
"A/B baseline pass",
|
||||
AB_BASELINE_LABEL + r" \*\(baseline\)\* \| (\d+) / \d+",
|
||||
"ab_old_pass",
|
||||
),
|
||||
(
|
||||
"README.md",
|
||||
"A/B suite denominator",
|
||||
AB_BASELINE_LABEL + r" \*\(baseline\)\* \| \d+ / (\d+)",
|
||||
"ab_total",
|
||||
),
|
||||
("benchmarks/README.md", "A/B suite size", r"(\d+) checks, ezdxf backend", "ab_total"),
|
||||
(
|
||||
"benchmarks/README.md",
|
||||
"A/B baseline pass",
|
||||
r"\*\*v1\.5\.0\*\* \(baseline\)\s*\| (\d+) / \d+",
|
||||
r"\*\*" + AB_BASELINE_LABEL + r"\*\* \(baseline\)\s*\| (\d+) / \d+",
|
||||
"ab_old_pass",
|
||||
),
|
||||
(
|
||||
"benchmarks/README.md",
|
||||
"A/B suite denominator",
|
||||
r"\*\*v1\.5\.0\*\* \(baseline\)\s*\| \d+ / (\d+)",
|
||||
r"\*\*" + AB_BASELINE_LABEL + r"\*\* \(baseline\)\s*\| \d+ / (\d+)",
|
||||
"ab_total",
|
||||
),
|
||||
(
|
||||
@@ -136,25 +153,26 @@ CHECKS: list[tuple[str, str, str, str]] = [
|
||||
(
|
||||
"README.md",
|
||||
"A/B baseline pass rate",
|
||||
r"v1\.5\.0 \*\(baseline\)\* \| \d+ / \d+ \| ([\d.]+) %",
|
||||
AB_BASELINE_LABEL + r" \*\(baseline\)\* \| \d+ / \d+ \| ([\d.]+) %",
|
||||
"ab_old_pct",
|
||||
),
|
||||
(
|
||||
"README.md",
|
||||
"A/B fixed count",
|
||||
r"v1\.5\.1\*\* \*\(this release\)\* \| \*\*\d+ / \d+\*\* \| \*\*\d+ %\*\* \| (\d+)",
|
||||
AB_CURRENT_LABEL
|
||||
+ r"\*\* \*\(this branch\)\* \| \*\*\d+ / \d+\*\* \| \*\*\d+ %\*\* \| (\d+)",
|
||||
"ab_fixed",
|
||||
),
|
||||
(
|
||||
"benchmarks/README.md",
|
||||
"A/B baseline pass rate",
|
||||
r"\*\*v1\.5\.0\*\* \(baseline\)\s*\| \d+ / \d+ \| ([\d.]+) %",
|
||||
r"\*\*" + AB_BASELINE_LABEL + r"\*\* \(baseline\)\s*\| \d+ / \d+ \| ([\d.]+) %",
|
||||
"ab_old_pct",
|
||||
),
|
||||
(
|
||||
"benchmarks/README.md",
|
||||
"A/B fixed count",
|
||||
r"\*\*v1\.5\.1\*\* \(this release\) \| \d+ / \d+ \| \d+ % \| (\d+)",
|
||||
r"\*\*" + AB_CURRENT_LABEL + r"\*\* \(this branch\) \| \d+ / \d+ \| \d+ % \| (\d+)",
|
||||
"ab_fixed",
|
||||
),
|
||||
("CLAUDE.md", "A/B suite size", r"correctness_suite\.py` \((\d+) checks\)", "ab_total"),
|
||||
@@ -169,8 +187,8 @@ def run(verbose: bool = False) -> int:
|
||||
print(
|
||||
f"DRIFT correctness_suite.py has {values['suite_checks']} checks but the published "
|
||||
f"A/B report was run over {values['ab_total']} -- regenerate the report:\n"
|
||||
f" python benchmarks/compare_versions.py v1.5.0 "
|
||||
f"--json benchmarks/results/published/ab-v1.5.0-vs-v1.5.1.json"
|
||||
f" python benchmarks/compare_versions.py {values['ab_baseline_ref']} "
|
||||
f"--json {AB_REPORT}"
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user