anatoly/autocad-mcp-pro
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

docs(pid): track A shipped — CLAUDE.md, README, changelog with the live COM smoke record

Browse Source 
scripts/smoke_pid_com.py runs the whole track against the live AutoCAD in
a new document. ComBackend.connect() is lazy by design, so the script reads
reachability from system_status()["connected"] rather than from connect()
as the plan sketched, and closes its document without saving unless --keep
is passed. On this machine (AutoCAD 2026, "25.1s (LMS Tech)") it exits 0:
5 nodes, 4 edges, 0 dangling, confidence_min 1.0, 0 critique issues,
2 XDATA chunks — identical to the headless engine on the same spec. The
first attempt exited 2 on a stale win32com gen_py cache (a package folder
holding only __pycache__); an environment fault, no backend change needed.

A/B against v1.5.1 published as ab-v1.5.1-vs-v1.6.0-dev.json: 26/29 ->
29/29, three miss -> pass, zero regressed. check_doc_numbers.py now names
the report and the two row labels in one place (AB_REPORT, AB_*_LABEL),
and README.md / benchmarks/README.md carry the new rows; the v1.5.0 and
v1.4.0 tables stay as history.

README: P&ID row, the showcase and catalogue images, the token table
re-measured (161 / 44,930 by default, 152 / 41,420 with TOOL_PACKS=core,
lean 50 / 14,007), every stale count reconciled (154 -> 166 registered,
149 -> 161 advertised, corpus 158/672 -> 162/731 by the same count, 19 ->
20 groups, 6 -> 8 resources, six-task matrix with pid_roundtrip), a
"Known limitations of the P&ID track" subsection, and the snapshot line at
this machine's 2423 collected tests (CI has not run on this tree; noted in
the changelog with the pre-existing 1369 vs 1363 drift).

CLAUDE.md: section 17, sections 5 and 7 counts, TOOL_PACKS, the two 1.6
contracts and the reader rules. Spec status -> implemented. The plan and
spec markdown were reformatted by ruff (fenced Python only) because
`ruff format --check .` was already red on them at HEAD and the gate
requires the whole tree clean.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
This commit is contained in:
Umut CAN
2026-09-16 13:34:06 +03:00
co-authored by Claude Opus 5
parent 8d9c444b76
commit 2be5e4180d
9 changed files with 2604 additions and 789 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/check_doc_numbers.py
+29 -11
View File
@@ -27,6 +27,12 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
#: The published A/B report the README and benchmarks/README quote. Bump this,
#: the two row labels below and the prose together, or the gate reports drift.
AB_REPORT = "benchmarks/results/published/ab-v1.5.1-vs-v1.6.0-dev.json"
AB_BASELINE_LABEL = r"v1\.5\.1"
AB_CURRENT_LABEL = r"v1\.6\.0-dev"
def _read(relative: str) -> str:
return (ROOT / relative).read_text(encoding="utf-8")
@@ -66,7 +72,7 @@ def derived() -> dict[str, object]:
"""Every number a document is allowed to quote, computed from the tree."""
sys.path.insert(0, str(ROOT))
inventory = _json("docs/tool-inventory.json")["totals"]
ab = _json("benchmarks/results/published/ab-v1.5.0-vs-v1.5.1.json")["summary"]
ab = _json(AB_REPORT)["summary"]
perf = {
w["workload_id"]: w["wall_ms"]
for w in _json("benchmarks/results/published/perf-ezdxf.json")["workloads"]
@@ -81,6 +87,7 @@ def derived() -> dict[str, object]:
"resources": inventory["resources"],
"prompts": inventory["prompts"],
"groups": inventory["groups"],
"ab_baseline_ref": ab["baseline_ref"],
"ab_total": ab["total"],
"ab_old_pass": ab["old_pass"],
"ab_old_pct": ab["old_pct"],
@@ -112,19 +119,29 @@ CHECKS: list[tuple[str, str, str, str]] = [
"prompts",
),
("README.md", "A/B suite size", r"(\d+) deterministic headless checks", "ab_total"),
("README.md", "A/B baseline pass", r"v1\.5\.0 \*\(baseline\)\* \| (\d+) / \d+", "ab_old_pass"),
("README.md", "A/B suite denominator", r"v1\.5\.0 \*\(baseline\)\* \| \d+ / (\d+)", "ab_total"),
(
"README.md",
"A/B baseline pass",
AB_BASELINE_LABEL + r" \*\(baseline\)\* \| (\d+) / \d+",
"ab_old_pass",
),
(
"README.md",
"A/B suite denominator",
AB_BASELINE_LABEL + r" \*\(baseline\)\* \| \d+ / (\d+)",
"ab_total",
),
("benchmarks/README.md", "A/B suite size", r"(\d+) checks, ezdxf backend", "ab_total"),
(
"benchmarks/README.md",
"A/B baseline pass",
r"\*\*v1\.5\.0\*\* \(baseline\)\s*\| (\d+) / \d+",
r"\*\*" + AB_BASELINE_LABEL + r"\*\* \(baseline\)\s*\| (\d+) / \d+",
"ab_old_pass",
),
(
"benchmarks/README.md",
"A/B suite denominator",
r"\*\*v1\.5\.0\*\* \(baseline\)\s*\| \d+ / (\d+)",
r"\*\*" + AB_BASELINE_LABEL + r"\*\* \(baseline\)\s*\| \d+ / (\d+)",
"ab_total",
),
(
@@ -136,25 +153,26 @@ CHECKS: list[tuple[str, str, str, str]] = [
(
"README.md",
"A/B baseline pass rate",
r"v1\.5\.0 \*\(baseline\)\* \| \d+ / \d+ \| ([\d.]+) %",
AB_BASELINE_LABEL + r" \*\(baseline\)\* \| \d+ / \d+ \| ([\d.]+) %",
"ab_old_pct",
),
(
"README.md",
"A/B fixed count",
r"v1\.5\.1\*\* \*\(this release\)\* \| \*\*\d+ / \d+\*\* \| \*\*\d+ %\*\* \| (\d+)",
AB_CURRENT_LABEL
+ r"\*\* \*\(this branch\)\* \| \*\*\d+ / \d+\*\* \| \*\*\d+ %\*\* \| (\d+)",
"ab_fixed",
),
(
"benchmarks/README.md",
"A/B baseline pass rate",
r"\*\*v1\.5\.0\*\* \(baseline\)\s*\| \d+ / \d+ \| ([\d.]+) %",
r"\*\*" + AB_BASELINE_LABEL + r"\*\* \(baseline\)\s*\| \d+ / \d+ \| ([\d.]+) %",
"ab_old_pct",
),
(
"benchmarks/README.md",
"A/B fixed count",
r"\*\*v1\.5\.1\*\* \(this release\) \| \d+ / \d+ \| \d+ % \| (\d+)",
r"\*\*" + AB_CURRENT_LABEL + r"\*\* \(this branch\) \| \d+ / \d+ \| \d+ % \| (\d+)",
"ab_fixed",
),
("CLAUDE.md", "A/B suite size", r"correctness_suite\.py` \((\d+) checks\)", "ab_total"),
@@ -169,8 +187,8 @@ def run(verbose: bool = False) -> int:
print(
f"DRIFT correctness_suite.py has {values['suite_checks']} checks but the published "
f"A/B report was run over {values['ab_total']} -- regenerate the report:\n"
f" python benchmarks/compare_versions.py v1.5.0 "
f"--json benchmarks/results/published/ab-v1.5.0-vs-v1.5.1.json"
f" python benchmarks/compare_versions.py {values['ab_baseline_ref']} "
f"--json {AB_REPORT}"
)
return 1
scripts/smoke_pid_com.py
+86
View File
@@ -0,0 +1,86 @@
"""Exercise the P&ID track once against the live AutoCAD on this machine.
Creates a NEW document (never touches the operator's open drawing), then runs
the whole track through the same code the tools call:
apply_layer_set("pid") -> run_spec(EXAMPLE_SPEC)
= block_define -> pid_symbol_insert -> ACADMCP_PID xdata
-> pid_line_draw -> pid_graph -> six pid_* critique focuses
-> entity_get_xdata on the pump
and prints the graph stats as JSON. Exit codes:
0 every line landed on a port (``dangling == 0``)
1 the sheet drew but the reader found a dangling end
2 no live CAD application reachable (pywin32 missing, or the ProgID
cannot be created) -- the COM paths then remain fake-tested only
The backend's ``connect()`` is lazy on purpose (the server must start before
AutoCAD is open), so reachability is read from ``system_status()`` rather than
from ``connect()``. Note that the backend's ``Dispatch`` fallback *launches*
the ProgID when nothing is running; a cold AutoCAD start can take longer than
``COM_CALL_TIMEOUT`` (60 s), so open AutoCAD first when the first call times
out.
AUTOCAD_MCP_BACKEND=com uv run --frozen python scripts/smoke_pid_com.py
uv run --frozen python scripts/smoke_pid_com.py --keep # leave the document open
"""
from __future__ import annotations
import asyncio
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
XDATA_APP = "ACADMCP_PID"
async def main(keep_document: bool = False) -> int:
try:
from backends.com_backend import ComBackend
except ImportError as exc:
print(f"pywin32 not available: {exc}")
return 2
backend = ComBackend()
try:
await backend.connect()
except Exception as exc:
print(f"no live AutoCAD reachable: {exc}")
return 2
try:
status = await backend.system_status()
if not status.get("connected"):
print(f"no live AutoCAD reachable: {status.get('error', status)}")
return 2
from engineering.layers import apply_layer_set
from engineering.pid.spec import EXAMPLE_SPEC, run_spec
await backend.drawing_new()
await apply_layer_set(backend, "pid")
result = await run_spec(backend, EXAMPLE_SPEC)
got = await backend.entity_get_xdata(result["handles"]["P-101"], XDATA_APP)
report = {
"autocad": status.get("autocad_version") or status.get("cad_progid"),
"active_document": (await backend.system_status()).get("active_document"),
"handles": result["handles"],
"graph_stats": result["graph"]["stats"],
"crossings_total": result["crossings_total"],
"critique_issues": len(result["critique"]),
"critique": result["critique"],
"xdata_chunks": len(got["xdata"].get(XDATA_APP, [])),
}
print(json.dumps(report, indent=2))
if not keep_document:
await backend.drawing_close(save=False)
return 0 if result["graph"]["stats"]["dangling"] == 0 else 1
finally:
await backend.disconnect()
if __name__ == "__main__":
raise SystemExit(asyncio.run(main(keep_document="--keep" in sys.argv[1:])))