Files
Umut CANandClaude Opus 5.5 0abbe55041 bench: matrix v6 adds arch_roundtrip; four architectural correctness checks; the live arch smoke
The evidence for track F, in the shape tracks A, E and B + G used.

- benchmarks/tasks_v6.py: 19 tasks, DEFAULT_MATRIX = "v6"; v5 to v2 stay
  addressable. arch_roundtrip draws the example plan with
  arch_plan_from_spec and reads it back without the spec: rooms_detect must
  find each labelled room within 0.1 % of its label and of the net floor
  computed by hand (4825 x 5750 and 3825 x 5750), the schedules must list
  D1, D2, W1, W2, and the plan is gated on a zero-issue critique and a
  finalize score of at least 90. A test removes one room label and the gate
  fails naming arch_room_unlabelled.
- Four correctness checks, every number worked out by hand in the docstring:
  L / T / X junction corners and wall areas, a 4 x 5 m room labelled 20.00 m2
  rather than its 21.84 m2 axis area, a door interrupting both faces, and a
  plan of plain lines read as 20 and 15 m2 at confidence 0.6. The published
  A/B report is re-run over 43 checks: 26 / 43 -> 43 / 43, zero regressed.
- scripts/smoke_arch_com.py runs the whole plan once on a live seat and
  closes every document it opens; the CHANGELOG records what it returned.
- The token suite measures a TOOL_PACKS=core,arch row; check_doc_numbers
  derives the task count from the v6 matrix; CLAUDE.md, both READMEs and
  the spec status line say what shipped.
- fix(com): the first live smoke exited 1 - the room reader found 0 wall
  segments. Measured on AutoCAD 2026 (2026-09-23): with a makepy cache,
  ModelSpace.Item(i) returns the gen_py IAcadEntity wrapper
  (type(item).__name__ == "IAcadEntity") and item.StartPoint on an
  AcDbLine raises AttributeError, so every entity_list row lost its
  type-specific properties (28 wall LINEs, 0 with start/end). _entity_info
  now re-dispatches exactly that wrapper through _com_unnarrow; after the
  fix 38 of 53 rows carry start/end and the smoke exits 0 with both rooms
  at 27 743 750 / 21 993 750 mm2 and 0 critique issues. Re-dispatch
  measured at ~1.7 ms per entity (dynamic vs gencache alike).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
2026-09-23 23:35:46 +03:00

193 lines
7.0 KiB
Python

"""Exercise track F once against the live AutoCAD on this machine.
Creates a NEW document (never touches the operator's open drawings), then runs
the whole plan through the same code the tools call:
draw_plan_from_spec(EXAMPLE_SPEC) (one transaction:)
drawing_apply_iso_layers("arch") -> draw_walls (junctions, cut
openings, poche, ACADMCP_ARCH records) -> draw_stair -> label_room x2
(areas measured off the walls just drawn) -> draw_chains
-> draw_schedule("doors" / "windows" / "rooms")
read_plan -> rooms_detect (wall layer) -> run_critique(focus=None)
and prints what came back as JSON. Exit codes:
0 the plan drew, both rooms read back within 0.1 % of their labels and of
the hand-computed net floor, and the critique returned nothing
1 it drew, but a room disagreed or the critique found something
2 no live CAD application reachable (pywin32 missing, or the ProgID
cannot be created) -- the COM paths then remain fake-tested only
The backend's ``connect()`` is lazy on purpose (the server must start before
AutoCAD is open), so reachability is read from ``system_status()`` rather than
from ``connect()``. Every document this run opens is closed on the way out,
also when a step fails, unless ``--keep`` is given.
AUTOCAD_MCP_BACKEND=com uv run --frozen python scripts/smoke_arch_com.py
uv run --frozen python scripts/smoke_arch_com.py --keep # leave it open
"""
from __future__ import annotations
import asyncio
import json
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
#: The net floor of EXAMPLE_SPEC's two rooms, by hand (see
#: benchmarks/adapters/autocad_mcp_pro.py::ROUNDTRIP_AREAS_MM2).
HAND_AREAS_MM2 = {"01": 4825.0 * 5750.0, "02": 3825.0 * 5750.0}
#: ``RPC_E_CALL_REJECTED``: AutoCAD's message filter refused the call because
#: it was busy (a document still initialising after ``Documents.Add`` is the
#: case measured on this seat by scripts/smoke_mech_com.py). A refused call
#: was not executed.
RPC_E_CALL_REJECTED = -2147418111
#: How long a refused call is retried before the smoke gives up on it.
REJECTED_PATIENCE_S = 90.0
#: Names of the documents this run opened and has not closed yet.
_OPENED: list[str] = []
def _patient(backend) -> None:
"""Retry any backend call AutoCAD refused with ``RPC_E_CALL_REJECTED``."""
run = backend._run
async def patient(func, *args, **kwargs):
deadline = time.monotonic() + REJECTED_PATIENCE_S
pause = 0.5
while True:
try:
return await run(func, *args, **kwargs)
except RuntimeError as exc:
cause = exc.__cause__
hr = cause.args[0] if cause is not None and cause.args else None
if hr != RPC_E_CALL_REJECTED or time.monotonic() + pause > deadline:
raise
await asyncio.sleep(pause)
pause = min(pause * 2.0, 5.0)
backend._run = patient
async def _names(backend) -> set[str]:
return {row["name"] for row in await backend.document_list()}
async def _new_document(backend) -> str:
"""``drawing_new``, recording every document it added (normally one)."""
before = await _names(backend)
await backend.drawing_new()
added = sorted((await _names(backend)) - before)
_OPENED.extend(name for name in added if name not in _OPENED)
return added[-1] if added else ""
async def _close_leftovers(backend) -> None:
open_names = {row["name"] for row in await backend.document_list()}
for name in list(_OPENED):
if name in open_names:
try:
await backend.document_close(name, discard=True)
except Exception as exc: # best effort on the way out
print(f"could not close {name}: {exc}", file=sys.stderr)
_OPENED.remove(name)
def _inside(point, loop) -> bool:
from engineering.arch.critique import point_in_loop
return point_in_loop(point, loop)
async def _plan(backend) -> dict:
from engineering.arch.draw import read_plan
from engineering.arch.layers import ARCH_ROLE_LAYER
from engineering.arch.rooms import rooms_detect
from engineering.arch.spec import EXAMPLE_SPEC, draw_plan_from_spec
from engineering.critique import run_critique
await _new_document(backend)
drawn = await draw_plan_from_spec(backend, EXAMPLE_SPEC)
plan = await read_plan(backend)
detected = await rooms_detect(backend, layers=[ARCH_ROLE_LAYER["wall"]])
faces = list(detected.get("rooms") or [])
rooms = {}
for room in plan["rooms"]:
face = next((f for f in faces if _inside(room.at, f["loop"])), None)
found = float(face["area"]) if face is not None else 0.0
hand = HAND_AREAS_MM2.get(str(room.number), 0.0)
rooms[str(room.number)] = {
"label_mm2": round(float(room.area), 3),
"detected_mm2": round(found, 3),
"hand_mm2": hand,
"agrees": face is not None
and abs(found - float(room.area)) <= 1e-3 * float(room.area)
and abs(found - hand) <= 1e-3 * hand,
}
issues = await run_critique(backend, None)
return {
"walls": drawn["walls"],
"openings": drawn["openings"],
"omitted": drawn["omitted"],
"stairs": len(drawn["stairs"]),
"schedules": [
{
"kind": item.get("kind"),
"representation": item.get("representation"),
"rows": item.get("row_count"),
}
for item in drawn["schedules"]
],
"rooms": rooms,
"faces_detected": len(faces),
"confidence_min": detected.get("confidence_min"),
"critique_issues": len(issues),
"critique": [issue.to_dict() for issue in issues],
}
async def main(keep_document: bool = False) -> int:
try:
from backends.com_backend import ComBackend
except ImportError as exc:
print(f"pywin32 not available: {exc}")
return 2
backend = ComBackend()
_patient(backend)
try:
await backend.connect()
except Exception as exc:
print(f"no live AutoCAD reachable: {exc}")
return 2
try:
status = await backend.system_status()
if not status.get("connected"):
print(f"no live AutoCAD reachable: {status.get('error', status)}")
return 2
report = {
"autocad": status.get("autocad_version") or status.get("cad_progid"),
**await _plan(backend),
}
print(json.dumps(report, indent=2, ensure_ascii=False))
ok = (
report["critique_issues"] == 0
and set(report["rooms"]) == set(HAND_AREAS_MM2)
and all(row["agrees"] for row in report["rooms"].values())
)
return 0 if ok else 1
finally:
if not keep_document:
await _close_leftovers(backend)
await backend.disconnect()
if __name__ == "__main__":
raise SystemExit(asyncio.run(main(keep_document="--keep" in sys.argv[1:])))