From 0abbe550419d61d8c5f40d9ae56e01ad2e7c9792 Mon Sep 17 00:00:00 2001 From: Umut CAN Date: Wed, 23 Sep 2026 23:35:46 +0300 Subject: [PATCH] bench: matrix v6 adds arch_roundtrip; four architectural correctness checks; the live arch smoke The evidence for track F, in the shape tracks A, E and B + G used. - benchmarks/tasks_v6.py: 19 tasks, DEFAULT_MATRIX = "v6"; v5 to v2 stay addressable. arch_roundtrip draws the example plan with arch_plan_from_spec and reads it back without the spec: rooms_detect must find each labelled room within 0.1 % of its label and of the net floor computed by hand (4825 x 5750 and 3825 x 5750), the schedules must list D1, D2, W1, W2, and the plan is gated on a zero-issue critique and a finalize score of at least 90. A test removes one room label and the gate fails naming arch_room_unlabelled. - Four correctness checks, every number worked out by hand in the docstring: L / T / X junction corners and wall areas, a 4 x 5 m room labelled 20.00 m2 rather than its 21.84 m2 axis area, a door interrupting both faces, and a plan of plain lines read as 20 and 15 m2 at confidence 0.6. The published A/B report is re-run over 43 checks: 26 / 43 -> 43 / 43, zero regressed. - scripts/smoke_arch_com.py runs the whole plan once on a live seat and closes every document it opens; the CHANGELOG records what it returned. - The token suite measures a TOOL_PACKS=core,arch row; check_doc_numbers derives the task count from the v6 matrix; CLAUDE.md, both READMEs and the spec status line say what shipped. - fix(com): the first live smoke exited 1 - the room reader found 0 wall segments. Measured on AutoCAD 2026 (2026-09-23): with a makepy cache, ModelSpace.Item(i) returns the gen_py IAcadEntity wrapper (type(item).__name__ == "IAcadEntity") and item.StartPoint on an AcDbLine raises AttributeError, so every entity_list row lost its type-specific properties (28 wall LINEs, 0 with start/end). _entity_info now re-dispatches exactly that wrapper through _com_unnarrow; after the fix 38 of 53 rows carry start/end and the smoke exits 0 with both rooms at 27 743 750 / 21 993 750 mm2 and 0 critique issues. Re-dispatch measured at ~1.7 ms per entity (dynamic vs gencache alike). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd --- CHANGELOG.md | 114 +++++++++++ CLAUDE.md | 14 +- README.md | 31 +-- backends/com_backend.py | 34 +++- benchmarks/README.md | 18 +- benchmarks/adapters/autocad_mcp_pro.py | 101 ++++++++- benchmarks/correctness_suite.py | 180 ++++++++++++++++ benchmarks/render_matrix_chart.py | 6 +- .../published/ab-v1.5.1-vs-v1.6.0-dev.json | 32 ++- benchmarks/run_competitors.py | 2 +- benchmarks/tasks_v6.py | 54 +++++ benchmarks/token_suite.py | 10 + .../specs/2026-09-23-v1.6-arch-design.md | 2 +- scripts/check_doc_numbers.py | 4 +- scripts/smoke_arch_com.py | 192 ++++++++++++++++++ tests/test_benchmark_v4.py | 2 +- tests/test_benchmark_v5.py | 2 +- tests/test_benchmark_v6.py | 119 +++++++++++ tests/test_com_backend.py | 53 +++++ tests/test_token_suite.py | 35 ++-- 20 files changed, 952 insertions(+), 53 deletions(-) create mode 100644 benchmarks/tasks_v6.py create mode 100644 scripts/smoke_arch_com.py create mode 100644 tests/test_benchmark_v6.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cda0d2f..a37783b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,16 @@ model and one view engine instead of a zoo of per-part generators, a new (`xref_attach`, `xref_manage`, `image_attach`, `drawing_export_dwg`). Spec: `docs/superpowers/specs/2026-09-22-v1.6-mech-design.md`. +Track F of 1.6: architectural 2D, the plan model. 12 tools in a new SECTION +25 — a wall network with hosted doors and windows drawn by one engine (L / T / +X junctions resolved, openings cut, poché by material), rooms computed as the +faces the walls enclose and read back from any plan, schedules read off the +drawing, a furniture and sanitary catalogue, the structural grid and plan +symbols, exterior dimension chains and the whole plan in one transaction — a +new `arch` pack, three `arch_*` critique focuses, and English or Turkish +labels. No new contract member. Spec: +`docs/superpowers/specs/2026-09-23-v1.6-arch-design.md`. + ### Added - **Styles — 10 tools, pack `core`.** `dimstyle_list/create/modify/set_current`, @@ -104,6 +114,31 @@ model and one view engine instead of a zoo of per-part generators, a new with its linked balloon, an xref attached and detached, and a real 31687-byte `.dwg` written by ActiveX `SaveAs` — `drawing_critique(focus=None)` returned 0 issues (record below). +- **Architecture — 12 tools, pack `arch`.** `arch_wall`, `arch_opening`, + `arch_stair`, `arch_room`, `arch_rooms_detect`, `arch_schedule`, + `arch_grid`, `arch_symbol`, `arch_catalogue_list`, `arch_catalogue_insert`, + `arch_dimension_chains`, `arch_plan_from_spec`. The plan travels as + `ACADMCP_ARCH` XDATA through the mechanical codec, generalised to take an + application id; `draw_prims` takes a role map. The `arch` layer set is + ISO 13567-style with ISO 128 lineweights; wall poché follows + `engineering/arch/materials.py`, whose rows each name their source. Areas + are measured, never typed. Lean gains four and reaches 64. +- **Benchmark evidence for track F.** `benchmarks/tasks_v6.py` (19 tasks, the + default matrix; v5 to v2 stay addressable) adds `arch_roundtrip`: the example + plan drawn by `arch_plan_from_spec` and read back by `arch_rooms_detect` + within 0.1 % of each label and of the hand-computed net floor, the schedules + listing the drawn tags, gated on a zero-issue `drawing_critique(focus=None)` + and a finalize score of at least 90; a test removes one room label and + watches the gate fail by name. The correctness suite grows to 43 with + `arch_junction_l_t_x`, `arch_room_area_net`, `arch_opening_cuts_wall` and + `arch_rooms_detect_foreign`. A/B against `v1.5.1`: **26 / 43 → 43 / 43, + seventeen `miss → pass`, zero regressed**. The token suite measures a + `TOOL_PACKS=core,arch` row: 191 tools / 60,568 tokens against 236 / 78,772 + by default. +- **Live COM smoke.** `scripts/smoke_arch_com.py` executed once on AutoCAD + 2026 (`25.1s (LMS Tech)`): the example plan drawn in one transaction, both + rooms read back within 0.1 % of their labels and of the hand-computed net + floor, and `drawing_critique(focus=None)` returned 0 issues (record below). - **P&ID — 9 tools, pack `pid`.** - `pid_symbol_list` / `pid_symbol_insert`: a 43-symbol ISO 10628-2 / @@ -268,6 +303,20 @@ model and one view engine instead of a zoo of per-part generators, a new excluded) and `mirrored` on both engines. The COM box is exact at right angles and measured off them; a member ActiveX cannot measure is carried by its corners and the box says `approximate: true`. +- **`entity_list` lost every type-specific property on a live seat with a + makepy cache.** `ModelSpace.Item` (and `SelectionSet.Item`) is declared + `IAcadEntity*`, so once `win32com.client.gencache` had run against AutoCAD + every listed entity arrived as the gen_py `IAcadEntity` wrapper: measured on + AutoCAD 2026, `item.StartPoint` on an AcDbLine raised `AttributeError`, and a + LINE row kept only its bounding box (no `start` / `end`, no polyline + `points`, no text). The room reader therefore saw 0 wall segments and + `arch_plan_from_spec` refused its first room label live, while every + headless test passed. `_entity_info` now re-dispatches exactly that wrapper + through its runtime class (`_com_concrete_entity`); a concrete wrapper from + `HandleToObject` is left alone. The same narrowing strips `start` from + every listed LINE, so it would equally explain the `pid_graph` + `KeyError: 'start'` recorded above as a busy-refused point read (not + re-measured). ### Changed @@ -835,6 +884,71 @@ for up to 90 s (a refused call was not executed) and tracks its documents by the document list rather than by an assumed name, because `SaveAs` renames the document it writes. +### Live COM smoke — architecture (track F) + +`scripts/smoke_arch_com.py`, AutoCAD 2026, Windows 11, in a new document only — +`draw_plan_from_spec(EXAMPLE_SPEC)` (walls, openings, stair, two room labels, +chains, three schedules in one transaction) → `read_plan` → `rooms_detect` → +`run_critique(focus=None)`, all through the COM engine, exit 0, the scratch +document closed afterwards: + +```json +{ + "autocad": "25.1s (LMS Tech)", + "walls": [ + "EXT", + "INT" + ], + "openings": [ + "D1", + "D2", + "W1", + "W2" + ], + "omitted": [], + "stairs": 1, + "schedules": [ + { + "kind": "doors", + "representation": "native", + "rows": 2 + }, + { + "kind": "windows", + "representation": "native", + "rows": 2 + }, + { + "kind": "rooms", + "representation": "native", + "rows": 3 + } + ], + "rooms": { + "01": { + "label_mm2": 27743750.0, + "detected_mm2": 27743750.0, + "hand_mm2": 27743750.0, + "agrees": true + }, + "02": { + "label_mm2": 21993750.0, + "detected_mm2": 21993750.0, + "hand_mm2": 21993750.0, + "agrees": true + } + }, + "faces_detected": 2, + "confidence_min": 1.0, + "critique_issues": 0, + "critique": [] +} +``` + +The first run exited 1: the first room label was refused because the room +reader found 0 wall segments on `A-WALL-E-N` — the `entity_list` narrowing +listed under **Fixed**. The run above is the one after the fix. + ## [1.5.1] — 2026-08-06 A patch for three defects found *after* 1.5.0 went to PyPI, two of them by the diff --git a/CLAUDE.md b/CLAUDE.md index 3abd404..7946502 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,10 +96,11 @@ Tools are organized into sections (counts are indicative — `system_about` is a 22. Standard parts (3 tools, pack `mech`): `std_part_list`, `std_part_insert`, `std_feature_draw`. ISO 4014 / 4017 / 4032 / 7089 / 4762 fasteners, ISO 15 bearings with ISO 8826-1 representation, DIN 471/472 retaining-ring grooves, DIN 509 undercuts, DIN 332 centre holes and ISO 3601-2 O-ring grooves — authored tables, inserted as real blocks with attributes and `ACADMCP_MECH` XDATA, which is what makes the parts list a *read* of the drawing. A size outside a table's transcribed coverage is refused by name, never interpolated. 23. Mechanical annotation (5 tools, pack `mech`): `surface_texture` (ISO 21920-1, ISO 1302's older Ra form accepted), `weld_symbol` (ISO 2553), `centre_marks` (ISO 128-23), `section_line` (ISO 128-40 — it returns the plane `mech_view_add(kind="section")` consumes, so the label and the view cannot disagree), `hatch_material` (the ISO 128-50 map; an unknown material is refused with the list, never hatched as steel). 24. Sheet and delivery (11 tools, pack `core`): `sheet_frame` (ISO 5457 A4-A0 — 20 mm filing margin, 10 mm elsewhere on A3, zone grid, centring and trimming marks), `titleblock_apply` (ISO 7200 for every size; `titleblock_apply_iso_a3` is now a thin alias), `revision_add`, `bom_extract` (never modifies the drawing), `bom_table` (ISO 7573), `balloon_add` (ISO 6433, linked by XDATA so a rerun renumbers instead of duplicating), `data_extract` (CSV always; XLSX refused with `xlsx_write` when `openpyxl` is absent), `xref_attach`, `xref_manage`, `image_attach`, `drawing_export_dwg` (COM `SaveAs` natively; headless only with the ODA File Converter, otherwise `dwg_write` refuses and names the install route). The COM paths of SECTIONS 21-24 were executed live once by `scripts/smoke_mech_com.py` (AutoCAD 2026): part, section, dimensions, an ISO 4014 block with its XDATA, the parts list read off the drawing, a linked balloon, an xref attached and detached, a real `.dwg`, and `drawing_critique(focus=None)` clean. +25. Architecture (12 tools, pack `arch`): `arch_wall`, `arch_opening`, `arch_stair`, `arch_room`, `arch_rooms_detect`, `arch_schedule`, `arch_grid`, `arch_symbol`, `arch_catalogue_list`, `arch_catalogue_insert`, `arch_dimension_chains`, `arch_plan_from_spec`. One plan model — a wall network (axis polyline, thickness, justification, material) with hosted doors and windows — and one engine: L / T / X junctions resolved, openings cut, cut faces hatched by material (`engineering/arch/materials.py`: ISO 128-50 general hatching for brick, the JIS A 0150 RC / stone / wood symbols, AutoCAD's AR-CONC / AR-SAND, AAC as a declared convention), and anything it cannot place listed in `omitted: [{element, reason}]`, never dropped. Rooms are *computed*: a label's area is the planar face its point lies in, and `arch_rooms_detect` reads rooms from any plan (confidence 1.0 for our walls, 0.6 for plain lines) without modifying it. Schedules are a read of the `ACADMCP_ARCH` records, drawn as real TABLEs. Labels and schedules speak English, or Turkish with `lang="tr"` (decimal comma). The COM paths were executed live once by `scripts/smoke_arch_com.py` (AutoCAD 2026). -**Tool profiles:** `TOOL_PROFILE=lean|full` (default `full`) controls the advertised surface — `lean` = 60 curated drafting tools (54 with `TOOL_PACKS=core`, which drops the three `pid_*` and the three `mech_*` lean members; the five settings essentials — `drawing_apply_standard`, `dimstyle_set_current`, `textstyle_set_current`, `page_setup_apply`, `batch_plot` — are core-pack tools and stay; including `cad_batch`, since a client with a tight tool cap is exactly the client paying most per turn). Applied in the lifespan; reported by `system_about`. The `core` profile was removed in v1.5.0 (the discovery layer replaced it); `TOOL_PROFILE=core` falls back to `full` with a warning. +**Tool profiles:** `TOOL_PROFILE=lean|full` (default `full`) controls the advertised surface — `lean` = 64 curated drafting tools (54 with `TOOL_PACKS=core`, which drops the three `pid_*`, the three `mech_*` and the four `arch_*` lean members; the five settings essentials — `drawing_apply_standard`, `dimstyle_set_current`, `textstyle_set_current`, `page_setup_apply`, `batch_plot` — are core-pack tools and stay; including `cad_batch`, since a client with a tight tool cap is exactly the client paying most per turn). Applied in the lifespan; reported by `system_about`. The `core` profile was removed in v1.5.0 (the discovery layer replaced it); `TOOL_PROFILE=core` falls back to `full` with a warning. -**Tool packs:** `TOOL_PACKS=all|core,pid,settings,mech` (default `all`) advertises only the vertical packs a client uses; `core` is always on and is everything no other pack claims; `settings` is the 22 SECTION 20 environment tools, `pid` the nine P&ID tools, `mech` the 14 SECTION 21-23 mechanical tools; SECTION 18/19/24 are core; unknown names are ignored with a warning; a `lean` profile intersects with the enabled packs. `system_about` reports `tool_packs`. +**Tool packs:** `TOOL_PACKS=all|core,pid,settings,mech,arch` (default `all`) advertises only the vertical packs a client uses; `core` is always on and is everything no other pack claims; `settings` is the 22 SECTION 20 environment tools, `pid` the nine P&ID tools, `mech` the 14 SECTION 21-23 mechanical tools, `arch` the 12 SECTION 25 architectural tools; SECTION 18/19/24 are core; unknown names are ignored with a warning; a `lean` profile intersects with the enabled packs. `system_about` reports `tool_packs`. **Tool discovery:** `DISCOVERY_MODE=off|search` (default `off`). `search` replaces the advertised catalog with `search_tools` + `call_tool`, ranking hits over an AutoCAD command / synonym corpus (`discovery/aliases.py`) layered on fastmcp's BM25 index. @@ -111,7 +112,7 @@ Tools are organized into sections (counts are indicative — `system_about` is a **Measuring:** never read vertices back and shoelace them — that loses 28.2% of the area on a semicircular edge, silently. `analysis_measure_entity(handle)` reads the real geometry, and its payload states its own accuracy: `exact`, `flatten_tolerance`, `assumed_closed`, `self_intersecting`. A HATCH reports the area it *fills* (outer loops minus islands) plus `hatch_style`; on a curved hatch edge `flatten_tolerance` is not the accuracy knob, because ezdxf hands boundaries over as cubic Beziers whose ~0.028% circle error is already baked in. `boundary_trace` / `boundary_from_entities` report the same number `analysis_measure_entity` gives for the polyline they just drew — if those ever diverge, one of them is approximating and neither says so. -**Benchmark matrix:** `benchmarks/tasks_v5.py` is the current set (18 tasks; `--matrix v4` / `v3` / `v2` reproduce the earlier sets). The published competitor reports are pinned v1.4 runs against v2 and are **never** back-filled for the five v3 tasks, the v4 `pid_roundtrip` / `page_setup_truth` or the v5 `mech_assembly` (a flange-coupling sheet gated on a zero-issue `drawing_critique(focus=None)` and a finalize score of at least 90) — the chart shows them `not run`, not zero. `benchmarks/correctness_suite.py` (39 checks) is the release A/B gate: `python benchmarks/compare_versions.py v1.5.1` (the published report is `benchmarks/results/published/ab-v1.5.1-vs-v1.6.0-dev.json`; `scripts/check_doc_numbers.py` names it in `AB_REPORT`). +**Benchmark matrix:** `benchmarks/tasks_v6.py` is the current set (19 tasks; `--matrix v5` / `v4` / `v3` / `v2` reproduce the earlier sets). The published competitor reports are pinned v1.4 runs against v2 and are **never** back-filled for the five v3 tasks, the v4 `pid_roundtrip` / `page_setup_truth`, the v5 `mech_assembly` (a flange-coupling sheet gated on a zero-issue `drawing_critique(focus=None)` and a finalize score of at least 90) or the v6 `arch_roundtrip` (a two-room plan drawn by `arch_plan_from_spec`, its rooms read back by `arch_rooms_detect` within 0.1 % of the label and of the hand-computed net floor, the schedules listing the drawn tags, gated like `mech_assembly`) — the chart shows them `not run`, not zero. `benchmarks/correctness_suite.py` (43 checks) is the release A/B gate: `python benchmarks/compare_versions.py v1.5.1` (the published report is `benchmarks/results/published/ab-v1.5.1-vs-v1.6.0-dev.json`; `scripts/check_doc_numbers.py` names it in `AB_REPORT`). ### Adding a New Tool @@ -138,7 +139,7 @@ Tools are organized into sections (counts are indicative — `system_about` is a - **Engineering layer scaffold**: `drawing_new` auto-bootstraps standard linetypes (CENTER, HIDDEN, PHANTOM) and engineering layers (GEOMETRY, DIM, CENTER, HIDDEN, PHANTOM, HATCH, TEXT, TITLEBLOCK). Pass `bootstrap=False` to opt out. - **Generic contracts added in 1.6:** `block_define(name, entities, attdefs)` (typed primitives + ATTDEFs, both engines; `overwrite` replaces contents so INSERTs keep the name) and `entity_get/set_xdata` (typed by group code; 255-char strings, 16 KB per entity, enforced before writing). - **P&ID reader rules:** `pid_graph` never modifies the drawing, reads geometry from the drawing (an INSERT's real rotation/scale) rather than the catalogue, and reports `source`/`confidence` per node (catalog 1.0, xdata 0.95, heuristic 0.6, inferred 0.3) — never raised by agreement between heuristics. Read `stats.confidence_min` before trusting a foreign drawing. -- **`TOOL_PACKS=all|core,pid,settings,mech`** advertises only the packs a client uses; `core` is always on; `system_about` reports `tool_packs`. +- **`TOOL_PACKS=all|core,pid,settings,mech,arch`** advertises only the packs a client uses; `core` is always on; `system_about` reports `tool_packs`. - **Layer states are ours, not AutoCAD's.** `layer_state_*` stores a portable snapshot in an `ACADMCP_LAYERSTATES` XRECORD; it survives save/reopen on both engines and travels with the file, and it is not listed in AutoCAD's Layer States Manager. Said in the tool docstring, in `system_capabilities.layer_states` (`mode: "xrecord"`, reason `portable_acadmcp_xrecord;not_listed_in_autocad_layer_states_manager`) and in the README. - **Page-setup evidence is the PDF.** `batch_plot` reports `mediabox_mm` parsed from the file's `/MediaBox` (`engineering/standards/pdfinfo.py`); a page setup is verified by what a printer would get, not by the setter's return value. A style or setting write reports `changed` — re-setting a value to itself is not a change. - **Nothing interprets coordinates in UCS.** `ucs_set` / `ucs_restore` store and activate a UCS for the operator's benefit; every tool input and output stays WCS (the repository rule above). @@ -147,6 +148,11 @@ Tools are organized into sections (counts are indicative — `system_about` is a - **Six `mech_*` critique focuses** (`mech_missing_centreline`, `mech_unhatched_section`, `mech_view_misaligned`, `mech_duplicate_dimension`, `mech_thread_unrepresented`, `mech_bom_balloon_mismatch`) run under `focus=None` and inside `drawing_finalize`; they return nothing on a drawing with no `ACADMCP_MECH` payload. `mech_view_misaligned` checks the projection axis and not the quadrant, and `mech_duplicate_dimension` catches two dimensions in the same place rather than a semantic restatement — both narrowings are in the module docstring, because the payload carries no projection angle and no engine exposes what a dimension measured. - **Live ActiveX will not create a layer on assignment** (`entity.Layer = "X"` on a missing layer raises `Key not found`; ezdxf creates it silently). Every mechanical/annotation draw path ensures its target layers before the first entity — a new path must do the same, or it passes every headless test and dies live. - **A diameter's text stands beyond the FIRST chord point** on both engines (the ActiveX `AddDimDiametric` rule, measured); `dimension_diameter(x1, y1, x2, y2)` annotates on the `(x1, y1)` side. +- **The plan model, not lines**: a wall is `arch_wall` (an axis, a thickness, a justification, a material) and a door or window is hosted in it by `arch_opening` — never hand-drawn lines. The engine resolves the junctions and cuts the openings; what it cannot place is in `omitted`. +- **Areas are measured, never typed**: a room's area is the planar face its label point lies in, written into the label and the `room` record by `arch_room`; no tool accepts an area from its caller, and `arch_plan_from_spec` refuses a typed `area` by path. +- **Plan text follows the plot scale**: `scale` is a denominator (50 = 1:50) and a paper height `h` is drawn at `h × scale` drawing units — no annotative objects. `lang="tr"` switches words and the decimal separator; any other language is refused with the list. +- **Three `arch_*` critique focuses** (`arch_room_unlabelled`, `arch_wall_gap` — two wall ends within 50 mm that do not join, `arch_opening_clash`) run under `focus=None` and inside `drawing_finalize`; they return nothing on a drawing with no `ACADMCP_ARCH` record. +- **Nominal, not standard**: the furniture and sanitary blocks carry nominal catalogue dimensions and say so; the stair tool reports the Blondel relation (2R + G within 600–650 mm) as a design rule, never as a building code; the `arch` layer colours are a repository convention (ISO 13567 fixes names, not colours). - **Production drawings**: For real engineering output, use the `engineering/` package primitives via the `gear_*` / `keyway_*` / `titleblock_*` MCP tools — do NOT hand-draw teeth/keyways/sections with raw `entity_create_*` calls. Always end with `drawing_finalize` for the 8-step validator. ### Premium Drawing Rules diff --git a/README.md b/README.md index d447317..455f68e 100644 --- a/README.md +++ b/README.md @@ -27,23 +27,24 @@ Live through COM on Windows, or headless through ezdxf anywhere — one typed co -> **v1.5 release snapshot:** 241 tools · 8 resources · 5 prompt templates · 4507 collected tests. +> **v1.5 release snapshot:** 241 tools · 8 resources · 5 prompt templates · 4516 collected tests. > 241 is the **registered** count; a default install advertises 236 over `tools/list`, > because `ENABLE_3D` is unset. `system_about` is the runtime authority. ## Why this exists -**A big MCP server is expensive to be connected to.** The full catalog costs a client **72,163 tokens** before it has asked for anything. Discovery mode replaces it with two tools and costs **356**. +**A big MCP server is expensive to be connected to.** The full catalog costs a client **78,772 tokens** before it has asked for anything. Discovery mode replaces it with two tools and costs **356**. **A drafter searches for `FILLET`, not `entity_fillet`.** Those command names appeared in no tool name or description — `df = 0` against a stock index, not badly ranked but *absent*. The fix was data: an authored corpus of **205 AutoCAD command names and 1318 synonym phrases** covering all 241 tools. A test refuses to let a tool exist without one. | Advertised surface | Tools seen | Idle cost | |---|---:|---:| -| `TOOL_PROFILE=full` (default) | 224 | 72,163 tokens | -| `TOOL_PACKS=core,settings` (full profile) | 201 | 60,727 tokens | -| `TOOL_PACKS=core,mech` (full profile) | 193 | 61,673 tokens | -| `TOOL_PACKS=core` (full profile) | 179 | 53,959 tokens | -| `TOOL_PROFILE=lean` | 60 | 19,484 tokens | +| `TOOL_PROFILE=full` (default) | 236 | 78,772 tokens | +| `TOOL_PACKS=core,settings` (full profile) | 201 | 60,926 tokens | +| `TOOL_PACKS=core,mech` (full profile) | 193 | 61,872 tokens | +| `TOOL_PACKS=core,arch` (full profile) | 191 | 60,568 tokens | +| `TOOL_PACKS=core` (full profile) | 179 | 54,158 tokens | +| `TOOL_PROFILE=lean` | 64 | 21,939 tokens | | `DISCOVERY_MODE=search` | 2 | **356 tokens** | > [!NOTE] @@ -104,6 +105,7 @@ Claude Desktop, Cursor, or any stdio MCP host. For HTTP: `autocad-mcp --transpor | Engineering generators | involute gears (front + section A-A), DIN 6885 keyed bores, ISO A3 title block | | Mechanical parts | one part model (segments or outline + typed features) and one view engine: front / side / top / section / detail with hidden lines, ISO 128-50 cut faces and ISO 129 dimensions; ISO 4014/4017/4032/7089/4762 fasteners, ISO 15 bearings, DIN 471/472, DIN 509, DIN 332 and ISO 3601-2 as real blocks with attributes; ISO 21920-1 surface texture, ISO 2553 welds, ISO 128-40 section lines | | Sheet & delivery | ISO 5457 frames A4-A0 with zones and trim marks, ISO 7200 title blocks for every size, revision blocks with clouds, ISO 7573 parts lists with ISO 6433 balloons linked by XDATA, CSV/XLSX extraction, xrefs and images on both engines, real `.dwg` on a live seat | +| Architecture | one plan model — walls with hosted doors and windows — and one engine: L / T / X junctions resolved, openings cut, poché by material; rooms measured from the faces the walls enclose (never typed) and read back from any plan with a confidence; door / window / room schedules as real TABLEs; a furniture and sanitary catalogue at nominal sizes; structural grid, north arrow, section / level / elevation marks; exterior dimension chains; English or Turkish labels (`lang="tr"`, decimal comma) | | P&ID | catalogue blocks with ports and tags (ISO 10628-2 / ISA-5.1), port-to-port lines with ISA-5.1 classes and line numbers, `pid_graph` reads any P&ID back with confidence, instrument index / line list / equipment list, `pid_from_spec` one-call sheets | | Styles & standards | ISO-25 / ANSI dimension styles, ISOCP / ROMANS text styles, ISO / ANSI leader styles from authored presets; `drawing_apply_standard("iso")` sets all of it plus units and the `mech` layers in one call; `changed` reports only what moved | | Discovery | `search_tools` ranked over an AutoCAD command and synonym corpus — `FILLET`, `BPOLY`, `QSELECT`, `WBLOCK`, `OVERKILL`, `CHSPACE` each rank **#1** of the 236-tool advertised catalog | @@ -160,29 +162,30 @@ Self-measurement, produced by scripts in [`benchmarks/`](https://github.com/U-C4 ### Correctness — every release re-proves itself -39 deterministic headless checks against the previous tag and the current tree, each in its own subprocess so a hard crash counts as a miss rather than killing the run. +43 deterministic headless checks against the previous tag and the current tree, each in its own subprocess so a hard crash counts as a miss rather than killing the run. | Version | Checks passing | Pass rate | Fixed | Regressed | |---|---:|---:|---:|---:| -| v1.5.1 *(baseline)* | 26 / 39 | 66.7 % | — | — | -| **v1.6.0-dev** *(this branch)* | **39 / 39** | **100 %** | 13 | **0** | +| v1.5.1 *(baseline)* | 26 / 43 | 60.5 % | — | — | +| **v1.6.0-dev** *(this branch)* | **43 / 43** | **100 %** | 17 | **0** | -The thirteen fixed rows are new capability (`miss → pass`). Tracks B and G add seven: `mech_part_roundtrip` (a part read back out of its own `ACADMCP_MECH` XDATA equals the part drawn), `mech_section_hatch_area` (a 60 x ⌀40 sleeve with a ⌀20 bore cuts 1200 mm² exactly, measured out of the drawing by `analysis_measure_entity`), `mech_iso286_on_dimension` (40 H7 is +0.025 / 0 and survives the layout), `mech_thread_unrepresented_is_caught` (the ISO 6410 focus fires on a thread drawn as a plain circle — a gate that never fires is not a gate), `std_part_iso4014_m12` (s = 18, k = 7.5, read by value), `sheet_frame_iso5457_a3` (the frame runs 20,10 to 410,287 inside the 420 × 297 sheet) and `bom_balloon_link` (two identical bolts are one row of quantity 2, and its balloon carries the row's item number). The other six are the three P&ID checks of track A and, from track E, `settings_dimstyle_iso25_values` (the ISO-25 preset lands in the DIMSTYLE table with ISO 129-1's numbers), `settings_layer_state_roundtrip` (save → change → restore puts the layer table back, and the state survives save/reopen because it is an XRECORD in the file) and `settings_pdf_mediabox_a3` (the plotted PDF's own `/MediaBox` reads 420 × 297). The 26 checks v1.5.1 was gated on all still pass. Against the older `v1.4.0` baseline the same 26 reported **21 / 26 → 26 / 26, five fixed, zero regressed** — two of those were repaired defects (`fail → pass`), the diameter and radius callouts, which measured the leader as geometry and dimensioned a 40 mm bore as 60 at default settings. +The seventeen fixed rows are new capability (`miss → pass`). Track F adds four: `arch_junction_l_t_x` (an L corner mitres at (4100, −100) / (3900, 100), a T stem stops on the near face at (2950, 100) / (3050, 100), an X cuts all four faces, and the wall areas are 1 400 000, 1 490 000 and 2 360 000 mm² — computed by hand), `arch_room_area_net` (a 4 × 5 m room between 200 mm walls measures 20.00 m² in its label and its record, not the 21.84 m² its axes enclose), `arch_opening_cuts_wall` (both faces are interrupted across a 900 door and two jambs close it) and `arch_rooms_detect_foreign` (a plan of plain lines on a `WALLS` layer yields 20 and 15 m² at confidence 0.6, and nothing is written). Tracks B and G add seven: `mech_part_roundtrip` (a part read back out of its own `ACADMCP_MECH` XDATA equals the part drawn), `mech_section_hatch_area` (a 60 x ⌀40 sleeve with a ⌀20 bore cuts 1200 mm² exactly, measured out of the drawing by `analysis_measure_entity`), `mech_iso286_on_dimension` (40 H7 is +0.025 / 0 and survives the layout), `mech_thread_unrepresented_is_caught` (the ISO 6410 focus fires on a thread drawn as a plain circle — a gate that never fires is not a gate), `std_part_iso4014_m12` (s = 18, k = 7.5, read by value), `sheet_frame_iso5457_a3` (the frame runs 20,10 to 410,287 inside the 420 × 297 sheet) and `bom_balloon_link` (two identical bolts are one row of quantity 2, and its balloon carries the row's item number). The other six are the three P&ID checks of track A and, from track E, `settings_dimstyle_iso25_values` (the ISO-25 preset lands in the DIMSTYLE table with ISO 129-1's numbers), `settings_layer_state_roundtrip` (save → change → restore puts the layer table back, and the state survives save/reopen because it is an XRECORD in the file) and `settings_pdf_mediabox_a3` (the plotted PDF's own `/MediaBox` reads 420 × 297). The 26 checks v1.5.1 was gated on all still pass. Against the older `v1.4.0` baseline the same 26 reported **21 / 26 → 26 / 26, five fixed, zero regressed** — two of those were repaired defects (`fail → pass`), the diameter and radius callouts, which measured the leader as geometry and dimensioned a 40 mm bore as 60 at default settings. ### The task matrix — tasks that can fail -An earlier matrix scored this server 10/10, which carried no information: every task in it exercised something the server was built around. Five were added in 1.5 because they *can* fail, and three did while being written; 1.6 adds three more — one the P&ID reader can fail on its own, one where the PDF file, not the setter, is the witness, and one where a whole mechanical sheet has to come out clean. +An earlier matrix scored this server 10/10, which carried no information: every task in it exercised something the server was built around. Five were added in 1.5 because they *can* fail, and three did while being written; 1.6 adds four more — one the P&ID reader can fail on its own, one where the PDF file, not the setter, is the witness, one where a whole mechanical sheet has to come out clean, and one where a floor plan's rooms are read back off the drawing. | Task | Verified against | |---|---| | `tool_discovery` | six AutoCAD command names, each ranking #1 | -| `token_budget` | 72,163 → 356 tokens, against a ceiling fixed in advance | +| `token_budget` | 78,772 → 356 tokens, against a ceiling fixed in advance | | `hatch_islands` | 300 filled with the island, 400 ignoring it | | `selection_filter` | window 1, crossing 2, bounding box 3, polygon 1 | | `measure_from_handle` | 139.2699 against the 100.0 a vertex shoelace gives | | `pid_roundtrip` | the example sheet drawn by `pid_from_spec`, read back by `pid_graph`, which never sees the spec: 5 nodes, 4 edges, 0 dangling, `confidence_min` 1.0, `FIC-101` wired to `FCV-101` | | `page_setup_truth` | ANSI B then ISO A3 landscape applied to Layout1, each plotted through `batch_plot` and read back from its PDF's `/MediaBox`: 432 × 279 then 420 × 297 mm, not the setter's return value (a fresh layout is already A3, so the B sheet is what a no-op setter cannot fake) | | `mech_assembly` | a flange-coupling A3 sheet — hub with a full section, flange with its bolt-circle end view, two ISO 4014 bolts with ISO 4032 nuts, an ISO 7573 parts list read off the drawing, linked balloons, ISO 5457 frame, ISO 7200 title block — gated on `drawing_critique(focus=None)` returning **zero** issues and a finalize score of at least **90**; a test drops one balloon and watches the gate fail | +| `arch_roundtrip` | a two-room plan — an entrance door, an interior door, two windows, a stair — drawn by `arch_plan_from_spec` and read back by `arch_rooms_detect`, which never sees the spec: each room within 0.1 % of its label and of the net floor computed by hand (4825 × 5750 and 3825 × 5750 mm), the door and window schedules listing D1, D2, W1, W2, `drawing_critique(focus=None)` at **zero** and a finalize score of at least **90**; a test removes one room label and watches the gate fail | ### Headless performance @@ -231,7 +234,7 @@ Nothing loads a `.env` file — export these, or set them in your MCP client's ` |---|---|---| | `AUTOCAD_MCP_BACKEND` | `auto` | `auto`, `com`, or `ezdxf` | | `CAD_PROGID` | `AutoCAD.Application` | COM ProgID the live backend attaches to | -| `TOOL_PROFILE` | `full` | `lean` (60 curated tools, 54 with `TOOL_PACKS=core`) or `full` | +| `TOOL_PROFILE` | `full` | `lean` (64 curated tools, 54 with `TOOL_PACKS=core`) or `full` | | `TOOL_PACKS` | `all` | Vertical packs to advertise: `core,pid,settings` (`core` always on) | | `DISCOVERY_MODE` | `off` | `search` replaces the catalog with `search_tools` + `call_tool` | | `ENABLE_3D` | `false` | Expose the opt-in `solid_*` tools (COM) | diff --git a/backends/com_backend.py b/backends/com_backend.py index f224529..43a85dc 100644 --- a/backends/com_backend.py +++ b/backends/com_backend.py @@ -897,10 +897,10 @@ def _com_unnarrow(obj): such a return in the gen_py class of the *declared* type, and that wrapper exposes only ``IAcadObject`` members: ``.Count`` / ``.Item`` / ``.GetName`` on the dictionary and ``.ArrowSize`` / ``.TextStyle`` on a leader style all - raise ``AttributeError`` even though the live object answers them. (The - rest of this backend never meets the problem: ``HandleToObject`` is declared - ``IDispatch*``, which keeps the concrete class, and ``ModelSpace.Item`` is - read only through ``IAcadEntity`` members.) + raise ``AttributeError`` even though the live object answers them. + (``HandleToObject`` is declared ``IDispatch*``, which keeps the concrete + class; ``ModelSpace.Item`` is declared ``IAcadEntity*`` and meets the same + problem - ``_entity_info`` un-narrows it through ``_com_concrete_entity``.) ``win32com.client.dynamic.Dispatch`` on the raw interface builds a late-bound proxy from the object's own type info, so every member of the @@ -1358,8 +1358,34 @@ def _temporary_edge_entity(space, kind: str, edge: dict): return ellipse +#: The gen_py class names pywin32 gives a result whose *declared* type is an +#: ActiveX base interface. ``ModelSpace.Item``, ``SelectionSet.Item`` and +#: ``IAcadBlock.Item`` are declared ``IAcadEntity*`` (acax25enu.tlb). +_COM_NARROWED_WRAPPERS = frozenset({"IAcadEntity", "IAcadObject"}) + + +def _com_concrete_entity(entity): + """The entity through its runtime class when a makepy wrapper narrowed it. + + With a makepy cache for the AutoCAD type library, ``ModelSpace.Item(i)`` + comes back as the gen_py ``IAcadEntity`` class, which answers only the + entity base's members. Measured on AutoCAD 2026 (2026-09-23): on an + AcDbLine, ``type(item).__name__ == "IAcadEntity"`` and ``item.StartPoint`` + raises ``AttributeError`` - so every ``entity_list`` row lost its + type-specific properties (a LINE kept only its bounding box) and the room + reader saw no wall segments at all. Only that wrapper is re-dispatched: + ``_com_unnarrow`` walks the object's type info across the process + boundary, and a concrete wrapper (``HandleToObject`` is declared + ``IDispatch*``) needs none of it. + """ + if type(entity).__name__ in _COM_NARROWED_WRAPPERS: + return _com_unnarrow(entity) + return entity + + def _entity_info(entity) -> EntityInfo: """Convert a COM entity object to EntityInfo dataclass.""" + entity = _com_concrete_entity(entity) try: bb_min, bb_max = entity.GetBoundingBox() props = { diff --git a/benchmarks/README.md b/benchmarks/README.md index 4eeeb23..d6b2a40 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -117,6 +117,16 @@ the live seat a missing `MECH` layer and a parts list that could not see a `BLOCKREFERENCE`; each is fixed with a test. The competitor reports carry no result for it. +### Matrix v6 (v1.6) — the architectural plan + +`tasks_v6.py` keeps the v5 eighteen unchanged and adds track F's evidence: + +| Task | Category | Verified against | +|---|---|---| +| `arch_roundtrip` | arch | the plan of `engineering/arch/spec.py::EXAMPLE_SPEC` — a 9 × 6 m brick ring at 250 mm split by a 100 mm AAC wall, an entrance door, an interior door, two windows, a straight stair, two labelled rooms, the exterior chains and the door / window / room schedules — drawn in one transaction by `draw_plan_from_spec`, then read back without the spec: `rooms_detect` on the wall layer must find each labelled room within 0.1 % of the area its label carries **and** of the net floor computed by hand (4825 × 5750 = 27 743 750 and 3825 × 5750 = 21 993 750 mm²), `schedule_rows` must list exactly D1, D2 and W1, W2, and the saved plan is gated like `mech_assembly`: `run_critique(focus=None)` returns **zero** issues and `combine(validator, critique)` scores at least **90**. `tests/test_benchmark_v6.py` removes one room label to prove the gate fails and names `arch_room_unlabelled` | + +The competitor reports carry no result for it. + ## Live competitor lane (v1.4) Two competitor adapters now execute the exact same task matrix, black-box over @@ -234,15 +244,15 @@ python benchmarks/compare_versions.py --json results.json ### Result — this branch (1.6.0-dev) vs v1.5.1 (release gate) -39 checks, ezdxf backend, one subprocess per check. Machine-readable report: +43 checks, ezdxf backend, one subprocess per check. Machine-readable report: [`results/published/ab-v1.5.1-vs-v1.6.0-dev.json`](results/published/ab-v1.5.1-vs-v1.6.0-dev.json). | Version | Checks passing | Pass rate | Fixed | Regressed | |---------|----------------|-----------|-------|-----------| -| **v1.5.1** (baseline) | 26 / 39 | 66.7 % | — | — | -| **v1.6.0-dev** (this branch) | 39 / 39 | 100 % | 13 | **0** | +| **v1.5.1** (baseline) | 26 / 43 | 60.5 % | — | — | +| **v1.6.0-dev** (this branch) | 43 / 43 | 100 % | 17 | **0** | -The thirteen are all `miss → pass` — v1.5.1 has none of the methods. Seven +The seventeen are all `miss → pass` — v1.5.1 has none of the methods. Four are track F: `arch_junction_l_t_x` (L, T and X junction outlines against hand-computed corners and wall areas), `arch_room_area_net` (a 4 × 5 m room between 200 mm walls is labelled 20.00 m², not its 21.84 m² axis area), `arch_opening_cuts_wall` (both faces interrupted across a door, two jambs) and `arch_rooms_detect_foreign` (plain lines on a WALLS layer read as 20 and 15 m² at confidence 0.6, nothing written). Seven are tracks B and G: `mech_part_roundtrip` (a part read back from its own `ACADMCP_MECH` XDATA equals the part drawn), `mech_section_hatch_area` (a sleeve 60 long, ⌀40 outside, ⌀20 bore: a full section cuts two 60 × 10 faces, diff --git a/benchmarks/adapters/autocad_mcp_pro.py b/benchmarks/adapters/autocad_mcp_pro.py index 85bc442..19ca6d9 100644 --- a/benchmarks/adapters/autocad_mcp_pro.py +++ b/benchmarks/adapters/autocad_mcp_pro.py @@ -1,4 +1,4 @@ -"""Reference adapter for this repository, covering the v5 task matrix. +"""Reference adapter for this repository, covering the v6 task matrix. Every task verifies against a value worked out independently of the code under test — a closed-form area, a count of entities placed on purpose, a token @@ -12,7 +12,7 @@ import math from pathlib import Path from benchmarks.adapters.base import BenchmarkAdapter, TaskResult -from benchmarks.tasks_v5 import TaskSpec +from benchmarks.tasks_v6 import TaskSpec from engineering.delivery import deliver_drawing from engineering.layers import ensure_engineering_layers, ensure_standard_linetypes from engineering.refiner import refine_drawing @@ -40,6 +40,15 @@ DISCOVERY_TOP_N = 3 #: rather than derived from the measurement, so the measurement can miss it. TOKEN_CEILING = 2_000 +#: The net floor of the two rooms of ``engineering/arch/spec.py::EXAMPLE_SPEC``, +#: computed by hand: the 250 mm ring on axis (0,0)-(9000,6000) has its inner +#: faces at x = 125 / 8875 and y = 125 / 5875, the 100 mm wall on x = 5000 its +#: faces at 4950 / 5050. Room 01: 4825 x 5750; room 02: 3825 x 5750. +ROUNDTRIP_AREAS_MM2: dict[str, float] = { + "01": 4825.0 * 5750.0, + "02": 3825.0 * 5750.0, +} + class AutoCADMCPProAdapter(BenchmarkAdapter): name = "autocad-mcp-pro" @@ -682,6 +691,94 @@ class AutoCADMCPProAdapter(BenchmarkAdapter): [str(sheet)], ) + async def _task_arch_roundtrip(self): + """A two-room plan drawn from one spec and read back off the drawing. + + Track F's evidence (spec §10). ``arch_plan_from_spec`` draws the plan of + ``engineering/arch/spec.py::EXAMPLE_SPEC`` - a 9 x 6 m brick ring at 250 + mm split by a 100 mm AAC wall, an entrance door, an interior door, two + windows, a straight stair, two labelled rooms, the exterior chains and + the three schedules. Then the drawing is read, never the spec: + + * ``rooms_detect`` on the wall layer must find each labelled room, and + its area must agree with the area the label's record carries within + 0.1 %, and with the net floor computed by hand in + ``ROUNDTRIP_AREAS_MM2`` - so a label that typed the axis area fails; + * the door and window schedules read off the records must list exactly + the drawn tags; + * ``run_critique(focus=None)`` returns zero issues and the finalize + score - ``combine(validator.summary, critique)`` over the saved plan - + is at least 90. + + Any one slipping fails the task, and the metrics name what slipped. + """ + from engineering import DrawingValidator + from engineering.arch.critique import point_in_loop + from engineering.arch.draw import read_plan + from engineering.arch.layers import ARCH_ROLE_LAYER + from engineering.arch.rooms import rooms_detect + from engineering.arch.schedule import schedule_rows + from engineering.arch.spec import EXAMPLE_SPEC, draw_plan_from_spec + from engineering.critique import run_critique + from engineering.scoring import combine + + drawn = await draw_plan_from_spec(self.backend, EXAMPLE_SPEC) + plan = await read_plan(self.backend) + detected = await rooms_detect(self.backend, layers=[ARCH_ROLE_LAYER["wall"]]) + faces = list(detected.get("rooms") or []) + + areas: dict[str, dict] = {} + for room in plan["rooms"]: + face = next((f for f in faces if point_in_loop(room.at, f["loop"])), None) + detected_area = float(face["area"]) if face is not None else 0.0 + hand = ROUNDTRIP_AREAS_MM2.get(str(room.number)) + areas[str(room.number)] = { + "label_mm2": float(room.area), + "detected_mm2": detected_area, + "hand_mm2": hand, + "agrees": face is not None + and hand is not None + and abs(detected_area - float(room.area)) <= 1e-3 * float(room.area) + and abs(detected_area - hand) <= 1e-3 * hand, + } + rooms_ok = ( + set(areas) == set(ROUNDTRIP_AREAS_MM2) + and len(faces) == len(ROUNDTRIP_AREAS_MM2) + and all(row["agrees"] for row in areas.values()) + ) + + door_tags = sorted(str(row["tag"]) for row in schedule_rows("doors", plan, lang="en")) + window_tags = sorted(str(row["tag"]) for row in schedule_rows("windows", plan, lang="en")) + tags_ok = door_tags == ["D1", "D2"] and window_tags == ["W1", "W2"] + + sheet = self.artifact_dir / "arch_roundtrip.dxf" + await self.backend.drawing_save_as(str(sheet)) + issues = await run_critique(self.backend, None) + validation = await DrawingValidator().run(self.backend) + summary = {"error": 0, "warning": 0, "info": 0} + for issue in issues: + summary[issue.severity] = summary.get(issue.severity, 0) + 1 + score = combine(validation.summary, summary) + passed = rooms_ok and tags_ok and len(issues) == 0 and float(score["score"]) >= 90.0 + return ( + passed, + { + "rooms": areas, + "faces_detected": len(faces), + "confidence_min": detected.get("confidence_min"), + "door_tags": door_tags, + "window_tags": window_tags, + "omitted": drawn.get("omitted"), + "critique_issues": len(issues), + "critique_focuses": sorted({issue.focus for issue in issues}), + "score": float(score["score"]), + "grade": score["grade"], + "invalidity_ratio": score["invalidity_ratio"], + "validator_findings": sorted(f.code for f in validation.findings), + }, + [str(sheet)], + ) + async def _task_auditable_delivery(self): await ensure_engineering_layers(self.backend) await self.backend.entity_create_line(0, 0, 10, 0, layer="GEOMETRY") diff --git a/benchmarks/correctness_suite.py b/benchmarks/correctness_suite.py index d8b3875..b95c7ec 100644 --- a/benchmarks/correctness_suite.py +++ b/benchmarks/correctness_suite.py @@ -642,6 +642,182 @@ async def std_part_iso4014_m12(): return payload["kind"] == "std_part" and "M12" in str(payload["designation"]) +# ── Architecture (track F) ────────────────────────────────────────────────── + + +def _arch_points(geometry) -> set: + """Every endpoint of a drawn face or jamb, rounded to a micron.""" + return { + (round(float(x), 6), round(float(y), 6)) + for line in (*geometry.faces, *geometry.jambs) + for x, y in (line.p1, line.p2) + } + + +def _arch_passes_through(geometry, point) -> bool: + """Does a drawn face run *through* ``point`` (not merely end on it)?""" + px, py = point + for line in geometry.faces: + (x1, y1), (x2, y2) = line.p1, line.p2 + length = math.hypot(x2 - x1, y2 - y1) + cross = (x2 - x1) * (py - y1) - (y2 - y1) * (px - x1) + along = ((px - x1) * (x2 - x1) + (py - y1) * (y2 - y1)) / (length * length) + if abs(cross) <= 1e-6 * length and 1e-9 < along < 1.0 - 1e-9: + return True + return False + + +def _arch_region_area(geometry) -> float: + from engineering.measure import polygon_area_perimeter + + return sum( + polygon_area_perimeter([(x, y, 0.0) for x, y in loop], True)[0] for loop in geometry.regions + ) + + +async def arch_junction_l_t_x(): + """The three junction kinds, against coordinates worked out by hand. + + L: one 200 mm centre wall turning at (4000, 0) - the outer corner is + (4100, -100), the inner (3900, 100), and a mitred corner keeps the area at + axis length x thickness: 7000 x 200 = 1 400 000 mm². + T: a 100 mm stem on x = 3000 ending on the axis of a 200 mm wall - its faces + stop on the near face at (2950, 100) and (3050, 100), the near face is open + between them, the far face runs through; 6000 x 200 + 2900 x 100 = 1 490 000. + X: two 200 mm walls crossing at (3000, 0) - the first stays whole, the second + stops on its faces, all four faces are cut at x = 2900 / 3100 and y = +-100; + 6000 x 200 + 2 x 2900 x 200 = 2 360 000. + """ + from engineering.arch.model import wall_from_dict + from engineering.arch.walls import wall_geometry + + def wall(wall_id, axis, thickness): + return wall_from_dict({"id": wall_id, "axis": axis, "thickness": thickness}) + + l_geo = wall_geometry([wall("L", [[0, 0], [4000, 0], [4000, 3000]], 200)], poche=False) + t_geo = wall_geometry( + [wall("A", [[0, 0], [6000, 0]], 200), wall("B", [[3000, 0], [3000, 3000]], 100)], + poche=False, + ) + x_geo = wall_geometry( + [wall("A", [[0, 0], [6000, 0]], 200), wall("B", [[3000, -3000], [3000, 3000]], 200)], + poche=False, + ) + l_ok = ( + {(4100.0, -100.0), (3900.0, 100.0)} <= _arch_points(l_geo) + and not {(4100.0, 100.0), (3900.0, -100.0)} & _arch_points(l_geo) + and abs(_arch_region_area(l_geo) - 1_400_000.0) < 1e-6 + ) + t_ok = ( + {(2950.0, 100.0), (3050.0, 100.0)} <= _arch_points(t_geo) + and not _arch_passes_through(t_geo, (3000.0, 100.0)) + and _arch_passes_through(t_geo, (3000.0, -100.0)) + and abs(_arch_region_area(t_geo) - 1_490_000.0) < 1e-6 + ) + x_ok = ( + {(2900.0, 100.0), (3100.0, 100.0), (2900.0, -100.0), (3100.0, -100.0)} + <= _arch_points(x_geo) + and not any( + _arch_passes_through(x_geo, p) + for p in ((3000.0, 100.0), (3000.0, -100.0), (2900.0, 0.0), (3100.0, 0.0)) + ) + and abs(_arch_region_area(x_geo) - 2_360_000.0) < 1e-6 + ) + return l_ok and t_ok and x_ok + + +async def arch_room_area_net(): + """A 4 x 5 m room between 200 mm walls measures 20.00 m², not its axis area. + + The ring's axis runs (-100, -100) to (4100, 5100), so its inner faces are + x = 0 / 4000 and y = 0 / 5000: the floor is 4000 x 5000 = 20 000 000 mm², + while the axis encloses 4200 x 5200 = 21 840 000. The number is read back + out of the room record the label carries, and the label text must say it. + """ + from engineering.arch.draw import draw_walls, read_plan + from engineering.arch.lang import fmt_area_m2 + from engineering.arch.model import wall_from_dict + from engineering.arch.rooms import label_room + + b = await _b() + ring = wall_from_dict( + { + "id": "W1", + "axis": [[-100, -100], [4100, -100], [4100, 5100], [-100, 5100]], + "thickness": 200, + "closed": True, + } + ) + await draw_walls(b, [ring]) + await label_room(b, {"id": "R1", "name": "ROOM", "number": "01", "at": [2000, 2500]}) + (room,) = (await read_plan(b))["rooms"] + texts = [e.properties.get("text", "") for e in await b.entity_list(type_filter="TEXT")] + return ( + abs(float(room.area) - 20_000_000.0) < 0.5 + and fmt_area_m2(room.area, "en") == "20.00 m²" + and any("20.00 m²" in str(text) for text in texts) + ) + + +async def arch_opening_cuts_wall(): + """A 900 door at 1000 along a 5 m, 200 mm wall interrupts both faces. + + No face runs through x = 1450 (the middle of the door) on either face line + y = +-100 while one still runs through x = 500, and two jambs close the cut + at x = 1000 and x = 1900. + """ + from engineering.arch.model import opening_from_dict, wall_from_dict + from engineering.arch.walls import wall_geometry + + wall = wall_from_dict({"id": "W1", "axis": [[0, 0], [5000, 0]], "thickness": 200}) + door = opening_from_dict( + {"id": "D1", "wall": "W1", "kind": "door", "offset": 1000, "width": 900} + ) + geo = wall_geometry([wall], [door], poche=False) + jambs = { + round(float(line.p1[0]), 6) + for line in geo.jambs + if abs(line.p1[0] - line.p2[0]) < 1e-9 and abs(abs(line.p1[1] - line.p2[1]) - 200.0) < 1e-9 + } + return ( + not _arch_passes_through(geo, (1450.0, 100.0)) + and not _arch_passes_through(geo, (1450.0, -100.0)) + and _arch_passes_through(geo, (500.0, 100.0)) + and {1000.0, 1900.0} <= jambs + and tuple(geo.omitted) == () + ) + + +async def arch_rooms_detect_foreign(): + """A plan of plain lines - no record of ours - yields the net areas. + + Two rooms drawn as the inner outlines of 200 mm walls on a layer named + WALLS: 4000 x 5000 and 3000 x 5000, inside an outer outline (-200, -200) + to (7400, 5200). The reader must find exactly those two faces - 20 000 000 + and 15 000 000 mm² - at the foreign-plan confidence 0.6, and must not have + written anything. + """ + from engineering.arch.rooms import rooms_detect + + b = await _b() + for x0, y0, x1, y1 in ( + (-200.0, -200.0, 7400.0, 5200.0), + (0.0, 0.0, 4000.0, 5000.0), + (4200.0, 0.0, 7200.0, 5000.0), + ): + corners = [(x0, y0), (x1, y0), (x1, y1), (x0, y1)] + for a, c in zip(corners, corners[1:] + corners[:1], strict=True): + await b.entity_create_line(a[0], a[1], c[0], c[1], layer="WALLS") + before = len(await b.entity_list(limit=1000)) + found = await rooms_detect(b) + areas = sorted(round(float(room["area"]), 3) for room in found["rooms"]) + return ( + areas == [15_000_000.0, 20_000_000.0] + and all(float(room["confidence"]) == 0.6 for room in found["rooms"]) + and len(await b.entity_list(limit=1000)) == before + ) + + CHECKS = { "core_line_length": (core_line_length, "Core"), "core_circle_radius": (core_circle_radius, "Core"), @@ -682,6 +858,10 @@ CHECKS = { "std_part_iso4014_m12": (std_part_iso4014_m12, "Mechanical"), "sheet_frame_iso5457_a3": (sheet_frame_iso5457_a3, "Sheet"), "bom_balloon_link": (bom_balloon_link, "Sheet"), + "arch_junction_l_t_x": (arch_junction_l_t_x, "Architecture"), + "arch_room_area_net": (arch_room_area_net, "Architecture"), + "arch_opening_cuts_wall": (arch_opening_cuts_wall, "Architecture"), + "arch_rooms_detect_foreign": (arch_rooms_detect_foreign, "Architecture"), } diff --git a/benchmarks/render_matrix_chart.py b/benchmarks/render_matrix_chart.py index 7513d33..7ff2ba8 100644 --- a/benchmarks/render_matrix_chart.py +++ b/benchmarks/render_matrix_chart.py @@ -13,7 +13,7 @@ import json from pathlib import Path from typing import Any -from benchmarks.tasks_v5 import TASKS_V5 +from benchmarks.tasks_v6 import TASKS_V6 ROOT = Path(__file__).resolve().parents[1] DEFAULT_INPUT_DIR = ROOT / "benchmarks" / "results" / "published" @@ -66,8 +66,8 @@ def render_chart(reports: list[dict[str, Any]], output: Path) -> None: matplotlib.rcParams["svg.hashsalt"] = "autocad-mcp-taskmatrix-v1" - task_ids = [task.task_id for task in TASKS_V5] - task_labels = [f"{task.task_id} ({task.category})" for task in TASKS_V5] + task_ids = [task.task_id for task in TASKS_V6] + task_labels = [f"{task.task_id} ({task.category})" for task in TASKS_V6] servers = [report["adapter"] for report in reports] status_by_server = { report["adapter"]: {item["task_id"]: item["status"] for item in report["results"]} diff --git a/benchmarks/results/published/ab-v1.5.1-vs-v1.6.0-dev.json b/benchmarks/results/published/ab-v1.5.1-vs-v1.6.0-dev.json index 7ad938c..ecb1407 100644 --- a/benchmarks/results/published/ab-v1.5.1-vs-v1.6.0-dev.json +++ b/benchmarks/results/published/ab-v1.5.1-vs-v1.6.0-dev.json @@ -1,12 +1,12 @@ { "summary": { "baseline_ref": "v1.5.1", - "total": 39, + "total": 43, "old_pass": 26, - "new_pass": 39, - "old_pct": 66.7, + "new_pass": 43, + "old_pct": 60.5, "new_pct": 100.0, - "fixed": 13, + "fixed": 17, "regressed": 0 }, "rows": [ @@ -243,6 +243,30 @@ "category": "Sheet", "old": "miss", "new": "pass" + }, + { + "check": "arch_junction_l_t_x", + "category": "Architecture", + "old": "miss", + "new": "pass" + }, + { + "check": "arch_room_area_net", + "category": "Architecture", + "old": "miss", + "new": "pass" + }, + { + "check": "arch_opening_cuts_wall", + "category": "Architecture", + "old": "miss", + "new": "pass" + }, + { + "check": "arch_rooms_detect_foreign", + "category": "Architecture", + "old": "miss", + "new": "pass" } ] } \ No newline at end of file diff --git a/benchmarks/run_competitors.py b/benchmarks/run_competitors.py index ee25a8c..7e5106b 100644 --- a/benchmarks/run_competitors.py +++ b/benchmarks/run_competitors.py @@ -16,7 +16,7 @@ from pathlib import Path from backends.capability import UnsupportedCapabilityError from benchmarks.adapters.base import BenchmarkAdapter, TaskResult -from benchmarks.tasks_v5 import DEFAULT_MATRIX, MATRICES, TaskSpec, task_by_id +from benchmarks.tasks_v6 import DEFAULT_MATRIX, MATRICES, TaskSpec, task_by_id def _load_registry() -> dict[str, dict]: diff --git a/benchmarks/tasks_v6.py b/benchmarks/tasks_v6.py new file mode 100644 index 0000000..2c2ac96 --- /dev/null +++ b/benchmarks/tasks_v6.py @@ -0,0 +1,54 @@ +"""The v6 task matrix: v5 plus what track F added. + +``arch_roundtrip`` (track F) is the roadmap evidence for the plan model: a +two-room plan - an entrance door, an interior door, two windows and a stair - +drawn by ``arch_plan_from_spec``, then read back by ``arch_rooms_detect``, +which never sees the spec. Each room's detected area has to agree with the area +its label carries within 0.1 %, and both with the net floor computed by hand; +the door and window schedules have to list the drawn tags; and the sheet is +gated on the same two numbers as ``mech_assembly``: ``drawing_critique +(focus=None)`` returns zero issues and the finalize score is at least 90. + +The published competitor reports were never asked this question; the chart +shows ``not_run`` for them, not zero. +""" + +from __future__ import annotations + +from benchmarks.tasks_v2 import TASKS_V2 +from benchmarks.tasks_v3 import TASKS_V3 +from benchmarks.tasks_v4 import TASKS_V4 +from benchmarks.tasks_v5 import TASKS_V5, TaskSpec + +#: Added in v1.6.0 by track F. +NEW_TASKS_V6: tuple[TaskSpec, ...] = ( + TaskSpec( + "arch_roundtrip", + "arch", + "Draw a two-room plan from one spec, read its rooms back from the drawing " + "within 0.1 %, schedule its tags, with a clean critique and a finalize score " + "of at least 90", + 1.0, + ), +) + +TASKS_V6: tuple[TaskSpec, ...] = TASKS_V5 + NEW_TASKS_V6 + +#: Runner-selectable task sets. v2 to v5 stay addressable so an earlier report +#: can be reproduced exactly rather than only described. +MATRICES: dict[str, tuple[TaskSpec, ...]] = { + "v2": TASKS_V2, + "v3": TASKS_V3, + "v4": TASKS_V4, + "v5": TASKS_V5, + "v6": TASKS_V6, +} + +DEFAULT_MATRIX = "v6" + + +def task_by_id(task_id: str, matrix: str = DEFAULT_MATRIX) -> TaskSpec: + for task in MATRICES[matrix]: + if task.task_id == task_id: + return task + raise KeyError(f"{task_id} is not in matrix {matrix}") diff --git a/benchmarks/token_suite.py b/benchmarks/token_suite.py index 48c2b97..1287c52 100644 --- a/benchmarks/token_suite.py +++ b/benchmarks/token_suite.py @@ -521,6 +521,16 @@ async def idle_lane(counter: Any, turns: int = SESSION_TURNS) -> list[dict[str, turns, ) ) + config.settings.tool_packs = "core,arch" + rows.append( + await _idle_row( + "packs_core_arch", + "TOOL_PACKS=core,arch - an architectural client that never opens a " + "P&ID, a machine part or a live seat.", + counter, + turns, + ) + ) config.settings.tool_packs = previous_packs srv._apply_discovery_mode("search") rows.append( diff --git a/docs/superpowers/specs/2026-09-23-v1.6-arch-design.md b/docs/superpowers/specs/2026-09-23-v1.6-arch-design.md index 7d1af99..4aca547 100644 --- a/docs/superpowers/specs/2026-09-23-v1.6-arch-design.md +++ b/docs/superpowers/specs/2026-09-23-v1.6-arch-design.md @@ -1,6 +1,6 @@ # v1.6 Track F — architectural 2D: the plan model -**Date:** 2026-09-23 · **Status:** design approved in conversation · +**Date:** 2026-09-23 · **Status:** shipped (v1.6.0-dev, track F) · **Parent:** `2026-09-15-v1.6-roadmap-design.md` (track F) · **Baseline:** main after tracks B + G (`2bba18d`, 229 tools, 4005 tests) diff --git a/scripts/check_doc_numbers.py b/scripts/check_doc_numbers.py index 15c715a..1fc6837 100644 --- a/scripts/check_doc_numbers.py +++ b/scripts/check_doc_numbers.py @@ -79,7 +79,7 @@ def derived() -> dict[str, object]: } from benchmarks import correctness_suite - from benchmarks.tasks_v5 import TASKS_V5 + from benchmarks.tasks_v6 import TASKS_V6 return { "collected_tests": _collected_tests(), @@ -94,7 +94,7 @@ def derived() -> dict[str, object]: "ab_fixed": ab["fixed"], "ab_regressed": ab["regressed"], "suite_checks": len(correctness_suite.CHECKS), - "matrix_tasks": len(TASKS_V5), + "matrix_tasks": len(TASKS_V6), "perf_ms": perf, } diff --git a/scripts/smoke_arch_com.py b/scripts/smoke_arch_com.py new file mode 100644 index 0000000..4c03684 --- /dev/null +++ b/scripts/smoke_arch_com.py @@ -0,0 +1,192 @@ +"""Exercise track F once against the live AutoCAD on this machine. + +Creates a NEW document (never touches the operator's open drawings), then runs +the whole plan through the same code the tools call: + + draw_plan_from_spec(EXAMPLE_SPEC) (one transaction:) + drawing_apply_iso_layers("arch") -> draw_walls (junctions, cut + openings, poche, ACADMCP_ARCH records) -> draw_stair -> label_room x2 + (areas measured off the walls just drawn) -> draw_chains + -> draw_schedule("doors" / "windows" / "rooms") + read_plan -> rooms_detect (wall layer) -> run_critique(focus=None) + +and prints what came back as JSON. Exit codes: + + 0 the plan drew, both rooms read back within 0.1 % of their labels and of + the hand-computed net floor, and the critique returned nothing + 1 it drew, but a room disagreed or the critique found something + 2 no live CAD application reachable (pywin32 missing, or the ProgID + cannot be created) -- the COM paths then remain fake-tested only + +The backend's ``connect()`` is lazy on purpose (the server must start before +AutoCAD is open), so reachability is read from ``system_status()`` rather than +from ``connect()``. Every document this run opens is closed on the way out, +also when a step fails, unless ``--keep`` is given. + + AUTOCAD_MCP_BACKEND=com uv run --frozen python scripts/smoke_arch_com.py + uv run --frozen python scripts/smoke_arch_com.py --keep # leave it open +""" + +from __future__ import annotations + +import asyncio +import json +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +#: The net floor of EXAMPLE_SPEC's two rooms, by hand (see +#: benchmarks/adapters/autocad_mcp_pro.py::ROUNDTRIP_AREAS_MM2). +HAND_AREAS_MM2 = {"01": 4825.0 * 5750.0, "02": 3825.0 * 5750.0} + +#: ``RPC_E_CALL_REJECTED``: AutoCAD's message filter refused the call because +#: it was busy (a document still initialising after ``Documents.Add`` is the +#: case measured on this seat by scripts/smoke_mech_com.py). A refused call +#: was not executed. +RPC_E_CALL_REJECTED = -2147418111 + +#: How long a refused call is retried before the smoke gives up on it. +REJECTED_PATIENCE_S = 90.0 + +#: Names of the documents this run opened and has not closed yet. +_OPENED: list[str] = [] + + +def _patient(backend) -> None: + """Retry any backend call AutoCAD refused with ``RPC_E_CALL_REJECTED``.""" + run = backend._run + + async def patient(func, *args, **kwargs): + deadline = time.monotonic() + REJECTED_PATIENCE_S + pause = 0.5 + while True: + try: + return await run(func, *args, **kwargs) + except RuntimeError as exc: + cause = exc.__cause__ + hr = cause.args[0] if cause is not None and cause.args else None + if hr != RPC_E_CALL_REJECTED or time.monotonic() + pause > deadline: + raise + await asyncio.sleep(pause) + pause = min(pause * 2.0, 5.0) + + backend._run = patient + + +async def _names(backend) -> set[str]: + return {row["name"] for row in await backend.document_list()} + + +async def _new_document(backend) -> str: + """``drawing_new``, recording every document it added (normally one).""" + before = await _names(backend) + await backend.drawing_new() + added = sorted((await _names(backend)) - before) + _OPENED.extend(name for name in added if name not in _OPENED) + return added[-1] if added else "" + + +async def _close_leftovers(backend) -> None: + open_names = {row["name"] for row in await backend.document_list()} + for name in list(_OPENED): + if name in open_names: + try: + await backend.document_close(name, discard=True) + except Exception as exc: # best effort on the way out + print(f"could not close {name}: {exc}", file=sys.stderr) + _OPENED.remove(name) + + +def _inside(point, loop) -> bool: + from engineering.arch.critique import point_in_loop + + return point_in_loop(point, loop) + + +async def _plan(backend) -> dict: + from engineering.arch.draw import read_plan + from engineering.arch.layers import ARCH_ROLE_LAYER + from engineering.arch.rooms import rooms_detect + from engineering.arch.spec import EXAMPLE_SPEC, draw_plan_from_spec + from engineering.critique import run_critique + + await _new_document(backend) + drawn = await draw_plan_from_spec(backend, EXAMPLE_SPEC) + plan = await read_plan(backend) + detected = await rooms_detect(backend, layers=[ARCH_ROLE_LAYER["wall"]]) + faces = list(detected.get("rooms") or []) + rooms = {} + for room in plan["rooms"]: + face = next((f for f in faces if _inside(room.at, f["loop"])), None) + found = float(face["area"]) if face is not None else 0.0 + hand = HAND_AREAS_MM2.get(str(room.number), 0.0) + rooms[str(room.number)] = { + "label_mm2": round(float(room.area), 3), + "detected_mm2": round(found, 3), + "hand_mm2": hand, + "agrees": face is not None + and abs(found - float(room.area)) <= 1e-3 * float(room.area) + and abs(found - hand) <= 1e-3 * hand, + } + issues = await run_critique(backend, None) + return { + "walls": drawn["walls"], + "openings": drawn["openings"], + "omitted": drawn["omitted"], + "stairs": len(drawn["stairs"]), + "schedules": [ + { + "kind": item.get("kind"), + "representation": item.get("representation"), + "rows": item.get("row_count"), + } + for item in drawn["schedules"] + ], + "rooms": rooms, + "faces_detected": len(faces), + "confidence_min": detected.get("confidence_min"), + "critique_issues": len(issues), + "critique": [issue.to_dict() for issue in issues], + } + + +async def main(keep_document: bool = False) -> int: + try: + from backends.com_backend import ComBackend + except ImportError as exc: + print(f"pywin32 not available: {exc}") + return 2 + backend = ComBackend() + _patient(backend) + try: + await backend.connect() + except Exception as exc: + print(f"no live AutoCAD reachable: {exc}") + return 2 + try: + status = await backend.system_status() + if not status.get("connected"): + print(f"no live AutoCAD reachable: {status.get('error', status)}") + return 2 + report = { + "autocad": status.get("autocad_version") or status.get("cad_progid"), + **await _plan(backend), + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + ok = ( + report["critique_issues"] == 0 + and set(report["rooms"]) == set(HAND_AREAS_MM2) + and all(row["agrees"] for row in report["rooms"].values()) + ) + return 0 if ok else 1 + finally: + if not keep_document: + await _close_leftovers(backend) + await backend.disconnect() + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main(keep_document="--keep" in sys.argv[1:]))) diff --git a/tests/test_benchmark_v4.py b/tests/test_benchmark_v4.py index 3d84d12..8b9d48e 100644 --- a/tests/test_benchmark_v4.py +++ b/tests/test_benchmark_v4.py @@ -49,7 +49,7 @@ def test_correctness_suite_has_the_settings_checks(): ): assert CHECKS[name][1] == "Settings" # v4 left the suite at 32; tracks B + G grew it to 39 (tests/test_benchmark_v5.py). - assert len(CHECKS) == 39 + assert len(CHECKS) == 43 @pytest.mark.asyncio # asyncio_mode is strict; the four tests above are sync diff --git a/tests/test_benchmark_v5.py b/tests/test_benchmark_v5.py index 446753c..dd2ad28 100644 --- a/tests/test_benchmark_v5.py +++ b/tests/test_benchmark_v5.py @@ -48,7 +48,7 @@ def test_the_suite_carries_the_mechanical_and_sheet_checks(): assert CHECKS[name][1] == "Mechanical" assert CHECKS["sheet_frame_iso5457_a3"][1] == "Sheet" assert CHECKS["bom_balloon_link"][1] == "Sheet" - assert len(CHECKS) == 39 + assert len(CHECKS) == 43 @pytest.mark.asyncio diff --git a/tests/test_benchmark_v6.py b/tests/test_benchmark_v6.py new file mode 100644 index 0000000..1ef60f1 --- /dev/null +++ b/tests/test_benchmark_v6.py @@ -0,0 +1,119 @@ +"""The v6 task matrix: v5 plus what track F added. + +``arch_roundtrip`` is track F's evidence - a two-room plan drawn from one +spec, read back off the drawing by the room reader that never sees the spec, +and gated on the numbers the design spec fixed in advance: every room's +detected area within 0.1 % of its label's and of the hand-computed net floor, +the schedules listing the drawn tags, ``drawing_critique(focus=None)`` at +zero and the finalize score at least 90. The published competitor reports +were never asked it and the chart shows them ``not_run``. +""" + +from __future__ import annotations + +import pytest + +from benchmarks.tasks_v5 import TASKS_V5 +from benchmarks.tasks_v6 import DEFAULT_MATRIX, MATRICES, NEW_TASKS_V6, TASKS_V6, task_by_id + + +def test_v6_extends_v5_without_touching_it(): + assert TASKS_V6[: len(TASKS_V5)] == TASKS_V5 + assert [t.task_id for t in NEW_TASKS_V6] == ["arch_roundtrip"] + assert len(TASKS_V6) == 19 + assert MATRICES["v5"] == TASKS_V5 and MATRICES["v6"] == TASKS_V6 + assert DEFAULT_MATRIX == "v6" + assert task_by_id("arch_roundtrip").category == "arch" + + +def test_the_adapter_implements_every_v6_task(): + from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter + + for task in TASKS_V6: + assert hasattr(AutoCADMCPProAdapter, f"_task_{task.task_id}"), task.task_id + + +def test_the_hand_areas_are_the_net_floor_not_the_axis_area(): + from benchmarks.adapters.autocad_mcp_pro import ROUNDTRIP_AREAS_MM2 + + assert ROUNDTRIP_AREAS_MM2 == {"01": 27_743_750.0, "02": 21_993_750.0} + # the axis areas the rooms would report if the wall thickness were ignored + assert 5000.0 * 6000.0 not in ROUNDTRIP_AREAS_MM2.values() + assert 4000.0 * 6000.0 not in ROUNDTRIP_AREAS_MM2.values() + + +def test_the_suite_carries_the_four_architectural_checks(): + from benchmarks.correctness_suite import CHECKS + + for name in ( + "arch_junction_l_t_x", + "arch_room_area_net", + "arch_opening_cuts_wall", + "arch_rooms_detect_foreign", + ): + assert CHECKS[name][1] == "Architecture" + assert len(CHECKS) == 43 + + +@pytest.mark.asyncio +async def test_arch_roundtrip_passes_headlessly(tmp_path): + """The benchmark task itself, run once here so a regression in the wall + engine, the room reader, the schedules or the critique fails the suite and + not only the matrix.""" + from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter + + adapter = AutoCADMCPProAdapter(backend="ezdxf") + await adapter.setup(tmp_path) + try: + await adapter._reset() + passed, metrics, _artifacts = await adapter._task_arch_roundtrip() + finally: + await adapter.cleanup() + assert passed, metrics + assert metrics["critique_issues"] == 0, metrics["critique_focuses"] + assert metrics["score"] >= 90.0 + assert metrics["door_tags"] == ["D1", "D2"] and metrics["window_tags"] == ["W1", "W2"] + for number in ("01", "02"): + assert metrics["rooms"][number]["agrees"], metrics["rooms"] + + +@pytest.mark.asyncio +async def test_arch_roundtrip_fails_loudly_when_a_room_label_is_removed(tmp_path, monkeypatch): + """The gate has to be able to fail. One room left without its label is + enough: the reader still finds the face, the critique names + arch_room_unlabelled, and the room is missing from the area comparison.""" + from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter + from engineering.arch import rooms + + real = rooms.label_room + + async def skip_the_hall(backend, room_dict, **kw): + if room_dict.get("number") == "02": + return {"ok": True, "skipped": True} + return await real(backend, room_dict, **kw) + + monkeypatch.setattr(rooms, "label_room", skip_the_hall) + adapter = AutoCADMCPProAdapter(backend="ezdxf") + await adapter.setup(tmp_path) + try: + await adapter._reset() + passed, metrics, _artifacts = await adapter._task_arch_roundtrip() + finally: + await adapter.cleanup() + assert passed is False + assert "arch_room_unlabelled" in metrics["critique_focuses"] + assert "02" not in metrics["rooms"] + + +@pytest.mark.asyncio +async def test_the_net_area_check_measures_the_label(): + from benchmarks import correctness_suite + + assert await correctness_suite.arch_room_area_net() is True + + +@pytest.mark.asyncio +async def test_the_foreign_plan_check_passes(): + from benchmarks import correctness_suite + + assert await correctness_suite.arch_rooms_detect_foreign() is True diff --git a/tests/test_com_backend.py b/tests/test_com_backend.py index a993cda..f3de2cf 100644 --- a/tests/test_com_backend.py +++ b/tests/test_com_backend.py @@ -568,6 +568,59 @@ def test_a_dimension_reports_where_its_text_stands(): assert props["text_position"] == [181.0, 230.0] +class IAcadEntity: + """What pywin32 hands back for ``ModelSpace.Item(i)`` once a makepy cache + for the AutoCAD type library exists: acax25enu.tlb declares the result + ``IAcadEntity*``, so the wrapper is the gen_py class of that name and it + exposes only the entity base's members. Measured on AutoCAD 2026 + (2026-09-23, scripts/smoke_arch_com.py): ``type(item).__name__`` is + ``"IAcadEntity"`` and ``item.StartPoint`` on an AcDbLine raises + ``AttributeError: '' object has no attribute 'StartPoint'``. The concrete object is + reachable only through ``_oleobj_``.""" + + BASE_MEMBERS = frozenset( + {"ObjectName", "Handle", "Layer", "color", "Linetype", "Visible", "GetBoundingBox"} + ) + + def __init__(self, raw): + object.__setattr__(self, "_oleobj_", raw) + + def __getattr__(self, key): + if key in self.BASE_MEMBERS: + return getattr(self._oleobj_, key) + raise AttributeError( + "'' " + f"object has no attribute {key!r}" + ) + + +def test_a_line_listed_through_a_narrowed_wrapper_keeps_its_endpoints(): + """``entity_list`` reads every entity through ``ModelSpace.Item``. With a + makepy cache the LINE rows lost ``start``/``end`` (only the bounding box + survived), so on the live seat the room reader found 0 wall segments and + ``arch_plan_from_spec`` refused its first room label.""" + from backends.com_backend import _entity_info + + raw = _Entity( + "AcDbLine", + GetBoundingBox=lambda: ((125.0, 125.0, 0.0), (1000.0, 125.0, 0.0)), + StartPoint=(125.0, 125.0, 0.0), + EndPoint=(1000.0, 125.0, 0.0), + Length=875.0, + Handle="2B0", + Layer="A-WALL-E-N", + color=256, + Linetype="ByLayer", + Visible=True, + ) + info = _entity_info(IAcadEntity(raw)) + assert info.type == "LINE" and info.layer == "A-WALL-E-N" + assert info.properties["start"] == [125.0, 125.0] + assert info.properties["end"] == [1000.0, 125.0] + assert info.properties["length"] == 875.0 + + def test_com_geometry_bbox_is_the_drawn_body_without_the_attribute(): from backends.com_backend import _entity_info diff --git a/tests/test_token_suite.py b/tests/test_token_suite.py index 686d7d8..ee85259 100644 --- a/tests/test_token_suite.py +++ b/tests/test_token_suite.py @@ -161,25 +161,36 @@ def test_the_cache_arithmetic_follows_the_published_multipliers(): # -- lane 1: idle cost ------------------------------------------------------ -def test_the_idle_lane_covers_the_four_advertised_surfaces(report): - """Default, lean, search - and TOOL_PACKS=core,mech, the mechanical client - that never opens a P&ID, a live seat or a floor plan: the default surface - minus exactly the pid, settings and arch packs (track F added `arch`).""" +def test_the_idle_lane_covers_the_five_advertised_surfaces(report): + """Default, lean, search - and the two single-vertical clients: + TOOL_PACKS=core,mech (the default surface minus exactly the pid and settings + packs, and the arch pack) and TOOL_PACKS=core,arch (minus pid, settings and + mech).""" variants = {row["variant"]: row for row in report["idle"]} - assert set(variants) == {"default", "lean", "packs_core_mech", "search"} + assert set(variants) == {"default", "lean", "packs_core_mech", "packs_core_arch", "search"} assert variants["default"]["advertised_tools"] > 100 assert variants["lean"]["advertised_tools"] == len(server.LEAN_TOOL_NAMES) assert variants["search"]["advertised_tools"] == 2 - dropped = sum(len(server.PACK_TOOL_NAMES[pack]) for pack in ("pid", "settings", "arch")) + packs = server.PACK_TOOL_NAMES assert variants["packs_core_mech"]["advertised_tools"] == ( - variants["default"]["advertised_tools"] - dropped + variants["default"]["advertised_tools"] + - len(packs["pid"]) + - len(packs["settings"]) + - len(packs["arch"]) ) - assert ( - variants["default"]["chars"] - > variants["packs_core_mech"]["chars"] - > variants["lean"]["chars"] - > variants["search"]["chars"] + assert variants["packs_core_arch"]["advertised_tools"] == ( + variants["default"]["advertised_tools"] + - len(packs["pid"]) + - len(packs["settings"]) + - len(packs["mech"]) ) + for narrowed in ("packs_core_mech", "packs_core_arch"): + assert ( + variants["default"]["chars"] + > variants[narrowed]["chars"] + > variants["lean"]["chars"] + > variants["search"]["chars"] + ) def test_the_idle_lane_separates_the_wire_payload_from_what_the_model_pays(report):