Umut CANandClaude Opus 5.5
0abbe55041
The evidence for track F, in the shape tracks A, E and B + G used. - benchmarks/tasks_v6.py: 19 tasks, DEFAULT_MATRIX = "v6"; v5 to v2 stay addressable. arch_roundtrip draws the example plan with arch_plan_from_spec and reads it back without the spec: rooms_detect must find each labelled room within 0.1 % of its label and of the net floor computed by hand (4825 x 5750 and 3825 x 5750), the schedules must list D1, D2, W1, W2, and the plan is gated on a zero-issue critique and a finalize score of at least 90. A test removes one room label and the gate fails naming arch_room_unlabelled. - Four correctness checks, every number worked out by hand in the docstring: L / T / X junction corners and wall areas, a 4 x 5 m room labelled 20.00 m2 rather than its 21.84 m2 axis area, a door interrupting both faces, and a plan of plain lines read as 20 and 15 m2 at confidence 0.6. The published A/B report is re-run over 43 checks: 26 / 43 -> 43 / 43, zero regressed. - scripts/smoke_arch_com.py runs the whole plan once on a live seat and closes every document it opens; the CHANGELOG records what it returned. - The token suite measures a TOOL_PACKS=core,arch row; check_doc_numbers derives the task count from the v6 matrix; CLAUDE.md, both READMEs and the spec status line say what shipped. - fix(com): the first live smoke exited 1 - the room reader found 0 wall segments. Measured on AutoCAD 2026 (2026-09-23): with a makepy cache, ModelSpace.Item(i) returns the gen_py IAcadEntity wrapper (type(item).__name__ == "IAcadEntity") and item.StartPoint on an AcDbLine raises AttributeError, so every entity_list row lost its type-specific properties (28 wall LINEs, 0 with start/end). _entity_info now re-dispatches exactly that wrapper through _com_unnarrow; after the fix 38 of 53 rows carry start/end and the smoke exits 0 with both rooms at 27 743 750 / 21 993 750 mm2 and 0 critique issues. Re-dispatch measured at ~1.7 ms per entity (dynamic vs gencache alike). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
120 lines
4.4 KiB
Python
120 lines
4.4 KiB
Python
"""The v6 task matrix: v5 plus what track F added.
|
|
|
|
``arch_roundtrip`` is track F's evidence - a two-room plan drawn from one
|
|
spec, read back off the drawing by the room reader that never sees the spec,
|
|
and gated on the numbers the design spec fixed in advance: every room's
|
|
detected area within 0.1 % of its label's and of the hand-computed net floor,
|
|
the schedules listing the drawn tags, ``drawing_critique(focus=None)`` at
|
|
zero and the finalize score at least 90. The published competitor reports
|
|
were never asked it and the chart shows them ``not_run``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from benchmarks.tasks_v5 import TASKS_V5
|
|
from benchmarks.tasks_v6 import DEFAULT_MATRIX, MATRICES, NEW_TASKS_V6, TASKS_V6, task_by_id
|
|
|
|
|
|
def test_v6_extends_v5_without_touching_it():
|
|
assert TASKS_V6[: len(TASKS_V5)] == TASKS_V5
|
|
assert [t.task_id for t in NEW_TASKS_V6] == ["arch_roundtrip"]
|
|
assert len(TASKS_V6) == 19
|
|
assert MATRICES["v5"] == TASKS_V5 and MATRICES["v6"] == TASKS_V6
|
|
assert DEFAULT_MATRIX == "v6"
|
|
assert task_by_id("arch_roundtrip").category == "arch"
|
|
|
|
|
|
def test_the_adapter_implements_every_v6_task():
|
|
from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter
|
|
|
|
for task in TASKS_V6:
|
|
assert hasattr(AutoCADMCPProAdapter, f"_task_{task.task_id}"), task.task_id
|
|
|
|
|
|
def test_the_hand_areas_are_the_net_floor_not_the_axis_area():
|
|
from benchmarks.adapters.autocad_mcp_pro import ROUNDTRIP_AREAS_MM2
|
|
|
|
assert ROUNDTRIP_AREAS_MM2 == {"01": 27_743_750.0, "02": 21_993_750.0}
|
|
# the axis areas the rooms would report if the wall thickness were ignored
|
|
assert 5000.0 * 6000.0 not in ROUNDTRIP_AREAS_MM2.values()
|
|
assert 4000.0 * 6000.0 not in ROUNDTRIP_AREAS_MM2.values()
|
|
|
|
|
|
def test_the_suite_carries_the_four_architectural_checks():
|
|
from benchmarks.correctness_suite import CHECKS
|
|
|
|
for name in (
|
|
"arch_junction_l_t_x",
|
|
"arch_room_area_net",
|
|
"arch_opening_cuts_wall",
|
|
"arch_rooms_detect_foreign",
|
|
):
|
|
assert CHECKS[name][1] == "Architecture"
|
|
assert len(CHECKS) == 43
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_arch_roundtrip_passes_headlessly(tmp_path):
|
|
"""The benchmark task itself, run once here so a regression in the wall
|
|
engine, the room reader, the schedules or the critique fails the suite and
|
|
not only the matrix."""
|
|
from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter
|
|
|
|
adapter = AutoCADMCPProAdapter(backend="ezdxf")
|
|
await adapter.setup(tmp_path)
|
|
try:
|
|
await adapter._reset()
|
|
passed, metrics, _artifacts = await adapter._task_arch_roundtrip()
|
|
finally:
|
|
await adapter.cleanup()
|
|
assert passed, metrics
|
|
assert metrics["critique_issues"] == 0, metrics["critique_focuses"]
|
|
assert metrics["score"] >= 90.0
|
|
assert metrics["door_tags"] == ["D1", "D2"] and metrics["window_tags"] == ["W1", "W2"]
|
|
for number in ("01", "02"):
|
|
assert metrics["rooms"][number]["agrees"], metrics["rooms"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_arch_roundtrip_fails_loudly_when_a_room_label_is_removed(tmp_path, monkeypatch):
|
|
"""The gate has to be able to fail. One room left without its label is
|
|
enough: the reader still finds the face, the critique names
|
|
arch_room_unlabelled, and the room is missing from the area comparison."""
|
|
from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter
|
|
from engineering.arch import rooms
|
|
|
|
real = rooms.label_room
|
|
|
|
async def skip_the_hall(backend, room_dict, **kw):
|
|
if room_dict.get("number") == "02":
|
|
return {"ok": True, "skipped": True}
|
|
return await real(backend, room_dict, **kw)
|
|
|
|
monkeypatch.setattr(rooms, "label_room", skip_the_hall)
|
|
adapter = AutoCADMCPProAdapter(backend="ezdxf")
|
|
await adapter.setup(tmp_path)
|
|
try:
|
|
await adapter._reset()
|
|
passed, metrics, _artifacts = await adapter._task_arch_roundtrip()
|
|
finally:
|
|
await adapter.cleanup()
|
|
assert passed is False
|
|
assert "arch_room_unlabelled" in metrics["critique_focuses"]
|
|
assert "02" not in metrics["rooms"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_the_net_area_check_measures_the_label():
|
|
from benchmarks import correctness_suite
|
|
|
|
assert await correctness_suite.arch_room_area_net() is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_the_foreign_plan_check_passes():
|
|
from benchmarks import correctness_suite
|
|
|
|
assert await correctness_suite.arch_rooms_detect_foreign() is True
|