bench: matrix v6 adds arch_roundtrip; four architectural correctness checks; the live arch smoke
The evidence for track F, in the shape tracks A, E and B + G used. - benchmarks/tasks_v6.py: 19 tasks, DEFAULT_MATRIX = "v6"; v5 to v2 stay addressable. arch_roundtrip draws the example plan with arch_plan_from_spec and reads it back without the spec: rooms_detect must find each labelled room within 0.1 % of its label and of the net floor computed by hand (4825 x 5750 and 3825 x 5750), the schedules must list D1, D2, W1, W2, and the plan is gated on a zero-issue critique and a finalize score of at least 90. A test removes one room label and the gate fails naming arch_room_unlabelled. - Four correctness checks, every number worked out by hand in the docstring: L / T / X junction corners and wall areas, a 4 x 5 m room labelled 20.00 m2 rather than its 21.84 m2 axis area, a door interrupting both faces, and a plan of plain lines read as 20 and 15 m2 at confidence 0.6. The published A/B report is re-run over 43 checks: 26 / 43 -> 43 / 43, zero regressed. - scripts/smoke_arch_com.py runs the whole plan once on a live seat and closes every document it opens; the CHANGELOG records what it returned. - The token suite measures a TOOL_PACKS=core,arch row; check_doc_numbers derives the task count from the v6 matrix; CLAUDE.md, both READMEs and the spec status line say what shipped. - fix(com): the first live smoke exited 1 - the room reader found 0 wall segments. Measured on AutoCAD 2026 (2026-09-23): with a makepy cache, ModelSpace.Item(i) returns the gen_py IAcadEntity wrapper (type(item).__name__ == "IAcadEntity") and item.StartPoint on an AcDbLine raises AttributeError, so every entity_list row lost its type-specific properties (28 wall LINEs, 0 with start/end). _entity_info now re-dispatches exactly that wrapper through _com_unnarrow; after the fix 38 of 53 rows carry start/end and the smoke exits 0 with both rooms at 27 743 750 / 21 993 750 mm2 and 0 critique issues. Re-dispatch measured at ~1.7 ms per entity (dynamic vs gencache alike). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
This commit is contained in:
Umut CAN
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
6e004cc658
commit
0abbe55041
@@ -1,4 +1,4 @@
|
||||
"""Reference adapter for this repository, covering the v5 task matrix.
|
||||
"""Reference adapter for this repository, covering the v6 task matrix.
|
||||
|
||||
Every task verifies against a value worked out independently of the code under
|
||||
test — a closed-form area, a count of entities placed on purpose, a token
|
||||
@@ -12,7 +12,7 @@ import math
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.adapters.base import BenchmarkAdapter, TaskResult
|
||||
from benchmarks.tasks_v5 import TaskSpec
|
||||
from benchmarks.tasks_v6 import TaskSpec
|
||||
from engineering.delivery import deliver_drawing
|
||||
from engineering.layers import ensure_engineering_layers, ensure_standard_linetypes
|
||||
from engineering.refiner import refine_drawing
|
||||
@@ -40,6 +40,15 @@ DISCOVERY_TOP_N = 3
|
||||
#: rather than derived from the measurement, so the measurement can miss it.
|
||||
TOKEN_CEILING = 2_000
|
||||
|
||||
#: The net floor of the two rooms of ``engineering/arch/spec.py::EXAMPLE_SPEC``,
|
||||
#: computed by hand: the 250 mm ring on axis (0,0)-(9000,6000) has its inner
|
||||
#: faces at x = 125 / 8875 and y = 125 / 5875, the 100 mm wall on x = 5000 its
|
||||
#: faces at 4950 / 5050. Room 01: 4825 x 5750; room 02: 3825 x 5750.
|
||||
ROUNDTRIP_AREAS_MM2: dict[str, float] = {
|
||||
"01": 4825.0 * 5750.0,
|
||||
"02": 3825.0 * 5750.0,
|
||||
}
|
||||
|
||||
|
||||
class AutoCADMCPProAdapter(BenchmarkAdapter):
|
||||
name = "autocad-mcp-pro"
|
||||
@@ -682,6 +691,94 @@ class AutoCADMCPProAdapter(BenchmarkAdapter):
|
||||
[str(sheet)],
|
||||
)
|
||||
|
||||
async def _task_arch_roundtrip(self):
|
||||
"""A two-room plan drawn from one spec and read back off the drawing.
|
||||
|
||||
Track F's evidence (spec §10). ``arch_plan_from_spec`` draws the plan of
|
||||
``engineering/arch/spec.py::EXAMPLE_SPEC`` - a 9 x 6 m brick ring at 250
|
||||
mm split by a 100 mm AAC wall, an entrance door, an interior door, two
|
||||
windows, a straight stair, two labelled rooms, the exterior chains and
|
||||
the three schedules. Then the drawing is read, never the spec:
|
||||
|
||||
* ``rooms_detect`` on the wall layer must find each labelled room, and
|
||||
its area must agree with the area the label's record carries within
|
||||
0.1 %, and with the net floor computed by hand in
|
||||
``ROUNDTRIP_AREAS_MM2`` - so a label that typed the axis area fails;
|
||||
* the door and window schedules read off the records must list exactly
|
||||
the drawn tags;
|
||||
* ``run_critique(focus=None)`` returns zero issues and the finalize
|
||||
score - ``combine(validator.summary, critique)`` over the saved plan -
|
||||
is at least 90.
|
||||
|
||||
Any one slipping fails the task, and the metrics name what slipped.
|
||||
"""
|
||||
from engineering import DrawingValidator
|
||||
from engineering.arch.critique import point_in_loop
|
||||
from engineering.arch.draw import read_plan
|
||||
from engineering.arch.layers import ARCH_ROLE_LAYER
|
||||
from engineering.arch.rooms import rooms_detect
|
||||
from engineering.arch.schedule import schedule_rows
|
||||
from engineering.arch.spec import EXAMPLE_SPEC, draw_plan_from_spec
|
||||
from engineering.critique import run_critique
|
||||
from engineering.scoring import combine
|
||||
|
||||
drawn = await draw_plan_from_spec(self.backend, EXAMPLE_SPEC)
|
||||
plan = await read_plan(self.backend)
|
||||
detected = await rooms_detect(self.backend, layers=[ARCH_ROLE_LAYER["wall"]])
|
||||
faces = list(detected.get("rooms") or [])
|
||||
|
||||
areas: dict[str, dict] = {}
|
||||
for room in plan["rooms"]:
|
||||
face = next((f for f in faces if point_in_loop(room.at, f["loop"])), None)
|
||||
detected_area = float(face["area"]) if face is not None else 0.0
|
||||
hand = ROUNDTRIP_AREAS_MM2.get(str(room.number))
|
||||
areas[str(room.number)] = {
|
||||
"label_mm2": float(room.area),
|
||||
"detected_mm2": detected_area,
|
||||
"hand_mm2": hand,
|
||||
"agrees": face is not None
|
||||
and hand is not None
|
||||
and abs(detected_area - float(room.area)) <= 1e-3 * float(room.area)
|
||||
and abs(detected_area - hand) <= 1e-3 * hand,
|
||||
}
|
||||
rooms_ok = (
|
||||
set(areas) == set(ROUNDTRIP_AREAS_MM2)
|
||||
and len(faces) == len(ROUNDTRIP_AREAS_MM2)
|
||||
and all(row["agrees"] for row in areas.values())
|
||||
)
|
||||
|
||||
door_tags = sorted(str(row["tag"]) for row in schedule_rows("doors", plan, lang="en"))
|
||||
window_tags = sorted(str(row["tag"]) for row in schedule_rows("windows", plan, lang="en"))
|
||||
tags_ok = door_tags == ["D1", "D2"] and window_tags == ["W1", "W2"]
|
||||
|
||||
sheet = self.artifact_dir / "arch_roundtrip.dxf"
|
||||
await self.backend.drawing_save_as(str(sheet))
|
||||
issues = await run_critique(self.backend, None)
|
||||
validation = await DrawingValidator().run(self.backend)
|
||||
summary = {"error": 0, "warning": 0, "info": 0}
|
||||
for issue in issues:
|
||||
summary[issue.severity] = summary.get(issue.severity, 0) + 1
|
||||
score = combine(validation.summary, summary)
|
||||
passed = rooms_ok and tags_ok and len(issues) == 0 and float(score["score"]) >= 90.0
|
||||
return (
|
||||
passed,
|
||||
{
|
||||
"rooms": areas,
|
||||
"faces_detected": len(faces),
|
||||
"confidence_min": detected.get("confidence_min"),
|
||||
"door_tags": door_tags,
|
||||
"window_tags": window_tags,
|
||||
"omitted": drawn.get("omitted"),
|
||||
"critique_issues": len(issues),
|
||||
"critique_focuses": sorted({issue.focus for issue in issues}),
|
||||
"score": float(score["score"]),
|
||||
"grade": score["grade"],
|
||||
"invalidity_ratio": score["invalidity_ratio"],
|
||||
"validator_findings": sorted(f.code for f in validation.findings),
|
||||
},
|
||||
[str(sheet)],
|
||||
)
|
||||
|
||||
async def _task_auditable_delivery(self):
|
||||
await ensure_engineering_layers(self.backend)
|
||||
await self.backend.entity_create_line(0, 0, 10, 0, layer="GEOMETRY")
|
||||
|
||||
Reference in New Issue
Block a user