Files
autocad-mcp-pro/tests/test_engineering_validator.py
Umut CANandClaude Opus 5.5 898e775f93 bench: matrix v5 adds mech_assembly; the correctness suite grows with the mechanical and sheet checks
The evidence for tracks B and G, in the shape tracks A and E used.

- benchmarks/tasks_v5.py: 18 tasks, DEFAULT_MATRIX = "v5"; v4, v3 and v2 stay
  addressable so an earlier report reproduces exactly rather than being only
  described. mech_assembly is roadmap criterion 3 - a flange-coupling sheet
  built from the part model and the sheet standard, through the code the
  tools call (apply_titleblock, draw_part/add_view/dimension_part,
  insert_std_part, extract_records -> draw_bom_table, add_balloon), gated on
  two numbers that cannot be negotiated: drawing_critique(focus=None) returns
  zero issues and the finalize score is at least 90 (it scores 100, 18/18 on
  the v5 matrix). The adapter returns the focuses that fired, so a
  regression names itself, and a test drops one balloon to prove the gate
  fails and names mech_bom_balloon_mismatch.
- Seven correctness checks (32 -> 39). mech_section_hatch_area is the one
  that cannot be faked: a sleeve of length 60, outside diameter 40 and bore
  20 cuts two faces totalling 1200 mm^2 exactly, read back out of the drawing
  through entity_measure. sheet_frame_iso5457_a3 pins the A3 frame (20,10 to
  410,287 inside the 420 x 297 trimmed sheet); mech_iso286_on_dimension pins
  40 H7 at +0.025/0 through the layout; std_part_iso4014_m12 pins s = 18 and
  k = 7.5 by value; mech_thread_unrepresented_is_caught is the seventh: a
  quality gate that never fires is not a gate.
- scripts/smoke_mech_com.py executed every new COM path once on AutoCAD 2026
  (25.1s): frame, title block, a sectioned shaft with 12 dimensions, an ISO
  4014 block with 1 ACADMCP_MECH chunk, a weld symbol, the parts list read off
  the drawing, a linked balloon, an xref attached / listed / detached, a
  31687-byte .dwg (AC1032) through ActiveX SaveAs, critique 0 issues, exit 0,
  every scratch document closed. It retries RPC_E_CALL_REJECTED (measured:
  the first call after Documents.Add is refused) and tracks its documents by
  the document list, since SaveAs renames them.
- Defects the evidence found, each fixed with a test that fails on the old
  code:
  * ezdxf dimension_diameter put the text beyond the SECOND chord point;
    ActiveX AddDimDiametric puts it beyond the first (measured live: a d50
    from (x,170) to (x,220), leader 10, text at (x,160)). Headlessly every
    front-view diameter stood on its own chain dimension. ezdxf now follows
    ActiveX.
  * draw_part's anchor goes on MECH; ActiveX refuses entity.Layer on a
    missing layer (measured -0x7ffdfff7 'Key not found'). draw.py ensures
    every target layer before the first entity.
  * bom.extract_records filtered on INSERT; ActiveX names it BLOCKREFERENCE
    (ObjectName AcDbBlockReference) - the live parts list was empty.
  * COM _entity_info reported no dimension text_position: ModelSpace.Item
    narrows to IAcadEntity, which has no TextPosition (measured
    AttributeError); read through _com_unnarrow (TextPosition measured on
    AcDbDiametricDimension, WCS). dim_overlap compared bbox centres and a
    diameter collided with its own bore.
  * pid graph raised KeyError 'start' mid-critique on a LINE whose point read
    AutoCAD refused while busy; such a line is left out.
  * the validator read "ISO 4014 - M12x60" as fake dimension text.
- The published A/B report is re-run over the grown suite rather than
  re-typed (26/39 -> 39/39, 13 fixed, 0 regressed), check_doc_numbers derives
  the task count from the v5 matrix, the token table is remeasured with a
  TOOL_PACKS=core,mech row (193 tools / 61,673 tokens), and the README's
  collected-tests figure is retaken (3990).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
2026-09-23 13:30:02 +03:00

163 lines
6.0 KiB
Python

"""Tests for engineering.validator — 8-step pre-completion validator."""
import os
import pytest
from engineering.layers import ensure_engineering_layers, ensure_standard_linetypes
from engineering.validator import DrawingValidator
pytestmark = pytest.mark.asyncio
def _codes(result) -> set[str]:
return {f.code for f in result.findings}
async def _ensure_hidden_linetype(backend) -> None:
"""ezdxf doesn't ship a HIDDEN pattern; add a synthetic one so the validator passes."""
doc = backend._require_doc()
if "HIDDEN" not in doc.linetypes:
doc.linetypes.add(
"HIDDEN",
pattern=[6.35, 5.0, -1.27],
description="Hidden _ _ _ _ _ _",
)
async def _setup_clean(backend, tmp_path):
"""Bootstrap layers/linetypes, draw a bore + a real DIMENSION, save to disk."""
await ensure_standard_linetypes(backend)
await _ensure_hidden_linetype(backend)
await ensure_engineering_layers(backend)
# Bore on GEOMETRY layer
await backend.entity_create_circle(0, 0, 12.5, layer="GEOMETRY")
# Real dimension entity
await backend.dimension_linear(0, 0, 25, 0, dim_x=12.5, dim_y=20, layer="DIM")
save_path = str(tmp_path / "clean.dxf")
await backend.drawing_save_as(save_path)
return save_path
async def test_validator_passes_clean_drawing(backend, tmp_path):
await _setup_clean(backend, tmp_path)
result = await DrawingValidator().run(backend)
# Should have zero errors; warnings/info okay (e.g. screenshot info)
assert result.ok is True, f"unexpected findings: {[f.code for f in result.findings]}"
assert result.summary["error"] == 0
async def test_validator_catches_unsaved_drawing(backend):
await ensure_standard_linetypes(backend)
await ensure_engineering_layers(backend)
await backend.entity_create_circle(0, 0, 10, layer="GEOMETRY")
# Do not save
result = await DrawingValidator().run(backend)
assert "not_saved" in _codes(result)
assert result.ok is False
async def test_validator_catches_missing_file(backend, tmp_path):
await ensure_standard_linetypes(backend)
await ensure_engineering_layers(backend)
save_path = str(tmp_path / "vanish.dxf")
await backend.drawing_save_as(save_path)
# Delete the file out from under the backend
os.remove(save_path)
result = await DrawingValidator().run(backend)
assert "file_missing" in _codes(result)
assert result.ok is False
async def test_validator_catches_helical_spur_title(backend, tmp_path):
await ensure_standard_linetypes(backend)
await ensure_engineering_layers(backend)
await backend.entity_create_text(
text="HELICAL SPUR GEAR",
x=0,
y=0,
height=5,
layer="TEXT",
)
await backend.drawing_save_as(str(tmp_path / "tb.dxf"))
result = await DrawingValidator().run(backend, expected={"helix_angle": 20})
codes = _codes(result)
assert "title_helical_spur" in codes or "title_says_spur_for_helical" in codes
assert result.ok is False
async def test_validator_catches_fake_dimension_text(backend, tmp_path):
await ensure_standard_linetypes(backend)
await ensure_engineering_layers(backend)
await backend.entity_create_text(
text="Ø78",
x=10,
y=10,
height=3.5,
layer="DIM",
)
await backend.drawing_save_as(str(tmp_path / "fake.dxf"))
result = await DrawingValidator().run(backend)
assert "fake_dimension_text" in _codes(result)
async def test_a_standard_part_designation_is_not_a_fake_dimension():
"""A parts-list row names a part: "ISO 4014 - M12x60" is the ISO 4014
designation grammar, not a measurement. Found by the v5 mech_assembly
benchmark, where every text parts list cost the sheet a warning. The
exemption is the designation only - a measurement beside it still flags."""
from engineering.validator import looks_like_dimension
assert not looks_like_dimension("ISO 4014 - M12x60")
assert not looks_like_dimension("DIN 912 - M8x20")
assert not looks_like_dimension("ISO 4032 - M12")
assert looks_like_dimension("ISO 4014 - M12x60 Ø78")
assert looks_like_dimension("40x60")
assert looks_like_dimension("Ø78")
async def test_validator_catches_hidden_orphans(backend, tmp_path):
await ensure_standard_linetypes(backend)
await ensure_engineering_layers(backend)
info = await backend.entity_create_circle(0, 0, 10, layer="GEOMETRY")
await backend.entity_set_properties(info.handle, visible=False)
await backend.drawing_save_as(str(tmp_path / "hidden.dxf"))
result = await DrawingValidator().run(backend)
assert "hidden_orphans" in _codes(result)
assert result.ok is False
async def test_validator_catches_missing_linetypes(backend, tmp_path):
"""Fresh drawing without ensure_standard_linetypes lacks CENTER/HIDDEN/PHANTOM."""
# Skip the bootstrap step entirely
await backend.entity_create_line(0, 0, 100, 0)
await backend.drawing_save_as(str(tmp_path / "bare.dxf"))
result = await DrawingValidator().run(backend)
codes = _codes(result)
assert "linetypes_missing" in codes
# And the missing detail names them
missing_finding = next(f for f in result.findings if f.code == "linetypes_missing")
missing_set = set(missing_finding.detail.get("missing", []))
assert {"center", "hidden", "phantom"}.issubset(missing_set)
async def test_validator_bore_missing_when_expected(backend, tmp_path):
await ensure_standard_linetypes(backend)
await ensure_engineering_layers(backend)
# No circle drawn at all
await backend.drawing_save_as(str(tmp_path / "nobore.dxf"))
result = await DrawingValidator().run(
backend,
expected={"must_have_bore": True},
)
assert "bore_missing" in _codes(result)
async def test_validation_result_to_dict_serializes(backend):
result = await DrawingValidator().run(backend)
d = result.to_dict()
assert "ok" in d
assert "summary" in d
assert "findings" in d
assert isinstance(d["findings"], list)