bench: matrix v4 adds page_setup_truth; correctness suite grows to 32 with three settings checks (A/B vs v1.5.1: 26 -> 32 / 32)

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
This commit is contained in:
Umut CAN
2026-09-22 13:07:31 +03:00
co-authored by Claude Opus 5
parent 02dbb133b5
commit c2942d3c94
8 changed files with 230 additions and 40 deletions
+42 -8
View File
@@ -1,23 +1,28 @@
"""Track A — the v4 task matrix.
# tests/test_benchmark_v4.py
"""The v4 task matrix: v3 plus what 1.6 added.
v4 adds one task, ``pid_roundtrip``: draw a P&ID through the same code the
tools call and read it back through the graph builder, so the edge count, the
dangling count and the instrument index are each derived independently of the
drawing code. The published competitor reports were never asked this question
and the chart shows them ``not_run`` — the same rule v3 set for its five.
Track A added ``pid_roundtrip``; track E adds ``page_setup_truth`` — apply an
ISO A3 landscape page setup, plot through ``batch_plot`` and read the sheet
size back from the PDF's own ``/MediaBox``. The published competitor reports
were never asked either question and the chart shows them ``not_run`` — the
same rule v3 set for its five.
"""
from __future__ import annotations
import pytest
from benchmarks.tasks_v3 import TASKS_V3
from benchmarks.tasks_v4 import DEFAULT_MATRIX, MATRICES, NEW_TASKS_V4, TASKS_V4, task_by_id
def test_v4_extends_v3_without_touching_it():
assert TASKS_V4[: len(TASKS_V3)] == TASKS_V3
assert [t.task_id for t in NEW_TASKS_V4] == ["pid_roundtrip"]
assert [t.task_id for t in NEW_TASKS_V4] == ["pid_roundtrip", "page_setup_truth"]
assert len(TASKS_V4) == 17
assert MATRICES["v3"] == TASKS_V3 and MATRICES["v4"] == TASKS_V4 and DEFAULT_MATRIX == "v4"
assert task_by_id("pid_roundtrip").category == "pid"
assert task_by_id("page_setup_truth").category == "pagesetup"
def test_the_adapter_implements_every_v4_task():
@@ -32,4 +37,33 @@ def test_correctness_suite_has_the_pid_checks():
for name in ("pid_block_define_attdef_roundtrip", "pid_tag_parse_fic", "pid_graph_edge_count"):
assert CHECKS[name][1] == "P&ID"
assert len(CHECKS) == 29
def test_correctness_suite_has_the_settings_checks():
from benchmarks.correctness_suite import CHECKS
for name in (
"settings_dimstyle_iso25_values",
"settings_layer_state_roundtrip",
"settings_pdf_mediabox_a3",
):
assert CHECKS[name][1] == "Settings"
assert len(CHECKS) == 32
@pytest.mark.asyncio # asyncio_mode is strict; the four tests above are sync
async def test_page_setup_truth_passes_headlessly(tmp_path):
"""The benchmark task itself, run once here so a regression in page setup
or the PDF read-back fails the suite and not only the matrix."""
from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter
adapter = AutoCADMCPProAdapter(backend="ezdxf")
await adapter.setup(tmp_path)
try:
await adapter._reset()
passed, metrics, _artifacts = await adapter._task_page_setup_truth()
finally:
await adapter.cleanup()
assert passed, metrics
assert abs(metrics["mediabox_mm"][0] - 420.0) < 0.5
assert abs(metrics["mediabox_mm"][1] - 297.0) < 0.5