bench: matrix v4 adds page_setup_truth; correctness suite grows to 32 with three settings checks (A/B vs v1.5.1: 26 -> 32 / 32)
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
This commit is contained in:
co-authored by
Claude Opus 5
parent
02dbb133b5
commit
c2942d3c94
@@ -1,23 +1,28 @@
|
||||
"""Track A — the v4 task matrix.
|
||||
# tests/test_benchmark_v4.py
|
||||
"""The v4 task matrix: v3 plus what 1.6 added.
|
||||
|
||||
v4 adds one task, ``pid_roundtrip``: draw a P&ID through the same code the
|
||||
tools call and read it back through the graph builder, so the edge count, the
|
||||
dangling count and the instrument index are each derived independently of the
|
||||
drawing code. The published competitor reports were never asked this question
|
||||
and the chart shows them ``not_run`` — the same rule v3 set for its five.
|
||||
Track A added ``pid_roundtrip``; track E adds ``page_setup_truth`` — apply an
|
||||
ISO A3 landscape page setup, plot through ``batch_plot`` and read the sheet
|
||||
size back from the PDF's own ``/MediaBox``. The published competitor reports
|
||||
were never asked either question and the chart shows them ``not_run`` — the
|
||||
same rule v3 set for its five.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from benchmarks.tasks_v3 import TASKS_V3
|
||||
from benchmarks.tasks_v4 import DEFAULT_MATRIX, MATRICES, NEW_TASKS_V4, TASKS_V4, task_by_id
|
||||
|
||||
|
||||
def test_v4_extends_v3_without_touching_it():
|
||||
assert TASKS_V4[: len(TASKS_V3)] == TASKS_V3
|
||||
assert [t.task_id for t in NEW_TASKS_V4] == ["pid_roundtrip"]
|
||||
assert [t.task_id for t in NEW_TASKS_V4] == ["pid_roundtrip", "page_setup_truth"]
|
||||
assert len(TASKS_V4) == 17
|
||||
assert MATRICES["v3"] == TASKS_V3 and MATRICES["v4"] == TASKS_V4 and DEFAULT_MATRIX == "v4"
|
||||
assert task_by_id("pid_roundtrip").category == "pid"
|
||||
assert task_by_id("page_setup_truth").category == "pagesetup"
|
||||
|
||||
|
||||
def test_the_adapter_implements_every_v4_task():
|
||||
@@ -32,4 +37,33 @@ def test_correctness_suite_has_the_pid_checks():
|
||||
|
||||
for name in ("pid_block_define_attdef_roundtrip", "pid_tag_parse_fic", "pid_graph_edge_count"):
|
||||
assert CHECKS[name][1] == "P&ID"
|
||||
assert len(CHECKS) == 29
|
||||
|
||||
|
||||
def test_correctness_suite_has_the_settings_checks():
|
||||
from benchmarks.correctness_suite import CHECKS
|
||||
|
||||
for name in (
|
||||
"settings_dimstyle_iso25_values",
|
||||
"settings_layer_state_roundtrip",
|
||||
"settings_pdf_mediabox_a3",
|
||||
):
|
||||
assert CHECKS[name][1] == "Settings"
|
||||
assert len(CHECKS) == 32
|
||||
|
||||
|
||||
@pytest.mark.asyncio # asyncio_mode is strict; the four tests above are sync
|
||||
async def test_page_setup_truth_passes_headlessly(tmp_path):
|
||||
"""The benchmark task itself, run once here so a regression in page setup
|
||||
or the PDF read-back fails the suite and not only the matrix."""
|
||||
from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter
|
||||
|
||||
adapter = AutoCADMCPProAdapter(backend="ezdxf")
|
||||
await adapter.setup(tmp_path)
|
||||
try:
|
||||
await adapter._reset()
|
||||
passed, metrics, _artifacts = await adapter._task_page_setup_truth()
|
||||
finally:
|
||||
await adapter.cleanup()
|
||||
assert passed, metrics
|
||||
assert abs(metrics["mediabox_mm"][0] - 420.0) < 0.5
|
||||
assert abs(metrics["mediabox_mm"][1] - 297.0) < 0.5
|
||||
|
||||
Reference in New Issue
Block a user