Umut CANandClaude Opus 5
8d9c444b76
tasks_v4.py extends v3 with one task the reader can fail on its own: the example spec is drawn through pid_from_spec's code and read back by the graph builder, which never sees the spec. The runner, the chart and the doc-number gate now default to v4; --matrix v3 / v2 reproduce the earlier sets, and tests/test_benchmark_v3.py labels its full-matrix run "v3" explicitly now that the default has moved. The plan's adapter check expected the two unnumbered lines to come back with no line number; draw_line builds one from the default number_format (bare sequence when size/service/spec/insulation are empty), so the task asserts the two authored numbers on the two process lines and sequence- built XDATA numbers on the other two instead. The published A/B report still covers 26 checks; Task 21 re-runs it against v1.5.1, which is what clears the remaining doc-number drift. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
36 lines
1.4 KiB
Python
36 lines
1.4 KiB
Python
"""Track A — the v4 task matrix.
|
|
|
|
v4 adds one task, ``pid_roundtrip``: draw a P&ID through the same code the
|
|
tools call and read it back through the graph builder, so the edge count, the
|
|
dangling count and the instrument index are each derived independently of the
|
|
drawing code. The published competitor reports were never asked this question
|
|
and the chart shows them ``not_run`` — the same rule v3 set for its five.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from benchmarks.tasks_v3 import TASKS_V3
|
|
from benchmarks.tasks_v4 import DEFAULT_MATRIX, MATRICES, NEW_TASKS_V4, TASKS_V4, task_by_id
|
|
|
|
|
|
def test_v4_extends_v3_without_touching_it():
|
|
assert TASKS_V4[: len(TASKS_V3)] == TASKS_V3
|
|
assert [t.task_id for t in NEW_TASKS_V4] == ["pid_roundtrip"]
|
|
assert MATRICES["v3"] == TASKS_V3 and MATRICES["v4"] == TASKS_V4 and DEFAULT_MATRIX == "v4"
|
|
assert task_by_id("pid_roundtrip").category == "pid"
|
|
|
|
|
|
def test_the_adapter_implements_every_v4_task():
|
|
from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter
|
|
|
|
for task in TASKS_V4:
|
|
assert hasattr(AutoCADMCPProAdapter, f"_task_{task.task_id}"), task.task_id
|
|
|
|
|
|
def test_correctness_suite_has_the_pid_checks():
|
|
from benchmarks.correctness_suite import CHECKS
|
|
|
|
for name in ("pid_block_define_attdef_roundtrip", "pid_tag_parse_fic", "pid_graph_edge_count"):
|
|
assert CHECKS[name][1] == "P&ID"
|
|
assert len(CHECKS) == 29
|