bench: matrix v4 adds pid_roundtrip; correctness suite grows to 29 with three P&ID checks

tasks_v4.py extends v3 with one task the reader can fail on its own: the
example spec is drawn through pid_from_spec's code and read back by the
graph builder, which never sees the spec. The runner, the chart and the
doc-number gate now default to v4; --matrix v3 / v2 reproduce the earlier
sets, and tests/test_benchmark_v3.py labels its full-matrix run "v3"
explicitly now that the default has moved.

The plan's adapter check expected the two unnumbered lines to come back
with no line number; draw_line builds one from the default number_format
(bare sequence when size/service/spec/insulation are empty), so the task
asserts the two authored numbers on the two process lines and sequence-
built XDATA numbers on the other two instead.

The published A/B report still covers 26 checks; Task 21 re-runs it
against v1.5.1, which is what clears the remaining doc-number drift.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
This commit is contained in:
Umut CAN
2026-09-16 13:06:52 +03:00
co-authored by Claude Opus 5
parent e911a3a8fb
commit 8d9c444b76
11 changed files with 192 additions and 16 deletions
+61 -2
View File
@@ -1,4 +1,4 @@
"""Reference adapter for this repository, covering the v3 task matrix.
"""Reference adapter for this repository, covering the v4 task matrix.
Every task verifies against a value worked out independently of the code under
test — a closed-form area, a count of entities placed on purpose, a token
@@ -12,7 +12,7 @@ import math
from pathlib import Path
from benchmarks.adapters.base import BenchmarkAdapter, TaskResult
from benchmarks.tasks_v3 import TaskSpec
from benchmarks.tasks_v4 import TaskSpec
from engineering.delivery import deliver_drawing
from engineering.layers import ensure_engineering_layers, ensure_standard_linetypes
from engineering.refiner import refine_drawing
@@ -364,6 +364,65 @@ class AutoCADMCPProAdapter(BenchmarkAdapter):
[],
)
async def _task_pid_roundtrip(self):
"""Five symbols, four lines, read back with nothing dangling.
The spec is drawn through the same code ``pid_from_spec`` calls; the
graph, the instrument index and the line list are then read back from
the drawing by the reader, which never sees the spec. Every expected
number below is counted from the spec by hand.
"""
from engineering.layers import apply_layer_set
from engineering.pid.deliverables import instrument_index, line_list
from engineering.pid.spec import EXAMPLE_SPEC, run_spec
await apply_layer_set(self.backend, "pid")
result = await run_spec(self.backend, EXAMPLE_SPEC)
graph = result["graph"]
index = instrument_index(graph)
lines = line_list(graph)
# The spec numbers its two process lines; the signal line and the
# connector line are left to the default ``number_format``, which with
# no size/service/spec/insulation collapses to the bare sequence
# number. Both must come back through XDATA, not a label search.
authored = "100-P-1001-CS1"
numbers = {(r["from_tag"], r["to_tag"]): r["line_number"] for r in lines}
authored_ok = (
numbers.get(("P-101", "FCV-101")) == authored
and numbers.get(("FCV-101", "V-201")) == authored
)
sequenced = [r for r in lines if r["line_number"] != authored]
sequenced_ok = len(sequenced) == 2 and all(
r["line_number"].isdigit() and r["number_source"] == "xdata" for r in sequenced
)
passed = (
len(graph["nodes"]) == 5
and len(graph["edges"]) == 4
and graph["stats"]["dangling"] == 0
and graph["stats"]["confidence_min"] == 1.0
and result["critique"] == []
and [r["tag"] for r in index] == ["FIC-101"]
and index[0]["connected_to"] == "FCV-101"
and len(lines) == 4
and authored_ok
and sequenced_ok
)
return (
passed,
{
"nodes": len(graph["nodes"]),
"edges": len(graph["edges"]),
"dangling": graph["stats"]["dangling"],
"confidence_min": graph["stats"]["confidence_min"],
"critique_issues": len(result["critique"]),
"instrument_index_rows": len(index),
"line_list_rows": len(lines),
"authored_line_numbers_kept": authored_ok,
"unnumbered_lines_sequenced": sequenced_ok,
},
[],
)
async def _task_auditable_delivery(self):
await ensure_engineering_layers(self.backend)
await self.backend.entity_create_line(0, 0, 10, 0, layer="GEOMETRY")