bench: matrix v4 adds pid_roundtrip; correctness suite grows to 29 with three P&ID checks
tasks_v4.py extends v3 with one task the reader can fail on its own: the example spec is drawn through pid_from_spec's code and read back by the graph builder, which never sees the spec. The runner, the chart and the doc-number gate now default to v4; --matrix v3 / v2 reproduce the earlier sets, and tests/test_benchmark_v3.py labels its full-matrix run "v3" explicitly now that the default has moved. The plan's adapter check expected the two unnumbered lines to come back with no line number; draw_line builds one from the default number_format (bare sequence when size/service/spec/insulation are empty), so the task asserts the two authored numbers on the two process lines and sequence- built XDATA numbers on the other two instead. The published A/B report still covers 26 checks; Task 21 re-runs it against v1.5.1, which is what clears the remaining doc-number drift. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd
This commit is contained in:
co-authored by
Claude Opus 5
parent
e911a3a8fb
commit
8d9c444b76
@@ -1,4 +1,4 @@
|
||||
"""Reference adapter for this repository, covering the v3 task matrix.
|
||||
"""Reference adapter for this repository, covering the v4 task matrix.
|
||||
|
||||
Every task verifies against a value worked out independently of the code under
|
||||
test — a closed-form area, a count of entities placed on purpose, a token
|
||||
@@ -12,7 +12,7 @@ import math
|
||||
from pathlib import Path
|
||||
|
||||
from benchmarks.adapters.base import BenchmarkAdapter, TaskResult
|
||||
from benchmarks.tasks_v3 import TaskSpec
|
||||
from benchmarks.tasks_v4 import TaskSpec
|
||||
from engineering.delivery import deliver_drawing
|
||||
from engineering.layers import ensure_engineering_layers, ensure_standard_linetypes
|
||||
from engineering.refiner import refine_drawing
|
||||
@@ -364,6 +364,65 @@ class AutoCADMCPProAdapter(BenchmarkAdapter):
|
||||
[],
|
||||
)
|
||||
|
||||
async def _task_pid_roundtrip(self):
|
||||
"""Five symbols, four lines, read back with nothing dangling.
|
||||
|
||||
The spec is drawn through the same code ``pid_from_spec`` calls; the
|
||||
graph, the instrument index and the line list are then read back from
|
||||
the drawing by the reader, which never sees the spec. Every expected
|
||||
number below is counted from the spec by hand.
|
||||
"""
|
||||
from engineering.layers import apply_layer_set
|
||||
from engineering.pid.deliverables import instrument_index, line_list
|
||||
from engineering.pid.spec import EXAMPLE_SPEC, run_spec
|
||||
|
||||
await apply_layer_set(self.backend, "pid")
|
||||
result = await run_spec(self.backend, EXAMPLE_SPEC)
|
||||
graph = result["graph"]
|
||||
index = instrument_index(graph)
|
||||
lines = line_list(graph)
|
||||
# The spec numbers its two process lines; the signal line and the
|
||||
# connector line are left to the default ``number_format``, which with
|
||||
# no size/service/spec/insulation collapses to the bare sequence
|
||||
# number. Both must come back through XDATA, not a label search.
|
||||
authored = "100-P-1001-CS1"
|
||||
numbers = {(r["from_tag"], r["to_tag"]): r["line_number"] for r in lines}
|
||||
authored_ok = (
|
||||
numbers.get(("P-101", "FCV-101")) == authored
|
||||
and numbers.get(("FCV-101", "V-201")) == authored
|
||||
)
|
||||
sequenced = [r for r in lines if r["line_number"] != authored]
|
||||
sequenced_ok = len(sequenced) == 2 and all(
|
||||
r["line_number"].isdigit() and r["number_source"] == "xdata" for r in sequenced
|
||||
)
|
||||
passed = (
|
||||
len(graph["nodes"]) == 5
|
||||
and len(graph["edges"]) == 4
|
||||
and graph["stats"]["dangling"] == 0
|
||||
and graph["stats"]["confidence_min"] == 1.0
|
||||
and result["critique"] == []
|
||||
and [r["tag"] for r in index] == ["FIC-101"]
|
||||
and index[0]["connected_to"] == "FCV-101"
|
||||
and len(lines) == 4
|
||||
and authored_ok
|
||||
and sequenced_ok
|
||||
)
|
||||
return (
|
||||
passed,
|
||||
{
|
||||
"nodes": len(graph["nodes"]),
|
||||
"edges": len(graph["edges"]),
|
||||
"dangling": graph["stats"]["dangling"],
|
||||
"confidence_min": graph["stats"]["confidence_min"],
|
||||
"critique_issues": len(result["critique"]),
|
||||
"instrument_index_rows": len(index),
|
||||
"line_list_rows": len(lines),
|
||||
"authored_line_numbers_kept": authored_ok,
|
||||
"unnumbered_lines_sequenced": sequenced_ok,
|
||||
},
|
||||
[],
|
||||
)
|
||||
|
||||
async def _task_auditable_delivery(self):
|
||||
await ensure_engineering_layers(self.backend)
|
||||
await self.backend.entity_create_line(0, 0, 10, 0, layer="GEOMETRY")
|
||||
|
||||
Reference in New Issue
Block a user