From 8d9c444b7648eed398d3028e38bac0d8dc0d5d6d Mon Sep 17 00:00:00 2001 From: Umut CAN Date: Wed, 16 Sep 2026 13:06:52 +0300 Subject: [PATCH] bench: matrix v4 adds pid_roundtrip; correctness suite grows to 29 with three P&ID checks tasks_v4.py extends v3 with one task the reader can fail on its own: the example spec is drawn through pid_from_spec's code and read back by the graph builder, which never sees the spec. The runner, the chart and the doc-number gate now default to v4; --matrix v3 / v2 reproduce the earlier sets, and tests/test_benchmark_v3.py labels its full-matrix run "v3" explicitly now that the default has moved. The plan's adapter check expected the two unnumbered lines to come back with no line number; draw_line builds one from the default number_format (bare sequence when size/service/spec/insulation are empty), so the task asserts the two authored numbers on the two process lines and sequence- built XDATA numbers on the other two instead. The published A/B report still covers 26 checks; Task 21 re-runs it against v1.5.1, which is what clears the remaining doc-number drift. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd --- CLAUDE.md | 2 +- README.md | 2 +- benchmarks/README.md | 21 +++++++-- benchmarks/adapters/autocad_mcp_pro.py | 63 +++++++++++++++++++++++++- benchmarks/correctness_suite.py | 31 +++++++++++++ benchmarks/render_matrix_chart.py | 6 +-- benchmarks/run_competitors.py | 2 +- benchmarks/tasks_v4.py | 38 ++++++++++++++++ scripts/check_doc_numbers.py | 6 +-- tests/test_benchmark_v3.py | 2 +- tests/test_benchmark_v4.py | 35 ++++++++++++++ 11 files changed, 192 insertions(+), 16 deletions(-) create mode 100644 benchmarks/tasks_v4.py create mode 100644 tests/test_benchmark_v4.py diff --git a/CLAUDE.md b/CLAUDE.md index b2d688e..ada94d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,7 +101,7 @@ Tools are organized into sections (counts are indicative — `system_about` is a **Measuring:** never read vertices back and shoelace them — that loses 28.2% of the area on a semicircular edge, silently. `analysis_measure_entity(handle)` reads the real geometry, and its payload states its own accuracy: `exact`, `flatten_tolerance`, `assumed_closed`, `self_intersecting`. A HATCH reports the area it *fills* (outer loops minus islands) plus `hatch_style`; on a curved hatch edge `flatten_tolerance` is not the accuracy knob, because ezdxf hands boundaries over as cubic Beziers whose ~0.028% circle error is already baked in. `boundary_trace` / `boundary_from_entities` report the same number `analysis_measure_entity` gives for the polyline they just drew — if those ever diverge, one of them is approximating and neither says so. -**Benchmark matrix:** `benchmarks/tasks_v3.py` is the current set (15 tasks; `--matrix v2` reproduces the v1.4 ten). The published competitor reports are pinned v1.4 runs against v2 and are **never** back-filled for the five v3 tasks — the chart shows them `not run`, not zero. `benchmarks/correctness_suite.py` (26 checks) is the release A/B gate: `python benchmarks/compare_versions.py v1.4.0`. +**Benchmark matrix:** `benchmarks/tasks_v4.py` is the current set (16 tasks; `--matrix v3` / `--matrix v2` reproduce the earlier sets). The published competitor reports are pinned v1.4 runs against v2 and are **never** back-filled for the five v3 tasks or the v4 `pid_roundtrip` — the chart shows them `not run`, not zero. `benchmarks/correctness_suite.py` (29 checks) is the release A/B gate: `python benchmarks/compare_versions.py v1.4.0`. ### Adding a New Tool diff --git a/README.md b/README.md index 61adaec..c6c6e92 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Self-measurement, produced by scripts in [`benchmarks/`](https://github.com/U-C4 ### Correctness — every release re-proves itself -26 deterministic headless checks against the previous tag and the current tree, each in its own subprocess so a hard crash counts as a miss rather than killing the run. +29 deterministic headless checks against the previous tag and the current tree, each in its own subprocess so a hard crash counts as a miss rather than killing the run. | Version | Checks passing | Pass rate | Fixed | Regressed | |---|---:|---:|---:|---:| diff --git a/benchmarks/README.md b/benchmarks/README.md index 72d533b..e3a8c7a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -41,12 +41,13 @@ Unsupported tasks remain in the fixed-matrix denominator with score zero, preventing partial implementations from receiving an inflated score. ```bash -python -m benchmarks.run_competitors --list --matrix v3 -python -m benchmarks.run_competitors --server autocad-mcp-pro --backend ezdxf --matrix v3 --json +python -m benchmarks.run_competitors --list --matrix v4 +python -m benchmarks.run_competitors --server autocad-mcp-pro --backend ezdxf --matrix v4 --json python -m benchmarks.run_competitors --task table_mleader --task hatch_islands --json ``` -The release-machine ezdxf self-check is **15/15 (100.0)** on the v3 matrix. +The release-machine ezdxf self-check is **16/16 (100.0)** on the v4 matrix +(`--matrix v3` / `--matrix v2` reproduce the earlier sets). Repository stars and raw tool counts do not contribute to the score. Adapter registration lives in `competitors.yaml`. @@ -86,6 +87,18 @@ zero we invented would be indistinguishable from a zero we measured. only described, and `--publish` writes the artifact-path-free file that `results/published/` holds. +### Matrix v4 (v1.6) — the P&ID round trip + +`tasks_v4.py` keeps the v3 fifteen unchanged and adds one task the reader can +fail on its own: + +| Task | Category | Verified against | +|---|---|---| +| `pid_roundtrip` | pid | the example spec drawn through `pid_from_spec`'s code, then read back by the graph builder, which never sees the spec: 5 nodes, 4 edges, 0 dangling ends, `confidence_min` 1.0, zero critique issues, an instrument index of exactly `FIC-101` wired to `FCV-101`, and the two authored line numbers back on the two process lines while the two unnumbered lines carry sequence-built numbers from XDATA | + +The competitor reports carry no result for it, for the same reason they carry +none for the v3 five. `--matrix v3` reproduces the v1.5 set exactly. + ## Live competitor lane (v1.4) Two competitor adapters now execute the exact same task matrix, black-box over @@ -203,7 +216,7 @@ python benchmarks/compare_versions.py --json results.json ### Result — v1.5.1 vs v1.5.0 (release gate) -26 checks, ezdxf backend, one subprocess per check. Machine-readable report: +29 checks, ezdxf backend, one subprocess per check. Machine-readable report: [`results/published/ab-v1.5.0-vs-v1.5.1.json`](results/published/ab-v1.5.0-vs-v1.5.1.json). | Version | Checks passing | Pass rate | Fixed | Regressed | diff --git a/benchmarks/adapters/autocad_mcp_pro.py b/benchmarks/adapters/autocad_mcp_pro.py index fd8cc2c..6203458 100644 --- a/benchmarks/adapters/autocad_mcp_pro.py +++ b/benchmarks/adapters/autocad_mcp_pro.py @@ -1,4 +1,4 @@ -"""Reference adapter for this repository, covering the v3 task matrix. +"""Reference adapter for this repository, covering the v4 task matrix. Every task verifies against a value worked out independently of the code under test — a closed-form area, a count of entities placed on purpose, a token @@ -12,7 +12,7 @@ import math from pathlib import Path from benchmarks.adapters.base import BenchmarkAdapter, TaskResult -from benchmarks.tasks_v3 import TaskSpec +from benchmarks.tasks_v4 import TaskSpec from engineering.delivery import deliver_drawing from engineering.layers import ensure_engineering_layers, ensure_standard_linetypes from engineering.refiner import refine_drawing @@ -364,6 +364,65 @@ class AutoCADMCPProAdapter(BenchmarkAdapter): [], ) + async def _task_pid_roundtrip(self): + """Five symbols, four lines, read back with nothing dangling. + + The spec is drawn through the same code ``pid_from_spec`` calls; the + graph, the instrument index and the line list are then read back from + the drawing by the reader, which never sees the spec. Every expected + number below is counted from the spec by hand. + """ + from engineering.layers import apply_layer_set + from engineering.pid.deliverables import instrument_index, line_list + from engineering.pid.spec import EXAMPLE_SPEC, run_spec + + await apply_layer_set(self.backend, "pid") + result = await run_spec(self.backend, EXAMPLE_SPEC) + graph = result["graph"] + index = instrument_index(graph) + lines = line_list(graph) + # The spec numbers its two process lines; the signal line and the + # connector line are left to the default ``number_format``, which with + # no size/service/spec/insulation collapses to the bare sequence + # number. Both must come back through XDATA, not a label search. + authored = "100-P-1001-CS1" + numbers = {(r["from_tag"], r["to_tag"]): r["line_number"] for r in lines} + authored_ok = ( + numbers.get(("P-101", "FCV-101")) == authored + and numbers.get(("FCV-101", "V-201")) == authored + ) + sequenced = [r for r in lines if r["line_number"] != authored] + sequenced_ok = len(sequenced) == 2 and all( + r["line_number"].isdigit() and r["number_source"] == "xdata" for r in sequenced + ) + passed = ( + len(graph["nodes"]) == 5 + and len(graph["edges"]) == 4 + and graph["stats"]["dangling"] == 0 + and graph["stats"]["confidence_min"] == 1.0 + and result["critique"] == [] + and [r["tag"] for r in index] == ["FIC-101"] + and index[0]["connected_to"] == "FCV-101" + and len(lines) == 4 + and authored_ok + and sequenced_ok + ) + return ( + passed, + { + "nodes": len(graph["nodes"]), + "edges": len(graph["edges"]), + "dangling": graph["stats"]["dangling"], + "confidence_min": graph["stats"]["confidence_min"], + "critique_issues": len(result["critique"]), + "instrument_index_rows": len(index), + "line_list_rows": len(lines), + "authored_line_numbers_kept": authored_ok, + "unnumbered_lines_sequenced": sequenced_ok, + }, + [], + ) + async def _task_auditable_delivery(self): await ensure_engineering_layers(self.backend) await self.backend.entity_create_line(0, 0, 10, 0, layer="GEOMETRY") diff --git a/benchmarks/correctness_suite.py b/benchmarks/correctness_suite.py index 595d206..7196aac 100644 --- a/benchmarks/correctness_suite.py +++ b/benchmarks/correctness_suite.py @@ -293,6 +293,34 @@ async def radius_dim_ignores_the_leader_length(): return all(abs(v - 20.0) < 1e-6 for v in values) +# ── P&ID (v1.6, track A) ────────────────────────────────────────────────────── + + +async def pid_block_define_attdef_roundtrip(): + b = await _b() + await b.block_define( + "PID_T", + [{"type": "line", "x1": 0, "y1": 0, "x2": 8, "y2": 0}], + [{"tag": "TAG", "x": 0, "y": 3, "height": 2.5}], + ) + ref = await b.block_insert("PID_T", 5, 5, attributes={"TAG": "HV-1"}) + return await b.block_get_attributes(ref.handle) == {"TAG": "HV-1"} + + +async def pid_tag_parse_fic(): + from engineering.pid.tags import parse_tag + + return parse_tag("FIC-101")["description"] == "Flow Indicating Controller" + + +async def pid_graph_edge_count(): + from engineering.pid.spec import EXAMPLE_SPEC, run_spec + + b = await _b() + result = await run_spec(b, EXAMPLE_SPEC) + return len(result["graph"]["edges"]) == 4 and result["graph"]["stats"]["dangling"] == 0 + + CHECKS = { "core_line_length": (core_line_length, "Core"), "core_circle_radius": (core_circle_radius, "Core"), @@ -320,6 +348,9 @@ CHECKS = { "two_vertex_circle_has_area": (two_vertex_circle_has_area, "Measurement"), "diameter_dim_measures_the_diameter": (diameter_dim_measures_the_diameter, "Dimensions"), "radius_dim_ignores_the_leader_length": (radius_dim_ignores_the_leader_length, "Dimensions"), + "pid_block_define_attdef_roundtrip": (pid_block_define_attdef_roundtrip, "P&ID"), + "pid_tag_parse_fic": (pid_tag_parse_fic, "P&ID"), + "pid_graph_edge_count": (pid_graph_edge_count, "P&ID"), } diff --git a/benchmarks/render_matrix_chart.py b/benchmarks/render_matrix_chart.py index 770b387..ebd975d 100644 --- a/benchmarks/render_matrix_chart.py +++ b/benchmarks/render_matrix_chart.py @@ -13,7 +13,7 @@ import json from pathlib import Path from typing import Any -from benchmarks.tasks_v3 import TASKS_V3 +from benchmarks.tasks_v4 import TASKS_V4 ROOT = Path(__file__).resolve().parents[1] DEFAULT_INPUT_DIR = ROOT / "benchmarks" / "results" / "published" @@ -66,8 +66,8 @@ def render_chart(reports: list[dict[str, Any]], output: Path) -> None: matplotlib.rcParams["svg.hashsalt"] = "autocad-mcp-taskmatrix-v1" - task_ids = [task.task_id for task in TASKS_V3] - task_labels = [f"{task.task_id} ({task.category})" for task in TASKS_V3] + task_ids = [task.task_id for task in TASKS_V4] + task_labels = [f"{task.task_id} ({task.category})" for task in TASKS_V4] servers = [report["adapter"] for report in reports] status_by_server = { report["adapter"]: {item["task_id"]: item["status"] for item in report["results"]} diff --git a/benchmarks/run_competitors.py b/benchmarks/run_competitors.py index 78f1784..ed3256a 100644 --- a/benchmarks/run_competitors.py +++ b/benchmarks/run_competitors.py @@ -16,7 +16,7 @@ from pathlib import Path from backends.capability import UnsupportedCapabilityError from benchmarks.adapters.base import BenchmarkAdapter, TaskResult -from benchmarks.tasks_v3 import DEFAULT_MATRIX, MATRICES, TaskSpec, task_by_id +from benchmarks.tasks_v4 import DEFAULT_MATRIX, MATRICES, TaskSpec, task_by_id def _load_registry() -> dict[str, dict]: diff --git a/benchmarks/tasks_v4.py b/benchmarks/tasks_v4.py new file mode 100644 index 0000000..80b37ba --- /dev/null +++ b/benchmarks/tasks_v4.py @@ -0,0 +1,38 @@ +"""The v4 task matrix: v3 plus the P&ID round trip (v1.6, track A). + +``pid_roundtrip`` draws a five-item P&ID through the same code the tools call +and reads it back: the edge count, the absence of dangling ends and the +instrument index are each derived independently of the drawing code. The +published competitor reports were never asked this question; the chart shows +``not_run`` for them, not zero. +""" + +from __future__ import annotations + +from benchmarks.tasks_v2 import TASKS_V2, TaskSpec +from benchmarks.tasks_v3 import TASKS_V3 + +#: Added in v1.6.0. The reader can fail this one on its own. +NEW_TASKS_V4: tuple[TaskSpec, ...] = ( + TaskSpec( + "pid_roundtrip", + "pid", + "Draw a P&ID from a spec and read back a matching graph, index and line list", + 1.0, + ), +) + +TASKS_V4: tuple[TaskSpec, ...] = TASKS_V3 + NEW_TASKS_V4 + +#: Runner-selectable task sets. v2 and v3 stay addressable so an earlier +#: report can be reproduced exactly rather than only described. +MATRICES: dict[str, tuple[TaskSpec, ...]] = {"v2": TASKS_V2, "v3": TASKS_V3, "v4": TASKS_V4} + +DEFAULT_MATRIX = "v4" + + +def task_by_id(task_id: str, matrix: str = DEFAULT_MATRIX) -> TaskSpec: + for task in MATRICES[matrix]: + if task.task_id == task_id: + return task + raise KeyError(f"{task_id} is not in matrix {matrix}") diff --git a/scripts/check_doc_numbers.py b/scripts/check_doc_numbers.py index 15f5da0..f52b6ad 100644 --- a/scripts/check_doc_numbers.py +++ b/scripts/check_doc_numbers.py @@ -73,7 +73,7 @@ def derived() -> dict[str, object]: } from benchmarks import correctness_suite - from benchmarks.tasks_v3 import TASKS_V3 + from benchmarks.tasks_v4 import TASKS_V4 return { "collected_tests": _collected_tests(), @@ -87,7 +87,7 @@ def derived() -> dict[str, object]: "ab_fixed": ab["fixed"], "ab_regressed": ab["regressed"], "suite_checks": len(correctness_suite.CHECKS), - "v3_tasks": len(TASKS_V3), + "v4_tasks": len(TASKS_V4), "perf_ms": perf, } @@ -158,7 +158,7 @@ CHECKS: list[tuple[str, str, str, str]] = [ "ab_fixed", ), ("CLAUDE.md", "A/B suite size", r"correctness_suite\.py` \((\d+) checks\)", "ab_total"), - ("CLAUDE.md", "v3 task count", r"the current set \((\d+) tasks", "v3_tasks"), + ("CLAUDE.md", "v4 task count", r"the current set \((\d+) tasks", "v4_tasks"), ] diff --git a/tests/test_benchmark_v3.py b/tests/test_benchmark_v3.py index 4c29b87..370d79b 100644 --- a/tests/test_benchmark_v3.py +++ b/tests/test_benchmark_v3.py @@ -100,7 +100,7 @@ async def test_each_new_task_passes_on_the_headless_backend(task, tmp_path): @pytest.mark.asyncio async def test_the_full_v3_matrix_runs_green(tmp_path): - report = await run_tasks(AutoCADMCPProAdapter(), TASKS_V3, tmp_path, timeout=180.0) + report = await run_tasks(AutoCADMCPProAdapter(), TASKS_V3, tmp_path, timeout=180.0, matrix="v3") assert report["summary"]["attempted"] == 15 assert report["summary"]["passed"] == 15, [ diff --git a/tests/test_benchmark_v4.py b/tests/test_benchmark_v4.py new file mode 100644 index 0000000..240e0a6 --- /dev/null +++ b/tests/test_benchmark_v4.py @@ -0,0 +1,35 @@ +"""Track A — the v4 task matrix. + +v4 adds one task, ``pid_roundtrip``: draw a P&ID through the same code the +tools call and read it back through the graph builder, so the edge count, the +dangling count and the instrument index are each derived independently of the +drawing code. The published competitor reports were never asked this question +and the chart shows them ``not_run`` — the same rule v3 set for its five. +""" + +from __future__ import annotations + +from benchmarks.tasks_v3 import TASKS_V3 +from benchmarks.tasks_v4 import DEFAULT_MATRIX, MATRICES, NEW_TASKS_V4, TASKS_V4, task_by_id + + +def test_v4_extends_v3_without_touching_it(): + assert TASKS_V4[: len(TASKS_V3)] == TASKS_V3 + assert [t.task_id for t in NEW_TASKS_V4] == ["pid_roundtrip"] + assert MATRICES["v3"] == TASKS_V3 and MATRICES["v4"] == TASKS_V4 and DEFAULT_MATRIX == "v4" + assert task_by_id("pid_roundtrip").category == "pid" + + +def test_the_adapter_implements_every_v4_task(): + from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter + + for task in TASKS_V4: + assert hasattr(AutoCADMCPProAdapter, f"_task_{task.task_id}"), task.task_id + + +def test_correctness_suite_has_the_pid_checks(): + from benchmarks.correctness_suite import CHECKS + + for name in ("pid_block_define_attdef_roundtrip", "pid_tag_parse_fic", "pid_graph_edge_count"): + assert CHECKS[name][1] == "P&ID" + assert len(CHECKS) == 29