From 18c7c69da68450e3799fdd17bd73a3263eddf67d Mon Sep 17 00:00:00 2001 From: Umut CAN Date: Tue, 22 Sep 2026 13:27:17 +0300 Subject: [PATCH] fix(task-25): page_setup_truth and settings_pdf_mediabox_a3 move the sheet to ANSI B before proving A3 A fresh Layout1 is already ISO A3 landscape (ezdxf R2018 default: 420 x 297, plot_rotation 0), so an A3-only /MediaBox read-back passed with page_setup_apply stubbed to a no-op: the A/B gate row and the v4 matrix task could not fail on the setter. Both now apply ANSI B (432 x 279, a size no fresh document has; spec section 9 names it) and read that PDF first, then apply ISO A3, read its PDF, and require the setter's changed.size_mm to have moved from B to A3. The stub fails at the first PDF. The moved-sheet comparison is +/-0.5 mm, the spec's tolerance, not exact: measured live on AutoCAD 2026 (scratch Documents.Add(), closed unsaved), the COM engine reports ANSI B as 431.8 x 279.4 and the DWG To PDF.pc3 file's MediaBox as 431.8 x 279.4 then 420.158 x 297.039 for A3; the task passes there with changed.size_mm [[431.8, 279.4], [420.0, 297.0]]. Headless it is [[432.0, 279.0], [420.0, 297.0]]. tests/test_benchmark_v4.py gains the mutation test the finding describes (setter stubbed -> both rows must fail; both passed on the old code) and a run of the gate row itself; the headless task test asserts the B stage. README rows and the tasks_v4 docstring say what the task now does; the README collected-tests figure follows the two new tests (3282 -> 3284). The regenerated A/B report against v1.5.1 is byte-identical (26 -> 32/32). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FKFStjEFzpGyLq5bMpWxTd --- README.md | 4 +- benchmarks/README.md | 2 +- benchmarks/adapters/autocad_mcp_pro.py | 45 ++++++++++++++++++--- benchmarks/correctness_suite.py | 38 +++++++++++++++--- benchmarks/tasks_v4.py | 4 +- tests/test_benchmark_v4.py | 55 ++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 55af7a4..9ef720c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Live through COM on Windows, or headless through ezdxf anywhere — one typed co -> **v1.5 release snapshot:** 204 tools · 8 resources · 5 prompt templates · 3282 collected tests. +> **v1.5 release snapshot:** 204 tools · 8 resources · 5 prompt templates · 3284 collected tests. > 204 is the **registered** count; a default install advertises 199 over `tools/list`, > because `ENABLE_3D` is unset. `system_about` is the runtime authority. @@ -176,7 +176,7 @@ An earlier matrix scored this server 10/10, which carried no information: every | `selection_filter` | window 1, crossing 2, bounding box 3, polygon 1 | | `measure_from_handle` | 139.2699 against the 100.0 a vertex shoelace gives | | `pid_roundtrip` | the example sheet drawn by `pid_from_spec`, read back by `pid_graph`, which never sees the spec: 5 nodes, 4 edges, 0 dangling, `confidence_min` 1.0, `FIC-101` wired to `FCV-101` | -| `page_setup_truth` | ISO A3 landscape applied to Layout1, plotted through `batch_plot`, and the sheet read back from the PDF's `/MediaBox`: 420 × 297 mm, not the setter's return value | +| `page_setup_truth` | ANSI B then ISO A3 landscape applied to Layout1, each plotted through `batch_plot` and read back from its PDF's `/MediaBox`: 432 × 279 then 420 × 297 mm, not the setter's return value (a fresh layout is already A3, so the B sheet is what a no-op setter cannot fake) | ### Headless performance diff --git a/benchmarks/README.md b/benchmarks/README.md index ddd7f0d..03efb0c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -95,7 +95,7 @@ fail on their own: | Task | Category | Verified against | |---|---|---| | `pid_roundtrip` | pid | the example spec drawn through `pid_from_spec`'s code, then read back by the graph builder, which never sees the spec: 5 nodes, 4 edges, 0 dangling ends, `confidence_min` 1.0, zero critique issues, an instrument index of exactly `FIC-101` wired to `FCV-101`, and the two authored line numbers back on the two process lines while the two unnumbered lines carry sequence-built numbers from XDATA | -| `page_setup_truth` | pagesetup | `page_setup_apply("Layout1", ISO_A3 landscape 1:1)` then `batch_plot`; the PDF's own `/MediaBox` must parse to 420 × 297 mm (±0.5), the page-setup list must say `ISO_A3` / `landscape` / `[420, 297]`, and the ctb must be a known one | +| `page_setup_truth` | pagesetup | `page_setup_apply("Layout1", ANSI_B landscape 1:1)` then `batch_plot` — the PDF's own `/MediaBox` must parse to 432 × 279 mm (±0.5), a size no fresh document has; then `page_setup_apply("Layout1", ISO_A3 landscape 1:1)` and `batch_plot` again — 420 × 297, the setter's `changed.size_mm` must read `[[432, 279], [420, 297]]`, the page-setup list must say `ISO_A3` / `landscape` / `[420, 297]`, and the ctb must be a known one. A setter that writes nothing fails at the first PDF (`tests/test_benchmark_v4.py` stubs it to prove that) | The competitor reports carry no result for either, for the same reason they carry none for the v3 five. `--matrix v3` reproduces the v1.5 set exactly. diff --git a/benchmarks/adapters/autocad_mcp_pro.py b/benchmarks/adapters/autocad_mcp_pro.py index c88b523..7681965 100644 --- a/benchmarks/adapters/autocad_mcp_pro.py +++ b/benchmarks/adapters/autocad_mcp_pro.py @@ -438,10 +438,29 @@ class AutoCADMCPProAdapter(BenchmarkAdapter): setter's ``applied`` block is what we asked for, the page-setup list is what the LAYOUT now says, and the PDF is what a printer would get. All three have to agree with ISO 216's 420 × 297. + + A fresh document's Layout1 is already A3 landscape, so that read-back + alone would pass with the setter stubbed out. The task first moves the + sheet to ANSI B — 432 × 279, a size no fresh document has — and reads + that PDF, then applies A3 and requires the setter to report the layout + moved from B to A3. The stub fails at the first PDF. """ from engineering.standards.papers import resolve_page_setup from engineering.standards.plot import batch_plot + out_dir = self.artifact_dir / "page_setup_truth" + out_dir.mkdir(parents=True, exist_ok=True) + # Stage 1: a non-default sheet, proven by its PDF. + await self.backend.page_setup_apply( + "Layout1", + resolve_page_setup("ANSI_B", orientation="landscape", scale="1:1"), + ) + before = await batch_plot( + self.backend, ["Layout1"], str(out_dir), "{drawing}-{layout}-ansi_b.pdf" + ) + row_before = before["sheets"][0] + width_before, height_before = (float(v) for v in row_before["mediabox_mm"]) + # Stage 2: the sheet under test. setup = resolve_page_setup( "ISO_A3", orientation="landscape", @@ -450,15 +469,29 @@ class AutoCADMCPProAdapter(BenchmarkAdapter): ) applied = await self.backend.page_setup_apply("Layout1", setup) listed = (await self.backend.page_setup_list("Layout1"))[0] - out_dir = self.artifact_dir / "page_setup_truth" - out_dir.mkdir(parents=True, exist_ok=True) - report = await batch_plot(self.backend, ["Layout1"], str(out_dir), "{drawing}-{layout}.pdf") + report = await batch_plot( + self.backend, ["Layout1"], str(out_dir), "{drawing}-{layout}-iso_a3.pdf" + ) row = report["sheets"][0] width_mm, height_mm = (float(v) for v in row["mediabox_mm"]) size_listed = tuple(float(v) for v in listed["size_mm"]) + size_changed = applied["changed"].get("size_mm") + # ±0.5 mm, the spec's own tolerance: the live engine measures ANSI B + # as 431.8 × 279.4 (11 × 17 in), the headless one as 432 × 279. + moved = size_changed or [[0.0, 0.0], [0.0, 0.0]] + moved_from_b = ( + abs(float(moved[0][0]) - 432.0) < 0.5 and abs(float(moved[0][1]) - 279.0) < 0.5 + ) + moved_to_a3 = ( + abs(float(moved[1][0]) - 420.0) < 0.5 and abs(float(moved[1][1]) - 297.0) < 0.5 + ) passed = ( - applied["ok"] is True + abs(width_before - 432.0) < 0.5 + and abs(height_before - 279.0) < 0.5 + and applied["ok"] is True and applied["plot_style_known"] is True + and moved_from_b + and moved_to_a3 and listed["paper"] == "ISO_A3" and listed["orientation"] == "landscape" and size_listed == (420.0, 297.0) @@ -473,10 +506,12 @@ class AutoCADMCPProAdapter(BenchmarkAdapter): "orientation": listed["orientation"], "size_mm_listed": list(size_listed), "mediabox_mm": [width_mm, height_mm], + "mediabox_mm_before": [width_before, height_before], + "size_mm_changed": size_changed, "pdf_bytes": row["bytes"], "plot_style_known": applied["plot_style_known"], }, - [str(row["path"])], + [str(row_before["path"]), str(row["path"])], ) async def _task_auditable_delivery(self): diff --git a/benchmarks/correctness_suite.py b/benchmarks/correctness_suite.py index 3bc5020..d22cf6b 100644 --- a/benchmarks/correctness_suite.py +++ b/benchmarks/correctness_suite.py @@ -376,17 +376,45 @@ async def settings_layer_state_roundtrip(): ) +def _sheet_near(size_mm, width_mm: float, height_mm: float) -> bool: + """Within the spec's ±0.5 mm: COM measures ANSI B as 431.8 × 279.4.""" + try: + w, h = (float(v) for v in size_mm) + except (TypeError, ValueError): + return False + return abs(w - width_mm) < 0.5 and abs(h - height_mm) < 0.5 + + async def settings_pdf_mediabox_a3(): - """Page setup evidence is the PDF's own /MediaBox (spec §8.3).""" + """Page setup evidence is the PDF's own /MediaBox (spec §8.3). + + A fresh document's Layout1 is already ISO A3 landscape, so an A3-only + read-back would pass with the setter stubbed out. The sheet is therefore + moved to ANSI B first — 432 × 279, a size no fresh document has (spec §9 + names it) — and that PDF is read; then ISO A3 is applied and read again. + The A3 apply must also report the layout moved from B to A3, so the PDF + and the setter's own diff have to agree on where the sheet was. + """ from engineering.standards.papers import resolve_page_setup from engineering.standards.plot import batch_plot b = await _b() - await b.page_setup_apply("Layout1", resolve_page_setup("ISO_A3")) + await b.page_setup_apply("Layout1", resolve_page_setup("ANSI_B")) out_dir = tempfile.mkdtemp() - report = await batch_plot(b, ["Layout1"], out_dir, "{drawing}-{layout}.pdf") - width_mm, height_mm = report["sheets"][0]["mediabox_mm"] - return abs(width_mm - 420.0) < 0.5 and abs(height_mm - 297.0) < 0.5 + report_b = await batch_plot(b, ["Layout1"], out_dir, "{drawing}-{layout}-ansi_b.pdf") + width_b, height_b = report_b["sheets"][0]["mediabox_mm"] + applied = await b.page_setup_apply("Layout1", resolve_page_setup("ISO_A3")) + report_a3 = await batch_plot(b, ["Layout1"], out_dir, "{drawing}-{layout}-iso_a3.pdf") + width_a3, height_a3 = report_a3["sheets"][0]["mediabox_mm"] + moved = applied["changed"].get("size_mm") or [[0.0, 0.0], [0.0, 0.0]] + return ( + abs(width_b - 432.0) < 0.5 + and abs(height_b - 279.0) < 0.5 + and _sheet_near(moved[0], 432.0, 279.0) + and _sheet_near(moved[1], 420.0, 297.0) + and abs(width_a3 - 420.0) < 0.5 + and abs(height_a3 - 297.0) < 0.5 + ) CHECKS = { diff --git a/benchmarks/tasks_v4.py b/benchmarks/tasks_v4.py index e2b54e9..55c8852 100644 --- a/benchmarks/tasks_v4.py +++ b/benchmarks/tasks_v4.py @@ -7,7 +7,9 @@ and the instrument index are each derived independently of the drawing code. ``page_setup_truth`` (track E) applies an ISO A3 landscape page setup to a layout, plots it through ``batch_plot`` and reads the sheet size back from the PDF's own ``/MediaBox`` — the setup is verified by the file, not by the -setter's return value. +setter's return value. A fresh Layout1 is already A3 landscape, so the task +moves the sheet to ANSI B (432 × 279) and reads that PDF first; a setter that +writes nothing fails there, not at the A3 read-back. The published competitor reports were never asked either question; the chart shows ``not_run`` for them, not zero. diff --git a/tests/test_benchmark_v4.py b/tests/test_benchmark_v4.py index 4c33aa4..25b6f75 100644 --- a/tests/test_benchmark_v4.py +++ b/tests/test_benchmark_v4.py @@ -67,3 +67,58 @@ async def test_page_setup_truth_passes_headlessly(tmp_path): assert passed, metrics assert abs(metrics["mediabox_mm"][0] - 420.0) < 0.5 assert abs(metrics["mediabox_mm"][1] - 297.0) < 0.5 + # The stage that makes the task falsifiable: the sheet was ANSI B first. + assert abs(metrics["mediabox_mm_before"][0] - 432.0) < 0.5 + assert abs(metrics["mediabox_mm_before"][1] - 279.0) < 0.5 + assert metrics["size_mm_changed"] == [[432.0, 279.0], [420.0, 297.0]] + + +async def _noop_page_setup_apply(self, layout, setup): + """A setter that writes nothing and claims success.""" + return { + "ok": True, + "layout": layout, + "applied": dict(setup), + "changed": {}, + "warnings": [], + "plot_style_known": True, + "viewports_kept": True, + "backend": "ezdxf", + } + + +@pytest.mark.asyncio +async def test_page_setup_truth_fails_when_the_setter_writes_nothing(tmp_path, monkeypatch): + """A fresh Layout1 is already ISO A3 landscape (ezdxf's R2018 default: 420 x 297, + rotation 0), so a task that only reads A3 back from the PDF cannot tell a + working setter from a stub. Both gate rows must fail with the stub.""" + pytest.importorskip("matplotlib", reason="rendering needs the [pdf] extra") + from backends.ezdxf_backend import EzdxfBackend + from benchmarks import correctness_suite + from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter + + monkeypatch.setattr(EzdxfBackend, "page_setup_apply", _noop_page_setup_apply) + + assert await correctness_suite.settings_pdf_mediabox_a3() is False + + adapter = AutoCADMCPProAdapter(backend="ezdxf") + await adapter.setup(tmp_path) + try: + await adapter._reset() + passed, metrics, _artifacts = await adapter._task_page_setup_truth() + finally: + await adapter.cleanup() + assert passed is False, metrics + # The stub's own evidence: the "ANSI B" PDF is still the default A3 sheet. + assert abs(metrics["mediabox_mm_before"][0] - 420.0) < 0.5 + assert metrics["size_mm_changed"] is None + + +@pytest.mark.asyncio +async def test_correctness_check_settings_pdf_mediabox_a3_passes(): + """The A/B gate row itself, run once here: ANSI B plotted at 432 x 279, + then ISO A3 at 420 x 297, each read from its own PDF.""" + pytest.importorskip("matplotlib", reason="rendering needs the [pdf] extra") + from benchmarks import correctness_suite + + assert await correctness_suite.settings_pdf_mediabox_a3() is True