From 47a90dbc540af4e82df05a74ea1e2925947960ac Mon Sep 17 00:00:00 2001 From: Umutcan Edizsalan Date: Thu, 23 Jul 2026 16:00:58 +0300 Subject: [PATCH] docs(bench): add the v1.3.0 -> v1.4.0 correctness A/B gate to the README benchmark lanes Ran benchmarks/compare_versions.py against the v1.3.0 tag: 21/21 on both versions, 0 fixed / 0 regressed - the v1.4.0 feature+infra work introduced no correctness regressions. Main README gains lane 4 (version A/B, with the historical v1.0.0 -> v1.1.0 contrast); benchmarks/README records the release-gate result and drops the stale 'performance coming' stub. Co-authored-by: Cursor --- README.md | 30 +++- benchmarks/README.md | 21 ++- .../published/ab-v1.3.0-vs-v1.4.0.json | 140 ++++++++++++++++++ 3 files changed, 185 insertions(+), 6 deletions(-) create mode 100644 benchmarks/results/published/ab-v1.3.0-vs-v1.4.0.json diff --git a/README.md b/README.md index 8355a08..11ae7f3 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,35 @@ python -m benchmarks.perf_suite --out benchmarks/results/published/perf-ezdxf.js python -m benchmarks.render_perf_chart ``` -### 4 · Source-reviewed capability rubric (context) +### 4 · Version A/B — every release re-proves correctness + +`compare_versions.py` runs the same 21 deterministic, headless correctness +checks against the previous release tag and the current tree — each check in +its own subprocess, so a hard crash counts as a miss instead of killing the +run. + +**v1.4.0 release gate** (baseline `v1.3.0`, report: +[`ab-v1.3.0-vs-v1.4.0.json`](benchmarks/results/published/ab-v1.3.0-vs-v1.4.0.json)): + +| Version | Checks passing | Pass rate | Fixed | Regressed | +|---|---:|---:|---:|---:| +| v1.3.0 (baseline) | 21 / 21 | 100 % | — | — | +| **v1.4.0** (this release) | **21 / 21** | **100 %** | 0 | **0** | + +v1.4.0 added CI, packaging, benchmarks, tool profiles, paper space, ISO 286 +fits and opt-in 3D solids **without breaking a single correctness check**. +For contrast, the same suite caught the v1.0.0 → v1.1.0 jump: + +| Version | Checks passing | Pass rate | +|---|---:|---:| +| v1.0.0 (first public release) | 8 / 21 | 38.1 % | +| v1.1.0 | 21 / 21 | 100 % | + +```bash +python benchmarks/compare_versions.py v1.3.0 --json ab.json +``` + +### 5 · Source-reviewed capability rubric (context) ![Source-reviewed AutoCAD MCP capability benchmark](docs/assets/autocad-mcp-benchmark.svg) diff --git a/benchmarks/README.md b/benchmarks/README.md index c67e068..b0ae3bd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -121,6 +121,20 @@ python benchmarks/compare_versions.py v1.0.0 # vs a tag/ref python benchmarks/compare_versions.py --json results.json ``` +### Result — v1.4.0 vs v1.3.0 (release gate) + +21 checks, ezdxf backend, one subprocess per check. Machine-readable report: +[`results/published/ab-v1.3.0-vs-v1.4.0.json`](results/published/ab-v1.3.0-vs-v1.4.0.json). + +| Version | Checks passing | Pass rate | Fixed | Regressed | +|---------|----------------|-----------|-------|-----------| +| **v1.3.0** (baseline) | 21 / 21 | 100 % | — | — | +| **v1.4.0** (this release) | 21 / 21 | 100 % | 0 | **0** | + +The v1.4.0 feature and infrastructure work (CI, packaging, competitor lane, +tool profiles, paper space, ISO 286 fits, opt-in solids) introduced **zero +correctness regressions**. + ### Result — v1.1.0 vs v1.0.0 (`origin/main`, commit 15fa2bc) 21 checks, ezdxf backend, CPython 3.14, one subprocess per check. @@ -167,8 +181,5 @@ not a cherry-picked list of failures; the baseline genuinely does basic CAD work - COM-backend behaviour is not covered here (no live AutoCAD); see the mocked-COM suites in `tests/`. -## Performance (throughput) — coming - -Scale / throughput numbers (1k–100k entities, render timings, model-by-model -tool-call success rate) are tracked in the main README and will be published here -with a hardware footprint. +Model-by-model tool-call success rates remain future work; scale/throughput +numbers live in the headless performance lane above. diff --git a/benchmarks/results/published/ab-v1.3.0-vs-v1.4.0.json b/benchmarks/results/published/ab-v1.3.0-vs-v1.4.0.json new file mode 100644 index 0000000..92b89e4 --- /dev/null +++ b/benchmarks/results/published/ab-v1.3.0-vs-v1.4.0.json @@ -0,0 +1,140 @@ +{ + "summary": { + "baseline_ref": "v1.3.0", + "total": 21, + "old_pass": 21, + "new_pass": 21, + "old_pct": 100.0, + "new_pct": 100.0, + "fixed": 0, + "regressed": 0 + }, + "rows": [ + { + "check": "core_line_length", + "category": "Core", + "old": "pass", + "new": "pass" + }, + { + "check": "core_circle_radius", + "category": "Core", + "old": "pass", + "new": "pass" + }, + { + "check": "core_layer_create", + "category": "Core", + "old": "pass", + "new": "pass" + }, + { + "check": "core_polyline_closed", + "category": "Core", + "old": "pass", + "new": "pass" + }, + { + "check": "core_linear_dim", + "category": "Core", + "old": "pass", + "new": "pass" + }, + { + "check": "core_save_dxf_roundtrip", + "category": "Core", + "old": "pass", + "new": "pass" + }, + { + "check": "dim_aligned_no_error", + "category": "Dimensions", + "old": "pass", + "new": "pass" + }, + { + "check": "dim_angular_no_error", + "category": "Dimensions", + "old": "pass", + "new": "pass" + }, + { + "check": "array_polar_360_distinct", + "category": "Modify", + "old": "pass", + "new": "pass" + }, + { + "check": "point_intersection_line_line", + "category": "Geometry", + "old": "pass", + "new": "pass" + }, + { + "check": "point_tangent_external", + "category": "Geometry", + "old": "pass", + "new": "pass" + }, + { + "check": "arc_has_length", + "category": "Query", + "old": "pass", + "new": "pass" + }, + { + "check": "arc_select_by_length", + "category": "Query", + "old": "pass", + "new": "pass" + }, + { + "check": "ezdxf_bounding_box", + "category": "Query", + "old": "pass", + "new": "pass" + }, + { + "check": "mtext_rotation_roundtrip", + "category": "Entities", + "old": "pass", + "new": "pass" + }, + { + "check": "screenshot_png", + "category": "Render", + "old": "pass", + "new": "pass" + }, + { + "check": "dim_overlap_critique_fires", + "category": "Quality gate", + "old": "pass", + "new": "pass" + }, + { + "check": "iso13567_dim_layer", + "category": "Quality gate", + "old": "pass", + "new": "pass" + }, + { + "check": "construction_left_iso_caught", + "category": "Quality gate", + "old": "pass", + "new": "pass" + }, + { + "check": "gear_no_self_overlap", + "category": "Engineering", + "old": "pass", + "new": "pass" + }, + { + "check": "center_linetype_applied", + "category": "Entities", + "old": "pass", + "new": "pass" + } + ] +} \ No newline at end of file