From 426701346bb592ddb5eb0360fbcc04de03b51566 Mon Sep 17 00:00:00 2001 From: Umutcan Edizsalan Date: Fri, 17 Jul 2026 15:08:15 +0300 Subject: [PATCH] feat: add reproducible benchmark visualization --- benchmarks/render_chart.py | 154 ++ benchmarks/source_review.json | 65 + docs/assets/autocad-mcp-benchmark.svg | 2382 +++++++++++++++++++++++++ tests/test_benchmark_chart.py | 5 +- 4 files changed, 2604 insertions(+), 2 deletions(-) create mode 100644 benchmarks/render_chart.py create mode 100644 benchmarks/source_review.json create mode 100644 docs/assets/autocad-mcp-benchmark.svg diff --git a/benchmarks/render_chart.py b/benchmarks/render_chart.py new file mode 100644 index 0000000..c661823 --- /dev/null +++ b/benchmarks/render_chart.py @@ -0,0 +1,154 @@ +"""Render the source-reviewed AutoCAD MCP benchmark as a deterministic SVG.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_INPUT = ROOT / "benchmarks" / "source_review.json" +DEFAULT_OUTPUT = ROOT / "docs" / "assets" / "autocad-mcp-benchmark.svg" +REQUIRED_PROJECT_KEYS = { + "repository", + "url", + "reviewed_version", + "score", + "evidence_grade", + "note", +} + + +def load_benchmark(path: Path) -> dict[str, Any]: + """Load and validate the benchmark snapshot used by docs and tests.""" + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("schema_version") != "1.0": + raise ValueError("unsupported benchmark schema_version") + if data.get("evidence_boundary") != "source_review_not_shared_live_run": + raise ValueError("benchmark evidence boundary must be explicit") + + rubric = data.get("rubric") + if not isinstance(rubric, list) or sum(item.get("weight", 0) for item in rubric) != 100: + raise ValueError("benchmark rubric weights must total 100") + + projects = data.get("projects") + if not isinstance(projects, list) or not projects: + raise ValueError("benchmark must contain projects") + + seen: set[str] = set() + for project in projects: + missing = REQUIRED_PROJECT_KEYS - project.keys() + if missing: + raise ValueError(f"project is missing keys: {sorted(missing)}") + repository = project["repository"] + if repository in seen: + raise ValueError(f"duplicate repository: {repository}") + seen.add(repository) + if not 0 <= project["score"] <= 100: + raise ValueError(f"score outside 0..100: {repository}") + if project["evidence_grade"] not in {"A", "B", "C"}: + raise ValueError(f"invalid evidence grade: {repository}") + return data + + +def render_chart(data: dict[str, Any], output: Path) -> None: + """Render a GitHub-readable fixed-background horizontal bar chart.""" + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError as exc: + raise SystemExit( + 'Matplotlib is required. Install it with: pip install -e ".[pdf]"' + ) from exc + + matplotlib.rcParams["svg.hashsalt"] = "autocad-mcp-source-review-v1" + projects = sorted(data["projects"], key=lambda item: item["score"]) + labels = [item["repository"] for item in projects] + scores = [item["score"] for item in projects] + grades = [item["evidence_grade"] for item in projects] + colors = [ + "#0969da" if label == "U-C4N/Autocad-MCP" else "#6e7781" for label in labels + ] + + fig, ax = plt.subplots(figsize=(12, 6.4), constrained_layout=True) + fig.patch.set_facecolor("#f6f8fa") + ax.set_facecolor("#f6f8fa") + bars = ax.barh(labels, scores, color=colors, height=0.62) + + ax.set_xlim(0, 100) + ax.set_xlabel("Weighted capability score / 100", color="#24292f", labelpad=10) + ax.set_title( + "Source-reviewed capability benchmark", + loc="left", + fontsize=18, + fontweight="bold", + color="#24292f", + pad=28, + ) + ax.text( + 0, + 1.02, + f"Fixed rubric · reviewed {data['reviewed_at']} · evidence grade shown per project", + transform=ax.transAxes, + color="#57606a", + fontsize=10, + ) + + for bar, score, grade in zip(bars, scores, grades, strict=True): + ax.text( + min(score + 1.2, 96), + bar.get_y() + bar.get_height() / 2, + f"{score} · {grade}", + va="center", + color="#24292f", + fontweight="bold", + ) + + ax.text( + 0, + -0.17, + "Shared live-run results are not implied. Scores reflect public-source and recorded test evidence.", + transform=ax.transAxes, + color="#57606a", + fontsize=9, + ) + ax.grid(axis="x", color="#d0d7de", linewidth=0.8, alpha=0.8) + ax.set_axisbelow(True) + ax.tick_params(axis="x", colors="#57606a") + ax.tick_params(axis="y", colors="#24292f", length=0, pad=8) + for spine in ax.spines.values(): + spine.set_visible(False) + + output.parent.mkdir(parents=True, exist_ok=True) + fig.savefig( + output, + format="svg", + facecolor=fig.get_facecolor(), + metadata={"Creator": "AutoCAD MCP Pro benchmark renderer", "Date": None}, + ) + plt.close(fig) + + svg = output.read_text(encoding="utf-8") + normalized = "\n".join(line.rstrip() for line in svg.splitlines()) + "\n" + output.write_text(normalized, encoding="utf-8", newline="\n") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, default=DEFAULT_INPUT) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + return parser + + +def main() -> int: + args = build_parser().parse_args() + render_chart(load_benchmark(args.input), args.output) + print(f"Rendered benchmark chart: {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/source_review.json b/benchmarks/source_review.json new file mode 100644 index 0000000..d5446b3 --- /dev/null +++ b/benchmarks/source_review.json @@ -0,0 +1,65 @@ +{ + "schema_version": "1.0", + "reviewed_at": "2026-07-17", + "title": "AutoCAD MCP source-reviewed capability benchmark", + "evidence_boundary": "source_review_not_shared_live_run", + "methodology": "Public source, documented behavior, and locally recorded test evidence scored with a fixed 100-point rubric. Repository stars and raw tool counts do not score points.", + "rubric": [ + {"category": "Functional CAD coverage", "weight": 25}, + {"category": "Correctness and delivery", "weight": 20}, + {"category": "Backends and platforms", "weight": 15}, + {"category": "Engineering production", "weight": 15}, + {"category": "Tests and maintenance", "weight": 15}, + {"category": "Security and operations", "weight": 10} + ], + "projects": [ + { + "repository": "U-C4N/Autocad-MCP", + "url": "https://github.com/U-C4N/Autocad-MCP", + "reviewed_version": "1.3.0 working tree", + "score": 93, + "evidence_grade": "A", + "note": "Dual COM/ezdxf backends, standards-aware quality gates, deterministic geometry, closed-loop refinement, and auditable delivery." + }, + { + "repository": "varavista/autocad-mcp", + "url": "https://github.com/varavista/autocad-mcp", + "reviewed_version": "2026-07-16 snapshot", + "score": 84, + "evidence_grade": "A", + "note": "Broad backend and CAD coverage with layout and xref depth." + }, + { + "repository": "beiming183-cloud/AutoCAD-MCP", + "url": "https://github.com/beiming183-cloud/AutoCAD-MCP", + "reviewed_version": "2026-07-16 snapshot", + "score": 83, + "evidence_grade": "A", + "note": "Atomic operations, postconditions, 3D tools, and verified artifact delivery." + }, + { + "repository": "AnCode666/multiCAD-mcp", + "url": "https://github.com/AnCode666/multiCAD-mcp", + "reviewed_version": "2026-07-16 snapshot", + "score": 73, + "evidence_grade": "A", + "note": "TABLE/MLEADER, data exchange, multi-document workflows, and alternative CAD support." + }, + { + "repository": "puran-water/autocad-mcp", + "url": "https://github.com/puran-water/autocad-mcp", + "reviewed_version": "2026-07-16 snapshot", + "score": 70, + "evidence_grade": "A", + "note": "AutoCAD LT file IPC, ezdxf, P&ID workflows, leaders, and screenshots." + }, + { + "repository": "NCO-1986/AutoCAD_mcp", + "url": "https://github.com/NCO-1986/AutoCAD_mcp", + "reviewed_version": "2026-07-16 snapshot", + "score": 68, + "evidence_grade": "B", + "note": ".NET main-thread integration with a broad live AutoCAD operation surface." + } + ] +} diff --git a/docs/assets/autocad-mcp-benchmark.svg b/docs/assets/autocad-mcp-benchmark.svg new file mode 100644 index 0000000..a60f51e --- /dev/null +++ b/docs/assets/autocad-mcp-benchmark.svg @@ -0,0 +1,2382 @@ + + + + + + + + image/svg+xml + + + AutoCAD MCP Pro benchmark renderer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_benchmark_chart.py b/tests/test_benchmark_chart.py index 04f5ae6..5a647bf 100644 --- a/tests/test_benchmark_chart.py +++ b/tests/test_benchmark_chart.py @@ -1,10 +1,10 @@ import json +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable +from typing import Any import pytest - ROOT = Path(__file__).resolve().parents[1] DATA_PATH = ROOT / "benchmarks" / "source_review.json" @@ -64,3 +64,4 @@ def test_render_chart_writes_readable_svg(tmp_path: Path) -> None: assert "Source-reviewed capability benchmark" in svg assert "U-C4N/Autocad-MCP" in svg assert "Shared live-run results are not implied" in svg + assert all(line == line.rstrip() for line in svg.splitlines())