anatoly/autocad-mcp-pro
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: add reproducible benchmark visualization

Browse Source
This commit is contained in:
Umutcan Edizsalan
2026-07-17 15:08:15 +03:00
parent 67cd8010dd
commit 426701346b
4 changed files with 2604 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/render_chart.py
+154
View File
@@ -0,0 +1,154 @@
"""Render the source-reviewed AutoCAD MCP benchmark as a deterministic SVG."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_INPUT = ROOT / "benchmarks" / "source_review.json"
DEFAULT_OUTPUT = ROOT / "docs" / "assets" / "autocad-mcp-benchmark.svg"
REQUIRED_PROJECT_KEYS = {
"repository",
"url",
"reviewed_version",
"score",
"evidence_grade",
"note",
}
def load_benchmark(path: Path) -> dict[str, Any]:
"""Load and validate the benchmark snapshot used by docs and tests."""
data = json.loads(path.read_text(encoding="utf-8"))
if data.get("schema_version") != "1.0":
raise ValueError("unsupported benchmark schema_version")
if data.get("evidence_boundary") != "source_review_not_shared_live_run":
raise ValueError("benchmark evidence boundary must be explicit")
rubric = data.get("rubric")
if not isinstance(rubric, list) or sum(item.get("weight", 0) for item in rubric) != 100:
raise ValueError("benchmark rubric weights must total 100")
projects = data.get("projects")
if not isinstance(projects, list) or not projects:
raise ValueError("benchmark must contain projects")
seen: set[str] = set()
for project in projects:
missing = REQUIRED_PROJECT_KEYS - project.keys()
if missing:
raise ValueError(f"project is missing keys: {sorted(missing)}")
repository = project["repository"]
if repository in seen:
raise ValueError(f"duplicate repository: {repository}")
seen.add(repository)
if not 0 <= project["score"] <= 100:
raise ValueError(f"score outside 0..100: {repository}")
if project["evidence_grade"] not in {"A", "B", "C"}:
raise ValueError(f"invalid evidence grade: {repository}")
return data
def render_chart(data: dict[str, Any], output: Path) -> None:
"""Render a GitHub-readable fixed-background horizontal bar chart."""
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
except ImportError as exc:
raise SystemExit(
'Matplotlib is required. Install it with: pip install -e ".[pdf]"'
) from exc
matplotlib.rcParams["svg.hashsalt"] = "autocad-mcp-source-review-v1"
projects = sorted(data["projects"], key=lambda item: item["score"])
labels = [item["repository"] for item in projects]
scores = [item["score"] for item in projects]
grades = [item["evidence_grade"] for item in projects]
colors = [
"#0969da" if label == "U-C4N/Autocad-MCP" else "#6e7781" for label in labels
]
fig, ax = plt.subplots(figsize=(12, 6.4), constrained_layout=True)
fig.patch.set_facecolor("#f6f8fa")
ax.set_facecolor("#f6f8fa")
bars = ax.barh(labels, scores, color=colors, height=0.62)
ax.set_xlim(0, 100)
ax.set_xlabel("Weighted capability score / 100", color="#24292f", labelpad=10)
ax.set_title(
"Source-reviewed capability benchmark",
loc="left",
fontsize=18,
fontweight="bold",
color="#24292f",
pad=28,
)
ax.text(
0,
1.02,
f"Fixed rubric · reviewed {data['reviewed_at']} · evidence grade shown per project",
transform=ax.transAxes,
color="#57606a",
fontsize=10,
)
for bar, score, grade in zip(bars, scores, grades, strict=True):
ax.text(
min(score + 1.2, 96),
bar.get_y() + bar.get_height() / 2,
f"{score} · {grade}",
va="center",
color="#24292f",
fontweight="bold",
)
ax.text(
0,
-0.17,
"Shared live-run results are not implied. Scores reflect public-source and recorded test evidence.",
transform=ax.transAxes,
color="#57606a",
fontsize=9,
)
ax.grid(axis="x", color="#d0d7de", linewidth=0.8, alpha=0.8)
ax.set_axisbelow(True)
ax.tick_params(axis="x", colors="#57606a")
ax.tick_params(axis="y", colors="#24292f", length=0, pad=8)
for spine in ax.spines.values():
spine.set_visible(False)
output.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(
output,
format="svg",
facecolor=fig.get_facecolor(),
metadata={"Creator": "AutoCAD MCP Pro benchmark renderer", "Date": None},
)
plt.close(fig)
svg = output.read_text(encoding="utf-8")
normalized = "\n".join(line.rstrip() for line in svg.splitlines()) + "\n"
output.write_text(normalized, encoding="utf-8", newline="\n")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--input", type=Path, default=DEFAULT_INPUT)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
return parser
def main() -> int:
args = build_parser().parse_args()
render_chart(load_benchmark(args.input), args.output)
print(f"Rendered benchmark chart: {args.output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
benchmarks/source_review.json
+65
View File
@@ -0,0 +1,65 @@
{
"schema_version": "1.0",
"reviewed_at": "2026-07-17",
"title": "AutoCAD MCP source-reviewed capability benchmark",
"evidence_boundary": "source_review_not_shared_live_run",
"methodology": "Public source, documented behavior, and locally recorded test evidence scored with a fixed 100-point rubric. Repository stars and raw tool counts do not score points.",
"rubric": [
{"category": "Functional CAD coverage", "weight": 25},
{"category": "Correctness and delivery", "weight": 20},
{"category": "Backends and platforms", "weight": 15},
{"category": "Engineering production", "weight": 15},
{"category": "Tests and maintenance", "weight": 15},
{"category": "Security and operations", "weight": 10}
],
"projects": [
{
"repository": "U-C4N/Autocad-MCP",
"url": "https://github.com/U-C4N/Autocad-MCP",
"reviewed_version": "1.3.0 working tree",
"score": 93,
"evidence_grade": "A",
"note": "Dual COM/ezdxf backends, standards-aware quality gates, deterministic geometry, closed-loop refinement, and auditable delivery."
},
{
"repository": "varavista/autocad-mcp",
"url": "https://github.com/varavista/autocad-mcp",
"reviewed_version": "2026-07-16 snapshot",
"score": 84,
"evidence_grade": "A",
"note": "Broad backend and CAD coverage with layout and xref depth."
},
{
"repository": "beiming183-cloud/AutoCAD-MCP",
"url": "https://github.com/beiming183-cloud/AutoCAD-MCP",
"reviewed_version": "2026-07-16 snapshot",
"score": 83,
"evidence_grade": "A",
"note": "Atomic operations, postconditions, 3D tools, and verified artifact delivery."
},
{
"repository": "AnCode666/multiCAD-mcp",
"url": "https://github.com/AnCode666/multiCAD-mcp",
"reviewed_version": "2026-07-16 snapshot",
"score": 73,
"evidence_grade": "A",
"note": "TABLE/MLEADER, data exchange, multi-document workflows, and alternative CAD support."
},
{
"repository": "puran-water/autocad-mcp",
"url": "https://github.com/puran-water/autocad-mcp",
"reviewed_version": "2026-07-16 snapshot",
"score": 70,
"evidence_grade": "A",
"note": "AutoCAD LT file IPC, ezdxf, P&ID workflows, leaders, and screenshots."
},
{
"repository": "NCO-1986/AutoCAD_mcp",
"url": "https://github.com/NCO-1986/AutoCAD_mcp",
"reviewed_version": "2026-07-16 snapshot",
"score": 68,
"evidence_grade": "B",
"note": ".NET main-thread integration with a broad live AutoCAD operation surface."
}
]
}
docs/assets/autocad-mcp-benchmark.svg
+2382
View File
File diff suppressed because it is too large Load Diff
Side by Side

After

Width:  |  Height:  |  Size: 70 KiB

tests/test_benchmark_chart.py
+3 -2
View File
@@ -1,10 +1,10 @@
import json
from collections.abc import Callable
from pathlib import Path
from typing import Any, Callable
from typing import Any
import pytest
ROOT = Path(__file__).resolve().parents[1]
DATA_PATH = ROOT / "benchmarks" / "source_review.json"
@@ -64,3 +64,4 @@ def test_render_chart_writes_readable_svg(tmp_path: Path) -> None:
assert "Source-reviewed capability benchmark" in svg
assert "U-C4N/Autocad-MCP" in svg
assert "Shared live-run results are not implied" in svg
assert all(line == line.rstrip() for line in svg.splitlines())