feat: add reproducible benchmark visualization

benchmarks/render_chart.py

@@ -0,0 +1,154 @@
+"""Render the source-reviewed AutoCAD MCP benchmark as a deterministic SVG."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_INPUT = ROOT / "benchmarks" / "source_review.json"
+DEFAULT_OUTPUT = ROOT / "docs" / "assets" / "autocad-mcp-benchmark.svg"
+REQUIRED_PROJECT_KEYS = {
+    "repository",
+    "url",
+    "reviewed_version",
+    "score",
+    "evidence_grade",
+    "note",
+}
+
+
+def load_benchmark(path: Path) -> dict[str, Any]:
+    """Load and validate the benchmark snapshot used by docs and tests."""
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if data.get("schema_version") != "1.0":
+        raise ValueError("unsupported benchmark schema_version")
+    if data.get("evidence_boundary") != "source_review_not_shared_live_run":
+        raise ValueError("benchmark evidence boundary must be explicit")
+
+    rubric = data.get("rubric")
+    if not isinstance(rubric, list) or sum(item.get("weight", 0) for item in rubric) != 100:
+        raise ValueError("benchmark rubric weights must total 100")
+
+    projects = data.get("projects")
+    if not isinstance(projects, list) or not projects:
+        raise ValueError("benchmark must contain projects")
+
+    seen: set[str] = set()
+    for project in projects:
+        missing = REQUIRED_PROJECT_KEYS - project.keys()
+        if missing:
+            raise ValueError(f"project is missing keys: {sorted(missing)}")
+        repository = project["repository"]
+        if repository in seen:
+            raise ValueError(f"duplicate repository: {repository}")
+        seen.add(repository)
+        if not 0 <= project["score"] <= 100:
+            raise ValueError(f"score outside 0..100: {repository}")
+        if project["evidence_grade"] not in {"A", "B", "C"}:
+            raise ValueError(f"invalid evidence grade: {repository}")
+    return data
+
+
+def render_chart(data: dict[str, Any], output: Path) -> None:
+    """Render a GitHub-readable fixed-background horizontal bar chart."""
+    try:
+        import matplotlib
+
+        matplotlib.use("Agg")
+        import matplotlib.pyplot as plt
+    except ImportError as exc:
+        raise SystemExit(
+            'Matplotlib is required. Install it with: pip install -e ".[pdf]"'
+        ) from exc
+
+    matplotlib.rcParams["svg.hashsalt"] = "autocad-mcp-source-review-v1"
+    projects = sorted(data["projects"], key=lambda item: item["score"])
+    labels = [item["repository"] for item in projects]
+    scores = [item["score"] for item in projects]
+    grades = [item["evidence_grade"] for item in projects]
+    colors = [
+        "#0969da" if label == "U-C4N/Autocad-MCP" else "#6e7781" for label in labels
+    ]
+
+    fig, ax = plt.subplots(figsize=(12, 6.4), constrained_layout=True)
+    fig.patch.set_facecolor("#f6f8fa")
+    ax.set_facecolor("#f6f8fa")
+    bars = ax.barh(labels, scores, color=colors, height=0.62)
+
+    ax.set_xlim(0, 100)
+    ax.set_xlabel("Weighted capability score / 100", color="#24292f", labelpad=10)
+    ax.set_title(
+        "Source-reviewed capability benchmark",
+        loc="left",
+        fontsize=18,
+        fontweight="bold",
+        color="#24292f",
+        pad=28,
+    )
+    ax.text(
+        0,
+        1.02,
+        f"Fixed rubric · reviewed {data['reviewed_at']} · evidence grade shown per project",
+        transform=ax.transAxes,
+        color="#57606a",
+        fontsize=10,
+    )
+
+    for bar, score, grade in zip(bars, scores, grades, strict=True):
+        ax.text(
+            min(score + 1.2, 96),
+            bar.get_y() + bar.get_height() / 2,
+            f"{score}  ·  {grade}",
+            va="center",
+            color="#24292f",
+            fontweight="bold",
+        )
+
+    ax.text(
+        0,
+        -0.17,
+        "Shared live-run results are not implied. Scores reflect public-source and recorded test evidence.",
+        transform=ax.transAxes,
+        color="#57606a",
+        fontsize=9,
+    )
+    ax.grid(axis="x", color="#d0d7de", linewidth=0.8, alpha=0.8)
+    ax.set_axisbelow(True)
+    ax.tick_params(axis="x", colors="#57606a")
+    ax.tick_params(axis="y", colors="#24292f", length=0, pad=8)
+    for spine in ax.spines.values():
+        spine.set_visible(False)
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(
+        output,
+        format="svg",
+        facecolor=fig.get_facecolor(),
+        metadata={"Creator": "AutoCAD MCP Pro benchmark renderer", "Date": None},
+    )
+    plt.close(fig)
+
+    svg = output.read_text(encoding="utf-8")
+    normalized = "\n".join(line.rstrip() for line in svg.splitlines()) + "\n"
+    output.write_text(normalized, encoding="utf-8", newline="\n")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, default=DEFAULT_INPUT)
+    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
+    return parser
+
+
+def main() -> int:
+    args = build_parser().parse_args()
+    render_chart(load_benchmark(args.input), args.output)
+    print(f"Rendered benchmark chart: {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/source_review.json

@@ -0,0 +1,65 @@
+{
+  "schema_version": "1.0",
+  "reviewed_at": "2026-07-17",
+  "title": "AutoCAD MCP source-reviewed capability benchmark",
+  "evidence_boundary": "source_review_not_shared_live_run",
+  "methodology": "Public source, documented behavior, and locally recorded test evidence scored with a fixed 100-point rubric. Repository stars and raw tool counts do not score points.",
+  "rubric": [
+    {"category": "Functional CAD coverage", "weight": 25},
+    {"category": "Correctness and delivery", "weight": 20},
+    {"category": "Backends and platforms", "weight": 15},
+    {"category": "Engineering production", "weight": 15},
+    {"category": "Tests and maintenance", "weight": 15},
+    {"category": "Security and operations", "weight": 10}
+  ],
+  "projects": [
+    {
+      "repository": "U-C4N/Autocad-MCP",
+      "url": "https://github.com/U-C4N/Autocad-MCP",
+      "reviewed_version": "1.3.0 working tree",
+      "score": 93,
+      "evidence_grade": "A",
+      "note": "Dual COM/ezdxf backends, standards-aware quality gates, deterministic geometry, closed-loop refinement, and auditable delivery."
+    },
+    {
+      "repository": "varavista/autocad-mcp",
+      "url": "https://github.com/varavista/autocad-mcp",
+      "reviewed_version": "2026-07-16 snapshot",
+      "score": 84,
+      "evidence_grade": "A",
+      "note": "Broad backend and CAD coverage with layout and xref depth."
+    },
+    {
+      "repository": "beiming183-cloud/AutoCAD-MCP",
+      "url": "https://github.com/beiming183-cloud/AutoCAD-MCP",
+      "reviewed_version": "2026-07-16 snapshot",
+      "score": 83,
+      "evidence_grade": "A",
+      "note": "Atomic operations, postconditions, 3D tools, and verified artifact delivery."
+    },
+    {
+      "repository": "AnCode666/multiCAD-mcp",
+      "url": "https://github.com/AnCode666/multiCAD-mcp",
+      "reviewed_version": "2026-07-16 snapshot",
+      "score": 73,
+      "evidence_grade": "A",
+      "note": "TABLE/MLEADER, data exchange, multi-document workflows, and alternative CAD support."
+    },
+    {
+      "repository": "puran-water/autocad-mcp",
+      "url": "https://github.com/puran-water/autocad-mcp",
+      "reviewed_version": "2026-07-16 snapshot",
+      "score": 70,
+      "evidence_grade": "A",
+      "note": "AutoCAD LT file IPC, ezdxf, P&ID workflows, leaders, and screenshots."
+    },
+    {
+      "repository": "NCO-1986/AutoCAD_mcp",
+      "url": "https://github.com/NCO-1986/AutoCAD_mcp",
+      "reviewed_version": "2026-07-16 snapshot",
+      "score": 68,
+      "evidence_grade": "B",
+      "note": ".NET main-thread integration with a broad live AutoCAD operation surface."
+    }
+  ]
+}

tests/test_benchmark_chart.py

@@ -1,10 +1,10 @@
 import json
+from collections.abc import Callable
 from pathlib import Path
-from typing import Any, Callable
+from typing import Any
 
 import pytest
 
-
 ROOT = Path(__file__).resolve().parents[1]
 DATA_PATH = ROOT / "benchmarks" / "source_review.json"
 
@@ -64,3 +64,4 @@ def test_render_chart_writes_readable_svg(tmp_path: Path) -> None:
     assert "Source-reviewed capability benchmark" in svg
     assert "U-C4N/Autocad-MCP" in svg
     assert "Shared live-run results are not implied" in svg
+    assert all(line == line.rstrip() for line in svg.splitlines())