autocad-mcp-pro/benchmarks/render_chart.py

"""Render the source-reviewed AutoCAD MCP benchmark as a deterministic SVG."""

from __future__ import annotations

import argparse
import json
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_INPUT = ROOT / "benchmarks" / "source_review.json"
DEFAULT_OUTPUT = ROOT / "docs" / "assets" / "autocad-mcp-benchmark.svg"
REQUIRED_PROJECT_KEYS = {
    "repository",
    "url",
    "reviewed_version",
    "score",
    "evidence_grade",
    "note",
}


def load_benchmark(path: Path) -> dict[str, Any]:
    """Load and validate the benchmark snapshot used by docs and tests."""
    data = json.loads(path.read_text(encoding="utf-8"))
    if data.get("schema_version") != "1.0":
        raise ValueError("unsupported benchmark schema_version")
    if data.get("evidence_boundary") != "source_review_not_shared_live_run":
        raise ValueError("benchmark evidence boundary must be explicit")

    rubric = data.get("rubric")
    if not isinstance(rubric, list) or sum(item.get("weight", 0) for item in rubric) != 100:
        raise ValueError("benchmark rubric weights must total 100")

    projects = data.get("projects")
    if not isinstance(projects, list) or not projects:
        raise ValueError("benchmark must contain projects")

    seen: set[str] = set()
    for project in projects:
        missing = REQUIRED_PROJECT_KEYS - project.keys()
        if missing:
            raise ValueError(f"project is missing keys: {sorted(missing)}")
        repository = project["repository"]
        if repository in seen:
            raise ValueError(f"duplicate repository: {repository}")
        seen.add(repository)
        if not 0 <= project["score"] <= 100:
            raise ValueError(f"score outside 0..100: {repository}")
        if project["evidence_grade"] not in {"A", "B", "C"}:
            raise ValueError(f"invalid evidence grade: {repository}")
    return data


def render_chart(data: dict[str, Any], output: Path) -> None:
    """Render a GitHub-readable fixed-background horizontal bar chart."""
    try:
        import matplotlib

        matplotlib.use("Agg")
        import matplotlib.pyplot as plt
    except ImportError as exc:
        raise SystemExit(
            'Matplotlib is required. Install it with: pip install -e ".[pdf]"'
        ) from exc

    matplotlib.rcParams["svg.hashsalt"] = "autocad-mcp-source-review-v1"
    projects = sorted(data["projects"], key=lambda item: item["score"])
    total = len(projects)
    # rank 1 = highest score (drawn topmost by barh with ascending sort)
    labels = [
        f"{total - index}. {item['repository']}" for index, item in enumerate(projects)
    ]
    scores = [item["score"] for item in projects]
    grades = [item["evidence_grade"] for item in projects]
    # leaderboard palette by rank (rank 1 first), canvas/LLM-benchmark style
    rank_palette = [
        "#3fb950",  # 1 green
        "#58a6ff",  # 2 sky
        "#6e7ce0",  # 3 indigo
        "#d29922",  # 4 amber
        "#c4622d",  # 5 rust
        "#e3b341",  # 6 yellow
        "#bf4b8a",  # 7 magenta
        "#f0876e",  # 8 salmon
        "#8957e5",  # 9 purple
    ]
    colors = [
        rank_palette[(total - 1 - index) % len(rank_palette)]
        for index in range(total)
    ]

    fig, ax = plt.subplots(figsize=(12, 0.62 * total + 2.4), constrained_layout=True)
    fig.patch.set_facecolor("#0d1117")
    ax.set_facecolor("#0d1117")
    bars = ax.barh(labels, scores, color=colors, height=0.62)

    ax.set_xlim(0, 100)
    ax.set_xlabel("Weighted capability score / 100", color="#e6edf3", labelpad=10)
    ax.set_title(
        "Public AutoCAD MCP leaderboard — Source-reviewed capability benchmark",
        loc="left",
        fontsize=17,
        fontweight="bold",
        color="#e6edf3",
        pad=28,
    )
    ax.text(
        0,
        1.02,
        f"Fixed rubric · reviewed {data['reviewed_at']} · evidence grade shown per project",
        transform=ax.transAxes,
        color="#8b949e",
        fontsize=10,
    )

    for bar, score, grade in zip(bars, scores, grades, strict=True):
        ax.text(
            min(score + 1.2, 88),
            bar.get_y() + bar.get_height() / 2,
            f"{score} / 100  ·  {grade}",
            va="center",
            color="#e6edf3",
            fontweight="bold",
        )

    ax.text(
        0,
        -0.17,
        "Shared live-run results are not implied. Scores reflect public-source and recorded test evidence.",
        transform=ax.transAxes,
        color="#8b949e",
        fontsize=9,
    )
    ax.grid(axis="x", color="#30363d", linewidth=0.8, alpha=0.9)
    ax.set_axisbelow(True)
    ax.tick_params(axis="x", colors="#8b949e")
    ax.tick_params(axis="y", colors="#e6edf3", length=0, pad=8)
    for spine in ax.spines.values():
        spine.set_visible(False)

    output.parent.mkdir(parents=True, exist_ok=True)
    fig.savefig(
        output,
        format="svg",
        facecolor=fig.get_facecolor(),
        metadata={"Creator": "AutoCAD MCP Pro benchmark renderer", "Date": None},
    )
    plt.close(fig)

    svg = output.read_text(encoding="utf-8")
    normalized = "\n".join(line.rstrip() for line in svg.splitlines()) + "\n"
    output.write_text(normalized, encoding="utf-8", newline="\n")


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--input", type=Path, default=DEFAULT_INPUT)
    parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
    return parser


def main() -> int:
    args = build_parser().parse_args()
    render_chart(load_benchmark(args.input), args.output)
    print(f"Rendered benchmark chart: {args.output}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())