diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..9396c33 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,43 @@ +name: Benchmark + +# Headless competitor benchmark (v2 fixed-task matrix). Manual or weekly: +# clones the pinned competitor SHAs, runs the black-box MCP stdio lane and +# uploads the machine-readable reports as artifacts. Results committed to +# benchmarks/results/ are produced from these runs. + +on: + workflow_dispatch: + schedule: + - cron: "17 5 * * 1" + +jobs: + headless-matrix: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + server: + - autocad-mcp-pro + - puran-water-autocad-mcp + - beiming183-autocad-mcp + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install + run: pip install -e ".[pdf]" + - name: Run benchmark (ezdxf lane) + run: | + python -m benchmarks.run_competitors \ + --server "${{ matrix.server }}" \ + --backend ezdxf \ + --artifact-dir "benchmarks/results/ci/${{ matrix.server }}" \ + > "report-${{ matrix.server }}.json" + - name: Upload report + uses: actions/upload-artifact@v4 + with: + name: benchmark-${{ matrix.server }} + path: | + report-${{ matrix.server }}.json + benchmarks/results/ci/${{ matrix.server }} diff --git a/benchmarks/README.md b/benchmarks/README.md index eee269f..9dc4b8b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -42,10 +42,50 @@ python -m benchmarks.run_competitors --server autocad-mcp-pro --backend ezdxf -- python -m benchmarks.run_competitors --task table_mleader --task auditable_delivery --json ``` -The release-machine v1.3 ezdxf self-check is **10/10 (100.0)**. It is not a -competitor ranking. Cross-project scores become comparable only when another -adapter runs this exact task matrix; repository stars and raw tool counts do not -contribute to the score. Adapter registration lives in `competitors.yaml`. +The release-machine ezdxf self-check is **10/10 (100.0)**. Repository stars and +raw tool counts do not contribute to the score. Adapter registration lives in +`competitors.yaml`. + +## Live competitor lane (v1.4) + +Two competitor adapters now execute the exact same task matrix, black-box over +MCP stdio, against commits pinned in `competitors.yaml`: + +| Server | Pinned | Score | Pass | Coverage | +|---|---|---:|---:|---:| +| autocad-mcp-pro (reference) | working tree | 100.0 | 10/10 | 100% | +| beiming183-cloud/AutoCAD-MCP | `11f7c47e` | 50.0 | 5/10 | 50% | +| puran-water/autocad-mcp | `95476a33` | 45.0 | 4/10 | 50% | + +Method and boundaries: + +- `benchmarks/competitors_env.py` clones the pinned SHA into + `benchmarks/.competitors//` (gitignored), builds an isolated venv, and + installs the competitor from its own `pyproject.toml`. +- `benchmarks/adapters/mcp_stdio.py` drives the competitor exactly like an MCP + host (fastmcp `Client` over stdio). No competitor code is imported. +- Task playbooks call each server's **own documented tool contract**, read from + the pinned source (consolidated `operation` + payload tools; beiming's + `doc_id`/`expected_revision` discipline is honored via + `transaction(operation="context")`). +- **Verification never trusts the competitor's response**: every geometry claim + must survive `save`/`save_as_dxf` and re-opening the DXF with ezdxf inside + the harness. +- Tasks with no documented equivalent are reported `unsupported` with the + reason string; they stay in the fixed-matrix denominator at score zero. +- This lane is headless (ezdxf backends only). File-IPC / live-AutoCAD lanes + (including COM-only servers such as best-cad-mcp or daobataotie/CAD-MCP) + require a local AutoCAD session and are out of CI scope by design. + +Published machine-readable reports (artifact paths sanitized to filenames) +live under [`results/published/`](results/published/); the README chart is +regenerated from them: + +```bash +python -m benchmarks.run_competitors --server puran-water-autocad-mcp --backend ezdxf +python -m benchmarks.run_competitors --server beiming183-autocad-mcp --backend ezdxf +python -m benchmarks.render_live_chart +``` ## Correctness A/B suite diff --git a/benchmarks/adapters/beiming_autocad.py b/benchmarks/adapters/beiming_autocad.py new file mode 100644 index 0000000..c257010 --- /dev/null +++ b/benchmarks/adapters/beiming_autocad.py @@ -0,0 +1,301 @@ +"""Black-box adapter for beiming183-cloud/AutoCAD-MCP (headless ezdxf lane). + +Contract verified against the pinned checkout's ``server.py`` (v3.10 line): + +- document identity comes from ``transaction(operation="context")`` and every + mutation must carry ``doc_id`` + ``expected_revision`` — top-level for the + ``entity``/``annotation`` tools, inside ``data`` for ``drawing`` operations; +- ``entity create_line`` takes top-level ``x1..y2``; circle/dimension payloads + sit under ``data`` (same field spellings as the upstream puran-water base); +- ``transaction begin/commit/rollback`` are compatibility operations that the + headless backend may reject with ``E_UNSUPPORTED*`` — that outcome is + reported as ``unsupported`` rather than ``fail``. + +Verification is artifact-based: geometry claims are checked by exporting a DXF +(``drawing save_as_dxf``) and re-reading it with ezdxf in the harness. +""" + +from __future__ import annotations + +import math +from typing import Any + +from benchmarks.adapters.base import TaskResult +from benchmarks.adapters.mcp_stdio import ( + McpStdioAdapter, + ToolCallFailed, + extract_handle, +) +from benchmarks.competitors_env import CompetitorSpec + +PINNED_SHA = "11f7c47e5038796a20451b38b23032e625b5aa26" + + +def _find_key(payload: Any, *keys: str) -> Any: + if isinstance(payload, dict): + for key in keys: + if key in payload and payload[key] not in (None, ""): + return payload[key] + for value in payload.values(): + found = _find_key(value, *keys) + if found is not None: + return found + elif isinstance(payload, (list, tuple)): + for item in payload: + found = _find_key(item, *keys) + if found is not None: + return found + return None + + +class BeimingAutocadAdapter(McpStdioAdapter): + name = "beiming183-autocad-mcp" + spec = CompetitorSpec( + competitor_id="beiming183-autocad-mcp", + repo_url="https://github.com/beiming183-cloud/AutoCAD-MCP.git", + pinned_sha=PINNED_SHA, + entry_candidates=( + ("-m", "autocad_mcp"), + ("server.py",), + ("src/server.py",), + ), + pip_installs=(("-e", "."),), + launch_env={"AUTOCAD_MCP_BACKEND": "ezdxf"}, + pythonpath="src", + ) + + def extra_launch_env(self) -> dict[str, str]: + # The server redirects writes outside its managed output root; point + # the root at the benchmark artifact directory so exports land where + # the harness verifies them. + return { + "AUTOCAD_MCP_OUTPUT_ROOT": str(self.artifact_dir), + "AUTOCAD_MCP_ALLOW_EXTERNAL_OUTPUTS": "true", + } + + def warmup_calls(self): + # Absorb lazy backend initialization once during setup. + return ( + ("system", {"operation": "status"}), + ("drawing", {"operation": "create", "data": {}}), + ) + + async def _ctx(self) -> dict[str, Any]: + """Read doc_id + revision from ``transaction(operation="context")``.""" + try: + payload = await self.call("transaction", {"operation": "context"}) + except ToolCallFailed: + return {} + doc_id = _find_key(payload, "doc_id", "document_id") + revision = _find_key(payload, "revision", "expected_revision", "document_revision") + context: dict[str, Any] = {} + if doc_id is not None: + context["doc_id"] = doc_id + if revision is not None: + context["expected_revision"] = revision + return context + + async def _new_drawing(self) -> None: + await self.call("drawing", {"operation": "create", "data": {}}) + + async def _save_dxf(self, filename: str) -> str: + target = self.save_target(filename) + payload: dict[str, Any] = {"path": str(target)} + try: + await self.call("drawing", {"operation": "save_as_dxf", "data": payload}) + except ToolCallFailed: + context = await self._ctx() + await self.call("drawing", {"operation": "save_as_dxf", "data": {**payload, **context}}) + return str(target) + + async def _entity(self, operation: str, **fields: Any) -> Any: + context = await self._ctx() + return await self.call("entity", {"operation": operation, **context, **fields}) + + async def _create_line(self, x1: float, y1: float, x2: float, y2: float): + return await self._entity("create_line", x1=x1, y1=y1, x2=x2, y2=y2) + + # -- tasks ---------------------------------------------------------------- + + async def _task_core_geometry(self) -> TaskResult: + await self._new_drawing() + await self._create_line(0, 0, 3, 4) + await self._entity("create_circle", data={"cx": 10, "cy": 10, "radius": 2}) + artifact = await self._save_dxf("beiming_core_geometry.dxf") + _, doc = self.verify_dxf(self.save_target("beiming_core_geometry.dxf")) + lines = doc.modelspace().query("LINE") + circles = doc.modelspace().query("CIRCLE") + length = None + if lines: + start, end = lines[0].dxf.start, lines[0].dxf.end + length = math.dist((start.x, start.y), (end.x, end.y)) + passed = ( + len(lines) == 1 + and len(circles) == 1 + and length is not None + and abs(length - 5.0) < 1e-6 + and abs(circles[0].dxf.radius - 2.0) < 1e-6 + ) + return TaskResult( + "core_geometry", + "pass" if passed else "fail", + 100.0 if passed else 0.0, + metrics={"line_length": length, "lines": len(lines), "circles": len(circles)}, + artifacts=[artifact], + ) + + async def _task_modify_query(self) -> TaskResult: + await self._new_drawing() + created = await self._create_line(0, 0, 10, 0) + handle = extract_handle(created) + if not handle: + return TaskResult( + "modify_query", + "fail", + 0.0, + "could not extract created-entity handle from the create_line response", + ) + await self._entity("move", entity_id=handle, data={"dx": 5, "dy": 2}) + artifact = await self._save_dxf("beiming_modify_query.dxf") + _, doc = self.verify_dxf(self.save_target("beiming_modify_query.dxf")) + lines = doc.modelspace().query("LINE") + moved = ( + bool(lines) + and abs(lines[0].dxf.start.x - 5.0) < 1e-6 + and (abs(lines[0].dxf.start.y - 2.0) < 1e-6) + ) + return TaskResult( + "modify_query", + "pass" if moved else "fail", + 100.0 if moved else 0.0, + artifacts=[artifact], + ) + + async def _task_layers_linetypes(self) -> TaskResult: + await self._new_drawing() + context = await self._ctx() + await self.call("drawing", {"operation": "setup_mechanical", "data": {**context}}) + artifact = await self._save_dxf("beiming_layers.dxf") + _, doc = self.verify_dxf(self.save_target("beiming_layers.dxf")) + layers = {layer.dxf.name.upper() for layer in doc.layers} + linetypes = {lt.dxf.name.upper() for lt in doc.linetypes} + expected_layers = {"OUTLINE", "CENTER", "DIM"} + got_layers = expected_layers.issubset(layers) + got_linetype = "CENTER" in linetypes + if got_layers and got_linetype: + return TaskResult("layers_linetypes", "pass", 100.0, artifacts=[artifact]) + if got_layers: + return TaskResult( + "layers_linetypes", + "partial", + 50.0, + "GB/T layers created but CENTER linetype missing from the table", + artifacts=[artifact], + ) + return TaskResult( + "layers_linetypes", + "fail", + 0.0, + f"layers={sorted(layers)}", + artifacts=[artifact], + ) + + async def _task_dimensions(self) -> TaskResult: + await self._new_drawing() + context = await self._ctx() + await self.call( + "annotation", + { + "operation": "create_dimension_linear", + **context, + "data": {"x1": 0, "y1": 0, "x2": 50, "y2": 0, "dim_x": 25, "dim_y": 10}, + }, + ) + artifact = await self._save_dxf("beiming_dimensions.dxf") + _, doc = self.verify_dxf(self.save_target("beiming_dimensions.dxf")) + dims = [e for e in doc.modelspace() if e.dxftype() == "DIMENSION"] + return TaskResult( + "dimensions", + "pass" if dims else "fail", + 100.0 if dims else 0.0, + metrics={"dimension_count": len(dims)}, + artifacts=[artifact], + ) + + async def _task_transactions(self) -> TaskResult: + await self._new_drawing() + context = await self._ctx() + try: + begin = await self.call("transaction", {"operation": "begin", **context}) + except ToolCallFailed as exc: + if "E_UNSUPPORTED" in str(exc): + return TaskResult( + "transactions", + "unsupported", + 0.0, + "transaction begin is rejected as unsupported on the headless backend", + ) + raise + transaction_id = _find_key(begin, "transaction_id", "txn_id") + await self._entity("create_circle", data={"cx": 0, "cy": 0, "radius": 1}) + context = await self._ctx() + rollback_args: dict[str, Any] = {"operation": "rollback", **context} + if transaction_id is not None: + rollback_args["transaction_id"] = transaction_id + await self.call("transaction", rollback_args) + artifact = await self._save_dxf("beiming_transactions.dxf") + _, doc = self.verify_dxf(self.save_target("beiming_transactions.dxf")) + count = len(list(doc.modelspace())) + return TaskResult( + "transactions", + "pass" if count == 0 else "fail", + 100.0 if count == 0 else 0.0, + metrics={"entity_count_after_rollback": count}, + artifacts=[artifact], + ) + + async def _task_dxf_roundtrip(self) -> TaskResult: + await self._new_drawing() + await self._create_line(0, 0, 10, 10) + artifact = await self._save_dxf("beiming_roundtrip.dxf") + _, doc = self.verify_dxf(self.save_target("beiming_roundtrip.dxf")) + count = len(list(doc.modelspace())) + return TaskResult( + "dxf_roundtrip", + "pass" if count == 1 else "fail", + 100.0 if count == 1 else 0.0, + metrics={"entity_count": count}, + artifacts=[artifact], + ) + + # -- documented coverage gaps ---------------------------------------------- + + async def _task_table_mleader(self) -> TaskResult: + return TaskResult( + "table_mleader", + "unsupported", + 0.0, + "no TABLE or MLEADER operation in the annotation tool (v3.10)", + ) + + async def _task_preflight(self) -> TaskResult: + return TaskResult("preflight", "unsupported", 0.0, "no requirements-preflight equivalent") + + async def _task_quality_refiner(self) -> TaskResult: + return TaskResult( + "quality_refiner", + "unsupported", + 0.0, + "audits detect issues but there is no automated repair loop", + ) + + async def _task_auditable_delivery(self) -> TaskResult: + return TaskResult( + "auditable_delivery", + "unsupported", + 0.0, + "drawing.deliver is documented as File-IPC only (not headless)", + ) + + +__all__ = ["BeimingAutocadAdapter"] diff --git a/benchmarks/adapters/mcp_stdio.py b/benchmarks/adapters/mcp_stdio.py new file mode 100644 index 0000000..62fc1b0 --- /dev/null +++ b/benchmarks/adapters/mcp_stdio.py @@ -0,0 +1,304 @@ +"""Generic MCP-stdio benchmark adapter. + +Drives a *competitor* MCP server as a black box over the standard stdio +transport (fastmcp ``Client``), exactly the way an agent host would. No +competitor code is imported; verification never trusts the competitor's +response payload — every task that claims to have produced geometry must +save a DXF which the harness re-opens with ezdxf and inspects itself. + +Subclasses provide the launch runtime (via ``benchmarks.competitors_env``) +and per-task playbooks named ``_task_``; anything unmapped is +reported as ``unsupported`` (coverage-aware scoring already accounts for it). +""" + +from __future__ import annotations + +import dataclasses +import json +import re +from pathlib import Path +from typing import Any + +import ezdxf +from fastmcp import Client +from fastmcp.client.transports import StdioTransport + +from benchmarks.adapters.base import BenchmarkAdapter, TaskResult +from benchmarks.competitors_env import CompetitorRuntime, CompetitorSpec, ensure_competitor +from benchmarks.tasks_v2 import TaskSpec + +CALL_TIMEOUT_SECONDS = 15.0 +WARMUP_TIMEOUT_SECONDS = 90.0 + + +class ToolCallFailed(RuntimeError): + """A competitor tool call returned an error result.""" + + +def read_dxf(path: Path): + return ezdxf.readfile(str(path)) + + +def modelspace_types(doc) -> list[str]: + return [entity.dxftype() for entity in doc.modelspace()] + + +def find_dxf(artifact_dir: Path, preferred: Path | None = None) -> Path | None: + """Locate the DXF a competitor produced. + + Servers with managed output roots may relocate writes, so fall back to + the newest ``*.dxf`` anywhere under the artifact directory. + """ + if preferred is not None and preferred.is_file(): + return preferred + candidates = sorted( + artifact_dir.rglob("*.dxf"), key=lambda item: item.stat().st_mtime, reverse=True + ) + return candidates[0] if candidates else None + + +class McpStdioAdapter(BenchmarkAdapter): + """Black-box adapter base for competitor MCP servers.""" + + name = "mcp-stdio" + spec: CompetitorSpec # set by subclasses + + def __init__(self, *, backend: str = "ezdxf"): + if backend != "ezdxf": + raise ValueError( + f"{type(self).__name__} only benchmarks the headless ezdxf lane; " + f"got backend={backend!r}. COM lanes run locally, not in CI." + ) + self.backend_name = backend + self.artifact_dir = Path() + self.runtime: CompetitorRuntime | None = None + self._client: Client | None = None + self.tool_names: set[str] = set() + + # -- lifecycle ---------------------------------------------------------- + + def extra_launch_env(self) -> dict[str, str]: + """Environment computed at setup time (``self.artifact_dir`` is set).""" + return {} + + async def setup(self, artifact_dir: Path) -> None: + self.artifact_dir = Path(artifact_dir) + self.runtime = ensure_competitor(self.spec) + env = {**self.runtime.env, **self.extra_launch_env()} + env.setdefault("PYTHONIOENCODING", "utf-8") + transport = StdioTransport( + command=str(self.runtime.python_exe), + args=list(self.runtime.entry_args), + env=env, + cwd=str(self.runtime.repo_dir), + ) + self._client = Client(transport) + await self._client.__aenter__() + self.tool_names = {tool.name for tool in await self._client.list_tools()} + await self._warmup() + + def warmup_calls(self) -> tuple[tuple[str, dict[str, Any]], ...]: + """Calls issued once after connect to absorb the competitor's lazy + backend initialization (which can exceed the per-call timeout).""" + return () + + async def _warmup(self) -> None: + for tool, arguments in self.warmup_calls(): + if tool not in self.tool_names: + continue + try: + await self._client.call_tool( + tool, arguments, timeout=WARMUP_TIMEOUT_SECONDS, raise_on_error=False + ) + except Exception: # cold-start absorption is best effort + continue + + async def cleanup(self) -> None: + if self._client is not None: + await self._client.__aexit__(None, None, None) + self._client = None + + def metadata(self) -> dict: + return { + "backend": self.backend_name, + "kind": "mcp-stdio-blackbox", + "repo": self.spec.repo_url, + "pinned_sha": self.spec.pinned_sha, + "tools_discovered": sorted(self.tool_names), + } + + # -- call helpers --------------------------------------------------------- + + async def call(self, tool: str, arguments: dict[str, Any]) -> Any: + """Call one tool; raise ``ToolCallFailed`` on an error result. + + Competitor servers in this family report failures *inside* the payload + (``CommandResult(ok=False, error=...)`` serialized as text) rather than + via the MCP ``isError`` flag, so the payload is inspected as well. + """ + if tool not in self.tool_names: + raise ToolCallFailed(f"tool {tool!r} is not exposed by {self.name}") + result = await self._client.call_tool( + tool, arguments, timeout=CALL_TIMEOUT_SECONDS, raise_on_error=False + ) + if getattr(result, "is_error", False): + raise ToolCallFailed(f"{tool}({arguments}) -> {_result_text(result)}") + data = getattr(result, "data", None) + payload = data if data is not None else _result_text(result) + payload = _normalize_payload(payload) + error = _payload_error(payload) + if error: + raise ToolCallFailed(f"{tool}({arguments}) -> {error}") + return payload + + async def call_op( + self, + tool: str, + operation: str, + data: dict[str, Any] | None = None, + *, + shapes: tuple[dict[str, Any], ...] | None = None, + ) -> Any: + """Call a consolidated ``{tool}(operation=..., ...)`` style tool. + + Competitors disagree on whether payload fields live under ``data`` or + at the top level, and on field spellings. ``shapes`` supplies explicit + argument dicts to try in order; otherwise both common layouts of + ``data`` are attempted. The first non-error result wins. + """ + attempts: list[dict[str, Any]] = [] + if shapes: + attempts.extend({"operation": operation, **shape} for shape in shapes) + else: + payload = data or {} + attempts.append({"operation": operation, "data": payload}) + if payload: + attempts.append({"operation": operation, **payload}) + last_error: Exception | None = None + for arguments in attempts: + try: + return await self.call(tool, arguments) + except ToolCallFailed as exc: # try the next argument shape + last_error = exc + raise ToolCallFailed(str(last_error)) + + # -- task dispatch -------------------------------------------------------- + + async def run_task(self, task: TaskSpec) -> TaskResult: + handler = getattr(self, f"_task_{task.task_id}", None) + if handler is None: + return TaskResult( + task.task_id, + "unsupported", + 0.0, + f"{self.name} has no documented equivalent for this task", + ) + try: + return await handler() + except ToolCallFailed as exc: + return TaskResult(task.task_id, "fail", 0.0, str(exc)[:500]) + + # -- verification --------------------------------------------------------- + + def save_target(self, filename: str) -> Path: + return self.artifact_dir / filename + + def verify_dxf(self, preferred: Path | None = None): + located = find_dxf(self.artifact_dir, preferred) + if located is None: + raise ToolCallFailed("no DXF artifact was produced") + return located, read_dxf(located) + + +def _result_text(result: Any) -> str: + parts: list[str] = [] + for item in getattr(result, "content", None) or []: + text = getattr(item, "text", None) + if text: + parts.append(str(text)) + return " | ".join(parts)[:1000] if parts else "" + + +def _normalize_payload(payload: Any) -> Any: + """Unwrap fastmcp's dynamic output wrappers and JSON-in-text payloads. + + Legacy servers return plain text; the fastmcp client may deserialize it + into a generated *dataclass* (e.g. ``entityOutput(result='')``) or a + pydantic model. Normalize down to dict / list / str so verification and + handle extraction see real data. + """ + if dataclasses.is_dataclass(payload) and not isinstance(payload, type): + payload = dataclasses.asdict(payload) + elif hasattr(payload, "model_dump"): + payload = payload.model_dump() + elif not isinstance(payload, (dict, list, str)) and hasattr(payload, "result"): + payload = payload.result + if isinstance(payload, dict) and set(payload) == {"result"}: + payload = payload["result"] + if isinstance(payload, str): + try: + parsed = json.loads(payload) + except ValueError: + parsed = None + if isinstance(parsed, (dict, list)): + payload = parsed + return payload + + +_TEXT_ERROR_MARKERS = ('"ok": false', "'ok': false", '"ok":false', "ok=false") + + +def _payload_error(payload: Any) -> str | None: + """Detect failure reported inside a tool payload instead of MCP isError.""" + if isinstance(payload, dict): + if payload.get("ok") is False or payload.get("success") is False: + return str(payload.get("error") or payload.get("message") or "ok=false")[:300] + error = payload.get("error") + if isinstance(error, str) and error.strip(): + return error[:300] + return None + if isinstance(payload, str): + lowered = payload.lower() + if lowered.startswith("error") or any(m in lowered for m in _TEXT_ERROR_MARKERS): + return payload[:300] + return None + + +def wrapped_shapes(payload: dict[str, Any], **top_level: Any) -> tuple[dict[str, Any], ...]: + """Produce the two common consolidated-tool argument layouts for a payload: + nested under ``data`` and flattened at the top level.""" + return ( + {**top_level, "data": payload}, + {**top_level, **payload}, + ) + + +_HANDLE_KEYS = ("handle", "entity_id", "entity_handle", "id", "object_id") +_HANDLE_RE = re.compile( + r'"(?:handle|entity_id|entity_handle|object_id)"\s*:\s*"?([0-9A-Fa-f*][\w*-]*)"?' +) + + +def extract_handle(payload: Any) -> str | None: + """Best-effort extraction of a created-entity handle from a black-box + tool response (structured dict, list, or raw text).""" + if isinstance(payload, dict): + for key in _HANDLE_KEYS: + value = payload.get(key) + if isinstance(value, (str, int)) and str(value): + return str(value) + for value in payload.values(): + found = extract_handle(value) + if found: + return found + return None + if isinstance(payload, (list, tuple)): + for item in payload: + found = extract_handle(item) + if found: + return found + return None + if isinstance(payload, str): + match = _HANDLE_RE.search(payload) + return match.group(1) if match else None + return None diff --git a/benchmarks/adapters/puran_water.py b/benchmarks/adapters/puran_water.py new file mode 100644 index 0000000..23d65ea --- /dev/null +++ b/benchmarks/adapters/puran_water.py @@ -0,0 +1,224 @@ +"""Black-box adapter for puran-water/autocad-mcp (headless ezdxf lane). + +The competitor exposes 8 consolidated tools (``drawing``, ``entity``, +``layer``, ``block``, ``annotation``, ``pid``, ``view``, ``system``) whose +operations take an ``operation`` selector plus a payload. Argument spellings +are probed through ``wrapped_shapes`` fallbacks; every geometry claim is +verified by saving a DXF and re-reading it with ezdxf in the harness. + +Tasks with no documented equivalent on this server are reported as +``unsupported`` with an explicit reason so the published JSON explains the +coverage gap instead of hiding it. +""" + +from __future__ import annotations + +import math + +from benchmarks.adapters.base import TaskResult +from benchmarks.adapters.mcp_stdio import ( + McpStdioAdapter, + ToolCallFailed, + extract_handle, +) +from benchmarks.competitors_env import CompetitorSpec + +PINNED_SHA = "95476a33a1c246308326eb4709d6379ef2efdbc1" + + +class PuranWaterAdapter(McpStdioAdapter): + name = "puran-water-autocad-mcp" + spec = CompetitorSpec( + competitor_id="puran-water-autocad-mcp", + repo_url="https://github.com/puran-water/autocad-mcp.git", + pinned_sha=PINNED_SHA, + entry_candidates=( + ("-m", "autocad_mcp"), + ("server.py",), + ("src/server.py",), + ("main.py",), + ), + pip_installs=(("-e", "."),), + launch_env={"AUTOCAD_MCP_BACKEND": "ezdxf"}, + pythonpath="src", + ) + + # Exact v3.1 contract (verified against the pinned checkout's server.py): + # entity create_line takes top-level x1..y2; circle/dimension payloads sit + # under ``data``; modify ops take top-level ``entity_id`` + ``data``. + + def warmup_calls(self): + # The first backend touch lazily initializes and can exceed the + # per-call timeout; absorb it once during setup. + return ( + ("system", {"operation": "status"}), + ("drawing", {"operation": "create", "data": {}}), + ) + + async def _new_drawing(self) -> None: + await self.call_op("drawing", "create", {}) + + async def _save(self, filename: str) -> str: + target = self.save_target(filename) + await self.call_op("drawing", "save", {"path": str(target)}) + return str(target) + + async def _create_line(self, x1: float, y1: float, x2: float, y2: float): + return await self.call( + "entity", + {"operation": "create_line", "x1": x1, "y1": y1, "x2": x2, "y2": y2}, + ) + + # -- tasks ---------------------------------------------------------------- + + async def _task_core_geometry(self) -> TaskResult: + await self._new_drawing() + await self._create_line(0, 0, 3, 4) + await self.call_op("entity", "create_circle", {"cx": 10, "cy": 10, "radius": 2}) + preferred = await self._save("puran_core_geometry.dxf") + _, doc = self.verify_dxf(self.save_target("puran_core_geometry.dxf")) + lines = doc.modelspace().query("LINE") + circles = doc.modelspace().query("CIRCLE") + length = None + if lines: + start, end = lines[0].dxf.start, lines[0].dxf.end + length = math.dist((start.x, start.y), (end.x, end.y)) + passed = ( + len(lines) == 1 + and len(circles) == 1 + and length is not None + and abs(length - 5.0) < 1e-6 + and abs(circles[0].dxf.radius - 2.0) < 1e-6 + ) + return TaskResult( + "core_geometry", + "pass" if passed else "fail", + 100.0 if passed else 0.0, + metrics={"line_length": length, "lines": len(lines), "circles": len(circles)}, + artifacts=[preferred], + ) + + async def _task_modify_query(self) -> TaskResult: + await self._new_drawing() + created = await self._create_line(0, 0, 10, 0) + handle = extract_handle(created) + if not handle: + return TaskResult( + "modify_query", + "fail", + 0.0, + "could not extract created-entity handle from the create_line response", + ) + await self.call( + "entity", + {"operation": "move", "entity_id": handle, "data": {"dx": 5, "dy": 2}}, + ) + artifact = await self._save("puran_modify_query.dxf") + _, doc = self.verify_dxf(self.save_target("puran_modify_query.dxf")) + lines = doc.modelspace().query("LINE") + moved = ( + bool(lines) + and abs(lines[0].dxf.start.x - 5.0) < 1e-6 + and (abs(lines[0].dxf.start.y - 2.0) < 1e-6) + ) + return TaskResult( + "modify_query", + "pass" if moved else "fail", + 100.0 if moved else 0.0, + artifacts=[artifact], + ) + + async def _task_layers_linetypes(self) -> TaskResult: + await self._new_drawing() + try: + await self.call_op( + "layer", + "create", + {"name": "GEOMETRY", "color": 3, "linetype": "CENTER"}, + ) + except ToolCallFailed: + await self.call_op("layer", "create", {"name": "GEOMETRY", "color": 3}) + artifact = await self._save("puran_layers.dxf") + _, doc = self.verify_dxf(self.save_target("puran_layers.dxf")) + has_layer = "GEOMETRY" in {layer.dxf.name.upper() for layer in doc.layers} + has_center = "CENTER" in {lt.dxf.name.upper() for lt in doc.linetypes} + if has_layer and has_center: + return TaskResult("layers_linetypes", "pass", 100.0, artifacts=[artifact]) + if has_layer: + return TaskResult( + "layers_linetypes", + "partial", + 50.0, + "layer created; no linetype-loading operation is exposed", + artifacts=[artifact], + ) + return TaskResult("layers_linetypes", "fail", 0.0, artifacts=[artifact]) + + async def _task_dimensions(self) -> TaskResult: + await self._new_drawing() + await self.call_op( + "annotation", + "create_dimension_linear", + {"x1": 0, "y1": 0, "x2": 50, "y2": 0, "dim_x": 25, "dim_y": 10}, + ) + artifact = await self._save("puran_dimensions.dxf") + _, doc = self.verify_dxf(self.save_target("puran_dimensions.dxf")) + dims = [e for e in doc.modelspace() if e.dxftype() == "DIMENSION"] + return TaskResult( + "dimensions", + "pass" if dims else "fail", + 100.0 if dims else 0.0, + metrics={"dimension_count": len(dims)}, + artifacts=[artifact], + ) + + async def _task_dxf_roundtrip(self) -> TaskResult: + await self._new_drawing() + await self._create_line(0, 0, 10, 10) + artifact = await self._save("puran_roundtrip.dxf") + _, doc = self.verify_dxf(self.save_target("puran_roundtrip.dxf")) + count = len(list(doc.modelspace())) + return TaskResult( + "dxf_roundtrip", + "pass" if count == 1 else "fail", + 100.0 if count == 1 else 0.0, + metrics={"entity_count": count}, + artifacts=[artifact], + ) + + # -- documented coverage gaps ---------------------------------------------- + + async def _task_table_mleader(self) -> TaskResult: + return TaskResult( + "table_mleader", + "unsupported", + 0.0, + "no TABLE or MLEADER operation in the 8-tool surface (v3.1)", + ) + + async def _task_transactions(self) -> TaskResult: + return TaskResult( + "transactions", + "unsupported", + 0.0, + "no transaction begin/commit/rollback tool (undo/redo are File-IPC only)", + ) + + async def _task_preflight(self) -> TaskResult: + return TaskResult("preflight", "unsupported", 0.0, "no requirements-preflight equivalent") + + async def _task_quality_refiner(self) -> TaskResult: + return TaskResult( + "quality_refiner", "unsupported", 0.0, "no critique/repair loop equivalent" + ) + + async def _task_auditable_delivery(self) -> TaskResult: + return TaskResult( + "auditable_delivery", + "unsupported", + 0.0, + "no hashed delivery-manifest equivalent (plot_pdf is File-IPC only)", + ) + + +__all__ = ["PuranWaterAdapter", "ToolCallFailed"] diff --git a/benchmarks/competitors.yaml b/benchmarks/competitors.yaml index 8c564ad..0342f89 100644 --- a/benchmarks/competitors.yaml +++ b/benchmarks/competitors.yaml @@ -5,6 +5,22 @@ "id": "autocad-mcp-pro", "adapter": "benchmarks.adapters.autocad_mcp_pro:AutoCADMCPProAdapter", "backends": ["ezdxf", "com"] + }, + { + "id": "puran-water-autocad-mcp", + "adapter": "benchmarks.adapters.puran_water:PuranWaterAdapter", + "backends": ["ezdxf"], + "repo": "https://github.com/puran-water/autocad-mcp", + "pinned_sha": "95476a33a1c246308326eb4709d6379ef2efdbc1", + "notes": "Black-box MCP stdio lane; File-IPC (live AutoCAD) features run in the local COM lane only." + }, + { + "id": "beiming183-autocad-mcp", + "adapter": "benchmarks.adapters.beiming_autocad:BeimingAutocadAdapter", + "backends": ["ezdxf"], + "repo": "https://github.com/beiming183-cloud/AutoCAD-MCP", + "pinned_sha": "11f7c47e5038796a20451b38b23032e625b5aa26", + "notes": "Black-box MCP stdio lane; deliver/undo/redo are documented File-IPC only and reported unsupported." } ] } diff --git a/benchmarks/competitors_env.py b/benchmarks/competitors_env.py new file mode 100644 index 0000000..25eec76 --- /dev/null +++ b/benchmarks/competitors_env.py @@ -0,0 +1,121 @@ +"""Reproducible checkout + venv management for competitor MCP servers. + +Every competitor is pinned to an exact commit so a benchmark run is +reproducible: same tasks, same tree, same interpreter isolation. The cache +lives under ``benchmarks/.competitors//`` (gitignored) as:: + + repo/ the pinned git checkout + venv/ an isolated interpreter with the competitor installed + +Nothing here talks MCP — it only prepares a runnable ``python`` + entry args +for :class:`benchmarks.adapters.mcp_stdio.McpStdioAdapter`. +""" + +from __future__ import annotations + +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +CACHE_ROOT = Path(__file__).resolve().parent / ".competitors" + + +@dataclass(frozen=True) +class CompetitorSpec: + """Everything needed to fetch and launch one competitor server.""" + + competitor_id: str + repo_url: str + pinned_sha: str + # Candidate stdio entrypoints, resolved in order against the checkout. + # ("-m", "pkg") entries are accepted when repo/pkg or repo/src/pkg exists. + entry_candidates: tuple[tuple[str, ...], ...] + # pip arguments executed inside the venv, relative to the checkout. + pip_installs: tuple[tuple[str, ...], ...] = (("-e", "."),) + # Extra environment applied when launching the server. + launch_env: dict[str, str] = field(default_factory=dict) + # Set PYTHONPATH to this repo-relative directory (e.g. "src") if not None. + pythonpath: str | None = None + + +@dataclass(frozen=True) +class CompetitorRuntime: + """A resolved, launchable competitor installation.""" + + python_exe: Path + repo_dir: Path + entry_args: tuple[str, ...] + env: dict[str, str] + + +def _run(cmd: list[str], cwd: Path | None = None, timeout: float = 600.0) -> None: + completed = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=timeout) + if completed.returncode != 0: + raise RuntimeError( + f"command failed ({' '.join(cmd)}):\n{completed.stdout}\n{completed.stderr}" + ) + + +def _venv_python(venv_dir: Path) -> Path: + if sys.platform == "win32": + return venv_dir / "Scripts" / "python.exe" + return venv_dir / "bin" / "python" + + +def _ensure_checkout(spec: CompetitorSpec, repo_dir: Path) -> None: + if not (repo_dir / ".git").exists(): + repo_dir.parent.mkdir(parents=True, exist_ok=True) + _run(["git", "clone", "--quiet", spec.repo_url, str(repo_dir)]) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=repo_dir, capture_output=True, text=True + ).stdout.strip() + if head != spec.pinned_sha: + _run(["git", "fetch", "--quiet", "origin"], cwd=repo_dir) + _run(["git", "checkout", "--quiet", spec.pinned_sha], cwd=repo_dir) + + +def _ensure_venv(spec: CompetitorSpec, venv_dir: Path, repo_dir: Path) -> Path: + python_exe = _venv_python(venv_dir) + marker = venv_dir / ".installed-sha" + if python_exe.exists() and marker.exists() and marker.read_text().strip() == spec.pinned_sha: + return python_exe + _run([sys.executable, "-m", "venv", str(venv_dir)]) + _run([str(python_exe), "-m", "pip", "install", "--quiet", "--upgrade", "pip"]) + for install in spec.pip_installs: + _run([str(python_exe), "-m", "pip", "install", "--quiet", *install], cwd=repo_dir) + marker.write_text(spec.pinned_sha) + return python_exe + + +def _resolve_entry(spec: CompetitorSpec, repo_dir: Path) -> tuple[str, ...]: + for candidate in spec.entry_candidates: + if candidate[0] == "-m": + package = candidate[1].replace(".", "/") + roots = [repo_dir, repo_dir / "src"] + if any( + (root / package).is_dir() or (root / f"{package}.py").is_file() for root in roots + ): + return candidate + elif (repo_dir / candidate[0]).is_file(): + return tuple([str(repo_dir / candidate[0]), *candidate[1:]]) + raise RuntimeError( + f"{spec.competitor_id}: no stdio entrypoint found " + f"(tried {[c[0] for c in spec.entry_candidates]})" + ) + + +def ensure_competitor(spec: CompetitorSpec, cache_root: Path | None = None) -> CompetitorRuntime: + """Clone, pin, install and resolve one competitor. Idempotent.""" + root = (cache_root or CACHE_ROOT) / spec.competitor_id + repo_dir = root / "repo" + _ensure_checkout(spec, repo_dir) + python_exe = _ensure_venv(spec, root / "venv", repo_dir) + entry_args = _resolve_entry(spec, repo_dir) + + env = dict(spec.launch_env) + if spec.pythonpath is not None: + env["PYTHONPATH"] = str(repo_dir / spec.pythonpath) + return CompetitorRuntime( + python_exe=python_exe, repo_dir=repo_dir, entry_args=entry_args, env=env + ) diff --git a/benchmarks/render_live_chart.py b/benchmarks/render_live_chart.py new file mode 100644 index 0000000..6c83959 --- /dev/null +++ b/benchmarks/render_live_chart.py @@ -0,0 +1,136 @@ +"""Render the live-run benchmark lane (v2 fixed-task matrix) as a deterministic SVG. + +Input: the published per-server reports under ``benchmarks/results/published/`` +(produced by ``benchmarks.run_competitors``, artifact paths sanitized to +filenames). Output: a GitHub-readable horizontal bar chart showing the +weighted score and task coverage per server. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_INPUT_DIR = ROOT / "benchmarks" / "results" / "published" +DEFAULT_OUTPUT = ROOT / "docs" / "assets" / "autocad-mcp-livebench.svg" + + +def load_reports(input_dir: Path) -> list[dict[str, Any]]: + reports: list[dict[str, Any]] = [] + for path in sorted(input_dir.glob("*.json")): + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("schema_version") != "2.0": + raise ValueError(f"{path.name}: unsupported schema_version") + summary = data.get("summary") or {} + if "score" not in summary or "coverage_percent" not in summary: + raise ValueError(f"{path.name}: summary is missing score/coverage") + reports.append(data) + if not reports: + raise SystemExit(f"no published reports found in {input_dir}") + return reports + + +def render_chart(reports: list[dict[str, Any]], output: Path) -> None: + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError as exc: + raise SystemExit( + 'Matplotlib is required. Install it with: pip install -e ".[pdf]"' + ) from exc + + matplotlib.rcParams["svg.hashsalt"] = "autocad-mcp-livebench-v2" + reports = sorted(reports, key=lambda item: item["summary"]["score"]) + labels = [item["adapter"] for item in reports] + scores = [item["summary"]["score"] for item in reports] + coverage = [item["summary"]["coverage_percent"] for item in reports] + passed = [item["summary"]["passed"] for item in reports] + attempted = [item["summary"]["attempted"] for item in reports] + colors = ["#0969da" if label == "autocad-mcp-pro" else "#6e7781" for label in labels] + + fig, ax = plt.subplots(figsize=(12, 4.8), constrained_layout=True) + fig.patch.set_facecolor("#f6f8fa") + ax.set_facecolor("#f6f8fa") + bars = ax.barh(labels, scores, color=colors, height=0.58) + + ax.set_xlim(0, 100) + ax.set_xlabel( + "Weighted live-run score / 100 (fixed 10-task matrix)", color="#24292f", labelpad=10 + ) + ax.set_title( + "Live-run benchmark - headless ezdxf lane", + loc="left", + fontsize=18, + fontweight="bold", + color="#24292f", + pad=28, + ) + ax.text( + 0, + 1.02, + "Same 10 tasks, same harness, pinned competitor commits; artifacts verified by re-opening the DXF", + transform=ax.transAxes, + color="#57606a", + fontsize=10, + ) + + for bar, score, cov, ok, total in zip(bars, scores, coverage, passed, attempted, strict=True): + ax.text( + min(score + 1.2, 82), + bar.get_y() + bar.get_height() / 2, + f"{score:g} ({ok}/{total} pass, {cov:g}% coverage)", + va="center", + color="#24292f", + fontweight="bold", + ) + + ax.text( + 0, + -0.22, + "Unsupported tasks score 0 in the fixed matrix; per-task statuses and reasons are in benchmarks/results/published/.", + transform=ax.transAxes, + color="#57606a", + fontsize=9, + ) + ax.grid(axis="x", color="#d0d7de", linewidth=0.8, alpha=0.8) + ax.set_axisbelow(True) + ax.tick_params(axis="x", colors="#57606a") + ax.tick_params(axis="y", colors="#24292f", length=0, pad=8) + for spine in ax.spines.values(): + spine.set_visible(False) + + output.parent.mkdir(parents=True, exist_ok=True) + fig.savefig( + output, + format="svg", + facecolor=fig.get_facecolor(), + metadata={"Creator": "AutoCAD MCP Pro live benchmark renderer", "Date": None}, + ) + plt.close(fig) + + svg = output.read_text(encoding="utf-8") + normalized = "\n".join(line.rstrip() for line in svg.splitlines()) + "\n" + output.write_text(normalized, encoding="utf-8", newline="\n") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-dir", type=Path, default=DEFAULT_INPUT_DIR) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + return parser + + +def main() -> int: + args = build_parser().parse_args() + render_chart(load_reports(args.input_dir), args.output) + print(f"Rendered live benchmark chart: {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/results/published/autocad-mcp-pro.json b/benchmarks/results/published/autocad-mcp-pro.json new file mode 100644 index 0000000..c940240 --- /dev/null +++ b/benchmarks/results/published/autocad-mcp-pro.json @@ -0,0 +1,270 @@ +{ + "adapter": "autocad-mcp-pro", + "adapter_metadata": { + "backend": "ezdxf", + "capabilities": { + "backend": "ezdxf", + "features": { + "delivery": { + "mode": "shared", + "reason": null, + "supported": true + }, + "drawing_2d": { + "mode": "native", + "reason": null, + "supported": true + }, + "dxf": { + "mode": "native", + "reason": null, + "supported": true + }, + "lisp": { + "mode": null, + "reason": "live_com_only", + "supported": false + }, + "mleader": { + "mode": "composite", + "reason": null, + "supported": true + }, + "paper_space": { + "mode": null, + "reason": "planned_1.4", + "supported": false + }, + "pdf": { + "mode": "rendered", + "reason": null, + "supported": true + }, + "png": { + "mode": "rendered", + "reason": null, + "supported": true + }, + "preflight": { + "mode": "shared", + "reason": null, + "supported": true + }, + "refiner": { + "mode": "shared", + "reason": null, + "supported": true + }, + "solid_3d": { + "mode": null, + "reason": "planned_1.5", + "supported": false + }, + "table": { + "mode": "composite", + "reason": null, + "supported": true + }, + "transactions": { + "mode": "snapshot", + "reason": null, + "supported": true + } + } + } + }, + "environment": { + "implementation": "CPython", + "machine": "AMD64", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.14.0" + }, + "generated_at": "2026-07-23T11:39:04.277175+00:00", + "git_sha": "4a2e9d658f051cb756f92cb8fae945c64b8cc7d4", + "results": [ + { + "artifacts": [], + "duration_ms": 8.14, + "message": "", + "metrics": { + "line_length": 5.0 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "core_geometry" + }, + { + "artifacts": [], + "duration_ms": 3.39, + "message": "", + "metrics": {}, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "modify_query" + }, + { + "artifacts": [], + "duration_ms": 8.11, + "message": "", + "metrics": {}, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "layers_linetypes" + }, + { + "artifacts": [], + "duration_ms": 348.74, + "message": "", + "metrics": { + "entity_type": "DIMENSION" + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "dimensions" + }, + { + "artifacts": [], + "duration_ms": 15.74, + "message": "", + "metrics": { + "representations": [ + "composite", + "composite" + ] + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "table_mleader" + }, + { + "artifacts": [], + "duration_ms": 28.95, + "message": "", + "metrics": { + "entity_count": 0 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "transactions" + }, + { + "artifacts": [], + "duration_ms": 2.5, + "message": "", + "metrics": {}, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "preflight" + }, + { + "artifacts": [], + "duration_ms": 29.19, + "message": "", + "metrics": { + "final_score": 100.0, + "initial_score": 92.0, + "remaining_issues": [], + "rounds": [ + { + "actions": [ + { + "detail": { + "deleted_handle": "3E", + "kept_handle": "3D" + }, + "fingerprint": "c1b87eac3cfdf11312c9b18ccf4c062e672791f7a9a6dfa3c45b6db7f9e5835e", + "focus": "duplicate_entities", + "handles": [ + "3D", + "3E" + ], + "status": "applied" + } + ], + "error_count_after": 0, + "error_count_before": 0, + "number": 1, + "score_after": 100.0, + "score_before": 92.0, + "transaction": "committed" + } + ], + "status": "threshold_met" + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "quality_refiner" + }, + { + "artifacts": [ + { + "filename": "roundtrip.dxf", + "sha256": "6413204a5baec19a5fbf551418eb56e30b1488ca5bb755e6a8709ac8e8ab0b22", + "size_bytes": 18442 + } + ], + "duration_ms": 16.66, + "message": "", + "metrics": { + "entity_count": 1 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "dxf_roundtrip" + }, + { + "artifacts": [ + { + "filename": "manifest.json", + "sha256": "38c2ee819b49b9c66b2311b952e856eff9e3a7565194cce2edea60a3211d233b", + "size_bytes": 3581 + } + ], + "duration_ms": 4380.8, + "message": "", + "metrics": { + "score": { + "errors": 0, + "grade": "A", + "info": 0, + "invalidity_ratio": 0.0, + "score": 100.0, + "warnings": 0 + } + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "auditable_delivery" + } + ], + "schema_version": "2.0", + "summary": { + "attempted": 10, + "coverage_percent": 100.0, + "passed": 10, + "score": 100.0, + "supported": 10, + "unsupported": 0 + }, + "timeout_seconds": 30.0 +} \ No newline at end of file diff --git a/benchmarks/results/published/beiming183-autocad-mcp.json b/benchmarks/results/published/beiming183-autocad-mcp.json new file mode 100644 index 0000000..c137ba5 --- /dev/null +++ b/benchmarks/results/published/beiming183-autocad-mcp.json @@ -0,0 +1,191 @@ +{ + "adapter": "beiming183-autocad-mcp", + "adapter_metadata": { + "backend": "ezdxf", + "kind": "mcp-stdio-blackbox", + "pinned_sha": "11f7c47e5038796a20451b38b23032e625b5aa26", + "repo": "https://github.com/beiming183-cloud/AutoCAD-MCP.git", + "tools_discovered": [ + "annotation", + "block", + "drawing", + "entity", + "job", + "layer", + "pid", + "product", + "solid", + "system", + "transaction", + "view" + ] + }, + "environment": { + "implementation": "CPython", + "machine": "AMD64", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.14.0" + }, + "generated_at": "2026-07-23T11:38:36.969519+00:00", + "git_sha": "4a2e9d658f051cb756f92cb8fae945c64b8cc7d4", + "results": [ + { + "artifacts": [ + { + "filename": "beiming_core_geometry.dxf", + "sha256": "d972b29da994b1ea8f3887eef84f2ed892a486409385b56cd444b1fe9794e337", + "size_bytes": 18587 + } + ], + "duration_ms": 96.24, + "message": "", + "metrics": { + "circles": 1, + "line_length": 5.0, + "lines": 1 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "core_geometry" + }, + { + "artifacts": [ + { + "filename": "beiming_modify_query.dxf", + "sha256": "298534d8512ce62833551ab6e54c0c2a0baf18bdf7a96bd956cdf06280fb4fb8", + "size_bytes": 18473 + } + ], + "duration_ms": 108.09, + "message": "", + "metrics": {}, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "modify_query" + }, + { + "artifacts": [ + { + "filename": "beiming_layers.dxf", + "sha256": "b401e145e97aca7011d03097190b3d0ece0a89b47d38022a81c6c4c7e250ed3d", + "size_bytes": 19887 + } + ], + "duration_ms": 69.86, + "message": "", + "metrics": {}, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "layers_linetypes" + }, + { + "artifacts": [ + { + "filename": "beiming_dimensions.dxf", + "sha256": "8996d2ba42da66f46e79d1655405a4bf2ba72301861eeea58509cba44c408fb4", + "size_bytes": 20972 + } + ], + "duration_ms": 75.55, + "message": "", + "metrics": { + "dimension_count": 1 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "dimensions" + }, + { + "artifacts": [], + "duration_ms": 0.12, + "message": "no TABLE or MLEADER operation in the annotation tool (v3.10)", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "table_mleader" + }, + { + "artifacts": [], + "duration_ms": 18.99, + "message": "transaction begin is rejected as unsupported on the headless backend", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "transactions" + }, + { + "artifacts": [], + "duration_ms": 0.04, + "message": "no requirements-preflight equivalent", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "preflight" + }, + { + "artifacts": [], + "duration_ms": 0.02, + "message": "audits detect issues but there is no automated repair loop", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "quality_refiner" + }, + { + "artifacts": [ + { + "filename": "beiming_roundtrip.dxf", + "sha256": "79d7bdc1e66e506dd2ecf66fbf030839242c77870021ccbfe3c819ad394695ce", + "size_bytes": 18474 + } + ], + "duration_ms": 69.95, + "message": "", + "metrics": { + "entity_count": 1 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "dxf_roundtrip" + }, + { + "artifacts": [], + "duration_ms": 0.06, + "message": "drawing.deliver is documented as File-IPC only (not headless)", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "auditable_delivery" + } + ], + "schema_version": "2.0", + "summary": { + "attempted": 10, + "coverage_percent": 50.0, + "passed": 5, + "score": 50.0, + "supported": 5, + "unsupported": 5 + }, + "timeout_seconds": 30.0 +} \ No newline at end of file diff --git a/benchmarks/results/published/puran-water-autocad-mcp.json b/benchmarks/results/published/puran-water-autocad-mcp.json new file mode 100644 index 0000000..9358a0d --- /dev/null +++ b/benchmarks/results/published/puran-water-autocad-mcp.json @@ -0,0 +1,187 @@ +{ + "adapter": "puran-water-autocad-mcp", + "adapter_metadata": { + "backend": "ezdxf", + "kind": "mcp-stdio-blackbox", + "pinned_sha": "95476a33a1c246308326eb4709d6379ef2efdbc1", + "repo": "https://github.com/puran-water/autocad-mcp.git", + "tools_discovered": [ + "annotation", + "block", + "drawing", + "entity", + "layer", + "pid", + "system", + "view" + ] + }, + "environment": { + "implementation": "CPython", + "machine": "AMD64", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.14.0" + }, + "generated_at": "2026-07-23T11:32:07.977232+00:00", + "git_sha": "4a2e9d658f051cb756f92cb8fae945c64b8cc7d4", + "results": [ + { + "artifacts": [ + { + "filename": "puran_core_geometry.dxf", + "sha256": "bc8185404c8344134859293a0e28ebf821cd5968553cdd96b7029e46e1ca0c99", + "size_bytes": 18587 + } + ], + "duration_ms": 205.92, + "message": "", + "metrics": { + "circles": 1, + "line_length": 5.0, + "lines": 1 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "core_geometry" + }, + { + "artifacts": [ + { + "filename": "puran_modify_query.dxf", + "sha256": "2113a59d0a9f624b6134e89f992d153bc8e085495e956f4887b8ba7b54f3c005", + "size_bytes": 18473 + } + ], + "duration_ms": 156.79, + "message": "", + "metrics": {}, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "modify_query" + }, + { + "artifacts": [ + { + "filename": "puran_layers.dxf", + "sha256": "7144467e22e2e8c8d18f1fc4f177f078f9ce450bb73f4e9ca8f74c16d67e1b53", + "size_bytes": 18498 + } + ], + "duration_ms": 189.98, + "message": "layer created; no linetype-loading operation is exposed", + "metrics": {}, + "score": 50.0, + "status": "partial", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "layers_linetypes" + }, + { + "artifacts": [ + { + "filename": "puran_dimensions.dxf", + "sha256": "dde808c5e504c5e6b7837f5f5c14ef096711812a6239dd284906efed5c9e3614", + "size_bytes": 20972 + } + ], + "duration_ms": 215.28, + "message": "", + "metrics": { + "dimension_count": 1 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "dimensions" + }, + { + "artifacts": [], + "duration_ms": 0.12, + "message": "no TABLE or MLEADER operation in the 8-tool surface (v3.1)", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "table_mleader" + }, + { + "artifacts": [], + "duration_ms": 0.02, + "message": "no transaction begin/commit/rollback tool (undo/redo are File-IPC only)", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "transactions" + }, + { + "artifacts": [], + "duration_ms": 0.01, + "message": "no requirements-preflight equivalent", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "preflight" + }, + { + "artifacts": [], + "duration_ms": 0.03, + "message": "no critique/repair loop equivalent", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "quality_refiner" + }, + { + "artifacts": [ + { + "filename": "puran_roundtrip.dxf", + "sha256": "11335953641be32f3f47c23fbc13f79876111c6f84f8874af8f5e04d5e398cf6", + "size_bytes": 18474 + } + ], + "duration_ms": 143.4, + "message": "", + "metrics": { + "entity_count": 1 + }, + "score": 100.0, + "status": "pass", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "dxf_roundtrip" + }, + { + "artifacts": [], + "duration_ms": 0.06, + "message": "no hashed delivery-manifest equivalent (plot_pdf is File-IPC only)", + "metrics": {}, + "score": 0.0, + "status": "unsupported", + "stderr_summary": "", + "stdout_summary": "", + "task_id": "auditable_delivery" + } + ], + "schema_version": "2.0", + "summary": { + "attempted": 10, + "coverage_percent": 50.0, + "passed": 4, + "score": 45.0, + "supported": 5, + "unsupported": 5 + }, + "timeout_seconds": 30.0 +} \ No newline at end of file diff --git a/benchmarks/run_competitors.py b/benchmarks/run_competitors.py index 5450170..28e37c8 100644 --- a/benchmarks/run_competitors.py +++ b/benchmarks/run_competitors.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import asyncio import hashlib +import importlib import json import platform import subprocess @@ -17,6 +18,19 @@ from benchmarks.adapters.base import BenchmarkAdapter, TaskResult from benchmarks.tasks_v2 import TASKS_V2, TaskSpec, task_by_id +def _load_registry() -> dict[str, dict]: + """Read competitors.yaml (JSON content) into an id -> entry registry.""" + path = Path(__file__).with_name("competitors.yaml") + data = json.loads(path.read_text(encoding="utf-8")) + return {entry["id"]: entry for entry in data.get("competitors", [])} + + +def _make_adapter(entry: dict, backend: str) -> BenchmarkAdapter: + module_name, _, attr = entry["adapter"].partition(":") + adapter_cls = getattr(importlib.import_module(module_name), attr) + return adapter_cls(backend=backend) + + def _git_sha() -> str | None: try: result = subprocess.run( @@ -91,9 +105,7 @@ async def run_tasks( total_weight = sum(weight_by_id.values()) supported_weight = sum(weight_by_id[item.task_id] for item in supported) weighted_score = sum( - weight_by_id[item.task_id] * item.score - for item in results - if item.status != "unsupported" + weight_by_id[item.task_id] * item.score for item in results if item.status != "unsupported" ) return { "schema_version": "2.0", @@ -122,10 +134,15 @@ async def run_tasks( } -def _parser() -> argparse.ArgumentParser: +def _parser(registry: dict[str, dict]) -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--list", action="store_true", help="List the fixed benchmark tasks") - parser.add_argument("--server", default="autocad-mcp-pro", choices=["autocad-mcp-pro"]) + parser.add_argument( + "--server", + default="autocad-mcp-pro", + choices=sorted(registry), + help="Adapter id from benchmarks/competitors.yaml", + ) parser.add_argument("--backend", default="ezdxf", choices=["ezdxf", "com"]) parser.add_argument("--task", action="append", help="Run only this task id (repeatable)") parser.add_argument("--artifact-dir", default="benchmarks/results/latest") @@ -135,19 +152,22 @@ def _parser() -> argparse.ArgumentParser: def main() -> None: - args = _parser().parse_args() + registry = _load_registry() + args = _parser(registry).parse_args() if args.list: for task in TASKS_V2: print(f"{task.task_id}\t{task.category}\t{task.description}") return - from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter - + entry = registry[args.server] + supported_backends = entry.get("backends", ["ezdxf"]) + if args.backend not in supported_backends: + raise SystemExit( + f"{args.server} supports backends {supported_backends}, not {args.backend!r}" + ) tasks = [task_by_id(task_id) for task_id in args.task] if args.task else list(TASKS_V2) - adapter = AutoCADMCPProAdapter(backend=args.backend) - report = asyncio.run( - run_tasks(adapter, tasks, args.artifact_dir, timeout=args.timeout) - ) + adapter = _make_adapter(entry, args.backend) + report = asyncio.run(run_tasks(adapter, tasks, args.artifact_dir, timeout=args.timeout)) rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) print(rendered) diff --git a/docs/assets/autocad-mcp-livebench.svg b/docs/assets/autocad-mcp-livebench.svg new file mode 100644 index 0000000..6597f84 --- /dev/null +++ b/docs/assets/autocad-mcp-livebench.svg @@ -0,0 +1,2213 @@ + + + + + + + + image/svg+xml + + + AutoCAD MCP Pro live benchmark renderer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +