Umutcan EdizsalanandCursor
6d7c39e012
- adapters/mcp_stdio.py: drives competitors over MCP stdio like a real host; payload-level error detection, fastmcp output-wrapper normalization, cold-start warmup, artifact-based verification (harness re-opens every exported DXF with ezdxf) - competitors_env.py: reproducible pinned-SHA checkout + isolated venv per competitor - adapters for puran-water/autocad-mcp (95476a3) and beiming183-cloud/AutoCAD-MCP (11f7c47) mapped to their own documented tool contracts; unmapped tasks report 'unsupported' with reasons - run_competitors.py: registry-driven --server from competitors.yaml - published reports (pro 100.0 / beiming 50.0 / puran 45.0) + render_live_chart.py -> docs/assets/autocad-mcp-livebench.svg - benchmark.yml: weekly + manual dispatch matrix Co-authored-by: Cursor <cursoragent@cursor.com>
302 lines
12 KiB
Python
302 lines
12 KiB
Python
"""Black-box adapter for beiming183-cloud/AutoCAD-MCP (headless ezdxf lane).
|
|
|
|
Contract verified against the pinned checkout's ``server.py`` (v3.10 line):
|
|
|
|
- document identity comes from ``transaction(operation="context")`` and every
|
|
mutation must carry ``doc_id`` + ``expected_revision`` — top-level for the
|
|
``entity``/``annotation`` tools, inside ``data`` for ``drawing`` operations;
|
|
- ``entity create_line`` takes top-level ``x1..y2``; circle/dimension payloads
|
|
sit under ``data`` (same field spellings as the upstream puran-water base);
|
|
- ``transaction begin/commit/rollback`` are compatibility operations that the
|
|
headless backend may reject with ``E_UNSUPPORTED*`` — that outcome is
|
|
reported as ``unsupported`` rather than ``fail``.
|
|
|
|
Verification is artifact-based: geometry claims are checked by exporting a DXF
|
|
(``drawing save_as_dxf``) and re-reading it with ezdxf in the harness.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from typing import Any
|
|
|
|
from benchmarks.adapters.base import TaskResult
|
|
from benchmarks.adapters.mcp_stdio import (
|
|
McpStdioAdapter,
|
|
ToolCallFailed,
|
|
extract_handle,
|
|
)
|
|
from benchmarks.competitors_env import CompetitorSpec
|
|
|
|
PINNED_SHA = "11f7c47e5038796a20451b38b23032e625b5aa26"
|
|
|
|
|
|
def _find_key(payload: Any, *keys: str) -> Any:
|
|
if isinstance(payload, dict):
|
|
for key in keys:
|
|
if key in payload and payload[key] not in (None, ""):
|
|
return payload[key]
|
|
for value in payload.values():
|
|
found = _find_key(value, *keys)
|
|
if found is not None:
|
|
return found
|
|
elif isinstance(payload, (list, tuple)):
|
|
for item in payload:
|
|
found = _find_key(item, *keys)
|
|
if found is not None:
|
|
return found
|
|
return None
|
|
|
|
|
|
class BeimingAutocadAdapter(McpStdioAdapter):
|
|
name = "beiming183-autocad-mcp"
|
|
spec = CompetitorSpec(
|
|
competitor_id="beiming183-autocad-mcp",
|
|
repo_url="https://github.com/beiming183-cloud/AutoCAD-MCP.git",
|
|
pinned_sha=PINNED_SHA,
|
|
entry_candidates=(
|
|
("-m", "autocad_mcp"),
|
|
("server.py",),
|
|
("src/server.py",),
|
|
),
|
|
pip_installs=(("-e", "."),),
|
|
launch_env={"AUTOCAD_MCP_BACKEND": "ezdxf"},
|
|
pythonpath="src",
|
|
)
|
|
|
|
def extra_launch_env(self) -> dict[str, str]:
|
|
# The server redirects writes outside its managed output root; point
|
|
# the root at the benchmark artifact directory so exports land where
|
|
# the harness verifies them.
|
|
return {
|
|
"AUTOCAD_MCP_OUTPUT_ROOT": str(self.artifact_dir),
|
|
"AUTOCAD_MCP_ALLOW_EXTERNAL_OUTPUTS": "true",
|
|
}
|
|
|
|
def warmup_calls(self):
|
|
# Absorb lazy backend initialization once during setup.
|
|
return (
|
|
("system", {"operation": "status"}),
|
|
("drawing", {"operation": "create", "data": {}}),
|
|
)
|
|
|
|
async def _ctx(self) -> dict[str, Any]:
|
|
"""Read doc_id + revision from ``transaction(operation="context")``."""
|
|
try:
|
|
payload = await self.call("transaction", {"operation": "context"})
|
|
except ToolCallFailed:
|
|
return {}
|
|
doc_id = _find_key(payload, "doc_id", "document_id")
|
|
revision = _find_key(payload, "revision", "expected_revision", "document_revision")
|
|
context: dict[str, Any] = {}
|
|
if doc_id is not None:
|
|
context["doc_id"] = doc_id
|
|
if revision is not None:
|
|
context["expected_revision"] = revision
|
|
return context
|
|
|
|
async def _new_drawing(self) -> None:
|
|
await self.call("drawing", {"operation": "create", "data": {}})
|
|
|
|
async def _save_dxf(self, filename: str) -> str:
|
|
target = self.save_target(filename)
|
|
payload: dict[str, Any] = {"path": str(target)}
|
|
try:
|
|
await self.call("drawing", {"operation": "save_as_dxf", "data": payload})
|
|
except ToolCallFailed:
|
|
context = await self._ctx()
|
|
await self.call("drawing", {"operation": "save_as_dxf", "data": {**payload, **context}})
|
|
return str(target)
|
|
|
|
async def _entity(self, operation: str, **fields: Any) -> Any:
|
|
context = await self._ctx()
|
|
return await self.call("entity", {"operation": operation, **context, **fields})
|
|
|
|
async def _create_line(self, x1: float, y1: float, x2: float, y2: float):
|
|
return await self._entity("create_line", x1=x1, y1=y1, x2=x2, y2=y2)
|
|
|
|
# -- tasks ----------------------------------------------------------------
|
|
|
|
async def _task_core_geometry(self) -> TaskResult:
|
|
await self._new_drawing()
|
|
await self._create_line(0, 0, 3, 4)
|
|
await self._entity("create_circle", data={"cx": 10, "cy": 10, "radius": 2})
|
|
artifact = await self._save_dxf("beiming_core_geometry.dxf")
|
|
_, doc = self.verify_dxf(self.save_target("beiming_core_geometry.dxf"))
|
|
lines = doc.modelspace().query("LINE")
|
|
circles = doc.modelspace().query("CIRCLE")
|
|
length = None
|
|
if lines:
|
|
start, end = lines[0].dxf.start, lines[0].dxf.end
|
|
length = math.dist((start.x, start.y), (end.x, end.y))
|
|
passed = (
|
|
len(lines) == 1
|
|
and len(circles) == 1
|
|
and length is not None
|
|
and abs(length - 5.0) < 1e-6
|
|
and abs(circles[0].dxf.radius - 2.0) < 1e-6
|
|
)
|
|
return TaskResult(
|
|
"core_geometry",
|
|
"pass" if passed else "fail",
|
|
100.0 if passed else 0.0,
|
|
metrics={"line_length": length, "lines": len(lines), "circles": len(circles)},
|
|
artifacts=[artifact],
|
|
)
|
|
|
|
async def _task_modify_query(self) -> TaskResult:
|
|
await self._new_drawing()
|
|
created = await self._create_line(0, 0, 10, 0)
|
|
handle = extract_handle(created)
|
|
if not handle:
|
|
return TaskResult(
|
|
"modify_query",
|
|
"fail",
|
|
0.0,
|
|
"could not extract created-entity handle from the create_line response",
|
|
)
|
|
await self._entity("move", entity_id=handle, data={"dx": 5, "dy": 2})
|
|
artifact = await self._save_dxf("beiming_modify_query.dxf")
|
|
_, doc = self.verify_dxf(self.save_target("beiming_modify_query.dxf"))
|
|
lines = doc.modelspace().query("LINE")
|
|
moved = (
|
|
bool(lines)
|
|
and abs(lines[0].dxf.start.x - 5.0) < 1e-6
|
|
and (abs(lines[0].dxf.start.y - 2.0) < 1e-6)
|
|
)
|
|
return TaskResult(
|
|
"modify_query",
|
|
"pass" if moved else "fail",
|
|
100.0 if moved else 0.0,
|
|
artifacts=[artifact],
|
|
)
|
|
|
|
async def _task_layers_linetypes(self) -> TaskResult:
|
|
await self._new_drawing()
|
|
context = await self._ctx()
|
|
await self.call("drawing", {"operation": "setup_mechanical", "data": {**context}})
|
|
artifact = await self._save_dxf("beiming_layers.dxf")
|
|
_, doc = self.verify_dxf(self.save_target("beiming_layers.dxf"))
|
|
layers = {layer.dxf.name.upper() for layer in doc.layers}
|
|
linetypes = {lt.dxf.name.upper() for lt in doc.linetypes}
|
|
expected_layers = {"OUTLINE", "CENTER", "DIM"}
|
|
got_layers = expected_layers.issubset(layers)
|
|
got_linetype = "CENTER" in linetypes
|
|
if got_layers and got_linetype:
|
|
return TaskResult("layers_linetypes", "pass", 100.0, artifacts=[artifact])
|
|
if got_layers:
|
|
return TaskResult(
|
|
"layers_linetypes",
|
|
"partial",
|
|
50.0,
|
|
"GB/T layers created but CENTER linetype missing from the table",
|
|
artifacts=[artifact],
|
|
)
|
|
return TaskResult(
|
|
"layers_linetypes",
|
|
"fail",
|
|
0.0,
|
|
f"layers={sorted(layers)}",
|
|
artifacts=[artifact],
|
|
)
|
|
|
|
async def _task_dimensions(self) -> TaskResult:
|
|
await self._new_drawing()
|
|
context = await self._ctx()
|
|
await self.call(
|
|
"annotation",
|
|
{
|
|
"operation": "create_dimension_linear",
|
|
**context,
|
|
"data": {"x1": 0, "y1": 0, "x2": 50, "y2": 0, "dim_x": 25, "dim_y": 10},
|
|
},
|
|
)
|
|
artifact = await self._save_dxf("beiming_dimensions.dxf")
|
|
_, doc = self.verify_dxf(self.save_target("beiming_dimensions.dxf"))
|
|
dims = [e for e in doc.modelspace() if e.dxftype() == "DIMENSION"]
|
|
return TaskResult(
|
|
"dimensions",
|
|
"pass" if dims else "fail",
|
|
100.0 if dims else 0.0,
|
|
metrics={"dimension_count": len(dims)},
|
|
artifacts=[artifact],
|
|
)
|
|
|
|
async def _task_transactions(self) -> TaskResult:
|
|
await self._new_drawing()
|
|
context = await self._ctx()
|
|
try:
|
|
begin = await self.call("transaction", {"operation": "begin", **context})
|
|
except ToolCallFailed as exc:
|
|
if "E_UNSUPPORTED" in str(exc):
|
|
return TaskResult(
|
|
"transactions",
|
|
"unsupported",
|
|
0.0,
|
|
"transaction begin is rejected as unsupported on the headless backend",
|
|
)
|
|
raise
|
|
transaction_id = _find_key(begin, "transaction_id", "txn_id")
|
|
await self._entity("create_circle", data={"cx": 0, "cy": 0, "radius": 1})
|
|
context = await self._ctx()
|
|
rollback_args: dict[str, Any] = {"operation": "rollback", **context}
|
|
if transaction_id is not None:
|
|
rollback_args["transaction_id"] = transaction_id
|
|
await self.call("transaction", rollback_args)
|
|
artifact = await self._save_dxf("beiming_transactions.dxf")
|
|
_, doc = self.verify_dxf(self.save_target("beiming_transactions.dxf"))
|
|
count = len(list(doc.modelspace()))
|
|
return TaskResult(
|
|
"transactions",
|
|
"pass" if count == 0 else "fail",
|
|
100.0 if count == 0 else 0.0,
|
|
metrics={"entity_count_after_rollback": count},
|
|
artifacts=[artifact],
|
|
)
|
|
|
|
async def _task_dxf_roundtrip(self) -> TaskResult:
|
|
await self._new_drawing()
|
|
await self._create_line(0, 0, 10, 10)
|
|
artifact = await self._save_dxf("beiming_roundtrip.dxf")
|
|
_, doc = self.verify_dxf(self.save_target("beiming_roundtrip.dxf"))
|
|
count = len(list(doc.modelspace()))
|
|
return TaskResult(
|
|
"dxf_roundtrip",
|
|
"pass" if count == 1 else "fail",
|
|
100.0 if count == 1 else 0.0,
|
|
metrics={"entity_count": count},
|
|
artifacts=[artifact],
|
|
)
|
|
|
|
# -- documented coverage gaps ----------------------------------------------
|
|
|
|
async def _task_table_mleader(self) -> TaskResult:
|
|
return TaskResult(
|
|
"table_mleader",
|
|
"unsupported",
|
|
0.0,
|
|
"no TABLE or MLEADER operation in the annotation tool (v3.10)",
|
|
)
|
|
|
|
async def _task_preflight(self) -> TaskResult:
|
|
return TaskResult("preflight", "unsupported", 0.0, "no requirements-preflight equivalent")
|
|
|
|
async def _task_quality_refiner(self) -> TaskResult:
|
|
return TaskResult(
|
|
"quality_refiner",
|
|
"unsupported",
|
|
0.0,
|
|
"audits detect issues but there is no automated repair loop",
|
|
)
|
|
|
|
async def _task_auditable_delivery(self) -> TaskResult:
|
|
return TaskResult(
|
|
"auditable_delivery",
|
|
"unsupported",
|
|
0.0,
|
|
"drawing.deliver is documented as File-IPC only (not headless)",
|
|
)
|
|
|
|
|
|
__all__ = ["BeimingAutocadAdapter"]
|