feat(bench): live competitor lane - black-box MCP stdio adapters, pinned checkouts, published results
- adapters/mcp_stdio.py: drives competitors over MCP stdio like a real host; payload-level error detection, fastmcp output-wrapper normalization, cold-start warmup, artifact-based verification (harness re-opens every exported DXF with ezdxf) - competitors_env.py: reproducible pinned-SHA checkout + isolated venv per competitor - adapters for puran-water/autocad-mcp (95476a3) and beiming183-cloud/AutoCAD-MCP (11f7c47) mapped to their own documented tool contracts; unmapped tasks report 'unsupported' with reasons - run_competitors.py: registry-driven --server from competitors.yaml - published reports (pro 100.0 / beiming 50.0 / puran 45.0) + render_live_chart.py -> docs/assets/autocad-mcp-livebench.svg - benchmark.yml: weekly + manual dispatch matrix Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Umutcan Edizsalan
co-authored by
Cursor
Cursor
parent
2f56a73093
commit
6d7c39e012
@@ -0,0 +1,301 @@
|
||||
"""Black-box adapter for beiming183-cloud/AutoCAD-MCP (headless ezdxf lane).
|
||||
|
||||
Contract verified against the pinned checkout's ``server.py`` (v3.10 line):
|
||||
|
||||
- document identity comes from ``transaction(operation="context")`` and every
|
||||
mutation must carry ``doc_id`` + ``expected_revision`` — top-level for the
|
||||
``entity``/``annotation`` tools, inside ``data`` for ``drawing`` operations;
|
||||
- ``entity create_line`` takes top-level ``x1..y2``; circle/dimension payloads
|
||||
sit under ``data`` (same field spellings as the upstream puran-water base);
|
||||
- ``transaction begin/commit/rollback`` are compatibility operations that the
|
||||
headless backend may reject with ``E_UNSUPPORTED*`` — that outcome is
|
||||
reported as ``unsupported`` rather than ``fail``.
|
||||
|
||||
Verification is artifact-based: geometry claims are checked by exporting a DXF
|
||||
(``drawing save_as_dxf``) and re-reading it with ezdxf in the harness.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from benchmarks.adapters.base import TaskResult
|
||||
from benchmarks.adapters.mcp_stdio import (
|
||||
McpStdioAdapter,
|
||||
ToolCallFailed,
|
||||
extract_handle,
|
||||
)
|
||||
from benchmarks.competitors_env import CompetitorSpec
|
||||
|
||||
PINNED_SHA = "11f7c47e5038796a20451b38b23032e625b5aa26"
|
||||
|
||||
|
||||
def _find_key(payload: Any, *keys: str) -> Any:
|
||||
if isinstance(payload, dict):
|
||||
for key in keys:
|
||||
if key in payload and payload[key] not in (None, ""):
|
||||
return payload[key]
|
||||
for value in payload.values():
|
||||
found = _find_key(value, *keys)
|
||||
if found is not None:
|
||||
return found
|
||||
elif isinstance(payload, (list, tuple)):
|
||||
for item in payload:
|
||||
found = _find_key(item, *keys)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
class BeimingAutocadAdapter(McpStdioAdapter):
|
||||
name = "beiming183-autocad-mcp"
|
||||
spec = CompetitorSpec(
|
||||
competitor_id="beiming183-autocad-mcp",
|
||||
repo_url="https://github.com/beiming183-cloud/AutoCAD-MCP.git",
|
||||
pinned_sha=PINNED_SHA,
|
||||
entry_candidates=(
|
||||
("-m", "autocad_mcp"),
|
||||
("server.py",),
|
||||
("src/server.py",),
|
||||
),
|
||||
pip_installs=(("-e", "."),),
|
||||
launch_env={"AUTOCAD_MCP_BACKEND": "ezdxf"},
|
||||
pythonpath="src",
|
||||
)
|
||||
|
||||
def extra_launch_env(self) -> dict[str, str]:
|
||||
# The server redirects writes outside its managed output root; point
|
||||
# the root at the benchmark artifact directory so exports land where
|
||||
# the harness verifies them.
|
||||
return {
|
||||
"AUTOCAD_MCP_OUTPUT_ROOT": str(self.artifact_dir),
|
||||
"AUTOCAD_MCP_ALLOW_EXTERNAL_OUTPUTS": "true",
|
||||
}
|
||||
|
||||
def warmup_calls(self):
|
||||
# Absorb lazy backend initialization once during setup.
|
||||
return (
|
||||
("system", {"operation": "status"}),
|
||||
("drawing", {"operation": "create", "data": {}}),
|
||||
)
|
||||
|
||||
async def _ctx(self) -> dict[str, Any]:
|
||||
"""Read doc_id + revision from ``transaction(operation="context")``."""
|
||||
try:
|
||||
payload = await self.call("transaction", {"operation": "context"})
|
||||
except ToolCallFailed:
|
||||
return {}
|
||||
doc_id = _find_key(payload, "doc_id", "document_id")
|
||||
revision = _find_key(payload, "revision", "expected_revision", "document_revision")
|
||||
context: dict[str, Any] = {}
|
||||
if doc_id is not None:
|
||||
context["doc_id"] = doc_id
|
||||
if revision is not None:
|
||||
context["expected_revision"] = revision
|
||||
return context
|
||||
|
||||
async def _new_drawing(self) -> None:
|
||||
await self.call("drawing", {"operation": "create", "data": {}})
|
||||
|
||||
async def _save_dxf(self, filename: str) -> str:
|
||||
target = self.save_target(filename)
|
||||
payload: dict[str, Any] = {"path": str(target)}
|
||||
try:
|
||||
await self.call("drawing", {"operation": "save_as_dxf", "data": payload})
|
||||
except ToolCallFailed:
|
||||
context = await self._ctx()
|
||||
await self.call("drawing", {"operation": "save_as_dxf", "data": {**payload, **context}})
|
||||
return str(target)
|
||||
|
||||
async def _entity(self, operation: str, **fields: Any) -> Any:
|
||||
context = await self._ctx()
|
||||
return await self.call("entity", {"operation": operation, **context, **fields})
|
||||
|
||||
async def _create_line(self, x1: float, y1: float, x2: float, y2: float):
|
||||
return await self._entity("create_line", x1=x1, y1=y1, x2=x2, y2=y2)
|
||||
|
||||
# -- tasks ----------------------------------------------------------------
|
||||
|
||||
async def _task_core_geometry(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
await self._create_line(0, 0, 3, 4)
|
||||
await self._entity("create_circle", data={"cx": 10, "cy": 10, "radius": 2})
|
||||
artifact = await self._save_dxf("beiming_core_geometry.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("beiming_core_geometry.dxf"))
|
||||
lines = doc.modelspace().query("LINE")
|
||||
circles = doc.modelspace().query("CIRCLE")
|
||||
length = None
|
||||
if lines:
|
||||
start, end = lines[0].dxf.start, lines[0].dxf.end
|
||||
length = math.dist((start.x, start.y), (end.x, end.y))
|
||||
passed = (
|
||||
len(lines) == 1
|
||||
and len(circles) == 1
|
||||
and length is not None
|
||||
and abs(length - 5.0) < 1e-6
|
||||
and abs(circles[0].dxf.radius - 2.0) < 1e-6
|
||||
)
|
||||
return TaskResult(
|
||||
"core_geometry",
|
||||
"pass" if passed else "fail",
|
||||
100.0 if passed else 0.0,
|
||||
metrics={"line_length": length, "lines": len(lines), "circles": len(circles)},
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
async def _task_modify_query(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
created = await self._create_line(0, 0, 10, 0)
|
||||
handle = extract_handle(created)
|
||||
if not handle:
|
||||
return TaskResult(
|
||||
"modify_query",
|
||||
"fail",
|
||||
0.0,
|
||||
"could not extract created-entity handle from the create_line response",
|
||||
)
|
||||
await self._entity("move", entity_id=handle, data={"dx": 5, "dy": 2})
|
||||
artifact = await self._save_dxf("beiming_modify_query.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("beiming_modify_query.dxf"))
|
||||
lines = doc.modelspace().query("LINE")
|
||||
moved = (
|
||||
bool(lines)
|
||||
and abs(lines[0].dxf.start.x - 5.0) < 1e-6
|
||||
and (abs(lines[0].dxf.start.y - 2.0) < 1e-6)
|
||||
)
|
||||
return TaskResult(
|
||||
"modify_query",
|
||||
"pass" if moved else "fail",
|
||||
100.0 if moved else 0.0,
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
async def _task_layers_linetypes(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
context = await self._ctx()
|
||||
await self.call("drawing", {"operation": "setup_mechanical", "data": {**context}})
|
||||
artifact = await self._save_dxf("beiming_layers.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("beiming_layers.dxf"))
|
||||
layers = {layer.dxf.name.upper() for layer in doc.layers}
|
||||
linetypes = {lt.dxf.name.upper() for lt in doc.linetypes}
|
||||
expected_layers = {"OUTLINE", "CENTER", "DIM"}
|
||||
got_layers = expected_layers.issubset(layers)
|
||||
got_linetype = "CENTER" in linetypes
|
||||
if got_layers and got_linetype:
|
||||
return TaskResult("layers_linetypes", "pass", 100.0, artifacts=[artifact])
|
||||
if got_layers:
|
||||
return TaskResult(
|
||||
"layers_linetypes",
|
||||
"partial",
|
||||
50.0,
|
||||
"GB/T layers created but CENTER linetype missing from the table",
|
||||
artifacts=[artifact],
|
||||
)
|
||||
return TaskResult(
|
||||
"layers_linetypes",
|
||||
"fail",
|
||||
0.0,
|
||||
f"layers={sorted(layers)}",
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
async def _task_dimensions(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
context = await self._ctx()
|
||||
await self.call(
|
||||
"annotation",
|
||||
{
|
||||
"operation": "create_dimension_linear",
|
||||
**context,
|
||||
"data": {"x1": 0, "y1": 0, "x2": 50, "y2": 0, "dim_x": 25, "dim_y": 10},
|
||||
},
|
||||
)
|
||||
artifact = await self._save_dxf("beiming_dimensions.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("beiming_dimensions.dxf"))
|
||||
dims = [e for e in doc.modelspace() if e.dxftype() == "DIMENSION"]
|
||||
return TaskResult(
|
||||
"dimensions",
|
||||
"pass" if dims else "fail",
|
||||
100.0 if dims else 0.0,
|
||||
metrics={"dimension_count": len(dims)},
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
async def _task_transactions(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
context = await self._ctx()
|
||||
try:
|
||||
begin = await self.call("transaction", {"operation": "begin", **context})
|
||||
except ToolCallFailed as exc:
|
||||
if "E_UNSUPPORTED" in str(exc):
|
||||
return TaskResult(
|
||||
"transactions",
|
||||
"unsupported",
|
||||
0.0,
|
||||
"transaction begin is rejected as unsupported on the headless backend",
|
||||
)
|
||||
raise
|
||||
transaction_id = _find_key(begin, "transaction_id", "txn_id")
|
||||
await self._entity("create_circle", data={"cx": 0, "cy": 0, "radius": 1})
|
||||
context = await self._ctx()
|
||||
rollback_args: dict[str, Any] = {"operation": "rollback", **context}
|
||||
if transaction_id is not None:
|
||||
rollback_args["transaction_id"] = transaction_id
|
||||
await self.call("transaction", rollback_args)
|
||||
artifact = await self._save_dxf("beiming_transactions.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("beiming_transactions.dxf"))
|
||||
count = len(list(doc.modelspace()))
|
||||
return TaskResult(
|
||||
"transactions",
|
||||
"pass" if count == 0 else "fail",
|
||||
100.0 if count == 0 else 0.0,
|
||||
metrics={"entity_count_after_rollback": count},
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
async def _task_dxf_roundtrip(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
await self._create_line(0, 0, 10, 10)
|
||||
artifact = await self._save_dxf("beiming_roundtrip.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("beiming_roundtrip.dxf"))
|
||||
count = len(list(doc.modelspace()))
|
||||
return TaskResult(
|
||||
"dxf_roundtrip",
|
||||
"pass" if count == 1 else "fail",
|
||||
100.0 if count == 1 else 0.0,
|
||||
metrics={"entity_count": count},
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
# -- documented coverage gaps ----------------------------------------------
|
||||
|
||||
async def _task_table_mleader(self) -> TaskResult:
|
||||
return TaskResult(
|
||||
"table_mleader",
|
||||
"unsupported",
|
||||
0.0,
|
||||
"no TABLE or MLEADER operation in the annotation tool (v3.10)",
|
||||
)
|
||||
|
||||
async def _task_preflight(self) -> TaskResult:
|
||||
return TaskResult("preflight", "unsupported", 0.0, "no requirements-preflight equivalent")
|
||||
|
||||
async def _task_quality_refiner(self) -> TaskResult:
|
||||
return TaskResult(
|
||||
"quality_refiner",
|
||||
"unsupported",
|
||||
0.0,
|
||||
"audits detect issues but there is no automated repair loop",
|
||||
)
|
||||
|
||||
async def _task_auditable_delivery(self) -> TaskResult:
|
||||
return TaskResult(
|
||||
"auditable_delivery",
|
||||
"unsupported",
|
||||
0.0,
|
||||
"drawing.deliver is documented as File-IPC only (not headless)",
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["BeimingAutocadAdapter"]
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Generic MCP-stdio benchmark adapter.
|
||||
|
||||
Drives a *competitor* MCP server as a black box over the standard stdio
|
||||
transport (fastmcp ``Client``), exactly the way an agent host would. No
|
||||
competitor code is imported; verification never trusts the competitor's
|
||||
response payload — every task that claims to have produced geometry must
|
||||
save a DXF which the harness re-opens with ezdxf and inspects itself.
|
||||
|
||||
Subclasses provide the launch runtime (via ``benchmarks.competitors_env``)
|
||||
and per-task playbooks named ``_task_<task_id>``; anything unmapped is
|
||||
reported as ``unsupported`` (coverage-aware scoring already accounts for it).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import ezdxf
|
||||
from fastmcp import Client
|
||||
from fastmcp.client.transports import StdioTransport
|
||||
|
||||
from benchmarks.adapters.base import BenchmarkAdapter, TaskResult
|
||||
from benchmarks.competitors_env import CompetitorRuntime, CompetitorSpec, ensure_competitor
|
||||
from benchmarks.tasks_v2 import TaskSpec
|
||||
|
||||
CALL_TIMEOUT_SECONDS = 15.0
|
||||
WARMUP_TIMEOUT_SECONDS = 90.0
|
||||
|
||||
|
||||
class ToolCallFailed(RuntimeError):
|
||||
"""A competitor tool call returned an error result."""
|
||||
|
||||
|
||||
def read_dxf(path: Path):
|
||||
return ezdxf.readfile(str(path))
|
||||
|
||||
|
||||
def modelspace_types(doc) -> list[str]:
|
||||
return [entity.dxftype() for entity in doc.modelspace()]
|
||||
|
||||
|
||||
def find_dxf(artifact_dir: Path, preferred: Path | None = None) -> Path | None:
|
||||
"""Locate the DXF a competitor produced.
|
||||
|
||||
Servers with managed output roots may relocate writes, so fall back to
|
||||
the newest ``*.dxf`` anywhere under the artifact directory.
|
||||
"""
|
||||
if preferred is not None and preferred.is_file():
|
||||
return preferred
|
||||
candidates = sorted(
|
||||
artifact_dir.rglob("*.dxf"), key=lambda item: item.stat().st_mtime, reverse=True
|
||||
)
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
class McpStdioAdapter(BenchmarkAdapter):
|
||||
"""Black-box adapter base for competitor MCP servers."""
|
||||
|
||||
name = "mcp-stdio"
|
||||
spec: CompetitorSpec # set by subclasses
|
||||
|
||||
def __init__(self, *, backend: str = "ezdxf"):
|
||||
if backend != "ezdxf":
|
||||
raise ValueError(
|
||||
f"{type(self).__name__} only benchmarks the headless ezdxf lane; "
|
||||
f"got backend={backend!r}. COM lanes run locally, not in CI."
|
||||
)
|
||||
self.backend_name = backend
|
||||
self.artifact_dir = Path()
|
||||
self.runtime: CompetitorRuntime | None = None
|
||||
self._client: Client | None = None
|
||||
self.tool_names: set[str] = set()
|
||||
|
||||
# -- lifecycle ----------------------------------------------------------
|
||||
|
||||
def extra_launch_env(self) -> dict[str, str]:
|
||||
"""Environment computed at setup time (``self.artifact_dir`` is set)."""
|
||||
return {}
|
||||
|
||||
async def setup(self, artifact_dir: Path) -> None:
|
||||
self.artifact_dir = Path(artifact_dir)
|
||||
self.runtime = ensure_competitor(self.spec)
|
||||
env = {**self.runtime.env, **self.extra_launch_env()}
|
||||
env.setdefault("PYTHONIOENCODING", "utf-8")
|
||||
transport = StdioTransport(
|
||||
command=str(self.runtime.python_exe),
|
||||
args=list(self.runtime.entry_args),
|
||||
env=env,
|
||||
cwd=str(self.runtime.repo_dir),
|
||||
)
|
||||
self._client = Client(transport)
|
||||
await self._client.__aenter__()
|
||||
self.tool_names = {tool.name for tool in await self._client.list_tools()}
|
||||
await self._warmup()
|
||||
|
||||
def warmup_calls(self) -> tuple[tuple[str, dict[str, Any]], ...]:
|
||||
"""Calls issued once after connect to absorb the competitor's lazy
|
||||
backend initialization (which can exceed the per-call timeout)."""
|
||||
return ()
|
||||
|
||||
async def _warmup(self) -> None:
|
||||
for tool, arguments in self.warmup_calls():
|
||||
if tool not in self.tool_names:
|
||||
continue
|
||||
try:
|
||||
await self._client.call_tool(
|
||||
tool, arguments, timeout=WARMUP_TIMEOUT_SECONDS, raise_on_error=False
|
||||
)
|
||||
except Exception: # cold-start absorption is best effort
|
||||
continue
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.__aexit__(None, None, None)
|
||||
self._client = None
|
||||
|
||||
def metadata(self) -> dict:
|
||||
return {
|
||||
"backend": self.backend_name,
|
||||
"kind": "mcp-stdio-blackbox",
|
||||
"repo": self.spec.repo_url,
|
||||
"pinned_sha": self.spec.pinned_sha,
|
||||
"tools_discovered": sorted(self.tool_names),
|
||||
}
|
||||
|
||||
# -- call helpers ---------------------------------------------------------
|
||||
|
||||
async def call(self, tool: str, arguments: dict[str, Any]) -> Any:
|
||||
"""Call one tool; raise ``ToolCallFailed`` on an error result.
|
||||
|
||||
Competitor servers in this family report failures *inside* the payload
|
||||
(``CommandResult(ok=False, error=...)`` serialized as text) rather than
|
||||
via the MCP ``isError`` flag, so the payload is inspected as well.
|
||||
"""
|
||||
if tool not in self.tool_names:
|
||||
raise ToolCallFailed(f"tool {tool!r} is not exposed by {self.name}")
|
||||
result = await self._client.call_tool(
|
||||
tool, arguments, timeout=CALL_TIMEOUT_SECONDS, raise_on_error=False
|
||||
)
|
||||
if getattr(result, "is_error", False):
|
||||
raise ToolCallFailed(f"{tool}({arguments}) -> {_result_text(result)}")
|
||||
data = getattr(result, "data", None)
|
||||
payload = data if data is not None else _result_text(result)
|
||||
payload = _normalize_payload(payload)
|
||||
error = _payload_error(payload)
|
||||
if error:
|
||||
raise ToolCallFailed(f"{tool}({arguments}) -> {error}")
|
||||
return payload
|
||||
|
||||
async def call_op(
|
||||
self,
|
||||
tool: str,
|
||||
operation: str,
|
||||
data: dict[str, Any] | None = None,
|
||||
*,
|
||||
shapes: tuple[dict[str, Any], ...] | None = None,
|
||||
) -> Any:
|
||||
"""Call a consolidated ``{tool}(operation=..., ...)`` style tool.
|
||||
|
||||
Competitors disagree on whether payload fields live under ``data`` or
|
||||
at the top level, and on field spellings. ``shapes`` supplies explicit
|
||||
argument dicts to try in order; otherwise both common layouts of
|
||||
``data`` are attempted. The first non-error result wins.
|
||||
"""
|
||||
attempts: list[dict[str, Any]] = []
|
||||
if shapes:
|
||||
attempts.extend({"operation": operation, **shape} for shape in shapes)
|
||||
else:
|
||||
payload = data or {}
|
||||
attempts.append({"operation": operation, "data": payload})
|
||||
if payload:
|
||||
attempts.append({"operation": operation, **payload})
|
||||
last_error: Exception | None = None
|
||||
for arguments in attempts:
|
||||
try:
|
||||
return await self.call(tool, arguments)
|
||||
except ToolCallFailed as exc: # try the next argument shape
|
||||
last_error = exc
|
||||
raise ToolCallFailed(str(last_error))
|
||||
|
||||
# -- task dispatch --------------------------------------------------------
|
||||
|
||||
async def run_task(self, task: TaskSpec) -> TaskResult:
|
||||
handler = getattr(self, f"_task_{task.task_id}", None)
|
||||
if handler is None:
|
||||
return TaskResult(
|
||||
task.task_id,
|
||||
"unsupported",
|
||||
0.0,
|
||||
f"{self.name} has no documented equivalent for this task",
|
||||
)
|
||||
try:
|
||||
return await handler()
|
||||
except ToolCallFailed as exc:
|
||||
return TaskResult(task.task_id, "fail", 0.0, str(exc)[:500])
|
||||
|
||||
# -- verification ---------------------------------------------------------
|
||||
|
||||
def save_target(self, filename: str) -> Path:
|
||||
return self.artifact_dir / filename
|
||||
|
||||
def verify_dxf(self, preferred: Path | None = None):
|
||||
located = find_dxf(self.artifact_dir, preferred)
|
||||
if located is None:
|
||||
raise ToolCallFailed("no DXF artifact was produced")
|
||||
return located, read_dxf(located)
|
||||
|
||||
|
||||
def _result_text(result: Any) -> str:
|
||||
parts: list[str] = []
|
||||
for item in getattr(result, "content", None) or []:
|
||||
text = getattr(item, "text", None)
|
||||
if text:
|
||||
parts.append(str(text))
|
||||
return " | ".join(parts)[:1000] if parts else "<no text content>"
|
||||
|
||||
|
||||
def _normalize_payload(payload: Any) -> Any:
|
||||
"""Unwrap fastmcp's dynamic output wrappers and JSON-in-text payloads.
|
||||
|
||||
Legacy servers return plain text; the fastmcp client may deserialize it
|
||||
into a generated *dataclass* (e.g. ``entityOutput(result='<json>')``) or a
|
||||
pydantic model. Normalize down to dict / list / str so verification and
|
||||
handle extraction see real data.
|
||||
"""
|
||||
if dataclasses.is_dataclass(payload) and not isinstance(payload, type):
|
||||
payload = dataclasses.asdict(payload)
|
||||
elif hasattr(payload, "model_dump"):
|
||||
payload = payload.model_dump()
|
||||
elif not isinstance(payload, (dict, list, str)) and hasattr(payload, "result"):
|
||||
payload = payload.result
|
||||
if isinstance(payload, dict) and set(payload) == {"result"}:
|
||||
payload = payload["result"]
|
||||
if isinstance(payload, str):
|
||||
try:
|
||||
parsed = json.loads(payload)
|
||||
except ValueError:
|
||||
parsed = None
|
||||
if isinstance(parsed, (dict, list)):
|
||||
payload = parsed
|
||||
return payload
|
||||
|
||||
|
||||
_TEXT_ERROR_MARKERS = ('"ok": false', "'ok': false", '"ok":false', "ok=false")
|
||||
|
||||
|
||||
def _payload_error(payload: Any) -> str | None:
|
||||
"""Detect failure reported inside a tool payload instead of MCP isError."""
|
||||
if isinstance(payload, dict):
|
||||
if payload.get("ok") is False or payload.get("success") is False:
|
||||
return str(payload.get("error") or payload.get("message") or "ok=false")[:300]
|
||||
error = payload.get("error")
|
||||
if isinstance(error, str) and error.strip():
|
||||
return error[:300]
|
||||
return None
|
||||
if isinstance(payload, str):
|
||||
lowered = payload.lower()
|
||||
if lowered.startswith("error") or any(m in lowered for m in _TEXT_ERROR_MARKERS):
|
||||
return payload[:300]
|
||||
return None
|
||||
|
||||
|
||||
def wrapped_shapes(payload: dict[str, Any], **top_level: Any) -> tuple[dict[str, Any], ...]:
|
||||
"""Produce the two common consolidated-tool argument layouts for a payload:
|
||||
nested under ``data`` and flattened at the top level."""
|
||||
return (
|
||||
{**top_level, "data": payload},
|
||||
{**top_level, **payload},
|
||||
)
|
||||
|
||||
|
||||
_HANDLE_KEYS = ("handle", "entity_id", "entity_handle", "id", "object_id")
|
||||
_HANDLE_RE = re.compile(
|
||||
r'"(?:handle|entity_id|entity_handle|object_id)"\s*:\s*"?([0-9A-Fa-f*][\w*-]*)"?'
|
||||
)
|
||||
|
||||
|
||||
def extract_handle(payload: Any) -> str | None:
|
||||
"""Best-effort extraction of a created-entity handle from a black-box
|
||||
tool response (structured dict, list, or raw text)."""
|
||||
if isinstance(payload, dict):
|
||||
for key in _HANDLE_KEYS:
|
||||
value = payload.get(key)
|
||||
if isinstance(value, (str, int)) and str(value):
|
||||
return str(value)
|
||||
for value in payload.values():
|
||||
found = extract_handle(value)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
if isinstance(payload, (list, tuple)):
|
||||
for item in payload:
|
||||
found = extract_handle(item)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
if isinstance(payload, str):
|
||||
match = _HANDLE_RE.search(payload)
|
||||
return match.group(1) if match else None
|
||||
return None
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Black-box adapter for puran-water/autocad-mcp (headless ezdxf lane).
|
||||
|
||||
The competitor exposes 8 consolidated tools (``drawing``, ``entity``,
|
||||
``layer``, ``block``, ``annotation``, ``pid``, ``view``, ``system``) whose
|
||||
operations take an ``operation`` selector plus a payload. Argument spellings
|
||||
are probed through ``wrapped_shapes`` fallbacks; every geometry claim is
|
||||
verified by saving a DXF and re-reading it with ezdxf in the harness.
|
||||
|
||||
Tasks with no documented equivalent on this server are reported as
|
||||
``unsupported`` with an explicit reason so the published JSON explains the
|
||||
coverage gap instead of hiding it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
from benchmarks.adapters.base import TaskResult
|
||||
from benchmarks.adapters.mcp_stdio import (
|
||||
McpStdioAdapter,
|
||||
ToolCallFailed,
|
||||
extract_handle,
|
||||
)
|
||||
from benchmarks.competitors_env import CompetitorSpec
|
||||
|
||||
PINNED_SHA = "95476a33a1c246308326eb4709d6379ef2efdbc1"
|
||||
|
||||
|
||||
class PuranWaterAdapter(McpStdioAdapter):
|
||||
name = "puran-water-autocad-mcp"
|
||||
spec = CompetitorSpec(
|
||||
competitor_id="puran-water-autocad-mcp",
|
||||
repo_url="https://github.com/puran-water/autocad-mcp.git",
|
||||
pinned_sha=PINNED_SHA,
|
||||
entry_candidates=(
|
||||
("-m", "autocad_mcp"),
|
||||
("server.py",),
|
||||
("src/server.py",),
|
||||
("main.py",),
|
||||
),
|
||||
pip_installs=(("-e", "."),),
|
||||
launch_env={"AUTOCAD_MCP_BACKEND": "ezdxf"},
|
||||
pythonpath="src",
|
||||
)
|
||||
|
||||
# Exact v3.1 contract (verified against the pinned checkout's server.py):
|
||||
# entity create_line takes top-level x1..y2; circle/dimension payloads sit
|
||||
# under ``data``; modify ops take top-level ``entity_id`` + ``data``.
|
||||
|
||||
def warmup_calls(self):
|
||||
# The first backend touch lazily initializes and can exceed the
|
||||
# per-call timeout; absorb it once during setup.
|
||||
return (
|
||||
("system", {"operation": "status"}),
|
||||
("drawing", {"operation": "create", "data": {}}),
|
||||
)
|
||||
|
||||
async def _new_drawing(self) -> None:
|
||||
await self.call_op("drawing", "create", {})
|
||||
|
||||
async def _save(self, filename: str) -> str:
|
||||
target = self.save_target(filename)
|
||||
await self.call_op("drawing", "save", {"path": str(target)})
|
||||
return str(target)
|
||||
|
||||
async def _create_line(self, x1: float, y1: float, x2: float, y2: float):
|
||||
return await self.call(
|
||||
"entity",
|
||||
{"operation": "create_line", "x1": x1, "y1": y1, "x2": x2, "y2": y2},
|
||||
)
|
||||
|
||||
# -- tasks ----------------------------------------------------------------
|
||||
|
||||
async def _task_core_geometry(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
await self._create_line(0, 0, 3, 4)
|
||||
await self.call_op("entity", "create_circle", {"cx": 10, "cy": 10, "radius": 2})
|
||||
preferred = await self._save("puran_core_geometry.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("puran_core_geometry.dxf"))
|
||||
lines = doc.modelspace().query("LINE")
|
||||
circles = doc.modelspace().query("CIRCLE")
|
||||
length = None
|
||||
if lines:
|
||||
start, end = lines[0].dxf.start, lines[0].dxf.end
|
||||
length = math.dist((start.x, start.y), (end.x, end.y))
|
||||
passed = (
|
||||
len(lines) == 1
|
||||
and len(circles) == 1
|
||||
and length is not None
|
||||
and abs(length - 5.0) < 1e-6
|
||||
and abs(circles[0].dxf.radius - 2.0) < 1e-6
|
||||
)
|
||||
return TaskResult(
|
||||
"core_geometry",
|
||||
"pass" if passed else "fail",
|
||||
100.0 if passed else 0.0,
|
||||
metrics={"line_length": length, "lines": len(lines), "circles": len(circles)},
|
||||
artifacts=[preferred],
|
||||
)
|
||||
|
||||
async def _task_modify_query(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
created = await self._create_line(0, 0, 10, 0)
|
||||
handle = extract_handle(created)
|
||||
if not handle:
|
||||
return TaskResult(
|
||||
"modify_query",
|
||||
"fail",
|
||||
0.0,
|
||||
"could not extract created-entity handle from the create_line response",
|
||||
)
|
||||
await self.call(
|
||||
"entity",
|
||||
{"operation": "move", "entity_id": handle, "data": {"dx": 5, "dy": 2}},
|
||||
)
|
||||
artifact = await self._save("puran_modify_query.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("puran_modify_query.dxf"))
|
||||
lines = doc.modelspace().query("LINE")
|
||||
moved = (
|
||||
bool(lines)
|
||||
and abs(lines[0].dxf.start.x - 5.0) < 1e-6
|
||||
and (abs(lines[0].dxf.start.y - 2.0) < 1e-6)
|
||||
)
|
||||
return TaskResult(
|
||||
"modify_query",
|
||||
"pass" if moved else "fail",
|
||||
100.0 if moved else 0.0,
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
async def _task_layers_linetypes(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
try:
|
||||
await self.call_op(
|
||||
"layer",
|
||||
"create",
|
||||
{"name": "GEOMETRY", "color": 3, "linetype": "CENTER"},
|
||||
)
|
||||
except ToolCallFailed:
|
||||
await self.call_op("layer", "create", {"name": "GEOMETRY", "color": 3})
|
||||
artifact = await self._save("puran_layers.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("puran_layers.dxf"))
|
||||
has_layer = "GEOMETRY" in {layer.dxf.name.upper() for layer in doc.layers}
|
||||
has_center = "CENTER" in {lt.dxf.name.upper() for lt in doc.linetypes}
|
||||
if has_layer and has_center:
|
||||
return TaskResult("layers_linetypes", "pass", 100.0, artifacts=[artifact])
|
||||
if has_layer:
|
||||
return TaskResult(
|
||||
"layers_linetypes",
|
||||
"partial",
|
||||
50.0,
|
||||
"layer created; no linetype-loading operation is exposed",
|
||||
artifacts=[artifact],
|
||||
)
|
||||
return TaskResult("layers_linetypes", "fail", 0.0, artifacts=[artifact])
|
||||
|
||||
async def _task_dimensions(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
await self.call_op(
|
||||
"annotation",
|
||||
"create_dimension_linear",
|
||||
{"x1": 0, "y1": 0, "x2": 50, "y2": 0, "dim_x": 25, "dim_y": 10},
|
||||
)
|
||||
artifact = await self._save("puran_dimensions.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("puran_dimensions.dxf"))
|
||||
dims = [e for e in doc.modelspace() if e.dxftype() == "DIMENSION"]
|
||||
return TaskResult(
|
||||
"dimensions",
|
||||
"pass" if dims else "fail",
|
||||
100.0 if dims else 0.0,
|
||||
metrics={"dimension_count": len(dims)},
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
async def _task_dxf_roundtrip(self) -> TaskResult:
|
||||
await self._new_drawing()
|
||||
await self._create_line(0, 0, 10, 10)
|
||||
artifact = await self._save("puran_roundtrip.dxf")
|
||||
_, doc = self.verify_dxf(self.save_target("puran_roundtrip.dxf"))
|
||||
count = len(list(doc.modelspace()))
|
||||
return TaskResult(
|
||||
"dxf_roundtrip",
|
||||
"pass" if count == 1 else "fail",
|
||||
100.0 if count == 1 else 0.0,
|
||||
metrics={"entity_count": count},
|
||||
artifacts=[artifact],
|
||||
)
|
||||
|
||||
# -- documented coverage gaps ----------------------------------------------
|
||||
|
||||
async def _task_table_mleader(self) -> TaskResult:
|
||||
return TaskResult(
|
||||
"table_mleader",
|
||||
"unsupported",
|
||||
0.0,
|
||||
"no TABLE or MLEADER operation in the 8-tool surface (v3.1)",
|
||||
)
|
||||
|
||||
async def _task_transactions(self) -> TaskResult:
|
||||
return TaskResult(
|
||||
"transactions",
|
||||
"unsupported",
|
||||
0.0,
|
||||
"no transaction begin/commit/rollback tool (undo/redo are File-IPC only)",
|
||||
)
|
||||
|
||||
async def _task_preflight(self) -> TaskResult:
|
||||
return TaskResult("preflight", "unsupported", 0.0, "no requirements-preflight equivalent")
|
||||
|
||||
async def _task_quality_refiner(self) -> TaskResult:
|
||||
return TaskResult(
|
||||
"quality_refiner", "unsupported", 0.0, "no critique/repair loop equivalent"
|
||||
)
|
||||
|
||||
async def _task_auditable_delivery(self) -> TaskResult:
|
||||
return TaskResult(
|
||||
"auditable_delivery",
|
||||
"unsupported",
|
||||
0.0,
|
||||
"no hashed delivery-manifest equivalent (plot_pdf is File-IPC only)",
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["PuranWaterAdapter", "ToolCallFailed"]
|
||||
Reference in New Issue
Block a user