release: prepare v1.3.0

benchmarks/README.md

@@ -29,8 +29,12 @@ dates, evidence grades, project URLs, and the boundary are stored with the data.
 The v2 runner separates runtime evidence from the source-review rubric. Every
 adapter receives the same ten tasks and produces the same closed result enum:
 `pass`, `partial`, `unsupported`, `fail`, `timeout`, or `not_run`. A timeout is
-isolated to its task. Reports include commit SHA, Python/platform details,
-backend capability claims, durations, and hashes for returned artifacts.
+reported per task. Execution is in-process, so adapters that delegate blocking
+work must provide cancellation-safe task implementations or be wrapped by an
+external process supervisor. Reports include commit SHA, Python/platform
+details, backend capability claims, durations, coverage, and hashes for returned
+artifacts. Unsupported tasks remain in the fixed-matrix denominator with score
+zero, preventing partial implementations from receiving an inflated score.
 
 ```bash
 python -m benchmarks.run_competitors --list

benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark suites for AutoCAD MCP Pro."""

benchmarks/adapters/__init__.py

@@ -0,0 +1,5 @@
+"""Benchmark adapters."""
+
+from .base import BenchmarkAdapter, TaskResult
+
+__all__ = ["BenchmarkAdapter", "TaskResult"]

benchmarks/adapters/autocad_mcp_pro.py

@@ -0,0 +1,151 @@
+"""Reference v2 adapter for this repository."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from benchmarks.adapters.base import BenchmarkAdapter, TaskResult
+from benchmarks.tasks_v2 import TaskSpec
+from engineering.delivery import deliver_drawing
+from engineering.layers import ensure_engineering_layers, ensure_standard_linetypes
+from engineering.refiner import refine_drawing
+
+
+class AutoCADMCPProAdapter(BenchmarkAdapter):
+    name = "autocad-mcp-pro"
+
+    def __init__(self, *, backend: str = "ezdxf"):
+        self.backend_name = backend
+        self.backend = None
+        self.artifact_dir = Path()
+
+    async def setup(self, artifact_dir: Path) -> None:
+        if self.backend_name == "ezdxf":
+            from backends.ezdxf_backend import EzdxfBackend
+
+            self.backend = EzdxfBackend()
+        elif self.backend_name == "com":
+            from backends.com_backend import ComBackend
+
+            self.backend = ComBackend()
+        else:
+            raise ValueError(f"Unknown backend: {self.backend_name}")
+        self.artifact_dir = artifact_dir
+        await self.backend.connect()
+
+    async def cleanup(self) -> None:
+        if self.backend is not None:
+            await self.backend.disconnect()
+
+    def metadata(self) -> dict:
+        return {
+            "backend": self.backend_name,
+            "capabilities": self.backend.capabilities().to_dict() if self.backend else None,
+        }
+
+    async def _reset(self) -> None:
+        await self.backend.drawing_new()
+
+    async def run_task(self, task: TaskSpec) -> TaskResult:
+        await self._reset()
+        handler = getattr(self, f"_task_{task.task_id}", None)
+        if handler is None:
+            return TaskResult(task.task_id, "unsupported", 0.0, "No adapter implementation")
+        passed, metrics, artifacts = await handler()
+        return TaskResult(
+            task.task_id,
+            "pass" if passed else "fail",
+            100.0 if passed else 0.0,
+            metrics=metrics,
+            artifacts=artifacts,
+        )
+
+    async def _task_core_geometry(self):
+        line = await self.backend.entity_create_line(0, 0, 3, 4)
+        circle = await self.backend.entity_create_circle(10, 10, 2)
+        info = await self.backend.entity_get(line.handle)
+        passed = abs(float(info.properties["length"]) - 5.0) < 1e-6
+        return passed and bool(circle.handle), {"line_length": info.properties["length"]}, []
+
+    async def _task_modify_query(self):
+        line = await self.backend.entity_create_line(0, 0, 10, 0)
+        await self.backend.entity_move(line.handle, 5, 2)
+        moved = await self.backend.entity_get(line.handle)
+        return moved.properties["start"][:2] == [5.0, 2.0], {}, []
+
+    async def _task_layers_linetypes(self):
+        await ensure_standard_linetypes(self.backend)
+        await ensure_engineering_layers(self.backend)
+        layers = {item.name for item in await self.backend.layer_list()}
+        linetypes = {item.upper() for item in await self.backend.linetype_list()}
+        return "GEOMETRY" in layers and "CENTER" in linetypes, {}, []
+
+    async def _task_dimensions(self):
+        await ensure_engineering_layers(self.backend)
+        dimension = await self.backend.dimension_linear(0, 0, 50, 0, 25, 10, layer="DIM")
+        return "DIM" in dimension.type.upper(), {"entity_type": dimension.type}, []
+
+    async def _task_table_mleader(self):
+        await ensure_engineering_layers(self.backend)
+        table = await self.backend.entity_create_table(
+            0, 20, [["A", "1"]], headers=["ITEM", "QTY"], layer="TEXT"
+        )
+        leader = await self.backend.leader_create_mleader(
+            [[0, 0], [10, 10]], "NOTE", layer="DIM"
+        )
+        representations = [table.properties.get("representation"), leader.properties.get("representation")]
+        return all(representations), {"representations": representations}, []
+
+    async def _task_transactions(self):
+        await self.backend.transaction_begin()
+        await self.backend.entity_create_circle(0, 0, 1)
+        await self.backend.transaction_rollback()
+        entities = await self.backend.entity_list(limit=100)
+        return len(entities) == 0, {"entity_count": len(entities)}, []
+
+    async def _task_preflight(self):
+        result = await self.backend.drawing_preflight(
+            "Benchmark plate",
+            {
+                "units": "mm",
+                "part_type": "plate",
+                "dimensions": {"width": 50, "height": 25},
+                "tolerance_policy": "ISO 2768-m",
+            },
+        )
+        return result.ready and result.spec_hash.startswith("sha256:"), {}, []
+
+    async def _task_quality_refiner(self):
+        await ensure_engineering_layers(self.backend)
+        await self.backend.entity_create_line(0, 0, 10, 0, layer="GEOMETRY")
+        await self.backend.entity_create_line(0, 0, 10, 0, layer="GEOMETRY")
+        result = await refine_drawing(self.backend, focus=["duplicate_entities"])
+        return result.final_score > result.initial_score, result.to_dict(), []
+
+    async def _task_dxf_roundtrip(self):
+        await self.backend.entity_create_line(0, 0, 10, 10)
+        path = self.artifact_dir / "roundtrip.dxf"
+        await self.backend.drawing_export_dxf(str(path))
+        from backends.ezdxf_backend import EzdxfBackend
+
+        reopened = EzdxfBackend()
+        await reopened.connect()
+        try:
+            await reopened.drawing_open(str(path))
+            count = (await reopened.drawing_info()).entity_count
+        finally:
+            await reopened.disconnect()
+        return count == 1, {"entity_count": count}, [str(path)]
+
+    async def _task_auditable_delivery(self):
+        await ensure_engineering_layers(self.backend)
+        await self.backend.entity_create_line(0, 0, 10, 0, layer="GEOMETRY")
+        destination = self.artifact_dir / "delivery"
+        result = await deliver_drawing(
+            self.backend,
+            destination,
+            formats=["dxf"],
+            min_score=0,
+            strict_critique=False,
+        )
+        return result.status == "success", {"score": result.score}, [result.manifest_path]

benchmarks/adapters/base.py

@@ -0,0 +1,58 @@
+"""Common, vendor-neutral benchmark adapter contract."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from benchmarks.tasks_v2 import TaskSpec
+
+TaskStatus = Literal["pass", "partial", "unsupported", "fail", "timeout", "not_run"]
+
+
+@dataclass
+class TaskResult:
+    task_id: str
+    status: TaskStatus
+    score: float
+    message: str = ""
+    duration_ms: float = 0.0
+    artifacts: list[str | dict[str, Any]] = field(default_factory=list)
+    metrics: dict = field(default_factory=dict)
+    stdout_summary: str = ""
+    stderr_summary: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "task_id": self.task_id,
+            "status": self.status,
+            "score": float(self.score),
+            "message": self.message,
+            "duration_ms": float(self.duration_ms),
+            "artifacts": list(self.artifacts),
+            "metrics": dict(self.metrics),
+            "stdout_summary": self.stdout_summary,
+            "stderr_summary": self.stderr_summary,
+        }
+
+
+class BenchmarkAdapter(ABC):
+    """Minimum interface every competing MCP adapter must implement."""
+
+    name: str
+
+    def metadata(self) -> dict:
+        """Runtime/backend claims captured alongside every benchmark report."""
+        return {}
+
+    @abstractmethod
+    async def setup(self, artifact_dir: Path) -> None: ...
+
+    @abstractmethod
+    async def run_task(self, task: TaskSpec) -> TaskResult: ...
+
+    @abstractmethod
+    async def cleanup(self) -> None: ...

benchmarks/competitors.yaml

@@ -0,0 +1,10 @@
+{
+  "schema_version": "2.0",
+  "competitors": [
+    {
+      "id": "autocad-mcp-pro",
+      "adapter": "benchmarks.adapters.autocad_mcp_pro:AutoCADMCPProAdapter",
+      "backends": ["ezdxf", "com"]
+    }
+  ]
+}

benchmarks/correctness_suite.py

@@ -224,11 +224,18 @@ CHECKS = {
 
 
 def main():
+    if len(sys.argv) == 1 or sys.argv[1] in {"-h", "--help"}:
+        print("usage: correctness_suite.py <check_name> | --list")
+        print("Run one deterministic correctness check, or list available checks.")
+        return
     if len(sys.argv) == 2 and sys.argv[1] == "--list":
         for k, (_fn, cat) in CHECKS.items():
             print(f"{k}\t{cat}")
         return
     name = sys.argv[1]
+    if name not in CHECKS:
+        print(f"Unknown check: {name}", file=sys.stderr)
+        raise SystemExit(2)
     fn, _cat = CHECKS[name]
     ok = asyncio.run(fn())
     print("PASS" if ok else "FAIL")

benchmarks/results/.gitkeep

@@ -0,0 +1 @@
+

benchmarks/run_competitors.py

@@ -0,0 +1,156 @@
+"""Command line runner for the fixed-task benchmark v2 adapter contract."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import hashlib
+import json
+import platform
+import subprocess
+import sys
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+from benchmarks.adapters.base import BenchmarkAdapter, TaskResult
+from benchmarks.tasks_v2 import TASKS_V2, TaskSpec, task_by_id
+
+
+def _git_sha() -> str | None:
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=2,
+        )
+        return result.stdout.strip() or None
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def _hash_file(path: Path) -> dict:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return {
+        "path": str(path),
+        "filename": path.name,
+        "size_bytes": path.stat().st_size,
+        "sha256": digest.hexdigest(),
+    }
+
+
+def _normalize_artifacts(result: TaskResult) -> None:
+    normalized: list[dict] = []
+    for artifact in result.artifacts:
+        if isinstance(artifact, dict):
+            normalized.append(dict(artifact))
+            continue
+        path = Path(artifact).resolve()
+        if path.is_file():
+            normalized.append(_hash_file(path))
+        else:
+            normalized.append({"path": str(path), "missing": True})
+    result.artifacts = normalized
+
+
+async def run_tasks(
+    adapter: BenchmarkAdapter,
+    tasks: list[TaskSpec] | tuple[TaskSpec, ...],
+    artifact_dir: str | Path,
+    *,
+    timeout: float = 30.0,
+) -> dict:
+    destination = Path(artifact_dir).resolve()
+    destination.mkdir(parents=True, exist_ok=True)
+    results: list[TaskResult] = []
+    await adapter.setup(destination)
+    try:
+        for task in tasks:
+            started = time.perf_counter()
+            try:
+                result = await asyncio.wait_for(adapter.run_task(task), timeout=timeout)
+            except TimeoutError:
+                result = TaskResult(task.task_id, "timeout", 0.0, f"Exceeded {timeout}s")
+            except Exception as exc:
+                result = TaskResult(task.task_id, "fail", 0.0, str(exc), stderr_summary=str(exc))
+            result.duration_ms = round((time.perf_counter() - started) * 1000, 2)
+            _normalize_artifacts(result)
+            results.append(result)
+    finally:
+        await adapter.cleanup()
+
+    attempted = len(results)
+    passed = sum(item.status == "pass" for item in results)
+    supported = [item for item in results if item.status != "unsupported"]
+    weight_by_id = {task.task_id: task.weight for task in tasks}
+    total_weight = sum(weight_by_id.values())
+    supported_weight = sum(weight_by_id[item.task_id] for item in supported)
+    weighted_score = sum(
+        weight_by_id[item.task_id] * item.score
+        for item in results
+        if item.status != "unsupported"
+    )
+    return {
+        "schema_version": "2.0",
+        "adapter": adapter.name,
+        "adapter_metadata": adapter.metadata(),
+        "git_sha": _git_sha(),
+        "generated_at": datetime.now(UTC).isoformat(),
+        "environment": {
+            "python": sys.version.split()[0],
+            "implementation": platform.python_implementation(),
+            "platform": platform.platform(),
+            "machine": platform.machine(),
+        },
+        "timeout_seconds": timeout,
+        "summary": {
+            "attempted": attempted,
+            "passed": passed,
+            "supported": len(supported),
+            "unsupported": sum(item.status == "unsupported" for item in results),
+            "coverage_percent": (
+                round(supported_weight / total_weight * 100, 2) if total_weight else 0.0
+            ),
+            "score": round(weighted_score / total_weight, 2) if total_weight else 0.0,
+        },
+        "results": [item.to_dict() for item in results],
+    }
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--list", action="store_true", help="List the fixed benchmark tasks")
+    parser.add_argument("--server", default="autocad-mcp-pro", choices=["autocad-mcp-pro"])
+    parser.add_argument("--backend", default="ezdxf", choices=["ezdxf", "com"])
+    parser.add_argument("--task", action="append", help="Run only this task id (repeatable)")
+    parser.add_argument("--artifact-dir", default="benchmarks/results/latest")
+    parser.add_argument("--timeout", type=float, default=30.0)
+    parser.add_argument("--json", action="store_true", help="Print machine-readable JSON")
+    return parser
+
+
+def main() -> None:
+    args = _parser().parse_args()
+    if args.list:
+        for task in TASKS_V2:
+            print(f"{task.task_id}\t{task.category}\t{task.description}")
+        return
+
+    from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter
+
+    tasks = [task_by_id(task_id) for task_id in args.task] if args.task else list(TASKS_V2)
+    adapter = AutoCADMCPProAdapter(backend=args.backend)
+    report = asyncio.run(
+        run_tasks(adapter, tasks, args.artifact_dir, timeout=args.timeout)
+    )
+    rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)
+    print(rendered)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/tasks_v2.py

@@ -0,0 +1,34 @@
+"""Fixed v2 task matrix shared by all competitor adapters."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class TaskSpec:
+    task_id: str
+    category: str
+    description: str
+    weight: float
+
+
+TASKS_V2: tuple[TaskSpec, ...] = (
+    TaskSpec("core_geometry", "creation", "Create and inspect deterministic geometry", 1.0),
+    TaskSpec("modify_query", "editing", "Modify geometry and verify the result", 1.0),
+    TaskSpec("layers_linetypes", "standards", "Apply standard layers and linetypes", 1.0),
+    TaskSpec("dimensions", "annotation", "Create real associative dimensions", 1.0),
+    TaskSpec("table_mleader", "annotation", "Create TABLE and MLEADER semantics", 1.0),
+    TaskSpec("transactions", "reliability", "Rollback a bounded change", 1.0),
+    TaskSpec("preflight", "planning", "Resolve and hash drawing requirements", 1.0),
+    TaskSpec("quality_refiner", "quality", "Detect and repair a quality defect", 1.0),
+    TaskSpec("dxf_roundtrip", "interoperability", "Save and re-open with structural parity", 1.0),
+    TaskSpec("auditable_delivery", "delivery", "Produce a hashed validation manifest", 1.0),
+)
+
+
+def task_by_id(task_id: str) -> TaskSpec:
+    for task in TASKS_V2:
+        if task.task_id == task_id:
+            return task
+    raise KeyError(f"Unknown benchmark task: {task_id}")