release: prepare v1.3.0

benchmarks/run_competitors.py

@@ -0,0 +1,156 @@
+"""Command line runner for the fixed-task benchmark v2 adapter contract."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import hashlib
+import json
+import platform
+import subprocess
+import sys
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+from benchmarks.adapters.base import BenchmarkAdapter, TaskResult
+from benchmarks.tasks_v2 import TASKS_V2, TaskSpec, task_by_id
+
+
+def _git_sha() -> str | None:
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=2,
+        )
+        return result.stdout.strip() or None
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def _hash_file(path: Path) -> dict:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return {
+        "path": str(path),
+        "filename": path.name,
+        "size_bytes": path.stat().st_size,
+        "sha256": digest.hexdigest(),
+    }
+
+
+def _normalize_artifacts(result: TaskResult) -> None:
+    normalized: list[dict] = []
+    for artifact in result.artifacts:
+        if isinstance(artifact, dict):
+            normalized.append(dict(artifact))
+            continue
+        path = Path(artifact).resolve()
+        if path.is_file():
+            normalized.append(_hash_file(path))
+        else:
+            normalized.append({"path": str(path), "missing": True})
+    result.artifacts = normalized
+
+
+async def run_tasks(
+    adapter: BenchmarkAdapter,
+    tasks: list[TaskSpec] | tuple[TaskSpec, ...],
+    artifact_dir: str | Path,
+    *,
+    timeout: float = 30.0,
+) -> dict:
+    destination = Path(artifact_dir).resolve()
+    destination.mkdir(parents=True, exist_ok=True)
+    results: list[TaskResult] = []
+    await adapter.setup(destination)
+    try:
+        for task in tasks:
+            started = time.perf_counter()
+            try:
+                result = await asyncio.wait_for(adapter.run_task(task), timeout=timeout)
+            except TimeoutError:
+                result = TaskResult(task.task_id, "timeout", 0.0, f"Exceeded {timeout}s")
+            except Exception as exc:
+                result = TaskResult(task.task_id, "fail", 0.0, str(exc), stderr_summary=str(exc))
+            result.duration_ms = round((time.perf_counter() - started) * 1000, 2)
+            _normalize_artifacts(result)
+            results.append(result)
+    finally:
+        await adapter.cleanup()
+
+    attempted = len(results)
+    passed = sum(item.status == "pass" for item in results)
+    supported = [item for item in results if item.status != "unsupported"]
+    weight_by_id = {task.task_id: task.weight for task in tasks}
+    total_weight = sum(weight_by_id.values())
+    supported_weight = sum(weight_by_id[item.task_id] for item in supported)
+    weighted_score = sum(
+        weight_by_id[item.task_id] * item.score
+        for item in results
+        if item.status != "unsupported"
+    )
+    return {
+        "schema_version": "2.0",
+        "adapter": adapter.name,
+        "adapter_metadata": adapter.metadata(),
+        "git_sha": _git_sha(),
+        "generated_at": datetime.now(UTC).isoformat(),
+        "environment": {
+            "python": sys.version.split()[0],
+            "implementation": platform.python_implementation(),
+            "platform": platform.platform(),
+            "machine": platform.machine(),
+        },
+        "timeout_seconds": timeout,
+        "summary": {
+            "attempted": attempted,
+            "passed": passed,
+            "supported": len(supported),
+            "unsupported": sum(item.status == "unsupported" for item in results),
+            "coverage_percent": (
+                round(supported_weight / total_weight * 100, 2) if total_weight else 0.0
+            ),
+            "score": round(weighted_score / total_weight, 2) if total_weight else 0.0,
+        },
+        "results": [item.to_dict() for item in results],
+    }
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--list", action="store_true", help="List the fixed benchmark tasks")
+    parser.add_argument("--server", default="autocad-mcp-pro", choices=["autocad-mcp-pro"])
+    parser.add_argument("--backend", default="ezdxf", choices=["ezdxf", "com"])
+    parser.add_argument("--task", action="append", help="Run only this task id (repeatable)")
+    parser.add_argument("--artifact-dir", default="benchmarks/results/latest")
+    parser.add_argument("--timeout", type=float, default=30.0)
+    parser.add_argument("--json", action="store_true", help="Print machine-readable JSON")
+    return parser
+
+
+def main() -> None:
+    args = _parser().parse_args()
+    if args.list:
+        for task in TASKS_V2:
+            print(f"{task.task_id}\t{task.category}\t{task.description}")
+        return
+
+    from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter
+
+    tasks = [task_by_id(task_id) for task_id in args.task] if args.task else list(TASKS_V2)
+    adapter = AutoCADMCPProAdapter(backend=args.backend)
+    report = asyncio.run(
+        run_tasks(adapter, tasks, args.artifact_dir, timeout=args.timeout)
+    )
+    rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)
+    print(rendered)
+
+
+if __name__ == "__main__":
+    main()