autocad-mcp-pro/benchmarks/run_competitors.py

"""Command line runner for the fixed-task benchmark v2 adapter contract."""

from __future__ import annotations

import argparse
import asyncio
import hashlib
import json
import platform
import subprocess
import sys
import time
from datetime import UTC, datetime
from pathlib import Path

from benchmarks.adapters.base import BenchmarkAdapter, TaskResult
from benchmarks.tasks_v2 import TASKS_V2, TaskSpec, task_by_id


def _git_sha() -> str | None:
    try:
        result = subprocess.run(
            ["git", "rev-parse", "HEAD"],
            check=True,
            capture_output=True,
            text=True,
            timeout=2,
        )
        return result.stdout.strip() or None
    except (OSError, subprocess.SubprocessError):
        return None


def _hash_file(path: Path) -> dict:
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(chunk)
    return {
        "path": str(path),
        "filename": path.name,
        "size_bytes": path.stat().st_size,
        "sha256": digest.hexdigest(),
    }


def _normalize_artifacts(result: TaskResult) -> None:
    normalized: list[dict] = []
    for artifact in result.artifacts:
        if isinstance(artifact, dict):
            normalized.append(dict(artifact))
            continue
        path = Path(artifact).resolve()
        if path.is_file():
            normalized.append(_hash_file(path))
        else:
            normalized.append({"path": str(path), "missing": True})
    result.artifacts = normalized


async def run_tasks(
    adapter: BenchmarkAdapter,
    tasks: list[TaskSpec] | tuple[TaskSpec, ...],
    artifact_dir: str | Path,
    *,
    timeout: float = 30.0,
) -> dict:
    destination = Path(artifact_dir).resolve()
    destination.mkdir(parents=True, exist_ok=True)
    results: list[TaskResult] = []
    await adapter.setup(destination)
    try:
        for task in tasks:
            started = time.perf_counter()
            try:
                result = await asyncio.wait_for(adapter.run_task(task), timeout=timeout)
            except TimeoutError:
                result = TaskResult(task.task_id, "timeout", 0.0, f"Exceeded {timeout}s")
            except Exception as exc:
                result = TaskResult(task.task_id, "fail", 0.0, str(exc), stderr_summary=str(exc))
            result.duration_ms = round((time.perf_counter() - started) * 1000, 2)
            _normalize_artifacts(result)
            results.append(result)
    finally:
        await adapter.cleanup()

    attempted = len(results)
    passed = sum(item.status == "pass" for item in results)
    supported = [item for item in results if item.status != "unsupported"]
    weight_by_id = {task.task_id: task.weight for task in tasks}
    total_weight = sum(weight_by_id.values())
    supported_weight = sum(weight_by_id[item.task_id] for item in supported)
    weighted_score = sum(
        weight_by_id[item.task_id] * item.score
        for item in results
        if item.status != "unsupported"
    )
    return {
        "schema_version": "2.0",
        "adapter": adapter.name,
        "adapter_metadata": adapter.metadata(),
        "git_sha": _git_sha(),
        "generated_at": datetime.now(UTC).isoformat(),
        "environment": {
            "python": sys.version.split()[0],
            "implementation": platform.python_implementation(),
            "platform": platform.platform(),
            "machine": platform.machine(),
        },
        "timeout_seconds": timeout,
        "summary": {
            "attempted": attempted,
            "passed": passed,
            "supported": len(supported),
            "unsupported": sum(item.status == "unsupported" for item in results),
            "coverage_percent": (
                round(supported_weight / total_weight * 100, 2) if total_weight else 0.0
            ),
            "score": round(weighted_score / total_weight, 2) if total_weight else 0.0,
        },
        "results": [item.to_dict() for item in results],
    }


def _parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--list", action="store_true", help="List the fixed benchmark tasks")
    parser.add_argument("--server", default="autocad-mcp-pro", choices=["autocad-mcp-pro"])
    parser.add_argument("--backend", default="ezdxf", choices=["ezdxf", "com"])
    parser.add_argument("--task", action="append", help="Run only this task id (repeatable)")
    parser.add_argument("--artifact-dir", default="benchmarks/results/latest")
    parser.add_argument("--timeout", type=float, default=30.0)
    parser.add_argument("--json", action="store_true", help="Print machine-readable JSON")
    return parser


def main() -> None:
    args = _parser().parse_args()
    if args.list:
        for task in TASKS_V2:
            print(f"{task.task_id}\t{task.category}\t{task.description}")
        return

    from benchmarks.adapters.autocad_mcp_pro import AutoCADMCPProAdapter

    tasks = [task_by_id(task_id) for task_id in args.task] if args.task else list(TASKS_V2)
    adapter = AutoCADMCPProAdapter(backend=args.backend)
    report = asyncio.run(
        run_tasks(adapter, tasks, args.artifact_dir, timeout=args.timeout)
    )
    rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)
    print(rendered)


if __name__ == "__main__":
    main()