release: prepare v1.3.0

benchmarks/adapters/base.py

@@ -0,0 +1,58 @@
+"""Common, vendor-neutral benchmark adapter contract."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from benchmarks.tasks_v2 import TaskSpec
+
+TaskStatus = Literal["pass", "partial", "unsupported", "fail", "timeout", "not_run"]
+
+
+@dataclass
+class TaskResult:
+    task_id: str
+    status: TaskStatus
+    score: float
+    message: str = ""
+    duration_ms: float = 0.0
+    artifacts: list[str | dict[str, Any]] = field(default_factory=list)
+    metrics: dict = field(default_factory=dict)
+    stdout_summary: str = ""
+    stderr_summary: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "task_id": self.task_id,
+            "status": self.status,
+            "score": float(self.score),
+            "message": self.message,
+            "duration_ms": float(self.duration_ms),
+            "artifacts": list(self.artifacts),
+            "metrics": dict(self.metrics),
+            "stdout_summary": self.stdout_summary,
+            "stderr_summary": self.stderr_summary,
+        }
+
+
+class BenchmarkAdapter(ABC):
+    """Minimum interface every competing MCP adapter must implement."""
+
+    name: str
+
+    def metadata(self) -> dict:
+        """Runtime/backend claims captured alongside every benchmark report."""
+        return {}
+
+    @abstractmethod
+    async def setup(self, artifact_dir: Path) -> None: ...
+
+    @abstractmethod
+    async def run_task(self, task: TaskSpec) -> TaskResult: ...
+
+    @abstractmethod
+    async def cleanup(self) -> None: ...