feat(bench): live competitor lane - black-box MCP stdio adapters, pinned checkouts, published results
- adapters/mcp_stdio.py: drives competitors over MCP stdio like a real host; payload-level error detection, fastmcp output-wrapper normalization, cold-start warmup, artifact-based verification (harness re-opens every exported DXF with ezdxf) - competitors_env.py: reproducible pinned-SHA checkout + isolated venv per competitor - adapters for puran-water/autocad-mcp (95476a3) and beiming183-cloud/AutoCAD-MCP (11f7c47) mapped to their own documented tool contracts; unmapped tasks report 'unsupported' with reasons - run_competitors.py: registry-driven --server from competitors.yaml - published reports (pro 100.0 / beiming 50.0 / puran 45.0) + render_live_chart.py -> docs/assets/autocad-mcp-livebench.svg - benchmark.yml: weekly + manual dispatch matrix Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Umutcan Edizsalan
co-authored by
Cursor
Cursor
parent
2f56a73093
commit
6d7c39e012
@@ -0,0 +1,270 @@
|
||||
{
|
||||
"adapter": "autocad-mcp-pro",
|
||||
"adapter_metadata": {
|
||||
"backend": "ezdxf",
|
||||
"capabilities": {
|
||||
"backend": "ezdxf",
|
||||
"features": {
|
||||
"delivery": {
|
||||
"mode": "shared",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"drawing_2d": {
|
||||
"mode": "native",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"dxf": {
|
||||
"mode": "native",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"lisp": {
|
||||
"mode": null,
|
||||
"reason": "live_com_only",
|
||||
"supported": false
|
||||
},
|
||||
"mleader": {
|
||||
"mode": "composite",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"paper_space": {
|
||||
"mode": null,
|
||||
"reason": "planned_1.4",
|
||||
"supported": false
|
||||
},
|
||||
"pdf": {
|
||||
"mode": "rendered",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"png": {
|
||||
"mode": "rendered",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"preflight": {
|
||||
"mode": "shared",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"refiner": {
|
||||
"mode": "shared",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"solid_3d": {
|
||||
"mode": null,
|
||||
"reason": "planned_1.5",
|
||||
"supported": false
|
||||
},
|
||||
"table": {
|
||||
"mode": "composite",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
},
|
||||
"transactions": {
|
||||
"mode": "snapshot",
|
||||
"reason": null,
|
||||
"supported": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"environment": {
|
||||
"implementation": "CPython",
|
||||
"machine": "AMD64",
|
||||
"platform": "Windows-11-10.0.26200-SP0",
|
||||
"python": "3.14.0"
|
||||
},
|
||||
"generated_at": "2026-07-23T11:39:04.277175+00:00",
|
||||
"git_sha": "4a2e9d658f051cb756f92cb8fae945c64b8cc7d4",
|
||||
"results": [
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 8.14,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"line_length": 5.0
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "core_geometry"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 3.39,
|
||||
"message": "",
|
||||
"metrics": {},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "modify_query"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 8.11,
|
||||
"message": "",
|
||||
"metrics": {},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "layers_linetypes"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 348.74,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"entity_type": "DIMENSION"
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "dimensions"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 15.74,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"representations": [
|
||||
"composite",
|
||||
"composite"
|
||||
]
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "table_mleader"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 28.95,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"entity_count": 0
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "transactions"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 2.5,
|
||||
"message": "",
|
||||
"metrics": {},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "preflight"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 29.19,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"final_score": 100.0,
|
||||
"initial_score": 92.0,
|
||||
"remaining_issues": [],
|
||||
"rounds": [
|
||||
{
|
||||
"actions": [
|
||||
{
|
||||
"detail": {
|
||||
"deleted_handle": "3E",
|
||||
"kept_handle": "3D"
|
||||
},
|
||||
"fingerprint": "c1b87eac3cfdf11312c9b18ccf4c062e672791f7a9a6dfa3c45b6db7f9e5835e",
|
||||
"focus": "duplicate_entities",
|
||||
"handles": [
|
||||
"3D",
|
||||
"3E"
|
||||
],
|
||||
"status": "applied"
|
||||
}
|
||||
],
|
||||
"error_count_after": 0,
|
||||
"error_count_before": 0,
|
||||
"number": 1,
|
||||
"score_after": 100.0,
|
||||
"score_before": 92.0,
|
||||
"transaction": "committed"
|
||||
}
|
||||
],
|
||||
"status": "threshold_met"
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "quality_refiner"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "roundtrip.dxf",
|
||||
"sha256": "6413204a5baec19a5fbf551418eb56e30b1488ca5bb755e6a8709ac8e8ab0b22",
|
||||
"size_bytes": 18442
|
||||
}
|
||||
],
|
||||
"duration_ms": 16.66,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"entity_count": 1
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "dxf_roundtrip"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "manifest.json",
|
||||
"sha256": "38c2ee819b49b9c66b2311b952e856eff9e3a7565194cce2edea60a3211d233b",
|
||||
"size_bytes": 3581
|
||||
}
|
||||
],
|
||||
"duration_ms": 4380.8,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"score": {
|
||||
"errors": 0,
|
||||
"grade": "A",
|
||||
"info": 0,
|
||||
"invalidity_ratio": 0.0,
|
||||
"score": 100.0,
|
||||
"warnings": 0
|
||||
}
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "auditable_delivery"
|
||||
}
|
||||
],
|
||||
"schema_version": "2.0",
|
||||
"summary": {
|
||||
"attempted": 10,
|
||||
"coverage_percent": 100.0,
|
||||
"passed": 10,
|
||||
"score": 100.0,
|
||||
"supported": 10,
|
||||
"unsupported": 0
|
||||
},
|
||||
"timeout_seconds": 30.0
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"adapter": "beiming183-autocad-mcp",
|
||||
"adapter_metadata": {
|
||||
"backend": "ezdxf",
|
||||
"kind": "mcp-stdio-blackbox",
|
||||
"pinned_sha": "11f7c47e5038796a20451b38b23032e625b5aa26",
|
||||
"repo": "https://github.com/beiming183-cloud/AutoCAD-MCP.git",
|
||||
"tools_discovered": [
|
||||
"annotation",
|
||||
"block",
|
||||
"drawing",
|
||||
"entity",
|
||||
"job",
|
||||
"layer",
|
||||
"pid",
|
||||
"product",
|
||||
"solid",
|
||||
"system",
|
||||
"transaction",
|
||||
"view"
|
||||
]
|
||||
},
|
||||
"environment": {
|
||||
"implementation": "CPython",
|
||||
"machine": "AMD64",
|
||||
"platform": "Windows-11-10.0.26200-SP0",
|
||||
"python": "3.14.0"
|
||||
},
|
||||
"generated_at": "2026-07-23T11:38:36.969519+00:00",
|
||||
"git_sha": "4a2e9d658f051cb756f92cb8fae945c64b8cc7d4",
|
||||
"results": [
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "beiming_core_geometry.dxf",
|
||||
"sha256": "d972b29da994b1ea8f3887eef84f2ed892a486409385b56cd444b1fe9794e337",
|
||||
"size_bytes": 18587
|
||||
}
|
||||
],
|
||||
"duration_ms": 96.24,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"circles": 1,
|
||||
"line_length": 5.0,
|
||||
"lines": 1
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "core_geometry"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "beiming_modify_query.dxf",
|
||||
"sha256": "298534d8512ce62833551ab6e54c0c2a0baf18bdf7a96bd956cdf06280fb4fb8",
|
||||
"size_bytes": 18473
|
||||
}
|
||||
],
|
||||
"duration_ms": 108.09,
|
||||
"message": "",
|
||||
"metrics": {},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "modify_query"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "beiming_layers.dxf",
|
||||
"sha256": "b401e145e97aca7011d03097190b3d0ece0a89b47d38022a81c6c4c7e250ed3d",
|
||||
"size_bytes": 19887
|
||||
}
|
||||
],
|
||||
"duration_ms": 69.86,
|
||||
"message": "",
|
||||
"metrics": {},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "layers_linetypes"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "beiming_dimensions.dxf",
|
||||
"sha256": "8996d2ba42da66f46e79d1655405a4bf2ba72301861eeea58509cba44c408fb4",
|
||||
"size_bytes": 20972
|
||||
}
|
||||
],
|
||||
"duration_ms": 75.55,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"dimension_count": 1
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "dimensions"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.12,
|
||||
"message": "no TABLE or MLEADER operation in the annotation tool (v3.10)",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "table_mleader"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 18.99,
|
||||
"message": "transaction begin is rejected as unsupported on the headless backend",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "transactions"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.04,
|
||||
"message": "no requirements-preflight equivalent",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "preflight"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.02,
|
||||
"message": "audits detect issues but there is no automated repair loop",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "quality_refiner"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "beiming_roundtrip.dxf",
|
||||
"sha256": "79d7bdc1e66e506dd2ecf66fbf030839242c77870021ccbfe3c819ad394695ce",
|
||||
"size_bytes": 18474
|
||||
}
|
||||
],
|
||||
"duration_ms": 69.95,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"entity_count": 1
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "dxf_roundtrip"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.06,
|
||||
"message": "drawing.deliver is documented as File-IPC only (not headless)",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "auditable_delivery"
|
||||
}
|
||||
],
|
||||
"schema_version": "2.0",
|
||||
"summary": {
|
||||
"attempted": 10,
|
||||
"coverage_percent": 50.0,
|
||||
"passed": 5,
|
||||
"score": 50.0,
|
||||
"supported": 5,
|
||||
"unsupported": 5
|
||||
},
|
||||
"timeout_seconds": 30.0
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
{
|
||||
"adapter": "puran-water-autocad-mcp",
|
||||
"adapter_metadata": {
|
||||
"backend": "ezdxf",
|
||||
"kind": "mcp-stdio-blackbox",
|
||||
"pinned_sha": "95476a33a1c246308326eb4709d6379ef2efdbc1",
|
||||
"repo": "https://github.com/puran-water/autocad-mcp.git",
|
||||
"tools_discovered": [
|
||||
"annotation",
|
||||
"block",
|
||||
"drawing",
|
||||
"entity",
|
||||
"layer",
|
||||
"pid",
|
||||
"system",
|
||||
"view"
|
||||
]
|
||||
},
|
||||
"environment": {
|
||||
"implementation": "CPython",
|
||||
"machine": "AMD64",
|
||||
"platform": "Windows-11-10.0.26200-SP0",
|
||||
"python": "3.14.0"
|
||||
},
|
||||
"generated_at": "2026-07-23T11:32:07.977232+00:00",
|
||||
"git_sha": "4a2e9d658f051cb756f92cb8fae945c64b8cc7d4",
|
||||
"results": [
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "puran_core_geometry.dxf",
|
||||
"sha256": "bc8185404c8344134859293a0e28ebf821cd5968553cdd96b7029e46e1ca0c99",
|
||||
"size_bytes": 18587
|
||||
}
|
||||
],
|
||||
"duration_ms": 205.92,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"circles": 1,
|
||||
"line_length": 5.0,
|
||||
"lines": 1
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "core_geometry"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "puran_modify_query.dxf",
|
||||
"sha256": "2113a59d0a9f624b6134e89f992d153bc8e085495e956f4887b8ba7b54f3c005",
|
||||
"size_bytes": 18473
|
||||
}
|
||||
],
|
||||
"duration_ms": 156.79,
|
||||
"message": "",
|
||||
"metrics": {},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "modify_query"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "puran_layers.dxf",
|
||||
"sha256": "7144467e22e2e8c8d18f1fc4f177f078f9ce450bb73f4e9ca8f74c16d67e1b53",
|
||||
"size_bytes": 18498
|
||||
}
|
||||
],
|
||||
"duration_ms": 189.98,
|
||||
"message": "layer created; no linetype-loading operation is exposed",
|
||||
"metrics": {},
|
||||
"score": 50.0,
|
||||
"status": "partial",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "layers_linetypes"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "puran_dimensions.dxf",
|
||||
"sha256": "dde808c5e504c5e6b7837f5f5c14ef096711812a6239dd284906efed5c9e3614",
|
||||
"size_bytes": 20972
|
||||
}
|
||||
],
|
||||
"duration_ms": 215.28,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"dimension_count": 1
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "dimensions"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.12,
|
||||
"message": "no TABLE or MLEADER operation in the 8-tool surface (v3.1)",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "table_mleader"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.02,
|
||||
"message": "no transaction begin/commit/rollback tool (undo/redo are File-IPC only)",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "transactions"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.01,
|
||||
"message": "no requirements-preflight equivalent",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "preflight"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.03,
|
||||
"message": "no critique/repair loop equivalent",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "quality_refiner"
|
||||
},
|
||||
{
|
||||
"artifacts": [
|
||||
{
|
||||
"filename": "puran_roundtrip.dxf",
|
||||
"sha256": "11335953641be32f3f47c23fbc13f79876111c6f84f8874af8f5e04d5e398cf6",
|
||||
"size_bytes": 18474
|
||||
}
|
||||
],
|
||||
"duration_ms": 143.4,
|
||||
"message": "",
|
||||
"metrics": {
|
||||
"entity_count": 1
|
||||
},
|
||||
"score": 100.0,
|
||||
"status": "pass",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "dxf_roundtrip"
|
||||
},
|
||||
{
|
||||
"artifacts": [],
|
||||
"duration_ms": 0.06,
|
||||
"message": "no hashed delivery-manifest equivalent (plot_pdf is File-IPC only)",
|
||||
"metrics": {},
|
||||
"score": 0.0,
|
||||
"status": "unsupported",
|
||||
"stderr_summary": "",
|
||||
"stdout_summary": "",
|
||||
"task_id": "auditable_delivery"
|
||||
}
|
||||
],
|
||||
"schema_version": "2.0",
|
||||
"summary": {
|
||||
"attempted": 10,
|
||||
"coverage_percent": 50.0,
|
||||
"passed": 4,
|
||||
"score": 45.0,
|
||||
"supported": 5,
|
||||
"unsupported": 5
|
||||
},
|
||||
"timeout_seconds": 30.0
|
||||
}
|
||||
Reference in New Issue
Block a user