autocad-mcp-pro/benchmarks/perf_suite.py

"""Headless performance suite (v3 lane): wall-time on real drawing workloads.

Complements the correctness/coverage lanes with *scale* evidence: how long the
ezdxf engine takes to create, roundtrip, query and quality-check real geometry.
Workloads drive the backend exactly like the MCP tools do (same async methods),
so the numbers include the server-side call overhead, not just raw ezdxf.

Honesty boundaries:

- Self-measurement only — no competitor timing claims (their servers would pay
  an extra stdio serialization cost ours does not pay in-process).
- Wall-clock on the machine recorded in the report; absolute numbers move with
  hardware, the workload definitions do not.

Run:

    python -m benchmarks.perf_suite            # human summary
    python -m benchmarks.perf_suite --json     # machine-readable report
    python -m benchmarks.perf_suite --out benchmarks/results/published/perf-ezdxf.json
"""

from __future__ import annotations

import argparse
import asyncio
import json
import platform
import sys
import time
from dataclasses import asdict, dataclass
from pathlib import Path

from backends.ezdxf_backend import EzdxfBackend


@dataclass
class WorkloadResult:
    workload_id: str
    description: str
    wall_ms: float
    metric: str
    metric_value: float
    detail: dict


async def _timed(coro) -> tuple[float, object]:
    started = time.perf_counter()
    result = await coro
    return (time.perf_counter() - started) * 1000.0, result


async def workload_create_lines_2k(backend: EzdxfBackend) -> WorkloadResult:
    """2,000 individual entity_create_line calls (per-call tool overhead included)."""
    await backend.drawing_new()
    started = time.perf_counter()
    for index in range(2_000):
        y = float(index % 100)
        await backend.entity_create_line(0.0, y, 100.0, y + 1.0)
    wall_ms = (time.perf_counter() - started) * 1000.0
    count = (await backend.drawing_info()).entity_count
    return WorkloadResult(
        "create_lines_2k",
        "Create 2,000 lines through individual backend calls",
        round(wall_ms, 1),
        "entities_per_second",
        round(2_000 / (wall_ms / 1000.0), 1),
        {"entity_count": count},
    )


async def workload_roundtrip_10k(backend: EzdxfBackend, scratch: Path) -> WorkloadResult:
    """Build a 10,000-entity drawing, export DXF, reopen, read info."""
    await backend.drawing_new()
    build_started = time.perf_counter()
    for index in range(10_000):
        x = float(index % 500)
        y = float(index // 500)
        await backend.entity_create_line(x, y, x + 0.8, y + 0.8)
    build_ms = (time.perf_counter() - build_started) * 1000.0

    target = scratch / "perf_roundtrip_10k.dxf"
    export_ms, _ = await _timed(backend.drawing_export_dxf(str(target)))

    reopened = EzdxfBackend()
    await reopened.connect()
    try:
        reopen_ms, _ = await _timed(reopened.drawing_open(str(target)))
        count = (await reopened.drawing_info()).entity_count
    finally:
        await reopened.disconnect()

    total = build_ms + export_ms + reopen_ms
    return WorkloadResult(
        "roundtrip_10k",
        "Create 10,000 lines, export DXF, reopen and verify",
        round(total, 1),
        "entities_per_second",
        round(10_000 / (total / 1000.0), 1),
        {
            "build_ms": round(build_ms, 1),
            "export_ms": round(export_ms, 1),
            "reopen_ms": round(reopen_ms, 1),
            "file_bytes": target.stat().st_size,
            "entity_count": count,
        },
    )


async def workload_region_query_10k(backend: EzdxfBackend) -> WorkloadResult:
    """Rectangular region query against the 10,000-entity drawing."""
    # backend still holds the 10k drawing from the roundtrip workload builder;
    # rebuild deterministically to stay order-independent.
    await backend.drawing_new()
    for index in range(10_000):
        x = float(index % 500)
        y = float(index // 500)
        await backend.entity_create_line(x, y, x + 0.8, y + 0.8)
    wall_ms, found = await _timed(backend.analysis_entities_in_region(0.0, 0.0, 250.0, 10.0))
    return WorkloadResult(
        "region_query_10k",
        "Crossing-selection region query over 10,000 entities",
        round(wall_ms, 1),
        "entities_matched",
        float(len(found)),
        {"entity_count": 10_000},
    )


async def workload_premium_pass(backend: EzdxfBackend) -> WorkloadResult:
    """Small end-to-end quality loop: layers, geometry, dimensions, critique."""
    started = time.perf_counter()
    await backend.drawing_new()
    await backend.drawing_apply_iso_layers("mech")
    plate = [
        (0.0, 0.0, 120.0, 0.0),
        (120.0, 0.0, 120.0, 80.0),
        (120.0, 80.0, 0.0, 80.0),
        (0.0, 80.0, 0.0, 0.0),
    ]
    for x1, y1, x2, y2 in plate:
        await backend.entity_create_line(x1, y1, x2, y2, layer="GEOMETRY")
    await backend.entity_create_circle(60.0, 40.0, 12.0, layer="GEOMETRY")
    await backend.dimension_linear(0, 0, 120, 0, 60, -15, layer="DIM")
    await backend.dimension_linear(0, 0, 0, 80, -15, 40, 90, layer="DIM")
    issues = await backend.drawing_critique(None)
    wall_ms = (time.perf_counter() - started) * 1000.0
    return WorkloadResult(
        "premium_pass",
        "Plate with hole: ISO layers, geometry, dimensions and full critique",
        round(wall_ms, 1),
        "critique_issues",
        float(len(issues)),
        {"entities": (await backend.drawing_info()).entity_count},
    )


async def run_suite(scratch: Path) -> dict:
    backend = EzdxfBackend()
    await backend.connect()
    try:
        results = [
            await workload_create_lines_2k(backend),
            await workload_roundtrip_10k(backend, scratch),
            await workload_region_query_10k(backend),
            await workload_premium_pass(backend),
        ]
    finally:
        await backend.disconnect()
    return {
        "schema_version": "perf/1.0",
        "backend": "ezdxf",
        "environment": {
            "python": sys.version.split()[0],
            "implementation": platform.python_implementation(),
            "platform": platform.platform(),
            "machine": platform.machine(),
        },
        "workloads": [asdict(item) for item in results],
    }


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--json", action="store_true", help="Print machine-readable JSON")
    parser.add_argument("--out", type=Path, default=None, help="Also write the report to a file")
    parser.add_argument("--scratch", type=Path, default=Path("benchmarks/results/perf-scratch"))
    return parser


def main() -> int:
    args = build_parser().parse_args()
    args.scratch.mkdir(parents=True, exist_ok=True)
    report = asyncio.run(run_suite(args.scratch))
    rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)
    if args.out is not None:
        args.out.parent.mkdir(parents=True, exist_ok=True)
        args.out.write_text(rendered + "\n", encoding="utf-8")
    if args.json:
        print(rendered)
    else:
        for item in report["workloads"]:
            print(
                f"{item['workload_id']:>18}  {item['wall_ms']:>10.1f} ms   "
                f"{item['metric']}={item['metric_value']}"
            )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())