name: Benchmark

# Headless competitor benchmark (v2 fixed-task matrix). Manual or weekly:
# clones the pinned competitor SHAs, runs the black-box MCP stdio lane and
# uploads the machine-readable reports as artifacts. Results committed to
# benchmarks/results/ are produced from these runs.

on:
  workflow_dispatch:
  schedule:
    - cron: "17 5 * * 1"

env:
  # Keep in step with ci.yml; all three workflows read the same uv.lock.
  LOCKED_UV_VERSION: "0.12.1"

jobs:
  headless-matrix:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        server:
          - autocad-mcp-pro
          - puran-water-autocad-mcp
          - beiming183-autocad-mcp
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        id: python
        with:
          python-version: "3.12"
      - run: pip install "uv==${{ env.LOCKED_UV_VERSION }}"
      - name: Install the locked harness environment
        # This lane publishes evidence, so the harness must not float: an ezdxf
        # release would otherwise move our own numbers between runs and the
        # diff would be unattributable. Competitors keep their own pinned
        # checkouts and venvs (benchmarks/competitors_env.py) either way.
        run: uv sync --locked --extra pdf --python "${{ steps.python.outputs.python-path }}"
      - name: Run benchmark (ezdxf lane)
        run: |
          uv run --no-sync python -m benchmarks.run_competitors \
            --server "${{ matrix.server }}" \
            --backend ezdxf \
            --artifact-dir "benchmarks/results/ci/${{ matrix.server }}" \
            > "report-${{ matrix.server }}.json"
      - name: Upload report
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-${{ matrix.server }}
          path: |
            report-${{ matrix.server }}.json
            benchmarks/results/ci/${{ matrix.server }}