GitHub Actions

# scripts/eval_gate.py
import os
import sys
from agentx import AgentX, CIGateFailure
from your_package import my_agent  # import your agent: a function query -> answer string

client = AgentX.from_env()

try:
    result = client.tracer.run_eval(
        dataset_id=os.environ["AGENTX_DATASET_ID"],
        agent_fn=my_agent,
        agent_name="my-agent",
        concurrency=4,
        # Keys must be camelCase. The API's gitContext schema is strict
        # and silently drops unrecognized (e.g. snake_case) keys.
        git_context={
            "branch": os.environ.get("GITHUB_HEAD_REF", ""),
            "commitSha": os.environ.get("GITHUB_SHA", ""),
            "prNumber": os.environ.get("GITHUB_REF", "").split("/")[-2],
            "repoUrl": "https://github.com/" + os.environ.get("GITHUB_REPOSITORY", ""),
            "triggeredBy": "github-actions",
        },
        fail_on_gate=True,
    )
    print(f"Gate PASSED, {result.pass_rate:.0%} ({result.passed_questions}/{result.total_questions})")

except CIGateFailure as e:
    r = e.result
    print(f"Gate FAILED, {r.pass_rate:.0%} ({r.passed_questions}/{r.total_questions})")
    for v in r.violations:
        print(f"  Q{v.question_index}: {v.metric} = {v.actual:.2f} (threshold {v.threshold:.2f})")
    sys.exit(1)

name: AgentX Evaluation Gate

on:
  pull_request:
    branches: [main]

jobs:
  eval:
    name: Evaluate agent
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install agentx-python

      - name: Run AgentX evaluation gate
        run: python scripts/eval_gate.py
        env:
          AGENTX_API_KEY: ${{ secrets.AGENTX_API_KEY }}
          AGENTX_DATASET_ID: ${{ secrets.AGENTX_DATASET_ID }}

Gate PASSED, 88% (7/8)