GitHub Actions

name: AgentX Evaluation Gate

on:
  pull_request:
    branches: [main]

jobs:
  eval:
    name: Evaluate agent
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: pip install agentx-python

      - name: Run AgentX evaluation gate
        env:
          AGENTX_API_KEY: ${{ secrets.AGENTX_API_KEY }}
        run: |
          python -c "
          import os, sys
          from agentx import AgentX, CIGateFailure

          # Import your agent
          from your_package import my_agent

          client = AgentX.from_env()

          try:
              result = client.tracer.run_eval(
                  dataset_id=os.environ['AGENTX_DATASET_ID'],
                  agent_fn=my_agent,
                  agent_name='my-agent',
                  # Keys must be camelCase. The API's gitContext schema is strict
                  # and silently drops unrecognized (e.g. snake_case) keys.
                  git_context={
                      'branch': os.environ.get('GITHUB_HEAD_REF', ''),
                      'commitSha': os.environ.get('GITHUB_SHA', ''),
                      'prNumber': os.environ.get('GITHUB_REF', '').split('/')[-2],
                      'repoUrl': 'https://github.com/' + os.environ.get('GITHUB_REPOSITORY', ''),
                      'triggeredBy': 'github-actions',
                  },
                  fail_on_gate=True,
              )
              print(f'Gate PASSED, {result.pass_rate:.0%} passed')
          except CIGateFailure as e:
              print(f'Gate FAILED, {e.result.pass_rate:.0%} passed ({e.result.passed_questions}/{e.result.total_questions})')
              sys.exit(1)
          "
        env:
          AGENTX_DATASET_ID: ${{ secrets.AGENTX_DATASET_ID }}

# scripts/eval_gate.py
import os
import sys
from agentx import AgentX, CIGateFailure
from your_package import my_agent  # import your agent

client = AgentX.from_env()

try:
    result = client.tracer.run_eval(
        dataset_id=os.environ["AGENTX_DATASET_ID"],
        agent_fn=my_agent,
        agent_name="my-agent",
        concurrency=4,
        # Keys must be camelCase. The API's gitContext schema is strict
        # and silently drops unrecognized (e.g. snake_case) keys.
        git_context={
            "branch": os.environ.get("GITHUB_HEAD_REF", ""),
            "commitSha": os.environ.get("GITHUB_SHA", ""),
            "prNumber": os.environ.get("GITHUB_REF", "").split("/")[-2],
            "repoUrl": "https://github.com/" + os.environ.get("GITHUB_REPOSITORY", ""),
            "triggeredBy": "github-actions",
        },
        fail_on_gate=True,
    )
    print(f"Gate PASSED, {result.pass_rate:.0%} ({result.passed_questions}/{result.total_questions})")

except CIGateFailure as e:
    r = e.result
    print(f"Gate FAILED, {r.pass_rate:.0%} ({r.passed_questions}/{r.total_questions})")
    for v in r.violations:
        print(f"  Q{v.question_index}: {v.metric} = {v.actual:.2f} (threshold {v.threshold:.2f})")
    sys.exit(1)

- name: Run AgentX evaluation gate
  run: python scripts/eval_gate.py
  env:
    AGENTX_API_KEY: ${{ secrets.AGENTX_API_KEY }}
    AGENTX_DATASET_ID: ${{ secrets.AGENTX_DATASET_ID }}