OpenAI

pip install agentx-python openai

from agentx import AgentX
from openai import OpenAI

client = AgentX.from_env()
oai = OpenAI()

def openai_agent(case):
    resp = oai.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "You are a helpful support agent."},
            {"role": "user", "content": case.query},
        ],
    )
    return {"output": resp.choices[0].message.content, "metadata": {"model": resp.model}}

run_context = (
    client.evaluations
    .run(dataset_id="...", subject={"kind": "custom_agent", "displayName": "GPT-4o-mini", "framework": "openai"})
    .execute(openai_agent)
    .finalize()
)
print(f"Average rating: {run_context.average_rating:.2f}")

from agentx import AgentX
from openai import OpenAI

client = AgentX.from_env()
oai = OpenAI()

def openai_agent(case):
    with client.tracer.trace(
        "openai-agent-call",
        input={"query": case.query},
        framework="openai",
        model="gpt-4o-mini",
        sync=True,
        monitor=False,  # the run's judge already scores this case
    ) as span:
        resp = oai.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {"role": "system", "content": "You are a helpful support agent."},
                {"role": "user", "content": case.query},
            ],
        )
        span.output = resp.choices[0].message.content

    return {
        "output": resp.choices[0].message.content,
        "metadata": {"model": resp.model},
        "trace_id": span.trace_id,
    }

import os
from typing import Any, Dict

from openai import OpenAI
from agentx import AgentX
from agentx.evaluations.models import Dataset, EvaluationCase, EvaluationSettings, Report
from agentx.evaluations.runner import EvaluationRunContext

client = AgentX(
    api_key=os.environ["AGENTX_API_KEY"],
    workspace_id=os.environ.get("AGENTX_WORKSPACE_ID"),  # set if your key spans multiple workspaces
)
oai = OpenAI(api_key=os.environ["OPENAI_API_KEY"])

# 1. Build a dataset. Skip this and pass an existing dataset_id if you already have one.
dataset: Dataset = (
    client.evaluations.datasets.builder(
        name="Support Agent Eval Sample",
        number_of_requests=2,  # runs per case
        acceptance_criteria="Accurate, concise, grounded in the support policy.",
        rejection_criteria="No hallucinated policies or made-up steps.",
    )
    .add_case(
        query="How do I reset my password?",
        expected_results="Explain the password reset process step by step.",
    )
    .add_case(
        query="What payment methods do you accept?",
        expected_results="List supported payment methods clearly.",
    )
    .publish()
)
print(f"Published dataset: {dataset.id}")

# 2. Publish a standalone, reusable grading config, independent of this (or any) dataset's own
# twin config, with every similarity metric turned on. Reuse this same id across other datasets
# instead of rebuilding it each time; see /sdk/evaluations/overview#reusable-grading-configs.
evaluation_settings: EvaluationSettings = client.evaluations.settings.builder(
    name="Strict - two runs",
    number_of_requests=2,
    acceptance_criteria="Accurate, concise, grounded in the support policy.",
    rejection_criteria="No hallucinated policies or made-up steps.",
    vector_similarity=True,
    jaccard_similarity=True,
    bleu_score=True,
    rouge_score=True,
).publish()
print(f"Published evaluation settings: {evaluation_settings.id}")

# 3. The agent under test, traced so each result links to a full Execution Timeline.
def support_agent(case: EvaluationCase) -> Dict[str, Any]:
    with client.tracer.trace(
        "support-agent-call",
        input={"query": case.query},
        framework="openai",
        model="gpt-4o-mini",
        sync=True,
        monitor=False,  # the run's judge already scores this case
    ) as span:
        resp = oai.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {"role": "system", "content": "You are a helpful support agent."},
                {"role": "user", "content": case.query},
            ],
        )
        span.output = resp.choices[0].message.content

    return {
        "output": resp.choices[0].message.content,
        "metadata": {"model": resp.model},
        "trace_id": span.trace_id,
    }

# 4. Run: .execute() calls support_agent once per case and scores each response immediately;
# .finalize() closes the run. Both return EvaluationRunContext (self), not a Report. Only
# .analyze() returns one. Typing run_context/report as two separate variables means a type
# checker catches it immediately if .analyze() is ever skipped and something reads
# report.average_rating off the wrong type.
run_context: EvaluationRunContext = (
    client.evaluations.run(
        dataset_id=dataset.id,
        subject={
            "kind": "custom_agent",
            "displayName": "GPT-4o-mini Support Agent",
            "framework": "openai",
        },
        evaluation_settings_id=evaluation_settings.id,
    )
    .execute(support_agent)
    .finalize()
)

# Available immediately, no .analyze() call needed just to see the score.
print(f"Average rating: {run_context.average_rating:.2f} ({run_context.rated_count} rated)")

# 5. Analyze (optional): adds the qualitative report (strengths, weaknesses, recommendations).
report: Report = run_context.analyze()
print(f"Cosine similarity: {report.cosine_similarity:.3f}" if report.cosine_similarity is not None else "")
print(f"Dashboard: {report.dashboard_url}")