AutoGen / AG2

pip install agentx-python pyautogen

from agentx import AgentX
from autogen import ConversableAgent

client = AgentX.from_env()

assistant = ConversableAgent(
    name="assistant",
    system_message="You are a helpful customer support agent.",
    llm_config={"config_list": [{"model": "gpt-4o-mini"}]},
)
user_proxy = ConversableAgent(
    name="user_proxy",
    human_input_mode="NEVER",
    max_consecutive_auto_reply=1,
    llm_config=False,
)

def autogen_agent(case):
    chat_result = user_proxy.initiate_chat(
        assistant,
        message=case.query,
        max_turns=1,
    )
    output = chat_result.chat_history[-1]["content"]
    return {"output": output, "metadata": {"model": "gpt-4o-mini"}}

run_context = (
    client.evaluations
    .run(dataset_id="...", subject={"kind": "custom_agent", "displayName": "Support Assistant", "framework": "autogen"})
    .execute(autogen_agent)
    .finalize()
)
print(f"Average rating: {run_context.average_rating:.2f}")