curl "http://localhost:4700/api/v1/custom-agent-evaluations/runs/rK7dP2qWx9TzB4mV6nJcE/report" \
-H "x-api-key: agtx_local_0f3c9a17d2b84e6a5c01b9f4e7d8a2c6431b5f97a0e2d4c8"
report = client.evaluations.get_report("rK7dP2qWx9TzB4mV6nJcE")
print(report.summary)
for case in report.low_scoring_cases:
print(case)
{
"summary": "The agent performs well on direct how-to questions but struggles with multi-step billing queries.",
"consistencyScore": 8,
"instructionAdherence": {
"score": 9,
"analysis": "The agent generally follows the support tone guidelines.",
"deviations": ["Q3 run 2: agent gave a refund without checking eligibility"],
"rating": "high"
},
"responsePatterns": {
"similarities": ["Consistent greeting and closing"],
"differences": ["Detail level varies on billing answers"],
"outliers": ["Q3 run 2"],
"rating": "medium"
},
"reasoningAnalysis": {
"cotQuality": "Reasoning is visible and mostly sound.",
"reasoningPatterns": ["Looks up KB before answering"],
"reasoningGaps": ["Skips eligibility checks under time pressure"],
"rating": "medium"
},
"toolUsageAnalysis": {
"effectiveness": "KB search used appropriately on most cases.",
"patterns": ["search_kb before every answer"],
"issues": ["No billing tool call on refund cases"],
"rating": "medium"
},
"recommendations": [
{
"category": "instructions",
"priority": "high",
"recommendation": "Add a guardrail that requires an eligibility check before issuing refunds",
"reasoning": "One run issued a refund without confirming the customer's eligibility window"
}
],
"instructionChanges": [],
"overallAssessment": {
"strengths": ["Consistent tone", "Accurate password reset guidance"],
"weaknesses": ["Billing edge cases", "Multi-step reasoning"],
"rating": "medium"
},
"runId": "rK7dP2qWx9TzB4mV6nJcE",
"datasetId": "dS4tG7hNb2VxZ8kQ5wMyA",
"status": "completed",
"statistics": {
"numberOfRuns": 6,
"averageRating": 7.8,
"minRating": 5,
"maxRating": 10,
"ratingVariance": 1.96
},
"lowScoringCases": [
{
"query": "Why was I charged twice?",
"response": "You may have been charged twice due to a pending authorization.",
"rating": 5,
"justification": "Agent did not escalate to the billing team as required."
}
]
}
{
"error": "No analysis found for this run. POST /runs/:runId/analyze first."
}
Custom Evaluations
Get Report
Get the stored analysis report with statistics and low-scoring cases
GET
/
api
/
v1
/
custom-agent-evaluations
/
runs
/
{runId}
/
report
curl "http://localhost:4700/api/v1/custom-agent-evaluations/runs/rK7dP2qWx9TzB4mV6nJcE/report" \
-H "x-api-key: agtx_local_0f3c9a17d2b84e6a5c01b9f4e7d8a2c6431b5f97a0e2d4c8"
report = client.evaluations.get_report("rK7dP2qWx9TzB4mV6nJcE")
print(report.summary)
for case in report.low_scoring_cases:
print(case)
{
"summary": "The agent performs well on direct how-to questions but struggles with multi-step billing queries.",
"consistencyScore": 8,
"instructionAdherence": {
"score": 9,
"analysis": "The agent generally follows the support tone guidelines.",
"deviations": ["Q3 run 2: agent gave a refund without checking eligibility"],
"rating": "high"
},
"responsePatterns": {
"similarities": ["Consistent greeting and closing"],
"differences": ["Detail level varies on billing answers"],
"outliers": ["Q3 run 2"],
"rating": "medium"
},
"reasoningAnalysis": {
"cotQuality": "Reasoning is visible and mostly sound.",
"reasoningPatterns": ["Looks up KB before answering"],
"reasoningGaps": ["Skips eligibility checks under time pressure"],
"rating": "medium"
},
"toolUsageAnalysis": {
"effectiveness": "KB search used appropriately on most cases.",
"patterns": ["search_kb before every answer"],
"issues": ["No billing tool call on refund cases"],
"rating": "medium"
},
"recommendations": [
{
"category": "instructions",
"priority": "high",
"recommendation": "Add a guardrail that requires an eligibility check before issuing refunds",
"reasoning": "One run issued a refund without confirming the customer's eligibility window"
}
],
"instructionChanges": [],
"overallAssessment": {
"strengths": ["Consistent tone", "Accurate password reset guidance"],
"weaknesses": ["Billing edge cases", "Multi-step reasoning"],
"rating": "medium"
},
"runId": "rK7dP2qWx9TzB4mV6nJcE",
"datasetId": "dS4tG7hNb2VxZ8kQ5wMyA",
"status": "completed",
"statistics": {
"numberOfRuns": 6,
"averageRating": 7.8,
"minRating": 5,
"maxRating": 10,
"ratingVariance": 1.96
},
"lowScoringCases": [
{
"query": "Why was I charged twice?",
"response": "You may have been charged twice due to a pending authorization.",
"rating": 5,
"justification": "Agent did not escalate to the billing team as required."
}
]
}
{
"error": "No analysis found for this run. POST /runs/:runId/analyze first."
}
Returns the report Analyze Run stored for this run:
the narrative analysis hoisted to the top level, plus the run’s identifiers, the statistics
computed when the analysis ran, and the run’s low-scoring cases. Requires an analysis to exist -
a run that was never analyzed returns
Run fields (always present):
404, so an empty report is never mistaken for a run
that scored nothing.
Authentication
string
required
Project API key.
Path Parameters
string
required
Run ID.
Response
Narrative analysis fields (top level, from the analysis body; absent when the analysis failed):| Field | Type | Description |
|---|---|---|
summary | string | A few sentences on how the agent performed overall |
consistencyScore | number | 0-10: how consistent responses were across similar inputs |
instructionAdherence | object | { score (0-10), analysis, deviations[], rating } |
responsePatterns | object | { similarities[], differences[], outliers[], rating } |
reasoningAnalysis | object | { cotQuality, reasoningPatterns[], reasoningGaps[], rating } |
toolUsageAnalysis | object | { effectiveness, patterns[], issues[], rating } |
recommendations | array | [{ category, priority, recommendation, reasoning }] - category is one of "instructions", "tools", "knowledge", "reasoning", "consistency", "other"; priority and every rating above are "high" | "medium" | "low" |
instructionChanges | array | Always [] for external agents (the engine doesn’t own the agent’s instructions) |
overallAssessment | object | { strengths[], weaknesses[], rating } |
string
Run ID.
string
Dataset the run scored against.
string
The stored analysis status:
"completed" or "failed".object | null
Computed when the analysis ran:
{ numberOfRuns, averageRating, minRating, maxRating, ratingVariance }, computed over rated
rows with smoke-test variants excluded - the same population as
Get Run’s averageRating, so the two agree.array
Results with a rating of 5 or below (smoke-test variants excluded), derived fresh from the
run’s own rows:
[{ query, response, rating, justification }].Similarity aggregates (cosine/Jaccard/BLEU/ROUGE) are per-result, not report-level - read
them from Get Run’s
results rows. strengths,
weaknesses, and the overall rating live under overallAssessment, not at the top level.Errors
| Status | Body | Meaning |
|---|---|---|
404 | { "error": "Run not found" } | No run with this id in the project |
404 | { "error": "No analysis found for this run. POST /runs/:runId/analyze first." } | The run exists but was never analyzed |
curl "http://localhost:4700/api/v1/custom-agent-evaluations/runs/rK7dP2qWx9TzB4mV6nJcE/report" \
-H "x-api-key: agtx_local_0f3c9a17d2b84e6a5c01b9f4e7d8a2c6431b5f97a0e2d4c8"
report = client.evaluations.get_report("rK7dP2qWx9TzB4mV6nJcE")
print(report.summary)
for case in report.low_scoring_cases:
print(case)
{
"summary": "The agent performs well on direct how-to questions but struggles with multi-step billing queries.",
"consistencyScore": 8,
"instructionAdherence": {
"score": 9,
"analysis": "The agent generally follows the support tone guidelines.",
"deviations": ["Q3 run 2: agent gave a refund without checking eligibility"],
"rating": "high"
},
"responsePatterns": {
"similarities": ["Consistent greeting and closing"],
"differences": ["Detail level varies on billing answers"],
"outliers": ["Q3 run 2"],
"rating": "medium"
},
"reasoningAnalysis": {
"cotQuality": "Reasoning is visible and mostly sound.",
"reasoningPatterns": ["Looks up KB before answering"],
"reasoningGaps": ["Skips eligibility checks under time pressure"],
"rating": "medium"
},
"toolUsageAnalysis": {
"effectiveness": "KB search used appropriately on most cases.",
"patterns": ["search_kb before every answer"],
"issues": ["No billing tool call on refund cases"],
"rating": "medium"
},
"recommendations": [
{
"category": "instructions",
"priority": "high",
"recommendation": "Add a guardrail that requires an eligibility check before issuing refunds",
"reasoning": "One run issued a refund without confirming the customer's eligibility window"
}
],
"instructionChanges": [],
"overallAssessment": {
"strengths": ["Consistent tone", "Accurate password reset guidance"],
"weaknesses": ["Billing edge cases", "Multi-step reasoning"],
"rating": "medium"
},
"runId": "rK7dP2qWx9TzB4mV6nJcE",
"datasetId": "dS4tG7hNb2VxZ8kQ5wMyA",
"status": "completed",
"statistics": {
"numberOfRuns": 6,
"averageRating": 7.8,
"minRating": 5,
"maxRating": 10,
"ratingVariance": 1.96
},
"lowScoringCases": [
{
"query": "Why was I charged twice?",
"response": "You may have been charged twice due to a pending authorization.",
"rating": 5,
"justification": "Agent did not escalate to the billing team as required."
}
]
}
{
"error": "No analysis found for this run. POST /runs/:runId/analyze first."
}