Basic Usage
import asyncio
from upsonic import Agent
from upsonic.eval import AccuracyEvaluator
agent = Agent(
model="anthropic/claude-sonnet-4-5",
name="Assistant",
role="Knowledge Assistant",
goal="Answer questions accurately",
)
judge = Agent(
model="anthropic/claude-sonnet-4-5",
name="Judge",
)
evaluator = AccuracyEvaluator(
judge_agent=judge,
agent_under_test=agent,
query="What is the capital of Japan?",
expected_output="Tokyo is the capital of Japan.",
additional_guidelines="Check if the answer correctly identifies Tokyo.",
num_iterations=1,
)
result = asyncio.run(evaluator.run(print_results=True))
print(f"Score: {result.average_score}/10")
print(f"Passed: {result.evaluation_scores[0].is_met}")
print(f"Output: {result.generated_output}")
Multiple Iterations
Run the evaluation multiple times and average the scores for a more robust measurement.import asyncio
from upsonic import Agent
from upsonic.eval import AccuracyEvaluator
agent = Agent(
model="anthropic/claude-sonnet-4-5",
name="Assistant",
)
judge = Agent(
model="anthropic/claude-sonnet-4-5",
name="Judge",
)
evaluator = AccuracyEvaluator(
judge_agent=judge,
agent_under_test=agent,
query="Explain photosynthesis in one sentence.",
expected_output="Photosynthesis converts sunlight, water, and CO2 into glucose and oxygen.",
num_iterations=3,
)
result = asyncio.run(evaluator.run(print_results=True))
print(f"Average score: {result.average_score}/10")
for i, score in enumerate(result.evaluation_scores):
print(f" Iteration {i+1}: {score.score}/10 — {score.reasoning[:80]}...")
Evaluate Pre-existing Output
Skip execution and score a string you already have.import asyncio
from upsonic import Agent
from upsonic.eval import AccuracyEvaluator
agent = Agent(
model="anthropic/claude-sonnet-4-5",
name="Assistant",
)
judge = Agent(
model="anthropic/claude-sonnet-4-5",
name="Judge",
)
evaluator = AccuracyEvaluator(
judge_agent=judge,
agent_under_test=agent,
query="What is the speed of light?",
expected_output="Approximately 299,792 km/s.",
num_iterations=1,
)
result = asyncio.run(
evaluator.run_with_output(
output="The speed of light is roughly 300,000 km per second.",
print_results=True,
)
)
print(f"Score: {result.average_score}/10")