Print eval lifecycle progress to stdout. Subscribe to a run's stream to use it.
Source code in autogen/beta/eval/reporters.py
| def console_reporter(event: BaseEvent) -> None:
"""Print eval lifecycle progress to stdout. Subscribe to a run's stream to use it."""
if isinstance(event, EvalStarted):
tag = f"[{event.label}] " if event.label else ""
print(f"{tag}running {event.total} task-run(s) over {event.suite!r}...")
elif isinstance(event, VariantStarted):
print(f" → variant {event.variant} ({event.index}/{event.total})")
elif isinstance(event, TaskEvaluated):
passed = sum(1 for fb in event.feedback if fb.score is True)
where = f"[{event.variant}] " if event.variant else ""
print(f" {where}{event.task_id}: {passed}/{len(event.feedback)} passed")
elif isinstance(event, VariantCompleted):
print(f" ✓ variant {event.variant} done")
elif isinstance(event, PairwiseStarted):
tag = f"[{event.label}] " if event.label else ""
print(f"{tag}comparing {event.variant_b!r} (B) vs {event.variant_a!r} (A) over {event.total} pair(s)...")
elif isinstance(event, PairwiseCompared):
print(f" {event.task_id} [{event.key}]: {event.winner}")
elif isinstance(event, PairwiseCompleted):
print(f" ✓ pairwise {event.result.run_id} done")
|