Skip to content

VariantRunResult

autogen.beta.eval.runtime.variants.VariantRunResult dataclass #

VariantRunResult(run_id, axis, results, created_at, duration_ms)

Result of :func:run_variants: a per-variant :class:RunResult plus ranking helpers.

run_id instance-attribute #

run_id

axis instance-attribute #

axis

results instance-attribute #

results

created_at instance-attribute #

created_at

duration_ms instance-attribute #

duration_ms

leaderboard #

leaderboard(key)

Variants ranked best-first by scorer key (pass-rate for boolean, mean for numeric).

Tied scores share a rank (competition ranking — 1, 1, 1, 4).

Source code in autogen/beta/eval/runtime/variants.py
def leaderboard(self, key: str) -> list[LeaderboardRow]:
    """Variants ranked best-first by scorer ``key`` (pass-rate for boolean, mean for numeric).

    Tied scores share a rank (competition ranking — ``1, 1, 1, 4``).
    """
    scored = sorted(
        ((name, *self._metric(res, key)) for name, res in self.results.items()),
        key=itemgetter(1),
        reverse=True,
    )
    return [
        LeaderboardRow(variant=name, score=score, n=n, rank=1 + sum(1 for _, other, _ in scored if other > score))
        for name, score, n in scored
    ]

best #

best(key)

Top-ranked variant for key — or None when the top score is shared (no unique winner).

Source code in autogen/beta/eval/runtime/variants.py
def best(self, key: str) -> str | None:
    """Top-ranked variant for ``key`` — or ``None`` when the top score is shared (no unique winner)."""
    leaders = [row for row in self.leaderboard(key) if row.rank == 1]
    return leaders[0].variant if len(leaders) == 1 else None

summary #

summary(key)

Printable ranked leaderboard for key.

Source code in autogen/beta/eval/runtime/variants.py
def summary(self, key: str) -> str:
    """Printable ranked leaderboard for ``key``."""
    rows = self.leaderboard(key)
    is_rate = any(key in res.aggregates.pass_rate for res in self.results.values())
    width = max((len(row.variant) for row in rows), default=0)
    lines = [f"Variants (axis: {self.axis}) — ranked by {key}:"]
    for row in rows:
        value = f"{row.score * 100:5.1f}%" if is_rate else f"{row.score:.3f}"
        lines.append(f"  {row.rank}. {row.variant:<{width}}  {value}  (n={row.n})")
    return "\n".join(lines)