86 lines
4.1 KiB
Python
86 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from control_plane.agents.models import Agent, AgentPlan, AgentVersion, BenchmarkRun, PromotionStatus
|
|
from control_plane.events.bus import EventBus
|
|
from control_plane.events.models import EventType
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BenchmarkDecision:
|
|
decision: str
|
|
metrics: dict[str, float]
|
|
|
|
|
|
class ProgenyService:
|
|
def __init__(self, bus: EventBus | None = None) -> None:
|
|
self.bus = bus or EventBus()
|
|
|
|
def create_candidate_from_plan(self, plan: AgentPlan) -> AgentVersion:
|
|
agent = plan.agent or Agent.objects.create(name=plan.name, role=plan.role)
|
|
next_version = (agent.versions.order_by("-version").first().version + 1) if agent.versions.exists() else 1
|
|
version = AgentVersion.objects.create(
|
|
agent=agent,
|
|
version=next_version,
|
|
model=plan.model,
|
|
system_contract=plan.system_contract,
|
|
capabilities=plan.capabilities,
|
|
tools=plan.tools,
|
|
permissions=plan.permissions,
|
|
context_policy=plan.context_policy,
|
|
workflow=plan.workflow,
|
|
retry_policy={"max_retries": 2},
|
|
evaluator=plan.success_criteria,
|
|
promotion_status=PromotionStatus.CHALLENGER,
|
|
)
|
|
if agent.champion_version_id is None:
|
|
agent.champion_version = version
|
|
version.promotion_status = PromotionStatus.CHAMPION
|
|
version.save(update_fields=["promotion_status", "updated_at"])
|
|
agent.save(update_fields=["champion_version", "updated_at"])
|
|
self.bus.publish(EventType.AGENT_CREATED, actor="progeny", payload={"agent": agent.name, "version": version.version})
|
|
return version
|
|
|
|
def replay_benchmark(self, champion: AgentVersion, challenger: AgentVersion, benchmark_set: list[dict[str, object]]) -> BenchmarkRun:
|
|
metrics = self._score(challenger, benchmark_set)
|
|
champion_metrics = self._score(champion, benchmark_set)
|
|
decision = "PROMOTE" if metrics["completion_rate"] >= champion_metrics["completion_rate"] and metrics["test_pass_rate"] >= champion_metrics["test_pass_rate"] else "REJECT"
|
|
return BenchmarkRun.objects.create(
|
|
champion=champion,
|
|
challenger=challenger,
|
|
benchmark_set=benchmark_set,
|
|
metrics={"champion": champion_metrics, "challenger": metrics},
|
|
decision=decision,
|
|
)
|
|
|
|
def promote_or_reject(self, run: BenchmarkRun) -> BenchmarkDecision:
|
|
challenger = run.challenger
|
|
agent = challenger.agent
|
|
challenger_metrics = run.metrics["challenger"]
|
|
if run.decision == "PROMOTE":
|
|
if agent.champion_version_id:
|
|
old = agent.champion_version
|
|
old.promotion_status = PromotionStatus.CANDIDATE
|
|
old.save(update_fields=["promotion_status", "updated_at"])
|
|
challenger.promotion_status = PromotionStatus.CHAMPION
|
|
challenger.save(update_fields=["promotion_status", "updated_at"])
|
|
agent.champion_version = challenger
|
|
agent.save(update_fields=["champion_version", "updated_at"])
|
|
self.bus.publish(EventType.AGENT_PROMOTED, actor="progeny", payload={"agent": agent.name, "version": challenger.version})
|
|
return BenchmarkDecision("PROMOTED", challenger_metrics)
|
|
challenger.promotion_status = PromotionStatus.REJECTED
|
|
challenger.save(update_fields=["promotion_status", "updated_at"])
|
|
return BenchmarkDecision("REJECTED", challenger_metrics)
|
|
|
|
def _score(self, version: AgentVersion, benchmark_set: list[dict[str, object]]) -> dict[str, float]:
|
|
if not benchmark_set:
|
|
return {"completion_rate": 0.0, "test_pass_rate": 0.0, "review_acceptance": 0.0, "tokens": 0.0, "runtime": 0.0}
|
|
base = 1.0 if "do not self-certify" in version.system_contract.lower() else 0.8
|
|
return {
|
|
"completion_rate": base,
|
|
"test_pass_rate": base,
|
|
"review_acceptance": base,
|
|
"tokens": float(len(version.system_contract.split())),
|
|
"runtime": float(len(benchmark_set)),
|
|
}
|