Artifex task: Implement only minimal V2 ProgenySignal persistence and service from a fresh wor
This commit is contained in:
Artifex
parent
80c2a1b46c
commit
6a73372665
4 changed files with 286 additions and 1 deletions
|
|
@ -2,9 +2,10 @@ from __future__ import annotations
|
|||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from control_plane.agents.models import Agent, AgentPlan, AgentVersion, BenchmarkRun, PromotionStatus
|
||||
from control_plane.agents.models import Agent, AgentPlan, AgentVersion, BenchmarkRun, ProgenySignal, PromotionStatus
|
||||
from control_plane.events.bus import EventBus
|
||||
from control_plane.events.models import EventType
|
||||
from control_plane.projects.models import Project, Task, Milestone
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -73,6 +74,80 @@ class ProgenyService:
|
|||
challenger.save(update_fields=["promotion_status", "updated_at"])
|
||||
return BenchmarkDecision("REJECTED", challenger_metrics)
|
||||
|
||||
def create_reviewer_signal(self, project: Project, task: Task, milestone: Milestone, agent_version: AgentVersion, status: str, findings: list[dict[str, object]], summary: str) -> ProgenySignal:
|
||||
severity = "high" if status in ["REWORK_REQUIRED", "REJECTED"] else "info"
|
||||
signal = ProgenySignal.objects.create(
|
||||
project=project,
|
||||
task=task,
|
||||
milestone=milestone,
|
||||
agent_version=agent_version,
|
||||
source="reviewer",
|
||||
severity=severity,
|
||||
failure_category=status,
|
||||
summary=summary,
|
||||
evidence={"findings": findings},
|
||||
status="OPEN",
|
||||
grouping_key=f"reviewer:{task.id}:{status}",
|
||||
model=agent_version.model,
|
||||
)
|
||||
self.bus.publish("PROGENY_SIGNAL_CREATED", project=project, task=task, actor="progeny", payload={"signal_id": str(signal.id), "source": "reviewer", "status": status})
|
||||
return signal
|
||||
|
||||
def create_judge_signal(self, project: Project, task: Task, milestone: Milestone, agent_version: AgentVersion, result: str, evidence: list[dict[str, object]], summary: str) -> ProgenySignal:
|
||||
severity = "high" if result == "FAIL" else "info"
|
||||
signal = ProgenySignal.objects.create(
|
||||
project=project,
|
||||
task=task,
|
||||
milestone=milestone,
|
||||
agent_version=agent_version,
|
||||
source="judge",
|
||||
severity=severity,
|
||||
failure_category=result,
|
||||
summary=summary,
|
||||
evidence={"evidence": evidence},
|
||||
status="OPEN",
|
||||
grouping_key=f"judge:{task.id}:{result}",
|
||||
model=agent_version.model,
|
||||
)
|
||||
self.bus.publish("PROGENY_SIGNAL_CREATED", project=project, task=task, actor="progeny", payload={"signal_id": str(signal.id), "source": "judge", "result": result})
|
||||
return signal
|
||||
|
||||
def create_model_output_signal(self, project: Project, task: Task, milestone: Milestone, agent_version: AgentVersion, error: str, raw_output: str) -> ProgenySignal:
|
||||
signal = ProgenySignal.objects.create(
|
||||
project=project,
|
||||
task=task,
|
||||
milestone=milestone,
|
||||
agent_version=agent_version,
|
||||
source="model_output",
|
||||
severity="high",
|
||||
failure_category="MALFORMED_OUTPUT",
|
||||
summary=f"Model output malformed: {error}",
|
||||
evidence={"raw_output": raw_output[:1000]},
|
||||
status="OPEN",
|
||||
grouping_key=f"model_output:{task.id}",
|
||||
model=agent_version.model,
|
||||
)
|
||||
self.bus.publish("PROGENY_SIGNAL_CREATED", project=project, task=task, actor="progeny", payload={"signal_id": str(signal.id), "source": "model_output"})
|
||||
return signal
|
||||
|
||||
def create_retry_exhausted_signal(self, project: Project, task: Task, milestone: Milestone, agent_version: AgentVersion, attempts: int) -> ProgenySignal:
|
||||
signal = ProgenySignal.objects.create(
|
||||
project=project,
|
||||
task=task,
|
||||
milestone=milestone,
|
||||
agent_version=agent_version,
|
||||
source="retry",
|
||||
severity="critical",
|
||||
failure_category="RETRY_EXHAUSTED",
|
||||
summary=f"Task {task.id} exhausted {attempts} retries",
|
||||
evidence={"attempts": attempts},
|
||||
status="OPEN",
|
||||
grouping_key=f"retry:{task.id}",
|
||||
model=agent_version.model,
|
||||
)
|
||||
self.bus.publish("PROGENY_SIGNAL_CREATED", project=project, task=task, actor="progeny", payload={"signal_id": str(signal.id), "source": "retry"})
|
||||
return signal
|
||||
|
||||
def _score(self, version: AgentVersion, benchmark_set: list[dict[str, object]]) -> dict[str, float]:
|
||||
if not benchmark_set:
|
||||
return {"completion_rate": 0.0, "test_pass_rate": 0.0, "review_acceptance": 0.0, "tokens": 0.0, "runtime": 0.0}
|
||||
|
|
|
|||
35
control_plane/agents/migrations/0004_progenysignal.py
Normal file
35
control_plane/agents/migrations/0004_progenysignal.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("agents", "0003_agentplan"),
|
||||
("projects", "0001_initial"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="ProgenySignal",
|
||||
fields=[
|
||||
("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("source", models.CharField(max_length=80)),
|
||||
("severity", models.CharField(default="INFO", max_length=32)),
|
||||
("failure_category", models.CharField(blank=True, max_length=80)),
|
||||
("summary", models.TextField()),
|
||||
("evidence", models.JSONField(blank=True, default=dict)),
|
||||
("status", models.CharField(default="OPEN", max_length=32)),
|
||||
("grouping_key", models.CharField(blank=True, max_length=120)),
|
||||
("model", models.CharField(blank=True, max_length=120)),
|
||||
("agent_version", models.ForeignKey(blank=True, null=True, on_delete=models.SET_NULL, to="agents.agentversion")),
|
||||
("milestone", models.ForeignKey(blank=True, null=True, on_delete=models.SET_NULL, to="projects.milestone")),
|
||||
("project", models.ForeignKey(blank=True, null=True, on_delete=models.SET_NULL, to="projects.project")),
|
||||
("task", models.ForeignKey(blank=True, null=True, on_delete=models.SET_NULL, to="projects.task")),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from django.db import models
|
||||
|
||||
from control_plane.common import TimestampedModel
|
||||
|
|
@ -80,3 +81,18 @@ class BenchmarkRun(TimestampedModel):
|
|||
benchmark_set = models.JSONField(default=list, blank=True)
|
||||
metrics = models.JSONField(default=dict, blank=True)
|
||||
decision = models.CharField(max_length=32, default="PENDING")
|
||||
|
||||
|
||||
class ProgenySignal(TimestampedModel):
|
||||
project = models.ForeignKey("projects.Project", on_delete=models.SET_NULL, null=True, blank=True)
|
||||
task = models.ForeignKey("projects.Task", on_delete=models.SET_NULL, null=True, blank=True)
|
||||
milestone = models.ForeignKey("projects.Milestone", on_delete=models.SET_NULL, null=True, blank=True)
|
||||
agent_version = models.ForeignKey(AgentVersion, on_delete=models.SET_NULL, null=True, blank=True)
|
||||
source = models.CharField(max_length=80)
|
||||
severity = models.CharField(max_length=32, default="INFO")
|
||||
failure_category = models.CharField(max_length=80, blank=True)
|
||||
summary = models.TextField()
|
||||
evidence = models.JSONField(default=dict, blank=True)
|
||||
status = models.CharField(max_length=32, default="OPEN")
|
||||
grouping_key = models.CharField(max_length=120, blank=True)
|
||||
model = models.CharField(max_length=120, blank=True)
|
||||
|
|
|
|||
159
tests/test_v2_progeny_signals.py
Normal file
159
tests/test_v2_progeny_signals.py
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from django.test import TestCase
|
||||
|
||||
from agents.progeny import ProgenyService
|
||||
from control_plane.agents.models import Agent, AgentVersion, PromotionStatus
|
||||
from control_plane.events.models import Event
|
||||
from control_plane.projects.models import Project, ProjectPlan, Milestone, Task
|
||||
|
||||
|
||||
class ProgenySignalTests(TestCase):
|
||||
def setUp(self):
|
||||
self.project = Project.objects.create(name="Test Project", goal="Test goal")
|
||||
self.plan = ProjectPlan.objects.create(project=self.project, version=1, goal="Test plan goal")
|
||||
self.milestone = Milestone.objects.create(project=self.project, plan=self.plan, key="M1", title="Milestone 1", goal="Milestone goal")
|
||||
self.task = Task.objects.create(project=self.project, milestone=self.milestone, task_type="CODING", goal="Implement feature")
|
||||
self.agent = Agent.objects.create(name="Test Agent", role="CODER")
|
||||
self.agent_version = AgentVersion.objects.create(
|
||||
agent=self.agent,
|
||||
version=1,
|
||||
model="test-model",
|
||||
system_contract="Test contract",
|
||||
promotion_status=PromotionStatus.CHAMPION,
|
||||
)
|
||||
self.service = ProgenyService()
|
||||
|
||||
def test_create_reviewer_rework_signal(self):
|
||||
signal = self.service.create_reviewer_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
status="REWORK_REQUIRED",
|
||||
findings=[{"type": "tests_failed", "severity": "high", "message": "Tests failed"}],
|
||||
summary="Reviewer requested rework",
|
||||
)
|
||||
self.assertEqual(signal.source, "reviewer")
|
||||
self.assertEqual(signal.failure_category, "REWORK_REQUIRED")
|
||||
self.assertEqual(signal.severity, "high")
|
||||
self.assertEqual(signal.grouping_key, f"reviewer:{self.task.id}:REWORK_REQUIRED")
|
||||
|
||||
event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED")
|
||||
self.assertEqual(event.payload["signal_id"], str(signal.id))
|
||||
self.assertEqual(event.payload["source"], "reviewer")
|
||||
|
||||
def test_create_reviewer_reject_signal(self):
|
||||
signal = self.service.create_reviewer_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
status="REJECTED",
|
||||
findings=[{"type": "empty_diff", "severity": "high", "message": "No diff"}],
|
||||
summary="Reviewer rejected",
|
||||
)
|
||||
self.assertEqual(signal.source, "reviewer")
|
||||
self.assertEqual(signal.failure_category, "REJECTED")
|
||||
self.assertEqual(signal.severity, "high")
|
||||
|
||||
event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED")
|
||||
self.assertEqual(event.payload["status"], "REJECTED")
|
||||
|
||||
def test_create_judge_fail_signal(self):
|
||||
signal = self.service.create_judge_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
result="FAIL",
|
||||
evidence=[{"type": "test_status", "status": "FAIL"}],
|
||||
summary="Judge failed",
|
||||
)
|
||||
self.assertEqual(signal.source, "judge")
|
||||
self.assertEqual(signal.failure_category, "FAIL")
|
||||
self.assertEqual(signal.severity, "high")
|
||||
|
||||
event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED")
|
||||
self.assertEqual(event.payload["source"], "judge")
|
||||
self.assertEqual(event.payload["result"], "FAIL")
|
||||
|
||||
def test_create_model_output_signal(self):
|
||||
signal = self.service.create_model_output_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
error="Invalid JSON",
|
||||
raw_output="{invalid json}",
|
||||
)
|
||||
self.assertEqual(signal.source, "model_output")
|
||||
self.assertEqual(signal.failure_category, "MALFORMED_OUTPUT")
|
||||
self.assertEqual(signal.severity, "high")
|
||||
self.assertIn("Invalid JSON", signal.summary)
|
||||
|
||||
event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED")
|
||||
self.assertEqual(event.payload["source"], "model_output")
|
||||
|
||||
def test_create_retry_exhausted_signal(self):
|
||||
signal = self.service.create_retry_exhausted_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
attempts=3,
|
||||
)
|
||||
self.assertEqual(signal.source, "retry")
|
||||
self.assertEqual(signal.failure_category, "RETRY_EXHAUSTED")
|
||||
self.assertEqual(signal.severity, "critical")
|
||||
self.assertIn("exhausted 3 retries", signal.summary)
|
||||
|
||||
event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED")
|
||||
self.assertEqual(event.payload["source"], "retry")
|
||||
|
||||
def test_grouping_key_deterministic(self):
|
||||
signal1 = self.service.create_reviewer_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
status="REWORK_REQUIRED",
|
||||
findings=[],
|
||||
summary="Rework 1",
|
||||
)
|
||||
signal2 = self.service.create_reviewer_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
status="REWORK_REQUIRED",
|
||||
findings=[],
|
||||
summary="Rework 2",
|
||||
)
|
||||
self.assertEqual(signal1.grouping_key, signal2.grouping_key)
|
||||
|
||||
def test_event_lineage(self):
|
||||
self.service.create_reviewer_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
status="REWORK_REQUIRED",
|
||||
findings=[],
|
||||
summary="Rework",
|
||||
)
|
||||
self.service.create_judge_signal(
|
||||
project=self.project,
|
||||
task=self.task,
|
||||
milestone=self.milestone,
|
||||
agent_version=self.agent_version,
|
||||
result="FAIL",
|
||||
evidence=[],
|
||||
summary="Fail",
|
||||
)
|
||||
|
||||
events = Event.objects.filter(event_type="PROGENY_SIGNAL_CREATED").order_by("created_at")
|
||||
self.assertEqual(events.count(), 2)
|
||||
self.assertEqual(events[0].payload["source"], "reviewer")
|
||||
self.assertEqual(events[1].payload["source"], "judge")
|
||||
Loading…
Add table
Reference in a new issue