From 6a73372665715950aee8416a9d2b3ec79955f816 Mon Sep 17 00:00:00 2001 From: Artifex Date: Sat, 15 Aug 2026 09:14:44 +0000 Subject: [PATCH] Artifex task: Implement only minimal V2 ProgenySignal persistence and service from a fresh wor --- agents/progeny.py | 77 ++++++++- .../agents/migrations/0004_progenysignal.py | 35 ++++ control_plane/agents/models.py | 16 ++ tests/test_v2_progeny_signals.py | 159 ++++++++++++++++++ 4 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 control_plane/agents/migrations/0004_progenysignal.py create mode 100644 tests/test_v2_progeny_signals.py diff --git a/agents/progeny.py b/agents/progeny.py index 15928c7..e5b98c0 100644 --- a/agents/progeny.py +++ b/agents/progeny.py @@ -2,9 +2,10 @@ from __future__ import annotations from dataclasses import dataclass -from control_plane.agents.models import Agent, AgentPlan, AgentVersion, BenchmarkRun, PromotionStatus +from control_plane.agents.models import Agent, AgentPlan, AgentVersion, BenchmarkRun, ProgenySignal, PromotionStatus from control_plane.events.bus import EventBus from control_plane.events.models import EventType +from control_plane.projects.models import Project, Task, Milestone @dataclass(frozen=True) @@ -73,6 +74,80 @@ class ProgenyService: challenger.save(update_fields=["promotion_status", "updated_at"]) return BenchmarkDecision("REJECTED", challenger_metrics) + def create_reviewer_signal(self, project: Project, task: Task, milestone: Milestone, agent_version: AgentVersion, status: str, findings: list[dict[str, object]], summary: str) -> ProgenySignal: + severity = "high" if status in ["REWORK_REQUIRED", "REJECTED"] else "info" + signal = ProgenySignal.objects.create( + project=project, + task=task, + milestone=milestone, + agent_version=agent_version, + source="reviewer", + severity=severity, + failure_category=status, + summary=summary, + evidence={"findings": findings}, + status="OPEN", + grouping_key=f"reviewer:{task.id}:{status}", + model=agent_version.model, + ) + self.bus.publish("PROGENY_SIGNAL_CREATED", project=project, task=task, actor="progeny", payload={"signal_id": str(signal.id), "source": "reviewer", "status": status}) + return signal + + def create_judge_signal(self, project: Project, task: Task, milestone: Milestone, agent_version: AgentVersion, result: str, evidence: list[dict[str, object]], summary: str) -> ProgenySignal: + severity = "high" if result == "FAIL" else "info" + signal = ProgenySignal.objects.create( + project=project, + task=task, + milestone=milestone, + agent_version=agent_version, + source="judge", + severity=severity, + failure_category=result, + summary=summary, + evidence={"evidence": evidence}, + status="OPEN", + grouping_key=f"judge:{task.id}:{result}", + model=agent_version.model, + ) + self.bus.publish("PROGENY_SIGNAL_CREATED", project=project, task=task, actor="progeny", payload={"signal_id": str(signal.id), "source": "judge", "result": result}) + return signal + + def create_model_output_signal(self, project: Project, task: Task, milestone: Milestone, agent_version: AgentVersion, error: str, raw_output: str) -> ProgenySignal: + signal = ProgenySignal.objects.create( + project=project, + task=task, + milestone=milestone, + agent_version=agent_version, + source="model_output", + severity="high", + failure_category="MALFORMED_OUTPUT", + summary=f"Model output malformed: {error}", + evidence={"raw_output": raw_output[:1000]}, + status="OPEN", + grouping_key=f"model_output:{task.id}", + model=agent_version.model, + ) + self.bus.publish("PROGENY_SIGNAL_CREATED", project=project, task=task, actor="progeny", payload={"signal_id": str(signal.id), "source": "model_output"}) + return signal + + def create_retry_exhausted_signal(self, project: Project, task: Task, milestone: Milestone, agent_version: AgentVersion, attempts: int) -> ProgenySignal: + signal = ProgenySignal.objects.create( + project=project, + task=task, + milestone=milestone, + agent_version=agent_version, + source="retry", + severity="critical", + failure_category="RETRY_EXHAUSTED", + summary=f"Task {task.id} exhausted {attempts} retries", + evidence={"attempts": attempts}, + status="OPEN", + grouping_key=f"retry:{task.id}", + model=agent_version.model, + ) + self.bus.publish("PROGENY_SIGNAL_CREATED", project=project, task=task, actor="progeny", payload={"signal_id": str(signal.id), "source": "retry"}) + return signal + def _score(self, version: AgentVersion, benchmark_set: list[dict[str, object]]) -> dict[str, float]: if not benchmark_set: return {"completion_rate": 0.0, "test_pass_rate": 0.0, "review_acceptance": 0.0, "tokens": 0.0, "runtime": 0.0} diff --git a/control_plane/agents/migrations/0004_progenysignal.py b/control_plane/agents/migrations/0004_progenysignal.py new file mode 100644 index 0000000..258d7f4 --- /dev/null +++ b/control_plane/agents/migrations/0004_progenysignal.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import uuid + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("agents", "0003_agentplan"), + ("projects", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="ProgenySignal", + fields=[ + ("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("source", models.CharField(max_length=80)), + ("severity", models.CharField(default="INFO", max_length=32)), + ("failure_category", models.CharField(blank=True, max_length=80)), + ("summary", models.TextField()), + ("evidence", models.JSONField(blank=True, default=dict)), + ("status", models.CharField(default="OPEN", max_length=32)), + ("grouping_key", models.CharField(blank=True, max_length=120)), + ("model", models.CharField(blank=True, max_length=120)), + ("agent_version", models.ForeignKey(blank=True, null=True, on_delete=models.SET_NULL, to="agents.agentversion")), + ("milestone", models.ForeignKey(blank=True, null=True, on_delete=models.SET_NULL, to="projects.milestone")), + ("project", models.ForeignKey(blank=True, null=True, on_delete=models.SET_NULL, to="projects.project")), + ("task", models.ForeignKey(blank=True, null=True, on_delete=models.SET_NULL, to="projects.task")), + ], + ), + ] diff --git a/control_plane/agents/models.py b/control_plane/agents/models.py index 1d9edc5..5f88f5d 100644 --- a/control_plane/agents/models.py +++ b/control_plane/agents/models.py @@ -1,5 +1,6 @@ from __future__ import annotations +import uuid from django.db import models from control_plane.common import TimestampedModel @@ -80,3 +81,18 @@ class BenchmarkRun(TimestampedModel): benchmark_set = models.JSONField(default=list, blank=True) metrics = models.JSONField(default=dict, blank=True) decision = models.CharField(max_length=32, default="PENDING") + + +class ProgenySignal(TimestampedModel): + project = models.ForeignKey("projects.Project", on_delete=models.SET_NULL, null=True, blank=True) + task = models.ForeignKey("projects.Task", on_delete=models.SET_NULL, null=True, blank=True) + milestone = models.ForeignKey("projects.Milestone", on_delete=models.SET_NULL, null=True, blank=True) + agent_version = models.ForeignKey(AgentVersion, on_delete=models.SET_NULL, null=True, blank=True) + source = models.CharField(max_length=80) + severity = models.CharField(max_length=32, default="INFO") + failure_category = models.CharField(max_length=80, blank=True) + summary = models.TextField() + evidence = models.JSONField(default=dict, blank=True) + status = models.CharField(max_length=32, default="OPEN") + grouping_key = models.CharField(max_length=120, blank=True) + model = models.CharField(max_length=120, blank=True) diff --git a/tests/test_v2_progeny_signals.py b/tests/test_v2_progeny_signals.py new file mode 100644 index 0000000..7044472 --- /dev/null +++ b/tests/test_v2_progeny_signals.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import pytest +from django.test import TestCase + +from agents.progeny import ProgenyService +from control_plane.agents.models import Agent, AgentVersion, PromotionStatus +from control_plane.events.models import Event +from control_plane.projects.models import Project, ProjectPlan, Milestone, Task + + +class ProgenySignalTests(TestCase): + def setUp(self): + self.project = Project.objects.create(name="Test Project", goal="Test goal") + self.plan = ProjectPlan.objects.create(project=self.project, version=1, goal="Test plan goal") + self.milestone = Milestone.objects.create(project=self.project, plan=self.plan, key="M1", title="Milestone 1", goal="Milestone goal") + self.task = Task.objects.create(project=self.project, milestone=self.milestone, task_type="CODING", goal="Implement feature") + self.agent = Agent.objects.create(name="Test Agent", role="CODER") + self.agent_version = AgentVersion.objects.create( + agent=self.agent, + version=1, + model="test-model", + system_contract="Test contract", + promotion_status=PromotionStatus.CHAMPION, + ) + self.service = ProgenyService() + + def test_create_reviewer_rework_signal(self): + signal = self.service.create_reviewer_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + status="REWORK_REQUIRED", + findings=[{"type": "tests_failed", "severity": "high", "message": "Tests failed"}], + summary="Reviewer requested rework", + ) + self.assertEqual(signal.source, "reviewer") + self.assertEqual(signal.failure_category, "REWORK_REQUIRED") + self.assertEqual(signal.severity, "high") + self.assertEqual(signal.grouping_key, f"reviewer:{self.task.id}:REWORK_REQUIRED") + + event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED") + self.assertEqual(event.payload["signal_id"], str(signal.id)) + self.assertEqual(event.payload["source"], "reviewer") + + def test_create_reviewer_reject_signal(self): + signal = self.service.create_reviewer_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + status="REJECTED", + findings=[{"type": "empty_diff", "severity": "high", "message": "No diff"}], + summary="Reviewer rejected", + ) + self.assertEqual(signal.source, "reviewer") + self.assertEqual(signal.failure_category, "REJECTED") + self.assertEqual(signal.severity, "high") + + event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED") + self.assertEqual(event.payload["status"], "REJECTED") + + def test_create_judge_fail_signal(self): + signal = self.service.create_judge_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + result="FAIL", + evidence=[{"type": "test_status", "status": "FAIL"}], + summary="Judge failed", + ) + self.assertEqual(signal.source, "judge") + self.assertEqual(signal.failure_category, "FAIL") + self.assertEqual(signal.severity, "high") + + event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED") + self.assertEqual(event.payload["source"], "judge") + self.assertEqual(event.payload["result"], "FAIL") + + def test_create_model_output_signal(self): + signal = self.service.create_model_output_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + error="Invalid JSON", + raw_output="{invalid json}", + ) + self.assertEqual(signal.source, "model_output") + self.assertEqual(signal.failure_category, "MALFORMED_OUTPUT") + self.assertEqual(signal.severity, "high") + self.assertIn("Invalid JSON", signal.summary) + + event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED") + self.assertEqual(event.payload["source"], "model_output") + + def test_create_retry_exhausted_signal(self): + signal = self.service.create_retry_exhausted_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + attempts=3, + ) + self.assertEqual(signal.source, "retry") + self.assertEqual(signal.failure_category, "RETRY_EXHAUSTED") + self.assertEqual(signal.severity, "critical") + self.assertIn("exhausted 3 retries", signal.summary) + + event = Event.objects.get(event_type="PROGENY_SIGNAL_CREATED") + self.assertEqual(event.payload["source"], "retry") + + def test_grouping_key_deterministic(self): + signal1 = self.service.create_reviewer_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + status="REWORK_REQUIRED", + findings=[], + summary="Rework 1", + ) + signal2 = self.service.create_reviewer_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + status="REWORK_REQUIRED", + findings=[], + summary="Rework 2", + ) + self.assertEqual(signal1.grouping_key, signal2.grouping_key) + + def test_event_lineage(self): + self.service.create_reviewer_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + status="REWORK_REQUIRED", + findings=[], + summary="Rework", + ) + self.service.create_judge_signal( + project=self.project, + task=self.task, + milestone=self.milestone, + agent_version=self.agent_version, + result="FAIL", + evidence=[], + summary="Fail", + ) + + events = Event.objects.filter(event_type="PROGENY_SIGNAL_CREATED").order_by("created_at") + self.assertEqual(events.count(), 2) + self.assertEqual(events[0].payload["source"], "reviewer") + self.assertEqual(events[1].payload["source"], "judge")