239 lines
13 KiB
Python
239 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from django.core.exceptions import ValidationError
|
|
|
|
from agents.lifecycle import EvolutionService, ExplorerService, ExtensionService, LifecycleInspectionService
|
|
from agents.providers import DeterministicCodingProvider, DeterministicSolProvider
|
|
from agents.steward import StewardService
|
|
from control_plane.agents.management.commands.seed_core_agents import Command as SeedAgentsCommand
|
|
from control_plane.projects.models import ExtensionCandidate, ExplorationOpportunity, EvolutionCandidate, Project, StewardFinding, StewardPolicy, Task, TaskStatus
|
|
from graph.bootstrap import champion_project_evolution_graph_v1, champion_project_exploration_graph_v1, champion_project_extension_graph_v1
|
|
from graph.langgraph_runtime import LangGraphRuntime
|
|
from graph.lifecycle import evolution_registry, exploration_registry, extension_registry, project_evolution_graph_v1, project_exploration_graph_v1, project_extension_graph_v1
|
|
from graph.models import GraphApprovalStatus, GraphRun, GraphRunStatus
|
|
from model_router.router import ModelRouter
|
|
from tests.test_m2_autonomous_loop import create_disposable_django_repo
|
|
|
|
|
|
def sol_router(payload: dict[str, object]) -> ModelRouter:
|
|
return ModelRouter({"sol": DeterministicSolProvider(json.dumps(payload))}, persist_requests=True)
|
|
|
|
|
|
def qwen_router() -> ModelRouter:
|
|
SeedAgentsCommand().handle()
|
|
return ModelRouter({"qwen": DeterministicCodingProvider()}, persist_requests=True)
|
|
|
|
|
|
def extension_payload() -> dict[str, object]:
|
|
return {
|
|
"extension_plan": {
|
|
"strategy": "Add a bounded health endpoint to the existing Django app.",
|
|
"acceptance_criteria": ["/health returns ok", "tests pass"],
|
|
"project_plan": {
|
|
"goal": "Keep existing project intent while adding health endpoint.",
|
|
"scope": "Existing-project extension only.",
|
|
"acceptance_criteria": ["/health returns ok", "tests pass"],
|
|
"milestones": [
|
|
{
|
|
"key": "HEALTH",
|
|
"title": "Health Endpoint",
|
|
"goal": "Add health endpoint",
|
|
"verification_contract": {"type": "extension"},
|
|
"features": [
|
|
{
|
|
"key": "F1",
|
|
"title": "Health endpoint",
|
|
"description": "Expose /health.",
|
|
"acceptance_criteria": ["/health returns ok"],
|
|
"tasks": [{"id": "T1", "goal": 'Add a /health endpoint returning JSON {"status": "ok"} and add tests.', "type": "implementation", "acceptance_criteria": ["/health returns ok", "tests pass"], "priority": 50, "dependencies": []}],
|
|
}
|
|
],
|
|
}
|
|
],
|
|
},
|
|
}
|
|
}
|
|
|
|
|
|
def evolution_payload(candidate: EvolutionCandidate, *, candidate_value: int = 95) -> dict[str, object]:
|
|
return {
|
|
"evolution_plan": {
|
|
"baseline": candidate.baseline_measurement,
|
|
"hypothesis": "Health endpoint implementation will keep tests passing while we compare latency.",
|
|
"intervention": "Add small health endpoint fixture change.",
|
|
"measurement_method": {"type": "metadata"},
|
|
"success_threshold": {"metric": "duration", "minimum_improvement_percent": 10},
|
|
"regression_constraints": ["Deterministic tests pass"],
|
|
"affected_components": [candidate.target],
|
|
"candidate_measurement": {"metric": "duration", "duration": candidate_value},
|
|
"project_plan": extension_payload()["extension_plan"]["project_plan"],
|
|
}
|
|
}
|
|
|
|
|
|
def project(tmp_path: Path) -> Project:
|
|
return Project.objects.create(name="Lifecycle", goal="Existing product", repository_path=str(create_disposable_django_repo(tmp_path)))
|
|
|
|
|
|
def approve_and_resume(graph_run: GraphRun, registry) -> GraphRun:
|
|
approval = graph_run.approvals.get()
|
|
approval.status = GraphApprovalStatus.APPROVED
|
|
approval.decided_by = "tester"
|
|
approval.save(update_fields=["status", "decided_by", "updated_at"])
|
|
graph_run.status = GraphRunStatus.RUNNING
|
|
graph_run.save(update_fields=["status", "updated_at"])
|
|
LangGraphRuntime(registry).run_until_terminal_or_paused(graph_run)
|
|
graph_run.refresh_from_db()
|
|
return graph_run
|
|
|
|
|
|
def test_extension_candidate_plan_graph_execution_verification_and_lineage(tmp_path: Path) -> None:
|
|
proj = project(tmp_path)
|
|
service = ExtensionService(sol_router(extension_payload()))
|
|
candidate = service.create_candidate(proj, title="Add health endpoint", description="Add /health endpoint", source="user", expected_value="Operational health checks", affected_areas=["api"], confidence=0.8)
|
|
version = champion_project_extension_graph_v1()
|
|
graph_run = GraphRun.objects.create(execution_graph_version=version, project=proj, current_node=version.graph_spec["entry"], metadata={"extension_candidate_id": str(candidate.id)})
|
|
|
|
registry = extension_registry(service, qwen_router(), test_command=["python", "manage.py", "test"])
|
|
LangGraphRuntime(registry).run_until_terminal_or_paused(graph_run)
|
|
assert graph_run.status == GraphRunStatus.PAUSED
|
|
|
|
approve_and_resume(graph_run, registry)
|
|
|
|
candidate.refresh_from_db()
|
|
plan = candidate.plans.get()
|
|
task = Task.objects.get(milestone__plan=plan.project_plan)
|
|
commit = task.commits.get()
|
|
assert graph_run.status == GraphRunStatus.COMPLETE
|
|
assert candidate.status == "COMPLETE"
|
|
assert plan.status == "COMPLETE"
|
|
assert task.status == TaskStatus.COMPLETE
|
|
assert commit.extension_candidate == candidate
|
|
assert plan.verification.result == "PASS"
|
|
|
|
|
|
def test_extension_verification_requires_acceptance_not_just_commits(tmp_path: Path) -> None:
|
|
proj = project(tmp_path)
|
|
service = ExtensionService()
|
|
candidate = service.create_candidate(proj, title="Empty extension", description="No acceptance", source="user")
|
|
plan = service.plan_with_project_brain(candidate)
|
|
plan.acceptance_criteria = []
|
|
plan.save(update_fields=["acceptance_criteria", "updated_at"])
|
|
service.approve_plan(plan)
|
|
service.materialize_project_dag(plan)
|
|
|
|
verification = service.verify_extension(plan)
|
|
|
|
assert verification.result == "FAIL"
|
|
assert candidate.status != "COMPLETE"
|
|
|
|
|
|
def test_evolution_requires_baseline() -> None:
|
|
proj = Project.objects.create(name="No Baseline", goal="Improve existing system")
|
|
with pytest.raises(ValidationError):
|
|
EvolutionService().create_candidate(proj, target="api", objective="reduce latency", baseline_measurement={}, desired_direction="DECREASE")
|
|
|
|
|
|
def test_evolution_graph_can_reject_technical_pass_as_not_improved(tmp_path: Path) -> None:
|
|
proj = project(tmp_path)
|
|
service = EvolutionService()
|
|
candidate = service.create_candidate(proj, target="api latency", objective="reduce duration", baseline_measurement={"metric": "duration", "duration": 100}, desired_direction="DECREASE", target_measurement={"metric": "duration", "duration": 95}, confidence=0.8)
|
|
service = EvolutionService(sol_router(evolution_payload(candidate, candidate_value=95)))
|
|
version = champion_project_evolution_graph_v1()
|
|
graph_run = GraphRun.objects.create(execution_graph_version=version, project=proj, current_node=version.graph_spec["entry"], metadata={"evolution_candidate_id": str(candidate.id)})
|
|
registry = evolution_registry(service, qwen_router(), test_command=["python", "manage.py", "test"])
|
|
|
|
LangGraphRuntime(registry).run_until_terminal_or_paused(graph_run)
|
|
assert graph_run.status == GraphRunStatus.PAUSED
|
|
approve_and_resume(graph_run, registry)
|
|
|
|
plan = candidate.plans.get()
|
|
assert graph_run.status == GraphRunStatus.COMPLETE
|
|
assert plan.verdict == "NOT_IMPROVED"
|
|
assert plan.status == "NOT_IMPROVED"
|
|
assert plan.verification.result == "FAIL"
|
|
assert Task.objects.filter(milestone__plan=plan.project_plan, status=TaskStatus.COMPLETE).exists()
|
|
|
|
|
|
def test_evolution_service_accepts_measurable_success(tmp_path: Path) -> None:
|
|
proj = project(tmp_path)
|
|
candidate = EvolutionService().create_candidate(proj, target="analysis", objective="reduce duration", baseline_measurement={"metric": "duration", "duration": 100}, desired_direction="DECREASE", target_measurement={"metric": "duration", "duration": 80})
|
|
service = EvolutionService(sol_router(evolution_payload(candidate, candidate_value=80)))
|
|
plan = service.plan_with_project_brain(candidate)
|
|
plan.candidate_measurement = {"metric": "duration", "duration": 80}
|
|
plan.save(update_fields=["candidate_measurement", "updated_at"])
|
|
|
|
verification = service.judge_evolution(plan)
|
|
|
|
assert verification.result == "PASS"
|
|
assert plan.verdict == "PASS"
|
|
|
|
|
|
def test_exploration_graph_persists_scores_dedupes_and_does_not_execute(tmp_path: Path) -> None:
|
|
proj = project(tmp_path)
|
|
payload = {
|
|
"opportunities": [
|
|
{"title": "Add project dashboard", "description": "Show lifecycle status", "opportunity_type": "FEATURE", "evidence": {"files": ["app/urls.py"]}, "rationale": "visibility", "expected_value": "operators see status", "effort_estimate": "MEDIUM", "value": 0.8, "effort": 0.4, "risk_score": 0.2, "confidence": 0.7, "technical_fit": 0.8, "strategic_fit": 0.9, "recommended_action": "EXTEND"},
|
|
{"title": "Reduce analysis latency", "description": "Measure and reduce repeated reads", "opportunity_type": "PERFORMANCE", "evidence": ["duration metric"], "rationale": "speed", "expected_value": "faster runs", "effort_estimate": "LOW", "value": "faster repeated reads", "effort": 0.2, "risk_score": 0.2, "confidence": 0.6, "technical_fit": 0.7, "strategic_fit": 0.7, "recommended_action": "EVOLVE"},
|
|
]
|
|
}
|
|
service = ExplorerService(sol_router(payload))
|
|
exploration = service.start_exploration(proj)
|
|
version = champion_project_exploration_graph_v1()
|
|
graph_run = GraphRun.objects.create(execution_graph_version=version, project=proj, current_node=version.graph_spec["entry"], metadata={"exploration_id": str(exploration.id)})
|
|
|
|
LangGraphRuntime(exploration_registry(service)).run_until_terminal_or_paused(graph_run)
|
|
LangGraphRuntime(exploration_registry(service)).run_until_terminal_or_paused(GraphRun.objects.create(execution_graph_version=version, project=proj, current_node=version.graph_spec["entry"], metadata={"exploration_id": str(service.start_exploration(proj).id)}))
|
|
|
|
assert graph_run.status == GraphRunStatus.COMPLETE
|
|
assert ExplorationOpportunity.objects.filter(project=proj).count() == 2
|
|
assert Task.objects.filter(project=proj).count() == 0
|
|
assert all(item.evidence for item in ExplorationOpportunity.objects.filter(project=proj))
|
|
first = ExplorationOpportunity.objects.filter(project=proj, recommended_action="EXTEND").get()
|
|
second = ExplorationOpportunity.objects.filter(project=proj, recommended_action="EVOLVE").get()
|
|
assert second.evidence == {"raw": ["duration metric"]}
|
|
assert second.value_score == 0.5
|
|
extension = service.convert_to_extension(first)
|
|
service.defer(second)
|
|
assert extension.source_opportunity == first
|
|
assert first.status == "CONVERTED"
|
|
assert second.status == "DEFERRED"
|
|
assert first.composite_score > 0
|
|
|
|
|
|
def test_steward_routes_extend_and_evolve_to_project_lifecycle_candidates(tmp_path: Path) -> None:
|
|
proj = project(tmp_path)
|
|
policy = StewardPolicy.objects.create(name="Lifecycle Steward", enabled_checks=[], approval_requirements={"EXTEND": "CRITICAL", "EVOLVE": "CRITICAL"})
|
|
service = StewardService()
|
|
service.enroll_project(proj, policy)
|
|
steward_run = service.start_run(proj.steward_enrollments.get())
|
|
extend = StewardFinding.objects.create(project=proj, steward_run=steward_run, finding_type="PRODUCT_OPPORTUNITY", title="Add dashboard", summary="Dashboard missing", severity="MEDIUM", confidence=0.8, recommended_action="EXTEND", grouping_key="extend")
|
|
evolve = StewardFinding.objects.create(project=proj, steward_run=steward_run, finding_type="PERFORMANCE_REGRESSION", title="Latency high", summary="Reduce p95 latency", severity="MEDIUM", confidence=0.8, recommended_action="EVOLVE", evidence={"baseline_measurement": {"metric": "latency", "latency": 100}}, grouping_key="evolve")
|
|
|
|
extend_action = service.route_finding(extend, policy)
|
|
evolve_action = service.route_finding(evolve, policy)
|
|
|
|
assert extend_action.extension_candidate is not None
|
|
assert evolve_action.evolution_candidate is not None
|
|
assert ExtensionCandidate.objects.filter(source_steward_finding=extend).exists()
|
|
assert EvolutionCandidate.objects.filter(source_steward_finding=evolve).exists()
|
|
|
|
|
|
def test_lifecycle_inspection_exposes_machine_readable_view(tmp_path: Path) -> None:
|
|
proj = project(tmp_path)
|
|
ExplorerService().generate_opportunities(ExplorerService().start_exploration(proj))
|
|
|
|
view = LifecycleInspectionService().project_lifecycle_view(proj)
|
|
|
|
assert set(view) == {"project_id", "repairs", "extensions", "evolutions", "explorations"}
|
|
assert view["explorations"][0]["opportunities"]
|
|
|
|
|
|
def test_graph_specs_are_serializable_and_distinct() -> None:
|
|
assert project_extension_graph_v1().name == "project_extension"
|
|
assert project_evolution_graph_v1().name == "project_evolution"
|
|
assert project_exploration_graph_v1().name == "project_exploration"
|