157 lines
8.7 KiB
Python
157 lines
8.7 KiB
Python
from __future__ import annotations
|
|
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
from agents.providers import DeterministicCodingProvider
|
|
from control_plane.agents.management.commands.seed_core_agents import Command as SeedAgentsCommand
|
|
from control_plane.events.models import Event, EventType
|
|
from control_plane.agents.models import AgentRole, AgentVersion
|
|
from control_plane.projects.models import Artifact, CommitRecord, Milestone, Project, ProjectPlan, Task, TaskStatus
|
|
from control_plane.verification.models import Review, TestRun as ArtifexTestRun, Verification, VerificationResult
|
|
from model_router.router import ModelRouter
|
|
from runtime_loop.autonomous_task_loop import AutonomousTaskLoop
|
|
|
|
|
|
def run(command: list[str], cwd: Path) -> None:
|
|
completed = subprocess.run(command, cwd=cwd, capture_output=True, text=True, check=False)
|
|
assert completed.returncode == 0, completed.stderr or completed.stdout
|
|
|
|
|
|
def write(path: Path, content: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(content, encoding="utf-8")
|
|
|
|
|
|
def create_disposable_django_repo(root: Path, *, with_items: bool = False) -> Path:
|
|
repo = root / "subject"
|
|
repo.mkdir()
|
|
write(repo / "pytest.ini", "DJANGO_SETTINGS_MODULE = app.settings\npython_files = test_*.py\n")
|
|
write(repo / "manage.py", "#!/usr/bin/env python\nimport os, sys\nos.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings')\nfrom django.core.management import execute_from_command_line\nexecute_from_command_line(sys.argv)\n")
|
|
write(repo / "app" / "__init__.py", "")
|
|
write(repo / "app" / "settings.py", "SECRET_KEY='test'\nDEBUG=True\nALLOWED_HOSTS=['testserver','localhost']\nROOT_URLCONF='app.urls'\nUSE_TZ=True\nDEFAULT_AUTO_FIELD='django.db.models.BigAutoField'\nDATABASES={'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}\nINSTALLED_APPS=['django.contrib.auth','django.contrib.contenttypes','items']\nMIDDLEWARE=[]\n")
|
|
write(repo / "app" / "urls.py", "from django.urls import path\n\nurlpatterns = []\n")
|
|
write(repo / "items" / "__init__.py", "")
|
|
write(repo / "items" / "apps.py", "from django.apps import AppConfig\n\nclass ItemsConfig(AppConfig):\n default_auto_field = 'django.db.models.BigAutoField'\n name = 'items'\n")
|
|
write(repo / "items" / "models.py", "from django.db import models\n\n\nclass Item(models.Model):\n name = models.CharField(max_length=100)\n\n def __str__(self):\n return self.name\n")
|
|
write(repo / "items" / "admin.py", "from django.contrib import admin\n\nfrom items.models import Item\n\nadmin.site.register(Item)\n")
|
|
write(repo / "items" / "migrations" / "__init__.py", "")
|
|
write(repo / "items" / "migrations" / "0001_initial.py", "from django.db import migrations, models\n\n\nclass Migration(migrations.Migration):\n initial = True\n dependencies = []\n operations = [migrations.CreateModel(name='Item', fields=[('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=100))])]\n")
|
|
write(repo / "tests" / "__init__.py", "")
|
|
write(repo / "tests" / "test_baseline.py", "def test_baseline():\n assert True\n")
|
|
run(["git", "init", "-b", "main"], repo)
|
|
run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "add", "."], repo)
|
|
run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-m", "Initial disposable repo"], repo)
|
|
return repo
|
|
|
|
|
|
def create_task(repository_path: Path, goal: str, acceptance: list[str]) -> Task:
|
|
project = Project.objects.create(name=f"Project {goal[:12]}", goal=goal, repository_path=str(repository_path))
|
|
plan = ProjectPlan.objects.create(project=project, version=1, goal=goal)
|
|
milestone = Milestone.objects.create(project=project, plan=plan, key="M2", title="Loop", goal="Autonomous loop")
|
|
return Task.objects.create(
|
|
project=project,
|
|
milestone=milestone,
|
|
task_type="implementation",
|
|
status=TaskStatus.READY,
|
|
goal=goal,
|
|
acceptance_criteria=acceptance,
|
|
max_retries=1,
|
|
)
|
|
|
|
|
|
def loop() -> AutonomousTaskLoop:
|
|
SeedAgentsCommand().handle()
|
|
router = ModelRouter({"qwen": DeterministicCodingProvider()})
|
|
return AutonomousTaskLoop(router)
|
|
|
|
|
|
def assert_accepted_trace(task: Task) -> CommitRecord:
|
|
task.refresh_from_db()
|
|
diagnostics = {
|
|
"task_status": task.status,
|
|
"attempts": list(task.attempts.values("attempt_number", "status", "coder_result", "review_findings", "judge_findings")),
|
|
"tests": list(ArtifexTestRun.objects.filter(task=task).values("status", "output_artifact__content")),
|
|
"reviews": list(Review.objects.filter(task=task).values("status", "findings")),
|
|
"verifications": list(Verification.objects.filter(task=task).values("result", "evidence")),
|
|
}
|
|
assert task.status == TaskStatus.COMPLETE, diagnostics
|
|
assert task.worktree.status == "CLEANED"
|
|
test_run = ArtifexTestRun.objects.get(task=task)
|
|
review = Review.objects.get(task=task)
|
|
verification = Verification.objects.get(task=task)
|
|
commit = CommitRecord.objects.get(task=task)
|
|
assert test_run.status == "PASS"
|
|
assert review.status == "PASS"
|
|
assert verification.result == VerificationResult.PASS
|
|
assert commit.sha
|
|
assert commit.graph_run_id is not None
|
|
assert commit.graph_run.execution_graph_version.graph.name == "task_execution"
|
|
assert commit.coder.agent.role == "CODER"
|
|
assert commit.reviewer.agent.role == "REVIEWER"
|
|
assert commit.judge.agent.role == "PROJECT_JUDGE"
|
|
event_types = set(Event.objects.filter(task=task).values_list("event_type", flat=True))
|
|
assert EventType.TASK_STARTED in event_types
|
|
assert EventType.COMMIT_CREATED in event_types
|
|
assert EventType.TASK_COMPLETED in event_types
|
|
return commit
|
|
|
|
|
|
def test_m2_health_endpoint_end_to_end(tmp_path: Path) -> None:
|
|
repo = create_disposable_django_repo(tmp_path)
|
|
task = create_task(repo, "Add a /health endpoint returning JSON {\"status\": \"ok\"} and add tests.", ["/health returns JSON ok", "tests pass"])
|
|
|
|
loop().run_once(test_command=["python", "manage.py", "test"])
|
|
|
|
commit = assert_accepted_trace(task)
|
|
show = subprocess.run(["git", "show", "--stat", commit.sha], cwd=repo, capture_output=True, text=True, check=False)
|
|
assert show.returncode == 0
|
|
|
|
|
|
def test_m2_multifile_model_admin_migration_end_to_end(tmp_path: Path) -> None:
|
|
repo = create_disposable_django_repo(tmp_path, with_items=True)
|
|
task = create_task(repo, "Add a description field to a model, create the migration, expose it in Django admin, and add tests.", ["description field exists", "migration exists", "admin exposes field", "tests pass"])
|
|
|
|
loop().run_once(test_command=["python", "manage.py", "test"])
|
|
|
|
assert_accepted_trace(task)
|
|
assert (task.worktree.worktree_path and Path(task.worktree.worktree_path).exists()) is False
|
|
|
|
|
|
def test_m2_negative_path_rejected_without_commit(tmp_path: Path) -> None:
|
|
repo = create_disposable_django_repo(tmp_path)
|
|
task = create_task(repo, "FORCE_BAD_IMPLEMENTATION Add a /health endpoint returning JSON {\"status\": \"ok\"} and add tests.", ["/health returns JSON ok", "tests pass"])
|
|
|
|
loop().run_once(test_command=["python", "manage.py", "test"])
|
|
|
|
task.refresh_from_db()
|
|
assert task.status == TaskStatus.FAILED
|
|
assert task.retry_count == 2
|
|
assert CommitRecord.objects.filter(task=task).count() == 0
|
|
assert Review.objects.filter(task=task).exclude(status="PASS").exists()
|
|
assert task.attempts.filter(status="FAILED").exists()
|
|
assert Event.objects.filter(task=task, event_type=EventType.TASK_FAILED).exists()
|
|
|
|
|
|
def test_failed_test_output_is_attached_to_attempt_retry_context(tmp_path: Path) -> None:
|
|
repo = create_disposable_django_repo(tmp_path)
|
|
task = create_task(repo, "Add a feature that fails tests", ["tests pass"])
|
|
SeedAgentsCommand().handle()
|
|
coder = AgentVersion.objects.get(agent__role=AgentRole.CODER, promotion_status="CHAMPION")
|
|
attempt = task.attempts.create(task=task, attempt_number=1, coder=coder, status="RUNNING", coder_result={"metadata": {}})
|
|
artifact = Artifact.objects.create(
|
|
project=task.project,
|
|
task=task,
|
|
artifact_type="test_report",
|
|
name="failed-tests",
|
|
content={"stdout": "short traceback", "stderr": "error details", "returncode": 1},
|
|
)
|
|
test_run = ArtifexTestRun.objects.create(project=task.project, task=task, command="pytest", status="FAIL", output_artifact=artifact)
|
|
|
|
loop()._attach_test_failure_evidence(attempt, test_run)
|
|
|
|
attempt.refresh_from_db()
|
|
evidence = attempt.coder_result["metadata"]["test_failure_evidence"]
|
|
assert evidence["status"] == "FAIL"
|
|
assert evidence["stdout_excerpt"] == "short traceback"
|
|
assert evidence["stderr_excerpt"] == "error details"
|