Artifex/tests/test_m2_autonomous_loop.py

from __future__ import annotations

import subprocess
from pathlib import Path

from agents.providers import DeterministicCodingProvider
from control_plane.agents.management.commands.seed_core_agents import Command as SeedAgentsCommand
from control_plane.events.models import Event, EventType
from control_plane.agents.models import AgentRole, AgentVersion
from control_plane.projects.models import Artifact, CommitRecord, Milestone, Project, ProjectPlan, Task, TaskStatus
from control_plane.verification.models import Review, TestRun as ArtifexTestRun, Verification, VerificationResult
from model_router.router import ModelRouter
from runtime_loop.autonomous_task_loop import AutonomousTaskLoop


def run(command: list[str], cwd: Path) -> None:
    completed = subprocess.run(command, cwd=cwd, capture_output=True, text=True, check=False)
    assert completed.returncode == 0, completed.stderr or completed.stdout


def write(path: Path, content: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(content, encoding="utf-8")


def create_disposable_django_repo(root: Path, *, with_items: bool = False) -> Path:
    repo = root / "subject"
    repo.mkdir()
    write(repo / "pytest.ini", "DJANGO_SETTINGS_MODULE = app.settings\npython_files = test_*.py\n")
    write(repo / "manage.py", "#!/usr/bin/env python\nimport os, sys\nos.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings')\nfrom django.core.management import execute_from_command_line\nexecute_from_command_line(sys.argv)\n")
    write(repo / "app" / "__init__.py", "")
    write(repo / "app" / "settings.py", "SECRET_KEY='test'\nDEBUG=True\nALLOWED_HOSTS=['testserver','localhost']\nROOT_URLCONF='app.urls'\nUSE_TZ=True\nDEFAULT_AUTO_FIELD='django.db.models.BigAutoField'\nDATABASES={'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}\nINSTALLED_APPS=['django.contrib.auth','django.contrib.contenttypes','items']\nMIDDLEWARE=[]\n")
    write(repo / "app" / "urls.py", "from django.urls import path\n\nurlpatterns = []\n")
    write(repo / "items" / "__init__.py", "")
    write(repo / "items" / "apps.py", "from django.apps import AppConfig\n\nclass ItemsConfig(AppConfig):\n    default_auto_field = 'django.db.models.BigAutoField'\n    name = 'items'\n")
    write(repo / "items" / "models.py", "from django.db import models\n\n\nclass Item(models.Model):\n    name = models.CharField(max_length=100)\n\n    def __str__(self):\n        return self.name\n")
    write(repo / "items" / "admin.py", "from django.contrib import admin\n\nfrom items.models import Item\n\nadmin.site.register(Item)\n")
    write(repo / "items" / "migrations" / "__init__.py", "")
    write(repo / "items" / "migrations" / "0001_initial.py", "from django.db import migrations, models\n\n\nclass Migration(migrations.Migration):\n    initial = True\n    dependencies = []\n    operations = [migrations.CreateModel(name='Item', fields=[('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=100))])]\n")
    write(repo / "tests" / "__init__.py", "")
    write(repo / "tests" / "test_baseline.py", "def test_baseline():\n    assert True\n")
    run(["git", "init", "-b", "main"], repo)
    run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "add", "."], repo)
    run(["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-m", "Initial disposable repo"], repo)
    return repo


def create_task(repository_path: Path, goal: str, acceptance: list[str]) -> Task:
    project = Project.objects.create(name=f"Project {goal[:12]}", goal=goal, repository_path=str(repository_path))
    plan = ProjectPlan.objects.create(project=project, version=1, goal=goal)
    milestone = Milestone.objects.create(project=project, plan=plan, key="M2", title="Loop", goal="Autonomous loop")
    return Task.objects.create(
        project=project,
        milestone=milestone,
        task_type="implementation",
        status=TaskStatus.READY,
        goal=goal,
        acceptance_criteria=acceptance,
        max_retries=1,
    )


def loop() -> AutonomousTaskLoop:
    SeedAgentsCommand().handle()
    router = ModelRouter({"qwen": DeterministicCodingProvider()})
    return AutonomousTaskLoop(router)


def assert_accepted_trace(task: Task) -> CommitRecord:
    task.refresh_from_db()
    diagnostics = {
        "task_status": task.status,
        "attempts": list(task.attempts.values("attempt_number", "status", "coder_result", "review_findings", "judge_findings")),
        "tests": list(ArtifexTestRun.objects.filter(task=task).values("status", "output_artifact__content")),
        "reviews": list(Review.objects.filter(task=task).values("status", "findings")),
        "verifications": list(Verification.objects.filter(task=task).values("result", "evidence")),
    }
    assert task.status == TaskStatus.COMPLETE, diagnostics
    assert task.worktree.status == "CLEANED"
    test_run = ArtifexTestRun.objects.get(task=task)
    review = Review.objects.get(task=task)
    verification = Verification.objects.get(task=task)
    commit = CommitRecord.objects.get(task=task)
    assert test_run.status == "PASS"
    assert review.status == "PASS"
    assert verification.result == VerificationResult.PASS
    assert commit.sha
    assert commit.graph_run_id is not None
    assert commit.graph_run.execution_graph_version.graph.name == "task_execution"
    assert commit.coder.agent.role == "CODER"
    assert commit.reviewer.agent.role == "REVIEWER"
    assert commit.judge.agent.role == "PROJECT_JUDGE"
    event_types = set(Event.objects.filter(task=task).values_list("event_type", flat=True))
    assert EventType.TASK_STARTED in event_types
    assert EventType.COMMIT_CREATED in event_types
    assert EventType.TASK_COMPLETED in event_types
    return commit


def test_m2_health_endpoint_end_to_end(tmp_path: Path) -> None:
    repo = create_disposable_django_repo(tmp_path)
    task = create_task(repo, "Add a /health endpoint returning JSON {\"status\": \"ok\"} and add tests.", ["/health returns JSON ok", "tests pass"])

    loop().run_once(test_command=["python", "manage.py", "test"])

    commit = assert_accepted_trace(task)
    show = subprocess.run(["git", "show", "--stat", commit.sha], cwd=repo, capture_output=True, text=True, check=False)
    assert show.returncode == 0


def test_m2_multifile_model_admin_migration_end_to_end(tmp_path: Path) -> None:
    repo = create_disposable_django_repo(tmp_path, with_items=True)
    task = create_task(repo, "Add a description field to a model, create the migration, expose it in Django admin, and add tests.", ["description field exists", "migration exists", "admin exposes field", "tests pass"])

    loop().run_once(test_command=["python", "manage.py", "test"])

    assert_accepted_trace(task)
    assert (task.worktree.worktree_path and Path(task.worktree.worktree_path).exists()) is False


def test_m2_negative_path_rejected_without_commit(tmp_path: Path) -> None:
    repo = create_disposable_django_repo(tmp_path)
    task = create_task(repo, "FORCE_BAD_IMPLEMENTATION Add a /health endpoint returning JSON {\"status\": \"ok\"} and add tests.", ["/health returns JSON ok", "tests pass"])

    loop().run_once(test_command=["python", "manage.py", "test"])

    task.refresh_from_db()
    assert task.status == TaskStatus.FAILED
    assert task.retry_count == 2
    assert CommitRecord.objects.filter(task=task).count() == 0
    assert Review.objects.filter(task=task).exclude(status="PASS").exists()
    assert task.attempts.filter(status="FAILED").exists()
    assert Event.objects.filter(task=task, event_type=EventType.TASK_FAILED).exists()


def test_failed_test_output_is_attached_to_attempt_retry_context(tmp_path: Path) -> None:
    repo = create_disposable_django_repo(tmp_path)
    task = create_task(repo, "Add a feature that fails tests", ["tests pass"])
    SeedAgentsCommand().handle()
    coder = AgentVersion.objects.get(agent__role=AgentRole.CODER, promotion_status="CHAMPION")
    attempt = task.attempts.create(task=task, attempt_number=1, coder=coder, status="RUNNING", coder_result={"metadata": {}})
    artifact = Artifact.objects.create(
        project=task.project,
        task=task,
        artifact_type="test_report",
        name="failed-tests",
        content={"stdout": "short traceback", "stderr": "error details", "returncode": 1},
    )
    test_run = ArtifexTestRun.objects.create(project=task.project, task=task, command="pytest", status="FAIL", output_artifact=artifact)

    loop()._attach_test_failure_evidence(attempt, test_run)

    attempt.refresh_from_db()
    evidence = attempt.coder_result["metadata"]["test_failure_evidence"]
    assert evidence["status"] == "FAIL"
    assert evidence["stdout_excerpt"] == "short traceback"
    assert evidence["stderr_excerpt"] == "error details"