91 lines
4.7 KiB
Python
91 lines
4.7 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
from django.core.management import call_command
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
|
|
from agents.replay_arena import ReplayArena
|
|
from control_plane.agents.management.commands.seed_core_agents import Command as SeedAgentsCommand
|
|
from control_plane.agents.models import ImprovementCandidate
|
|
from control_plane.projects.models import Milestone, Project, ProjectPlan, Task, TaskStatus
|
|
from control_plane.resources.models import Resource
|
|
from graph.bootstrap import champion_task_execution_graph_v1
|
|
from model_router.providers import QwenProvider
|
|
from model_router.router import ModelRouter
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Opt-in smoke test: run a tiny real-Qwen Replay Arena champion/challenger experiment."
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument("--cases", type=int, default=2)
|
|
parser.add_argument("--root", default="/tmp/artifex-replay-arena-smoke")
|
|
|
|
def handle(self, *args, **options):
|
|
case_count = int(options["cases"])
|
|
if case_count < 1:
|
|
raise CommandError("--cases must be at least 1")
|
|
resource = Resource.objects.filter(provider="local_inference", is_active=True).first()
|
|
if resource is None:
|
|
raise CommandError("No Qwen/local_inference resource configured. Run seed_spark_resources first.")
|
|
SeedAgentsCommand().handle()
|
|
arena = ReplayArena(ModelRouter({"qwen": QwenProvider(resource)}, persist_requests=True), test_command=["python", "manage.py", "test"])
|
|
dataset = arena.create_dataset(
|
|
"qwen-replay-arena-smoke",
|
|
description="Real Qwen Replay Arena smoke",
|
|
selection_criteria={"source": "bounded smoke", "case_count": case_count},
|
|
)
|
|
root = Path(options["root"])
|
|
cases = []
|
|
for index in range(case_count):
|
|
repo = root / f"case-{index}"
|
|
if repo.exists():
|
|
shutil.rmtree(repo)
|
|
call_command("create_disposable_django_repo", str(repo), verbosity=0)
|
|
goal = 'Add a /health endpoint returning JSON {"status": "ok"} and add tests.'
|
|
project = Project.objects.create(name=f"Replay Source {index}", goal=goal, repository_path=str(repo))
|
|
plan = ProjectPlan.objects.create(project=project, version=1, goal=goal)
|
|
milestone = Milestone.objects.create(project=project, plan=plan, key="R", title="Replay", goal="Replay")
|
|
task = Task.objects.create(
|
|
project=project,
|
|
milestone=milestone,
|
|
task_type="implementation",
|
|
status=TaskStatus.COMPLETE,
|
|
goal=goal,
|
|
acceptance_criteria=["/health returns ok", "tests pass"],
|
|
)
|
|
cases.append(arena.add_case_from_task(dataset, task))
|
|
arena.freeze_dataset(dataset)
|
|
candidate = ImprovementCandidate.objects.create(
|
|
target_type="EXECUTION_GRAPH",
|
|
hypothesis="Insert deterministic static-analysis node before Reviewer to reduce rework risk",
|
|
recommended_route="Progeny Graph Evolution",
|
|
)
|
|
experiment = arena.create_experiment(candidate, dataset, success_criteria={"minimum_replay_cases": 3})
|
|
champion = arena.add_champion(experiment, graph_version=champion_task_execution_graph_v1())
|
|
challenger = arena.add_challenger(experiment, graph_version=arena.ensure_static_analysis_graph_challenger())
|
|
arena.run_experiment(experiment, max_cases=case_count)
|
|
comparison = experiment.comparison
|
|
payload = {
|
|
"dataset_id": str(dataset.id),
|
|
"dataset_version": dataset.version,
|
|
"case_count": dataset.cases.count(),
|
|
"frozen": dataset.status,
|
|
"experiment_id": str(experiment.id),
|
|
"hypothesis": experiment.hypothesis,
|
|
"target_type": experiment.target_type,
|
|
"champion": f"{champion.configuration_snapshot['graph']} v{champion.configuration_snapshot['version']}",
|
|
"challenger": f"{challenger.configuration_snapshot['graph']} v{challenger.configuration_snapshot['version']}",
|
|
"case_ids": [str(case.id) for case in cases],
|
|
"source_task_ids": [str(case.source_task_id) for case in cases],
|
|
"baseline_shas": [case.repository_baseline_ref for case in cases],
|
|
"metrics": comparison.aggregate_metrics,
|
|
"paired": comparison.paired_outcomes,
|
|
"regressions": comparison.regression_cases,
|
|
"verdict": comparison.verdict,
|
|
"reasons": comparison.reasons,
|
|
}
|
|
self.stdout.write(json.dumps(payload, indent=2, default=str))
|