diff --git a/control_plane/resources/management/commands/qwen_replay_arena_smoke.py b/control_plane/resources/management/commands/qwen_replay_arena_smoke.py new file mode 100644 index 0000000..0d815d7 --- /dev/null +++ b/control_plane/resources/management/commands/qwen_replay_arena_smoke.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +from django.core.management import call_command +from django.core.management.base import BaseCommand, CommandError + +from agents.replay_arena import ReplayArena +from control_plane.agents.management.commands.seed_core_agents import Command as SeedAgentsCommand +from control_plane.agents.models import ImprovementCandidate +from control_plane.projects.models import Milestone, Project, ProjectPlan, Task, TaskStatus +from control_plane.resources.models import Resource +from graph.bootstrap import champion_task_execution_graph_v1 +from model_router.providers import QwenProvider +from model_router.router import ModelRouter + + +class Command(BaseCommand): + help = "Opt-in smoke test: run a tiny real-Qwen Replay Arena champion/challenger experiment." + + def add_arguments(self, parser): + parser.add_argument("--cases", type=int, default=2) + parser.add_argument("--root", default="/tmp/artifex-replay-arena-smoke") + + def handle(self, *args, **options): + case_count = int(options["cases"]) + if case_count < 1: + raise CommandError("--cases must be at least 1") + resource = Resource.objects.filter(provider="local_inference", is_active=True).first() + if resource is None: + raise CommandError("No Qwen/local_inference resource configured. Run seed_spark_resources first.") + SeedAgentsCommand().handle() + arena = ReplayArena(ModelRouter({"qwen": QwenProvider(resource)}, persist_requests=True), test_command=["python", "manage.py", "test"]) + dataset = arena.create_dataset( + "qwen-replay-arena-smoke", + description="Real Qwen Replay Arena smoke", + selection_criteria={"source": "bounded smoke", "case_count": case_count}, + ) + root = Path(options["root"]) + cases = [] + for index in range(case_count): + repo = root / f"case-{index}" + if repo.exists(): + shutil.rmtree(repo) + call_command("create_disposable_django_repo", str(repo), verbosity=0) + goal = 'Add a /health endpoint returning JSON {"status": "ok"} and add tests.' + project = Project.objects.create(name=f"Replay Source {index}", goal=goal, repository_path=str(repo)) + plan = ProjectPlan.objects.create(project=project, version=1, goal=goal) + milestone = Milestone.objects.create(project=project, plan=plan, key="R", title="Replay", goal="Replay") + task = Task.objects.create( + project=project, + milestone=milestone, + task_type="implementation", + status=TaskStatus.COMPLETE, + goal=goal, + acceptance_criteria=["/health returns ok", "tests pass"], + ) + cases.append(arena.add_case_from_task(dataset, task)) + arena.freeze_dataset(dataset) + candidate = ImprovementCandidate.objects.create( + target_type="EXECUTION_GRAPH", + hypothesis="Insert deterministic static-analysis node before Reviewer to reduce rework risk", + recommended_route="Progeny Graph Evolution", + ) + experiment = arena.create_experiment(candidate, dataset, success_criteria={"minimum_replay_cases": 3}) + champion = arena.add_champion(experiment, graph_version=champion_task_execution_graph_v1()) + challenger = arena.add_challenger(experiment, graph_version=arena.ensure_static_analysis_graph_challenger()) + arena.run_experiment(experiment, max_cases=case_count) + comparison = experiment.comparison + payload = { + "dataset_id": str(dataset.id), + "dataset_version": dataset.version, + "case_count": dataset.cases.count(), + "frozen": dataset.status, + "experiment_id": str(experiment.id), + "hypothesis": experiment.hypothesis, + "target_type": experiment.target_type, + "champion": f"{champion.configuration_snapshot['graph']} v{champion.configuration_snapshot['version']}", + "challenger": f"{challenger.configuration_snapshot['graph']} v{challenger.configuration_snapshot['version']}", + "case_ids": [str(case.id) for case in cases], + "source_task_ids": [str(case.source_task_id) for case in cases], + "baseline_shas": [case.repository_baseline_ref for case in cases], + "metrics": comparison.aggregate_metrics, + "paired": comparison.paired_outcomes, + "regressions": comparison.regression_cases, + "verdict": comparison.verdict, + "reasons": comparison.reasons, + } + self.stdout.write(json.dumps(payload, indent=2, default=str))