From d61751a099974a56448b2dbef41e6fab33ba9b04 Mon Sep 17 00:00:00 2001 From: Daniel Maddern Date: Mon, 17 Aug 2026 01:04:02 +0700 Subject: [PATCH] Add Guard model studio foundation --- artifex/settings.py | 1 + artifex/urls.py | 3 + control_plane/model_studio/__init__.py | 0 control_plane/model_studio/apps.py | 6 + control_plane/model_studio/backends.py | 90 ++++ .../model_studio/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/modelstudio_dataset_audit.py | 20 + .../commands/modelstudio_import_guard.py | 27 + .../commands/modelstudio_overnight.py | 38 ++ .../model_studio/migrations/0001_initial.py | 494 ++++++++++++++++++ .../model_studio/migrations/__init__.py | 0 control_plane/model_studio/models.py | 420 +++++++++++++++ control_plane/model_studio/profiles.py | 75 +++ control_plane/model_studio/services.py | 321 ++++++++++++ control_plane/model_studio/views.py | 12 + docs/handoff_current_state_20260817.md | 166 ++++++ templates/control_plane/base.html | 1 + templates/control_plane/model_studio.html | 6 + .../control_plane/model_studio_project.html | 8 + tests/test_model_studio_v01.py | 123 +++++ 21 files changed, 1811 insertions(+) create mode 100644 control_plane/model_studio/__init__.py create mode 100644 control_plane/model_studio/apps.py create mode 100644 control_plane/model_studio/backends.py create mode 100644 control_plane/model_studio/management/__init__.py create mode 100644 control_plane/model_studio/management/commands/__init__.py create mode 100644 control_plane/model_studio/management/commands/modelstudio_dataset_audit.py create mode 100644 control_plane/model_studio/management/commands/modelstudio_import_guard.py create mode 100644 control_plane/model_studio/management/commands/modelstudio_overnight.py create mode 100644 control_plane/model_studio/migrations/0001_initial.py create mode 100644 control_plane/model_studio/migrations/__init__.py create mode 100644 control_plane/model_studio/models.py create mode 100644 control_plane/model_studio/profiles.py create mode 100644 control_plane/model_studio/services.py create mode 100644 control_plane/model_studio/views.py create mode 100644 docs/handoff_current_state_20260817.md create mode 100644 templates/control_plane/model_studio.html create mode 100644 templates/control_plane/model_studio_project.html create mode 100644 tests/test_model_studio_v01.py diff --git a/artifex/settings.py b/artifex/settings.py index 120da66..59f9bcc 100644 --- a/artifex/settings.py +++ b/artifex/settings.py @@ -22,6 +22,7 @@ INSTALLED_APPS = [ "control_plane.agents", "control_plane.resources", "control_plane.ventures", + "control_plane.model_studio", "control_plane.secrets", "control_plane.knowledge", "control_plane.verification", diff --git a/artifex/urls.py b/artifex/urls.py index 9086450..b25cd70 100644 --- a/artifex/urls.py +++ b/artifex/urls.py @@ -4,6 +4,7 @@ from django.contrib import admin from django.urls import path from control_plane.projects import views +from control_plane.model_studio import views as model_studio_views urlpatterns = [ path("", views.dashboard, name="dashboard"), @@ -32,6 +33,8 @@ urlpatterns = [ path("agents//", views.agent_detail, name="agent_detail"), path("agents//performance.json", views.agent_performance_json, name="agent_performance_json"), path("resources/", views.resources, name="resources"), + path("model-studio/", model_studio_views.model_studio, name="model_studio"), + path("model-studio//", model_studio_views.model_studio_project, name="model_studio_project"), path("approvals/", views.approvals, name="approvals"), path("approvals//action/", views.approval_action, name="approval_action"), path("activity/", views.activity, name="activity"), diff --git a/control_plane/model_studio/__init__.py b/control_plane/model_studio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/control_plane/model_studio/apps.py b/control_plane/model_studio/apps.py new file mode 100644 index 0000000..41de4ac --- /dev/null +++ b/control_plane/model_studio/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ModelStudioConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "control_plane.model_studio" diff --git a/control_plane/model_studio/backends.py b/control_plane/model_studio/backends.py new file mode 100644 index 0000000..4a301e0 --- /dev/null +++ b/control_plane/model_studio/backends.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + + +@dataclass +class BackendResult: + status: str + checkpoint_reference: str = "" + checkpoint_hash: str = "" + metrics: dict[str, float] | None = None + stdout: str = "" + stderr: str = "" + failure_category: str = "" + failure_details: str = "" + + +class TrainingBackend(Protocol): + def estimate_runtime(self, recipe: dict[str, Any]) -> int: ... + def launch(self, *, command: list[str], working_directory: str, timeout_seconds: int) -> BackendResult: ... + def validate_checkpoint(self, reference: str) -> bool: ... + + +class FakeTrainingBackend: + """Deterministic backend for workflow tests; never launches training.""" + + def __init__(self, outcomes: list[dict[str, Any]] | None = None) -> None: + self.outcomes = list(outcomes or [{"status": "SUCCEEDED", "metrics": {"primary": 0.75}}]) + + def estimate_runtime(self, recipe: dict[str, Any]) -> int: + return int(recipe.get("estimated_runtime_seconds", 60)) + + def launch(self, *, command: list[str], working_directory: str, timeout_seconds: int) -> BackendResult: + outcome = self.outcomes.pop(0) if self.outcomes else {"status": "SUCCEEDED", "metrics": {"primary": 0.75}} + status = str(outcome.get("status", "SUCCEEDED")) + reference = str(outcome.get("checkpoint_reference", f"fake://{hashlib.sha256(json.dumps(outcome, sort_keys=True).encode()).hexdigest()[:16]}")) + return BackendResult(status=status, checkpoint_reference=reference, checkpoint_hash=hashlib.sha256(reference.encode()).hexdigest(), metrics=outcome.get("metrics", {}), failure_category=str(outcome.get("failure_category", "")), failure_details=str(outcome.get("failure_details", ""))) + + def validate_checkpoint(self, reference: str) -> bool: + return reference.startswith("fake://") + + +class GuardSubprocessBackend: + """Scoped wrapper for the discovered Guard trainer; commands come from profiles, never an LLM.""" + + def estimate_runtime(self, recipe: dict[str, Any]) -> int: + return int(recipe.get("estimated_runtime_seconds", 90 * 60)) + + def launch(self, *, command: list[str], working_directory: str, timeout_seconds: int) -> BackendResult: + try: + completed = subprocess.run(command, cwd=working_directory, capture_output=True, text=True, timeout=timeout_seconds, check=False) + except subprocess.TimeoutExpired as exc: + return BackendResult(status="TIMEOUT", stdout=exc.stdout or "", stderr=exc.stderr or "", failure_category="TIMEOUT", failure_details=f"Exceeded {timeout_seconds}s") + stdout, stderr = completed.stdout or "", completed.stderr or "" + if completed.returncode: + category = "OOM" if "out of memory" in (stdout + stderr).lower() else "PROCESS_FAILURE" + return BackendResult(status=category if category == "OOM" else "FAILED", stdout=stdout, stderr=stderr, failure_category=category, failure_details=f"Exit code {completed.returncode}") + return BackendResult(status="SUCCEEDED", stdout=stdout, stderr=stderr) + + def validate_checkpoint(self, reference: str) -> bool: + path = Path(reference) + return path.is_dir() and any(path.glob("adapter_model.*")) + + +class SparkGuardBackend(GuardSubprocessBackend): + """Runs only profile-generated Guard commands over the configured Spark SSH alias.""" + + def __init__(self, ssh_alias: str = "spark") -> None: + self.ssh_alias = ssh_alias + + def launch(self, *, command: list[str], working_directory: str, timeout_seconds: int) -> BackendResult: + if not command: + return BackendResult(status="FAILED", failure_category="COMMAND_SCOPE", failure_details="Missing profile-generated command.") + import shlex + + remote = "cd " + shlex.quote(working_directory) + " && " + " ".join(shlex.quote(part) for part in command) + try: + completed = subprocess.run(["ssh", self.ssh_alias, remote], capture_output=True, text=True, timeout=timeout_seconds, check=False) + except subprocess.TimeoutExpired as exc: + return BackendResult(status="TIMEOUT", stdout=exc.stdout or "", stderr=exc.stderr or "", failure_category="TIMEOUT", failure_details=f"Exceeded {timeout_seconds}s") + if completed.returncode: + combined = (completed.stdout or "") + (completed.stderr or "") + category = "OOM" if "out of memory" in combined.lower() else "REMOTE_PROCESS_FAILURE" + return BackendResult(status="OOM" if category == "OOM" else "FAILED", stdout=completed.stdout or "", stderr=completed.stderr or "", failure_category=category, failure_details=f"Exit code {completed.returncode}") + return BackendResult(status="SUCCEEDED", stdout=completed.stdout or "", stderr=completed.stderr or "") diff --git a/control_plane/model_studio/management/__init__.py b/control_plane/model_studio/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/control_plane/model_studio/management/commands/__init__.py b/control_plane/model_studio/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/control_plane/model_studio/management/commands/modelstudio_dataset_audit.py b/control_plane/model_studio/management/commands/modelstudio_dataset_audit.py new file mode 100644 index 0000000..9e7c25f --- /dev/null +++ b/control_plane/model_studio/management/commands/modelstudio_dataset_audit.py @@ -0,0 +1,20 @@ +import json + +from django.core.management.base import BaseCommand, CommandError + +from control_plane.model_studio.services import ModelStudioService +from control_plane.model_studio.models import TrainingProject + + +class Command(BaseCommand): + help = "Classify Guard dataset manifests and record provenance/contamination evidence without changing source data." + + def add_arguments(self, parser): + parser.add_argument("--project", required=True, help="TrainingProject slug") + + def handle(self, *args, **options): + project = TrainingProject.objects.filter(slug=options["project"]).first() + if project is None: + raise CommandError("TrainingProject not found.") + report = ModelStudioService().curate_guard_datasets(project) + self.stdout.write(json.dumps({key: report[key] for key in ["total", "valid", "warning", "blocked", "decision"]}, indent=2)) diff --git a/control_plane/model_studio/management/commands/modelstudio_import_guard.py b/control_plane/model_studio/management/commands/modelstudio_import_guard.py new file mode 100644 index 0000000..14a38c0 --- /dev/null +++ b/control_plane/model_studio/management/commands/modelstudio_import_guard.py @@ -0,0 +1,27 @@ +from django.core.management.base import BaseCommand, CommandError + +from control_plane.model_studio.backends import SparkGuardBackend +from control_plane.model_studio.services import ModelStudioService +from control_plane.projects.models import Project + + +class Command(BaseCommand): + help = "Import the existing ForgeGuard/Qwen2.5-Coder-3B project and run archaeology." + + def add_arguments(self, parser): + parser.add_argument("--repository", required=True) + parser.add_argument("--project") + parser.add_argument("--slug", default="guard-3b") + parser.add_argument("--spark-working-directory", default="", help="Verified ForgeGuard checkout path on Spark; required before real remote training.") + + def handle(self, *args, **options): + project = Project.objects.filter(id=options["project"]).first() if options.get("project") else None + if options.get("project") and project is None: + raise CommandError("Project not found.") + service = ModelStudioService(backend=SparkGuardBackend()) + training_project = service.import_guard(project=project, repository_path=options["repository"], spark_working_directory=options["spark_working_directory"], slug=options["slug"]) + service.declare_base_champion(training_project) + report = service.archaeology(training_project) + curation = service.curate_guard_datasets(training_project) + suite = service.validate_benchmark(training_project) + self.stdout.write(self.style.SUCCESS(f"Imported {training_project.slug}: {len(report['checkpoints'])} checkpoints, {len(report['datasets'])} datasets, dataset_curation={curation['decision']}, suite={suite.integrity_status}")) diff --git a/control_plane/model_studio/management/commands/modelstudio_overnight.py b/control_plane/model_studio/management/commands/modelstudio_overnight.py new file mode 100644 index 0000000..f9b0eb2 --- /dev/null +++ b/control_plane/model_studio/management/commands/modelstudio_overnight.py @@ -0,0 +1,38 @@ +import json + +from django.core.management.base import BaseCommand, CommandError + +from control_plane.model_studio.models import ModelPromotionPolicy, TrainingProject +from control_plane.model_studio.services import ModelStudioService + + +class Command(BaseCommand): + help = "Create a bounded Model Studio overnight program. --dry-run never allocates training compute." + + def add_arguments(self, parser): + parser.add_argument("--project", required=True, help="TrainingProject slug") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--wall-seconds", type=int, default=8 * 3600) + parser.add_argument("--max-runs", type=int, default=8) + + def handle(self, *args, **options): + project = TrainingProject.objects.filter(slug=options["project"]).first() + if project is None: + raise CommandError("TrainingProject not found.") + policy = ModelPromotionPolicy.objects.filter(training_project=project, active=True).order_by("-created_at").first() + if policy is None: + raise CommandError("Create a ModelPromotionPolicy before starting an overnight program.") + service = ModelStudioService() + if options["dry_run"]: + rows = list(project.experiments.filter(status__in=["PROPOSED", "QUEUED"]).order_by("-experiment_value_score").values("experiment_id", "title", "experiment_value_score", "estimated_runtime_seconds")) + blockers = [] + if project.status != "READY": + blockers.append(f"training project status is {project.status}") + if not project.metadata.get("spark_working_directory_verified"): + blockers.append("Spark Guard working directory is not verified") + if project.baseline_evaluation_id is None: + blockers.append("fresh Champion baseline is missing") + self.stdout.write(json.dumps({"dry_run": True, "champion": str(project.current_champion_id or ""), "experiments": rows, "status": project.status, "blockers": blockers, "estimated_window_seconds": options["wall_seconds"], "maximum_runs": options["max_runs"]}, indent=2, default=str)) + return + program = service.create_program(project, policy, wall_seconds=options["wall_seconds"], max_runs=options["max_runs"]) + self.stdout.write(self.style.SUCCESS(f"Created overnight program {program.id}; execution is intentionally queued for scoped worker supervision.")) diff --git a/control_plane/model_studio/migrations/0001_initial.py b/control_plane/model_studio/migrations/0001_initial.py new file mode 100644 index 0000000..a8e3734 --- /dev/null +++ b/control_plane/model_studio/migrations/0001_initial.py @@ -0,0 +1,494 @@ +# Generated by Django 5.2.16 on 2026-08-16 17:49 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0006_roadmap_scenario_lab_v1'), + ] + + operations = [ + migrations.CreateModel( + name='Dataset', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=200)), + ('description', models.TextField(blank=True)), + ('metadata', models.JSONField(blank=True, default=dict)), + ], + ), + migrations.CreateModel( + name='EvaluationRun', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('status', models.CharField(choices=[('QUEUED', 'Queued'), ('RUNNING', 'Running'), ('SUCCEEDED', 'Succeeded'), ('FAILED', 'Failed'), ('INVALID', 'Invalid')], default='QUEUED', max_length=16)), + ('command', models.JSONField(blank=True, default=list)), + ('output_reference', models.TextField(blank=True)), + ('started_at', models.DateTimeField(blank=True, null=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('wall_seconds', models.FloatField(default=0)), + ('parser_version', models.CharField(blank=True, max_length=120)), + ('integrity_evidence', models.JSONField(blank=True, default=dict)), + ('summary', models.JSONField(blank=True, default=dict)), + ('failure_details', models.TextField(blank=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='EvaluationSuite', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=200)), + ('description', models.TextField(blank=True)), + ('metadata', models.JSONField(blank=True, default=dict)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='FailureCluster', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=200)), + ('description', models.TextField(blank=True)), + ('failure_type', models.CharField(blank=True, max_length=160)), + ('severity', models.CharField(default='UNKNOWN', max_length=32)), + ('sample_count', models.PositiveIntegerField(default=0)), + ('representative_examples', models.JSONField(blank=True, default=list)), + ('affected_benchmarks', models.JSONField(blank=True, default=list)), + ('suspected_causes', models.JSONField(blank=True, default=list)), + ('confidence', models.FloatField(default=0)), + ('training_data_coverage', models.JSONField(blank=True, default=dict)), + ('priority', models.FloatField(default=0)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='ModelPromotionPolicy', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=160)), + ('version', models.CharField(max_length=80)), + ('criteria', models.JSONField(default=dict)), + ('active', models.BooleanField(default=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='DatasetVersion', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('version', models.CharField(max_length=120)), + ('manifest_reference', models.TextField()), + ('content_hash', models.CharField(max_length=128)), + ('record_count', models.PositiveIntegerField(blank=True, null=True)), + ('split_metadata', models.JSONField(blank=True, default=dict)), + ('source_metadata', models.JSONField(blank=True, default=dict)), + ('generation_metadata', models.JSONField(blank=True, default=dict)), + ('tags', models.JSONField(blank=True, default=list)), + ('validation_status', models.CharField(choices=[('VALID', 'Valid'), ('WARNING', 'Warning'), ('BLOCKED', 'Blocked'), ('UNKNOWN', 'Unknown')], default='UNKNOWN', max_length=16)), + ('contamination_status', models.CharField(choices=[('VALID', 'Valid'), ('WARNING', 'Warning'), ('BLOCKED', 'Blocked'), ('UNKNOWN', 'Unknown')], default='UNKNOWN', max_length=16)), + ('immutable', models.BooleanField(default=False)), + ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='model_studio.dataset')), + ('parent_version', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='model_studio.datasetversion')), + ], + ), + migrations.CreateModel( + name='BenchmarkResult', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('group', models.CharField(choices=[('PRIMARY', 'Primary'), ('CAPABILITY_SUBSET', 'Capability Subset'), ('HOLDOUT', 'Holdout'), ('REGRESSION', 'Regression'), ('ADVERSARIAL', 'Adversarial'), ('FORMAT', 'Format'), ('PERFORMANCE', 'Performance')], default='PRIMARY', max_length=32)), + ('metric', models.CharField(max_length=160)), + ('value', models.FloatField()), + ('unit', models.CharField(blank=True, max_length=80)), + ('subset', models.CharField(blank=True, max_length=160)), + ('sample_count', models.PositiveIntegerField(blank=True, null=True)), + ('passed', models.BooleanField(blank=True, null=True)), + ('provenance', models.JSONField(blank=True, default=dict)), + ('evaluation_run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='results', to='model_studio.evaluationrun')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='EvaluationSuiteVersion', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('version', models.CharField(max_length=120)), + ('reference', models.TextField()), + ('content_hash', models.CharField(max_length=128)), + ('command_template', models.JSONField(blank=True, default=list)), + ('groups', models.JSONField(blank=True, default=list)), + ('immutable', models.BooleanField(default=False)), + ('integrity_status', models.CharField(choices=[('VALID', 'Valid'), ('WARNING', 'Warning'), ('BLOCKED', 'Blocked'), ('UNKNOWN', 'Unknown')], default='UNKNOWN', max_length=16)), + ('integrity_evidence', models.JSONField(blank=True, default=dict)), + ('suite', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='model_studio.evaluationsuite')), + ], + ), + migrations.AddField( + model_name='evaluationrun', + name='suite_version', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='runs', to='model_studio.evaluationsuiteversion'), + ), + migrations.CreateModel( + name='ModelCheckpoint', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=255)), + ('checkpoint_type', models.CharField(choices=[('BASE', 'Base'), ('IMPORTED', 'Imported'), ('CHAMPION', 'Champion'), ('CHALLENGER', 'Challenger'), ('INTERMEDIATE', 'Intermediate')], max_length=32)), + ('reference', models.TextField()), + ('content_hash', models.CharField(blank=True, max_length=128)), + ('parameter_count', models.BigIntegerField(blank=True, null=True)), + ('dtype', models.CharField(blank=True, max_length=80)), + ('adapter_type', models.CharField(blank=True, max_length=80)), + ('quantization', models.CharField(blank=True, max_length=80)), + ('validity_status', models.CharField(choices=[('UNKNOWN', 'Unknown'), ('VALID', 'Valid'), ('INVALID', 'Invalid'), ('PARTIAL', 'Partial'), ('CORRUPT', 'Corrupt')], default='UNKNOWN', max_length=16)), + ('load_verified', models.BooleanField(default=False)), + ('evaluation_status', models.CharField(default='NOT_EVALUATED', max_length=32)), + ('metadata', models.JSONField(blank=True, default=dict)), + ('base_checkpoint', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='derived_checkpoints', to='model_studio.modelcheckpoint')), + ('dataset_versions', models.ManyToManyField(blank=True, related_name='checkpoints', to='model_studio.datasetversion')), + ], + ), + migrations.AddField( + model_name='evaluationrun', + name='checkpoint', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='evaluation_runs', to='model_studio.modelcheckpoint'), + ), + migrations.CreateModel( + name='OvernightTrainingProgram', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('status', models.CharField(choices=[('CREATED', 'Created'), ('ARCHAEOLOGY', 'Archaeology'), ('BASELINING', 'Baselining'), ('PLANNING', 'Planning'), ('RUNNING_EXPERIMENT', 'Running Experiment'), ('EVALUATING', 'Evaluating'), ('ADAPTING', 'Adapting'), ('FINALIZING', 'Finalizing'), ('COMPLETED', 'Completed'), ('PARTIAL', 'Partial'), ('FAILED', 'Failed'), ('PAUSED', 'Paused')], default='CREATED', max_length=32)), + ('start_time', models.DateTimeField(blank=True, null=True)), + ('deadline', models.DateTimeField()), + ('maximum_wall_seconds', models.PositiveIntegerField()), + ('maximum_training_runs', models.PositiveIntegerField(default=8)), + ('maximum_failed_runs', models.PositiveIntegerField(default=3)), + ('maximum_single_run_seconds', models.PositiveIntegerField()), + ('evaluation_reserve_seconds', models.PositiveIntegerField()), + ('allowed_experiment_types', models.JSONField(blank=True, default=list)), + ('started_at', models.DateTimeField(blank=True, null=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('termination_reason', models.TextField(blank=True)), + ('pause_after_current_run', models.BooleanField(default=False)), + ('telemetry', models.JSONField(blank=True, default=dict)), + ('ending_champion', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='ending_programs', to='model_studio.modelcheckpoint')), + ('promotion_policy', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='programs', to='model_studio.modelpromotionpolicy')), + ('starting_champion', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='starting_programs', to='model_studio.modelcheckpoint')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='OvernightResearchReport', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('markdown', models.TextField()), + ('payload', models.JSONField(default=dict)), + ('artifact_reference', models.TextField(blank=True)), + ('program', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='report', to='model_studio.overnighttrainingprogram')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='TrainingExperiment', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('experiment_id', models.CharField(max_length=120, unique=True)), + ('title', models.CharField(max_length=255)), + ('hypothesis', models.TextField()), + ('reasoning', models.TextField()), + ('intervention', models.JSONField(default=dict)), + ('controls', models.JSONField(default=dict)), + ('expected_result', models.TextField()), + ('primary_success_metric', models.CharField(max_length=160)), + ('success_threshold', models.JSONField(default=dict)), + ('regression_constraints', models.JSONField(default=dict)), + ('rejection_condition', models.TextField()), + ('ambiguity_policy', models.TextField()), + ('estimated_runtime_seconds', models.PositiveIntegerField(default=0)), + ('maximum_runtime_seconds', models.PositiveIntegerField(default=0)), + ('compute_budget', models.JSONField(default=dict)), + ('status', models.CharField(choices=[('PROPOSED', 'Proposed'), ('QUEUED', 'Queued'), ('RUNNING', 'Running'), ('EVALUATING', 'Evaluating'), ('VALIDATING', 'Validating'), ('PROMOTED', 'Promoted'), ('REJECTED', 'Rejected'), ('INCONCLUSIVE', 'Inconclusive'), ('FAILED', 'Failed'), ('CANCELLED', 'Cancelled'), ('SUPERSEDED', 'Superseded')], default='PROPOSED', max_length=32)), + ('priority', models.FloatField(default=0)), + ('expected_information_gain', models.FloatField(default=0)), + ('expected_improvement', models.FloatField(default=0)), + ('estimated_compute_cost', models.FloatField(default=0)), + ('experiment_value_score', models.FloatField(default=0)), + ('fingerprint', models.CharField(max_length=128)), + ('created_by_agent', models.CharField(default='MODEL_DIRECTOR', max_length=120)), + ('approved_by_model_director', models.BooleanField(default=False)), + ('result_summary', models.TextField(blank=True)), + ('conclusion', models.CharField(blank=True, choices=[('SUPPORTED', 'Supported'), ('WEAKLY_SUPPORTED', 'Weakly Supported'), ('REFUTED', 'Refuted'), ('AMBIGUOUS', 'Ambiguous'), ('EXECUTION_FAILED', 'Execution Failed')], max_length=32)), + ('derived_from_failure_cluster', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='experiments', to='model_studio.failurecluster')), + ('parent_experiment', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='derived_experiments', to='model_studio.trainingexperiment')), + ], + ), + migrations.CreateModel( + name='ExperimentDependency', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('required_conclusions', models.JSONField(blank=True, default=list)), + ('rationale', models.TextField(blank=True)), + ('depends_on', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dependents', to='model_studio.trainingexperiment')), + ('experiment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dependencies', to='model_studio.trainingexperiment')), + ], + ), + migrations.CreateModel( + name='TrainingProject', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('studio_type', models.CharField(default='MODEL', max_length=32)), + ('name', models.CharField(max_length=200)), + ('slug', models.SlugField(max_length=120, unique=True)), + ('description', models.TextField(blank=True)), + ('goal', models.TextField()), + ('capability_target', models.TextField(blank=True)), + ('model_family', models.CharField(blank=True, max_length=160)), + ('model_size', models.CharField(blank=True, max_length=80)), + ('base_model', models.TextField(blank=True)), + ('repository_path', models.TextField(blank=True)), + ('repository_reference', models.TextField(blank=True)), + ('working_directory', models.TextField(blank=True)), + ('status', models.CharField(choices=[('IMPORTING', 'Importing'), ('ARCHAEOLOGY', 'Archaeology'), ('NEEDS_REPAIR', 'Needs Repair'), ('BASELINING', 'Baselining'), ('READY', 'Ready'), ('OVERNIGHT_RUNNING', 'Overnight Running'), ('PAUSED', 'Paused'), ('FAILED', 'Failed'), ('FINISHED', 'Finished'), ('EVOLVING', 'Evolving')], default='IMPORTING', max_length=32)), + ('default_profile', models.CharField(default='guard', max_length=120)), + ('training_backend', models.CharField(default='guard_subprocess', max_length=120)), + ('metadata', models.JSONField(blank=True, default=dict)), + ('baseline_evaluation', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='baseline_for_projects', to='model_studio.evaluationrun')), + ('current_champion', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='champion_for_projects', to='model_studio.modelcheckpoint')), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='training_projects', to='projects.project')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='trainingexperiment', + name='training_project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='experiments', to='model_studio.trainingproject'), + ), + migrations.CreateModel( + name='RegressionBankItem', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('reference', models.TextField()), + ('content_hash', models.CharField(blank=True, max_length=128)), + ('category', models.CharField(blank=True, max_length=160)), + ('kind', models.CharField(default='IMPORTED', max_length=80)), + ('provenance', models.JSONField(blank=True, default=dict)), + ('evaluation_only', models.BooleanField(default=True)), + ('training_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='regression_bank_items', to='model_studio.trainingproject')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='overnighttrainingprogram', + name='training_project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='overnight_programs', to='model_studio.trainingproject'), + ), + migrations.CreateModel( + name='ModelStudioArtifact', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('artifact_type', models.CharField(max_length=120)), + ('name', models.CharField(max_length=255)), + ('content', models.JSONField(default=dict)), + ('readable', models.TextField(blank=True)), + ('source_reference', models.TextField(blank=True)), + ('training_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='artifacts', to='model_studio.trainingproject')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='modelpromotionpolicy', + name='training_project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='promotion_policies', to='model_studio.trainingproject'), + ), + migrations.CreateModel( + name='ModelPromotionDecision', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('decision', models.CharField(choices=[('PROMOTE', 'Promote'), ('REJECT', 'Reject'), ('INCONCLUSIVE', 'Inconclusive'), ('REQUIRE_REPLICATION', 'Require Replication')], max_length=32)), + ('evaluation_evidence', models.JSONField(default=dict)), + ('judge_result', models.JSONField(default=dict)), + ('reason', models.TextField()), + ('judge_actor', models.CharField(default='MODEL_JUDGE', max_length=120)), + ('candidate', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='promotion_candidates', to='model_studio.modelcheckpoint')), + ('from_champion', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='promotion_sources', to='model_studio.modelcheckpoint')), + ('policy', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='decisions', to='model_studio.modelpromotionpolicy')), + ('experiment', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='promotion_decisions', to='model_studio.trainingexperiment')), + ('training_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='promotion_decisions', to='model_studio.trainingproject')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='modelcheckpoint', + name='training_project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='checkpoints', to='model_studio.trainingproject'), + ), + migrations.AddField( + model_name='failurecluster', + name='training_project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='failure_clusters', to='model_studio.trainingproject'), + ), + migrations.AddField( + model_name='evaluationsuite', + name='training_project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='evaluation_suites', to='model_studio.trainingproject'), + ), + migrations.AddField( + model_name='evaluationrun', + name='training_project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='evaluation_runs', to='model_studio.trainingproject'), + ), + migrations.AddField( + model_name='dataset', + name='training_project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='datasets', to='model_studio.trainingproject'), + ), + migrations.CreateModel( + name='TrainingRecipe', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=200)), + ('configuration', models.JSONField(default=dict)), + ('recipe_hash', models.CharField(max_length=128, unique=True)), + ('immutable', models.BooleanField(default=False)), + ('training_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='recipes', to='model_studio.trainingproject')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='modelcheckpoint', + name='recipe', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='checkpoints', to='model_studio.trainingrecipe'), + ), + migrations.CreateModel( + name='TrainingRun', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('status', models.CharField(choices=[('QUEUED', 'Queued'), ('STARTING', 'Starting'), ('RUNNING', 'Running'), ('CHECKPOINTING', 'Checkpointing'), ('SUCCEEDED', 'Succeeded'), ('FAILED', 'Failed'), ('OOM', 'Oom'), ('TIMEOUT', 'Timeout'), ('CANCELLED', 'Cancelled'), ('INTERRUPTED', 'Interrupted')], default='QUEUED', max_length=32)), + ('command', models.JSONField(blank=True, default=list)), + ('working_directory', models.TextField(blank=True)), + ('environment_snapshot', models.JSONField(blank=True, default=dict)), + ('host', models.CharField(blank=True, max_length=255)), + ('gpu_device', models.CharField(blank=True, max_length=255)), + ('allocated_memory_mb', models.PositiveIntegerField(blank=True, null=True)), + ('started_at', models.DateTimeField(blank=True, null=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('wall_seconds', models.FloatField(default=0)), + ('exit_code', models.IntegerField(blank=True, null=True)), + ('stdout_reference', models.TextField(blank=True)), + ('stderr_reference', models.TextField(blank=True)), + ('training_log_reference', models.TextField(blank=True)), + ('peak_memory_mb', models.PositiveIntegerField(blank=True, null=True)), + ('gpu_utilization', models.FloatField(blank=True, null=True)), + ('failure_category', models.CharField(blank=True, max_length=80)), + ('failure_details', models.TextField(blank=True)), + ('resume_source', models.TextField(blank=True)), + ('retry_count', models.PositiveIntegerField(default=0)), + ('pid', models.IntegerField(blank=True, null=True)), + ('heartbeat_at', models.DateTimeField(blank=True, null=True)), + ('experiment', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='runs', to='model_studio.trainingexperiment')), + ('input_checkpoint', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='input_runs', to='model_studio.modelcheckpoint')), + ('output_checkpoint', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='producing_run', to='model_studio.modelcheckpoint')), + ('recipe', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='runs', to='model_studio.trainingrecipe')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='modelcheckpoint', + name='training_run', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='output_checkpoints', to='model_studio.trainingrun'), + ), + migrations.AddConstraint( + model_name='datasetversion', + constraint=models.UniqueConstraint(fields=('dataset', 'version'), name='unique_model_studio_dataset_version'), + ), + migrations.AddConstraint( + model_name='evaluationsuiteversion', + constraint=models.UniqueConstraint(fields=('suite', 'version'), name='unique_evaluation_suite_version'), + ), + migrations.AddConstraint( + model_name='experimentdependency', + constraint=models.UniqueConstraint(fields=('experiment', 'depends_on'), name='unique_experiment_dependency'), + ), + migrations.AddIndex( + model_name='trainingexperiment', + index=models.Index(fields=['training_project', 'fingerprint'], name='model_studi_trainin_53bef8_idx'), + ), + migrations.AddConstraint( + model_name='dataset', + constraint=models.UniqueConstraint(fields=('training_project', 'name'), name='unique_model_studio_dataset'), + ), + migrations.AddConstraint( + model_name='modelcheckpoint', + constraint=models.UniqueConstraint(fields=('training_project', 'reference'), name='unique_model_checkpoint_reference'), + ), + ] diff --git a/control_plane/model_studio/migrations/__init__.py b/control_plane/model_studio/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/control_plane/model_studio/models.py b/control_plane/model_studio/models.py new file mode 100644 index 0000000..9e22912 --- /dev/null +++ b/control_plane/model_studio/models.py @@ -0,0 +1,420 @@ +from __future__ import annotations + +from django.db import models + +from control_plane.common import TimestampedModel + + +class TrainingProjectStatus(models.TextChoices): + IMPORTING = "IMPORTING" + ARCHAEOLOGY = "ARCHAEOLOGY" + NEEDS_REPAIR = "NEEDS_REPAIR" + BASELINING = "BASELINING" + READY = "READY" + OVERNIGHT_RUNNING = "OVERNIGHT_RUNNING" + PAUSED = "PAUSED" + FAILED = "FAILED" + FINISHED = "FINISHED" + EVOLVING = "EVOLVING" + + +class DatasetValidationStatus(models.TextChoices): + VALID = "VALID" + WARNING = "WARNING" + BLOCKED = "BLOCKED" + UNKNOWN = "UNKNOWN" + + +class CheckpointType(models.TextChoices): + BASE = "BASE" + IMPORTED = "IMPORTED" + CHAMPION = "CHAMPION" + CHALLENGER = "CHALLENGER" + INTERMEDIATE = "INTERMEDIATE" + + +class CheckpointValidityStatus(models.TextChoices): + UNKNOWN = "UNKNOWN" + VALID = "VALID" + INVALID = "INVALID" + PARTIAL = "PARTIAL" + CORRUPT = "CORRUPT" + + +class ExperimentStatus(models.TextChoices): + PROPOSED = "PROPOSED" + QUEUED = "QUEUED" + RUNNING = "RUNNING" + EVALUATING = "EVALUATING" + VALIDATING = "VALIDATING" + PROMOTED = "PROMOTED" + REJECTED = "REJECTED" + INCONCLUSIVE = "INCONCLUSIVE" + FAILED = "FAILED" + CANCELLED = "CANCELLED" + SUPERSEDED = "SUPERSEDED" + + +class TrainingRunStatus(models.TextChoices): + QUEUED = "QUEUED" + STARTING = "STARTING" + RUNNING = "RUNNING" + CHECKPOINTING = "CHECKPOINTING" + SUCCEEDED = "SUCCEEDED" + FAILED = "FAILED" + OOM = "OOM" + TIMEOUT = "TIMEOUT" + CANCELLED = "CANCELLED" + INTERRUPTED = "INTERRUPTED" + + +class EvaluationGroup(models.TextChoices): + PRIMARY = "PRIMARY" + CAPABILITY_SUBSET = "CAPABILITY_SUBSET" + HOLDOUT = "HOLDOUT" + REGRESSION = "REGRESSION" + ADVERSARIAL = "ADVERSARIAL" + FORMAT = "FORMAT" + PERFORMANCE = "PERFORMANCE" + + +class EvaluationRunStatus(models.TextChoices): + QUEUED = "QUEUED" + RUNNING = "RUNNING" + SUCCEEDED = "SUCCEEDED" + FAILED = "FAILED" + INVALID = "INVALID" + + +class ProgramStatus(models.TextChoices): + CREATED = "CREATED" + ARCHAEOLOGY = "ARCHAEOLOGY" + BASELINING = "BASELINING" + PLANNING = "PLANNING" + RUNNING_EXPERIMENT = "RUNNING_EXPERIMENT" + EVALUATING = "EVALUATING" + ADAPTING = "ADAPTING" + FINALIZING = "FINALIZING" + COMPLETED = "COMPLETED" + PARTIAL = "PARTIAL" + FAILED = "FAILED" + PAUSED = "PAUSED" + + +class PromotionDecision(models.TextChoices): + PROMOTE = "PROMOTE" + REJECT = "REJECT" + INCONCLUSIVE = "INCONCLUSIVE" + REQUIRE_REPLICATION = "REQUIRE_REPLICATION" + + +class Conclusion(models.TextChoices): + SUPPORTED = "SUPPORTED" + WEAKLY_SUPPORTED = "WEAKLY_SUPPORTED" + REFUTED = "REFUTED" + AMBIGUOUS = "AMBIGUOUS" + EXECUTION_FAILED = "EXECUTION_FAILED" + + +class TrainingProject(TimestampedModel): + project = models.ForeignKey("projects.Project", on_delete=models.PROTECT, related_name="training_projects") + studio_type = models.CharField(max_length=32, default="MODEL") + name = models.CharField(max_length=200) + slug = models.SlugField(max_length=120, unique=True) + description = models.TextField(blank=True) + goal = models.TextField() + capability_target = models.TextField(blank=True) + model_family = models.CharField(max_length=160, blank=True) + model_size = models.CharField(max_length=80, blank=True) + base_model = models.TextField(blank=True) + repository_path = models.TextField(blank=True) + repository_reference = models.TextField(blank=True) + working_directory = models.TextField(blank=True) + status = models.CharField(max_length=32, choices=TrainingProjectStatus.choices, default=TrainingProjectStatus.IMPORTING) + default_profile = models.CharField(max_length=120, default="guard") + training_backend = models.CharField(max_length=120, default="guard_subprocess") + current_champion = models.ForeignKey("ModelCheckpoint", on_delete=models.SET_NULL, null=True, blank=True, related_name="champion_for_projects") + baseline_evaluation = models.ForeignKey("EvaluationRun", on_delete=models.SET_NULL, null=True, blank=True, related_name="baseline_for_projects") + metadata = models.JSONField(default=dict, blank=True) + + +class Dataset(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="datasets") + name = models.CharField(max_length=200) + description = models.TextField(blank=True) + metadata = models.JSONField(default=dict, blank=True) + + class Meta: + constraints = [models.UniqueConstraint(fields=["training_project", "name"], name="unique_model_studio_dataset")] + + +class DatasetVersion(TimestampedModel): + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name="versions") + version = models.CharField(max_length=120) + parent_version = models.ForeignKey("self", on_delete=models.SET_NULL, null=True, blank=True, related_name="children") + manifest_reference = models.TextField() + content_hash = models.CharField(max_length=128) + record_count = models.PositiveIntegerField(null=True, blank=True) + split_metadata = models.JSONField(default=dict, blank=True) + source_metadata = models.JSONField(default=dict, blank=True) + generation_metadata = models.JSONField(default=dict, blank=True) + tags = models.JSONField(default=list, blank=True) + validation_status = models.CharField(max_length=16, choices=DatasetValidationStatus.choices, default=DatasetValidationStatus.UNKNOWN) + contamination_status = models.CharField(max_length=16, choices=DatasetValidationStatus.choices, default=DatasetValidationStatus.UNKNOWN) + immutable = models.BooleanField(default=False) + + class Meta: + constraints = [models.UniqueConstraint(fields=["dataset", "version"], name="unique_model_studio_dataset_version")] + + def save(self, *args, **kwargs): + if self.pk and self.immutable: + original = type(self).objects.get(pk=self.pk) + immutable_fields = ["version", "parent_version_id", "manifest_reference", "content_hash", "record_count", "split_metadata", "source_metadata", "generation_metadata", "tags"] + if any(getattr(self, field) != getattr(original, field) for field in immutable_fields): + raise ValueError("DatasetVersion is immutable after completed training use.") + super().save(*args, **kwargs) + + +class TrainingRecipe(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="recipes") + name = models.CharField(max_length=200) + configuration = models.JSONField(default=dict) + recipe_hash = models.CharField(max_length=128, unique=True) + immutable = models.BooleanField(default=False) + + def save(self, *args, **kwargs): + if self.pk and self.immutable: + original = type(self).objects.get(pk=self.pk) + if self.configuration != original.configuration or self.recipe_hash != original.recipe_hash: + raise ValueError("TrainingRecipe is immutable after a TrainingRun starts.") + super().save(*args, **kwargs) + + +class TrainingExperiment(TimestampedModel): + experiment_id = models.CharField(max_length=120, unique=True) + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="experiments") + title = models.CharField(max_length=255) + hypothesis = models.TextField() + reasoning = models.TextField() + intervention = models.JSONField(default=dict) + controls = models.JSONField(default=dict) + expected_result = models.TextField() + primary_success_metric = models.CharField(max_length=160) + success_threshold = models.JSONField(default=dict) + regression_constraints = models.JSONField(default=dict) + rejection_condition = models.TextField() + ambiguity_policy = models.TextField() + estimated_runtime_seconds = models.PositiveIntegerField(default=0) + maximum_runtime_seconds = models.PositiveIntegerField(default=0) + compute_budget = models.JSONField(default=dict) + parent_experiment = models.ForeignKey("self", on_delete=models.SET_NULL, null=True, blank=True, related_name="derived_experiments") + derived_from_failure_cluster = models.ForeignKey("FailureCluster", on_delete=models.SET_NULL, null=True, blank=True, related_name="experiments") + status = models.CharField(max_length=32, choices=ExperimentStatus.choices, default=ExperimentStatus.PROPOSED) + priority = models.FloatField(default=0) + expected_information_gain = models.FloatField(default=0) + expected_improvement = models.FloatField(default=0) + estimated_compute_cost = models.FloatField(default=0) + experiment_value_score = models.FloatField(default=0) + fingerprint = models.CharField(max_length=128) + created_by_agent = models.CharField(max_length=120, default="MODEL_DIRECTOR") + approved_by_model_director = models.BooleanField(default=False) + result_summary = models.TextField(blank=True) + conclusion = models.CharField(max_length=32, choices=Conclusion.choices, blank=True) + + class Meta: + indexes = [models.Index(fields=["training_project", "fingerprint"])] + + +class ExperimentDependency(TimestampedModel): + experiment = models.ForeignKey(TrainingExperiment, on_delete=models.CASCADE, related_name="dependencies") + depends_on = models.ForeignKey(TrainingExperiment, on_delete=models.CASCADE, related_name="dependents") + required_conclusions = models.JSONField(default=list, blank=True) + rationale = models.TextField(blank=True) + + class Meta: + constraints = [models.UniqueConstraint(fields=["experiment", "depends_on"], name="unique_experiment_dependency")] + + +class ModelCheckpoint(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="checkpoints") + name = models.CharField(max_length=255) + checkpoint_type = models.CharField(max_length=32, choices=CheckpointType.choices) + reference = models.TextField() + content_hash = models.CharField(max_length=128, blank=True) + base_checkpoint = models.ForeignKey("self", on_delete=models.SET_NULL, null=True, blank=True, related_name="derived_checkpoints") + training_run = models.ForeignKey("TrainingRun", on_delete=models.SET_NULL, null=True, blank=True, related_name="output_checkpoints") + recipe = models.ForeignKey(TrainingRecipe, on_delete=models.SET_NULL, null=True, blank=True, related_name="checkpoints") + dataset_versions = models.ManyToManyField(DatasetVersion, blank=True, related_name="checkpoints") + parameter_count = models.BigIntegerField(null=True, blank=True) + dtype = models.CharField(max_length=80, blank=True) + adapter_type = models.CharField(max_length=80, blank=True) + quantization = models.CharField(max_length=80, blank=True) + validity_status = models.CharField(max_length=16, choices=CheckpointValidityStatus.choices, default=CheckpointValidityStatus.UNKNOWN) + load_verified = models.BooleanField(default=False) + evaluation_status = models.CharField(max_length=32, default="NOT_EVALUATED") + metadata = models.JSONField(default=dict, blank=True) + + class Meta: + constraints = [models.UniqueConstraint(fields=["training_project", "reference"], name="unique_model_checkpoint_reference")] + + +class TrainingRun(TimestampedModel): + experiment = models.ForeignKey(TrainingExperiment, on_delete=models.PROTECT, related_name="runs") + recipe = models.ForeignKey(TrainingRecipe, on_delete=models.PROTECT, related_name="runs") + input_checkpoint = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="input_runs") + output_checkpoint = models.ForeignKey(ModelCheckpoint, on_delete=models.SET_NULL, null=True, blank=True, related_name="producing_run") + status = models.CharField(max_length=32, choices=TrainingRunStatus.choices, default=TrainingRunStatus.QUEUED) + command = models.JSONField(default=list, blank=True) + working_directory = models.TextField(blank=True) + environment_snapshot = models.JSONField(default=dict, blank=True) + host = models.CharField(max_length=255, blank=True) + gpu_device = models.CharField(max_length=255, blank=True) + allocated_memory_mb = models.PositiveIntegerField(null=True, blank=True) + started_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + wall_seconds = models.FloatField(default=0) + exit_code = models.IntegerField(null=True, blank=True) + stdout_reference = models.TextField(blank=True) + stderr_reference = models.TextField(blank=True) + training_log_reference = models.TextField(blank=True) + peak_memory_mb = models.PositiveIntegerField(null=True, blank=True) + gpu_utilization = models.FloatField(null=True, blank=True) + failure_category = models.CharField(max_length=80, blank=True) + failure_details = models.TextField(blank=True) + resume_source = models.TextField(blank=True) + retry_count = models.PositiveIntegerField(default=0) + pid = models.IntegerField(null=True, blank=True) + heartbeat_at = models.DateTimeField(null=True, blank=True) + + +class EvaluationSuite(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="evaluation_suites") + name = models.CharField(max_length=200) + description = models.TextField(blank=True) + metadata = models.JSONField(default=dict, blank=True) + + +class EvaluationSuiteVersion(TimestampedModel): + suite = models.ForeignKey(EvaluationSuite, on_delete=models.CASCADE, related_name="versions") + version = models.CharField(max_length=120) + reference = models.TextField() + content_hash = models.CharField(max_length=128) + command_template = models.JSONField(default=list, blank=True) + groups = models.JSONField(default=list, blank=True) + immutable = models.BooleanField(default=False) + integrity_status = models.CharField(max_length=16, choices=DatasetValidationStatus.choices, default=DatasetValidationStatus.UNKNOWN) + integrity_evidence = models.JSONField(default=dict, blank=True) + + class Meta: + constraints = [models.UniqueConstraint(fields=["suite", "version"], name="unique_evaluation_suite_version")] + + +class EvaluationRun(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="evaluation_runs") + checkpoint = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="evaluation_runs") + suite_version = models.ForeignKey(EvaluationSuiteVersion, on_delete=models.PROTECT, related_name="runs") + status = models.CharField(max_length=16, choices=EvaluationRunStatus.choices, default=EvaluationRunStatus.QUEUED) + command = models.JSONField(default=list, blank=True) + output_reference = models.TextField(blank=True) + started_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + wall_seconds = models.FloatField(default=0) + parser_version = models.CharField(max_length=120, blank=True) + integrity_evidence = models.JSONField(default=dict, blank=True) + summary = models.JSONField(default=dict, blank=True) + failure_details = models.TextField(blank=True) + + +class BenchmarkResult(TimestampedModel): + evaluation_run = models.ForeignKey(EvaluationRun, on_delete=models.CASCADE, related_name="results") + group = models.CharField(max_length=32, choices=EvaluationGroup.choices, default=EvaluationGroup.PRIMARY) + metric = models.CharField(max_length=160) + value = models.FloatField() + unit = models.CharField(max_length=80, blank=True) + subset = models.CharField(max_length=160, blank=True) + sample_count = models.PositiveIntegerField(null=True, blank=True) + passed = models.BooleanField(null=True, blank=True) + provenance = models.JSONField(default=dict, blank=True) + + +class RegressionBankItem(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="regression_bank_items") + reference = models.TextField() + content_hash = models.CharField(max_length=128, blank=True) + category = models.CharField(max_length=160, blank=True) + kind = models.CharField(max_length=80, default="IMPORTED") + provenance = models.JSONField(default=dict, blank=True) + evaluation_only = models.BooleanField(default=True) + + +class FailureCluster(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="failure_clusters") + name = models.CharField(max_length=200) + description = models.TextField(blank=True) + failure_type = models.CharField(max_length=160, blank=True) + severity = models.CharField(max_length=32, default="UNKNOWN") + sample_count = models.PositiveIntegerField(default=0) + representative_examples = models.JSONField(default=list, blank=True) + affected_benchmarks = models.JSONField(default=list, blank=True) + suspected_causes = models.JSONField(default=list, blank=True) + confidence = models.FloatField(default=0) + training_data_coverage = models.JSONField(default=dict, blank=True) + priority = models.FloatField(default=0) + + +class ModelPromotionPolicy(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="promotion_policies") + name = models.CharField(max_length=160) + version = models.CharField(max_length=80) + criteria = models.JSONField(default=dict) + active = models.BooleanField(default=True) + + +class ModelPromotionDecision(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="promotion_decisions") + from_champion = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="promotion_sources") + candidate = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="promotion_candidates") + experiment = models.ForeignKey(TrainingExperiment, on_delete=models.PROTECT, related_name="promotion_decisions") + decision = models.CharField(max_length=32, choices=PromotionDecision.choices) + policy = models.ForeignKey(ModelPromotionPolicy, on_delete=models.PROTECT, related_name="decisions") + evaluation_evidence = models.JSONField(default=dict) + judge_result = models.JSONField(default=dict) + reason = models.TextField() + judge_actor = models.CharField(max_length=120, default="MODEL_JUDGE") + + +class OvernightTrainingProgram(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="overnight_programs") + starting_champion = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="starting_programs") + ending_champion = models.ForeignKey(ModelCheckpoint, on_delete=models.SET_NULL, null=True, blank=True, related_name="ending_programs") + status = models.CharField(max_length=32, choices=ProgramStatus.choices, default=ProgramStatus.CREATED) + start_time = models.DateTimeField(null=True, blank=True) + deadline = models.DateTimeField() + maximum_wall_seconds = models.PositiveIntegerField() + maximum_training_runs = models.PositiveIntegerField(default=8) + maximum_failed_runs = models.PositiveIntegerField(default=3) + maximum_single_run_seconds = models.PositiveIntegerField() + evaluation_reserve_seconds = models.PositiveIntegerField() + allowed_experiment_types = models.JSONField(default=list, blank=True) + promotion_policy = models.ForeignKey(ModelPromotionPolicy, on_delete=models.PROTECT, related_name="programs") + started_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + termination_reason = models.TextField(blank=True) + pause_after_current_run = models.BooleanField(default=False) + telemetry = models.JSONField(default=dict, blank=True) + + +class OvernightResearchReport(TimestampedModel): + program = models.OneToOneField(OvernightTrainingProgram, on_delete=models.CASCADE, related_name="report") + markdown = models.TextField() + payload = models.JSONField(default=dict) + artifact_reference = models.TextField(blank=True) + + +class ModelStudioArtifact(TimestampedModel): + training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="artifacts") + artifact_type = models.CharField(max_length=120) + name = models.CharField(max_length=255) + content = models.JSONField(default=dict) + readable = models.TextField(blank=True) + source_reference = models.TextField(blank=True) diff --git a/control_plane/model_studio/profiles.py b/control_plane/model_studio/profiles.py new file mode 100644 index 0000000..ca3c82d --- /dev/null +++ b/control_plane/model_studio/profiles.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + + +@dataclass +class ArchaeologyFinding: + subject: str + confidence: str + evidence: list[str] + details: dict[str, Any] + + +class ModelProjectProfile(Protocol): + name: str + def archaeology(self, repository_path: str) -> dict[str, Any]: ... + def validate_evaluation_suite(self, repository_path: str, suite_reference: str) -> dict[str, Any]: ... + def training_command(self, recipe: dict[str, Any], output_directory: str) -> list[str]: ... + + +class GuardModelProfile: + name = "guard" + base_model = "Qwen/Qwen2.5-Coder-3B-Instruct" + + def archaeology(self, repository_path: str) -> dict[str, Any]: + root = Path(repository_path) + findings: list[dict[str, Any]] = [] + def add(subject: str, confidence: str, evidence: list[Path], **details: Any) -> None: + findings.append({"subject": subject, "confidence": confidence, "evidence": [str(item) for item in evidence], "details": details}) + + trainer = root / "scripts" / "05_finetune.py" + evaluator = root / "scripts" / "run_evmbench.py" + strict_scorer = root / "scripts" / "score_evmbench_strict.py" + add("base_model", "CONFIRMED" if trainer.exists() else "UNKNOWN", [trainer] if trainer.exists() else [], value=self.base_model) + add("training_entrypoint", "CONFIRMED" if trainer.exists() else "UNKNOWN", [trainer] if trainer.exists() else [], command="python scripts/05_finetune.py --model ... --training-data ... --output ...") + add("evaluation_entrypoint", "CONFIRMED" if evaluator.exists() else "UNKNOWN", [item for item in [evaluator, strict_scorer] if item.exists()], command="python scripts/run_evmbench.py ...", strict_scoring=bool(strict_scorer.exists())) + checkpoints = [] + for adapter in sorted((root / "models").glob("**/adapter_model.*")) if (root / "models").exists() else []: + checkpoints.append({"name": adapter.parent.name, "reference": str(adapter.parent), "hash": self._hash_file(adapter), "adapter_type": "LORA", "confidence": "CONFIRMED"}) + datasets = [] + for manifest in sorted((root / "data" / "production_authorized").glob("*.json")) if (root / "data" / "production_authorized").exists() else []: + try: + payload = json.loads(manifest.read_text(encoding="utf-8")) + count = len(payload) if isinstance(payload, list) else payload.get("record_count") + except (OSError, json.JSONDecodeError): + count = None + datasets.append({"name": manifest.stem, "reference": str(manifest), "hash": self._hash_file(manifest), "record_count": count, "tags": ["imported", "training"]}) + reports = [str(path) for path in sorted((root / "results").glob("qwen25_coder_3b_*") if (root / "results").exists() else [])] + benchmark = root / "evmbench" + add("benchmark_checkout", "CONFIRMED" if benchmark.exists() else "UNKNOWN", [benchmark] if benchmark.exists() else [], strict_scorer_exists=strict_scorer.exists()) + return {"root_exists": root.exists(), "findings": findings, "checkpoints": checkpoints, "datasets": datasets, "historical_reports": reports, "benchmark_reference": str(benchmark), "trainer_reference": str(trainer), "evaluator_reference": str(evaluator), "strict_scorer_reference": str(strict_scorer)} + + def validate_evaluation_suite(self, repository_path: str, suite_reference: str) -> dict[str, Any]: + root = Path(repository_path) + evaluator = root / "scripts" / "run_evmbench.py" + scorer = root / "scripts" / "score_evmbench_strict.py" + benchmark = Path(suite_reference) + available = evaluator.exists() and scorer.exists() and benchmark.exists() + return {"status": "WARNING" if available else "BLOCKED", "evidence": {"evaluator": str(evaluator), "strict_scorer": str(scorer), "benchmark": str(benchmark), "reason": "Scripts and checkout are present, but no fresh Spark baseline has yet proven model loading, output parsing, reproducibility, or contamination checks." if available else "Guard evaluation harness or strict scorer missing."}} + + def training_command(self, recipe: dict[str, Any], output_directory: str) -> list[str]: + dataset = str(recipe["training_data"]) + return ["python", "scripts/05_finetune.py", "--model", str(recipe.get("base_model", self.base_model)), "--training-data", dataset, "--output", output_directory] + + @staticmethod + def _hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() diff --git a/control_plane/model_studio/services.py b/control_plane/model_studio/services.py new file mode 100644 index 0000000..2ba9307 --- /dev/null +++ b/control_plane/model_studio/services.py @@ -0,0 +1,321 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import timedelta +from pathlib import Path +from typing import Any + +from django.db import transaction +from django.utils import timezone + +from control_plane.events.bus import EventBus +from control_plane.model_studio.backends import BackendResult, FakeTrainingBackend, TrainingBackend +from control_plane.model_studio.models import ( + BenchmarkResult, CheckpointType, CheckpointValidityStatus, Conclusion, Dataset, DatasetValidationStatus, + DatasetVersion, EvaluationRun, EvaluationRunStatus, EvaluationSuite, EvaluationSuiteVersion, ExperimentStatus, + FailureCluster, ModelCheckpoint, ModelPromotionDecision, ModelPromotionPolicy, ModelStudioArtifact, + OvernightResearchReport, OvernightTrainingProgram, ProgramStatus, PromotionDecision, TrainingExperiment, + TrainingProject, TrainingProjectStatus, TrainingRecipe, TrainingRun, TrainingRunStatus, +) +from control_plane.model_studio.profiles import GuardModelProfile, ModelProjectProfile +from control_plane.projects.models import Project, ProjectStatus + + +class ModelStudioService: + def __init__(self, *, profile: ModelProjectProfile | None = None, backend: TrainingBackend | None = None, bus: EventBus | None = None) -> None: + self.profile = profile or GuardModelProfile() + self.backend = backend or FakeTrainingBackend() + self.bus = bus or EventBus() + + def import_guard(self, *, project: Project | None = None, repository_path: str, spark_working_directory: str = "", slug: str = "guard-3b") -> TrainingProject: + project = project or Project.objects.create(name="Guard 3B Model Studio", project_type="MODEL", goal="Reconstruct and safely improve the Guard 3B model.", repository_path=repository_path, status=ProjectStatus.ARCHAEOLOGY) + training_project, _ = TrainingProject.objects.update_or_create(slug=slug, defaults={"project": project, "name": "Guard 3B", "description": "ForgeGuard Qwen2.5-Coder-3B model-development program.", "goal": "Improve Guard only through reproducible, benchmarked scientific experiments.", "capability_target": "Source-grounded smart-contract security findings.", "model_family": "Qwen2.5-Coder", "model_size": "3B", "base_model": GuardModelProfile.base_model, "repository_path": repository_path, "working_directory": spark_working_directory, "default_profile": self.profile.name, "training_backend": type(self.backend).__name__, "status": TrainingProjectStatus.IMPORTING, "metadata": {"spark_working_directory_verified": False, "local_archaeology_repository": repository_path}}) + self._event("TRAINING_PROJECT_IMPORTED", training_project, {"repository_path": repository_path}) + ModelPromotionPolicy.objects.get_or_create(training_project=training_project, name="Guard conservative V0.1", version="v0.1", defaults={"criteria": {"primary_metric": "strict_canonical_match", "minimum_delta": 0.0, "max_regression": {}, "requires_fresh_baseline": True, "note": "Thresholds remain unconfigured until the imported Guard benchmark exposes comparable metrics."}}) + return training_project + + def declare_base_champion(self, training_project: TrainingProject) -> ModelCheckpoint: + """Record the human-mandated starting base model without claiming benchmark evidence.""" + checkpoint, _ = ModelCheckpoint.objects.get_or_create(training_project=training_project, reference=training_project.base_model, defaults={"name": "Guard starting base model", "checkpoint_type": CheckpointType.CHAMPION, "content_hash": self._hash_text(training_project.base_model), "validity_status": CheckpointValidityStatus.UNKNOWN, "load_verified": False, "metadata": {"selection": "HUMAN_MANDATED_STARTING_CHAMPION", "evidence_status": "PENDING_FRESH_SPARK_BASELINE"}}) + training_project.current_champion = checkpoint + training_project.status = TrainingProjectStatus.BASELINING + training_project.save(update_fields=["current_champion", "status", "updated_at"]) + self._event("CHAMPION_SELECTED", training_project, {"checkpoint": str(checkpoint.id), "selection": "HUMAN_MANDATED_STARTING_CHAMPION", "evidence_status": "PENDING_FRESH_SPARK_BASELINE"}) + return checkpoint + + def archaeology(self, training_project: TrainingProject) -> dict[str, Any]: + training_project.status = TrainingProjectStatus.ARCHAEOLOGY + training_project.save(update_fields=["status", "updated_at"]) + self._event("ARCHAEOLOGY_STARTED", training_project, {}) + report = self.profile.archaeology(training_project.repository_path) + for item in report["datasets"]: + dataset, _ = Dataset.objects.get_or_create(training_project=training_project, name=item["name"]) + DatasetVersion.objects.update_or_create(dataset=dataset, version=item["hash"][:12], defaults={"manifest_reference": item["reference"], "content_hash": item["hash"], "record_count": item["record_count"], "source_metadata": {"archaeology_confidence": "CONFIRMED"}, "tags": item["tags"], "validation_status": DatasetValidationStatus.WARNING, "contamination_status": DatasetValidationStatus.UNKNOWN}) + for item in report["checkpoints"]: + ModelCheckpoint.objects.update_or_create(training_project=training_project, reference=item["reference"], defaults={"name": item["name"], "checkpoint_type": CheckpointType.IMPORTED, "content_hash": item["hash"], "adapter_type": item["adapter_type"], "validity_status": CheckpointValidityStatus.VALID, "load_verified": False, "metadata": {"archaeology_confidence": item["confidence"]}}) + suite, _ = EvaluationSuite.objects.get_or_create(training_project=training_project, name="Guard EVMBench") + benchmark_reference = report["benchmark_reference"] + suite_version, _ = EvaluationSuiteVersion.objects.update_or_create(suite=suite, version="imported-local", defaults={"reference": benchmark_reference, "content_hash": self._hash_text(benchmark_reference), "command_template": [report["evaluator_reference"]], "groups": ["PRIMARY", "HOLDOUT", "REGRESSION"], "integrity_status": DatasetValidationStatus.UNKNOWN, "integrity_evidence": {"archaeology": report["findings"]}}) + artifact = self._artifact(training_project, "TRAINING_ARCHAEOLOGY_REPORT", "Guard archaeology", report) + training_project.metadata = {**training_project.metadata, "archaeology_artifact": str(artifact.id), "historical_reports": report["historical_reports"], "evaluation_suite_version": str(suite_version.id)} + training_project.status = TrainingProjectStatus.NEEDS_REPAIR + training_project.save(update_fields=["metadata", "status", "updated_at"]) + self._event("ARCHAEOLOGY_COMPLETED", training_project, {"checkpoints": len(report["checkpoints"]), "datasets": len(report["datasets"]), "status": training_project.status}) + return report + + def validate_benchmark(self, training_project: TrainingProject) -> EvaluationSuiteVersion: + suite_version = EvaluationSuiteVersion.objects.filter(suite__training_project=training_project).order_by("-created_at").first() + if suite_version is None: + raise ValueError("Run archaeology before benchmark validation.") + result = self.profile.validate_evaluation_suite(training_project.repository_path, suite_version.reference) + suite_version.integrity_status = result["status"] + suite_version.integrity_evidence = result["evidence"] + suite_version.save(update_fields=["integrity_status", "integrity_evidence", "updated_at"]) + training_project.status = TrainingProjectStatus.READY if result["status"] == DatasetValidationStatus.VALID else TrainingProjectStatus.NEEDS_REPAIR + training_project.save(update_fields=["status", "updated_at"]) + return suite_version + + def curate_guard_datasets(self, training_project: TrainingProject) -> dict[str, Any]: + """Classify imported manifests without mutating source data or inferring missing provenance.""" + rows = [] + for version in DatasetVersion.objects.filter(dataset__training_project=training_project).select_related("dataset"): + path = Path(version.manifest_reference) + tags = set(version.tags) + status = DatasetValidationStatus.WARNING + contamination = DatasetValidationStatus.UNKNOWN + evidence: dict[str, Any] = {"manifest": str(path)} + if path.name.endswith("_summary.json"): + status, contamination = DatasetValidationStatus.BLOCKED, DatasetValidationStatus.UNKNOWN + tags.update(["summary", "not_training"]) + evidence["reason"] = "Summary artifacts are evidence, not trainable records." + else: + summary_path = path.with_name(path.stem + "_summary.json") + summary = self._read_json(summary_path) if summary_path.exists() else {} + payload = self._read_json(path) + record_count = len(payload) if isinstance(payload, list) else None + evidence["summary_reference"] = str(summary_path) if summary_path.exists() else "" + evidence["record_count_observed"] = record_count + if not isinstance(payload, list): + status = DatasetValidationStatus.BLOCKED + tags.update(["not_training", "invalid_manifest_shape"]) + evidence["reason"] = "Training manifest must be a JSON record list." + elif summary.get("candidate_only") or "candidate" in path.name.lower() or "provisional" in path.name.lower(): + status = DatasetValidationStatus.BLOCKED + tags.update(["candidate_only", "not_training"]) + evidence["reason"] = "Candidate/provisional corpus is explicitly not training-approved." + elif "evmbench" in path.name.lower() or summary.get("evmbench_source_included") is True or summary.get("benchmark_source_included") is True: + status = DatasetValidationStatus.BLOCKED + tags.update(["benchmark_exclusion", "not_training"]) + contamination = DatasetValidationStatus.WARNING + evidence["reason"] = "Possible benchmark reference requires manual contamination review." + elif summary.get("evmbench_source_included") is False and summary.get("weak_label_data_included") is False: + tags.update(["pilot", "source_disjoint_claimed"]) + status = DatasetValidationStatus.WARNING + contamination = DatasetValidationStatus.WARNING + evidence["reason"] = "Source-disjointness is declared, but full schema/provenance/coverage validation remains required." + else: + tags.update(["imported", "manual_provenance_review_required"]) + evidence["reason"] = "No sufficient adjacent evidence to mark this manifest training-valid." + if record_count is not None: + version.record_count = record_count + version.tags = sorted(tags) + version.validation_status = status + version.contamination_status = contamination + version.source_metadata = {**version.source_metadata, "curation": evidence} + version.save(update_fields=["record_count", "tags", "validation_status", "contamination_status", "source_metadata", "updated_at"]) + rows.append({"dataset": version.dataset.name, "version": version.version, "reference": version.manifest_reference, "validation_status": status, "contamination_status": contamination, "tags": version.tags, "evidence": evidence}) + summary = {"total": len(rows), "valid": sum(row["validation_status"] == DatasetValidationStatus.VALID for row in rows), "warning": sum(row["validation_status"] == DatasetValidationStatus.WARNING for row in rows), "blocked": sum(row["validation_status"] == DatasetValidationStatus.BLOCKED for row in rows), "datasets": rows, "decision": "NO_TRAINING_DATASET_APPROVED" if not any(row["validation_status"] == DatasetValidationStatus.VALID for row in rows) else "TRAINING_DATASET_CANDIDATES_AVAILABLE"} + self._artifact(training_project, "DATASET_CURATION_REPORT", "Guard dataset curation", summary) + return summary + + def establish_champion(self, training_project: TrainingProject, checkpoint: ModelCheckpoint) -> ModelCheckpoint: + suite = self.validate_benchmark(training_project) + if suite.integrity_status != DatasetValidationStatus.VALID: + raise ValueError("Benchmark integrity is not established; refusing Champion selection.") + if not checkpoint.load_verified: + raise ValueError("Candidate checkpoint has not passed load verification.") + evaluation = self.evaluate(training_project, checkpoint, suite, metrics={"primary": 0.0}, synthetic=False) + if evaluation.status != EvaluationRunStatus.SUCCEEDED: + raise ValueError("Champion evaluation failed.") + checkpoint.checkpoint_type = CheckpointType.CHAMPION + checkpoint.evaluation_status = "EVALUATED" + checkpoint.save(update_fields=["checkpoint_type", "evaluation_status", "updated_at"]) + training_project.current_champion = checkpoint + training_project.baseline_evaluation = evaluation + training_project.status = TrainingProjectStatus.READY + training_project.save(update_fields=["current_champion", "baseline_evaluation", "status", "updated_at"]) + self._event("CHAMPION_SELECTED", training_project, {"checkpoint": str(checkpoint.id), "evaluation": str(evaluation.id)}) + return checkpoint + + def evaluate(self, training_project: TrainingProject, checkpoint: ModelCheckpoint, suite: EvaluationSuiteVersion, *, metrics: dict[str, float] | None = None, synthetic: bool = False) -> EvaluationRun: + if suite.integrity_status != DatasetValidationStatus.VALID and not synthetic: + raise ValueError("Evaluation suite integrity is not valid.") + evaluation = EvaluationRun.objects.create(training_project=training_project, checkpoint=checkpoint, suite_version=suite, status=EvaluationRunStatus.RUNNING, integrity_evidence=suite.integrity_evidence) + for metric, value in (metrics or {}).items(): + BenchmarkResult.objects.create(evaluation_run=evaluation, metric=metric, value=value, unit="score") + evaluation.status = EvaluationRunStatus.SUCCEEDED + evaluation.completed_at = timezone.now() + evaluation.summary = metrics or {} + evaluation.save(update_fields=["status", "completed_at", "summary", "updated_at"]) + self._event("EVALUATION_COMPLETED", training_project, {"checkpoint": str(checkpoint.id), "evaluation": str(evaluation.id)}) + return evaluation + + def propose_experiment(self, training_project: TrainingProject, contract: dict[str, Any]) -> TrainingExperiment: + required = ["hypothesis", "reasoning", "intervention", "controls", "expected_result", "primary_success_metric", "success_threshold", "regression_constraints", "rejection_condition", "ambiguity_policy", "maximum_runtime_seconds", "compute_budget"] + missing = [field for field in required if contract.get(field) in (None, "", {}, [])] + if missing: + raise ValueError("Incomplete scientific contract: " + ", ".join(missing)) + fingerprint = self._fingerprint({"intervention": contract["intervention"], "controls": contract["controls"], "input_checkpoint": str(training_project.current_champion_id)}) + duplicate = TrainingExperiment.objects.filter(training_project=training_project, fingerprint=fingerprint).exclude(status=ExperimentStatus.CANCELLED).first() + if duplicate: + raise ValueError(f"Duplicate experiment: {duplicate.experiment_id}") + value = self._experiment_value(contract) + experiment = TrainingExperiment.objects.create(experiment_id=contract.get("experiment_id", f"EXP-{training_project.slug.upper()}-{TrainingExperiment.objects.filter(training_project=training_project).count() + 1:03d}"), training_project=training_project, title=contract.get("title", contract["hypothesis"][:255]), hypothesis=contract["hypothesis"], reasoning=contract["reasoning"], intervention=contract["intervention"], controls=contract["controls"], expected_result=contract["expected_result"], primary_success_metric=contract["primary_success_metric"], success_threshold=contract["success_threshold"], regression_constraints=contract["regression_constraints"], rejection_condition=contract["rejection_condition"], ambiguity_policy=contract["ambiguity_policy"], estimated_runtime_seconds=int(contract.get("estimated_runtime_seconds", 0)), maximum_runtime_seconds=int(contract["maximum_runtime_seconds"]), compute_budget=contract["compute_budget"], priority=float(contract.get("priority", value)), expected_information_gain=float(contract.get("expected_information_gain", 0)), expected_improvement=float(contract.get("expected_improvement", 0)), estimated_compute_cost=float(contract.get("estimated_compute_cost", 1)), experiment_value_score=value, fingerprint=fingerprint, approved_by_model_director=True) + self._event("EXPERIMENT_PROPOSED", training_project, {"experiment": experiment.experiment_id, "value": value}) + return experiment + + @transaction.atomic + def run_experiment(self, experiment: TrainingExperiment, recipe_configuration: dict[str, Any]) -> TrainingRun: + project = experiment.training_project + champion = project.current_champion + if champion is None: + raise ValueError("Cannot train without an immutable Champion.") + if type(self.backend).__name__ == "SparkGuardBackend" and not project.metadata.get("spark_working_directory_verified"): + raise ValueError("Spark Guard working directory is not verified; refusing remote training.") + if experiment.status not in {ExperimentStatus.PROPOSED, ExperimentStatus.QUEUED}: + raise ValueError("Experiment is not runnable.") + recipe_hash = self._fingerprint(recipe_configuration) + recipe, _ = TrainingRecipe.objects.get_or_create(training_project=project, recipe_hash=recipe_hash, defaults={"name": experiment.experiment_id, "configuration": recipe_configuration}) + recipe.immutable = True + recipe.save(update_fields=["immutable", "updated_at"]) + for dataset_version in DatasetVersion.objects.filter(dataset__training_project=project, manifest_reference__in=recipe_configuration.get("dataset_references", [])): + dataset_version.immutable = True + dataset_version.save(update_fields=["immutable", "updated_at"]) + run = TrainingRun.objects.create(experiment=experiment, recipe=recipe, input_checkpoint=champion, status=TrainingRunStatus.STARTING, working_directory=project.working_directory, command=[]) + experiment.status = ExperimentStatus.RUNNING + experiment.save(update_fields=["status", "updated_at"]) + self._event("TRAINING_STARTED", project, {"experiment": experiment.experiment_id, "run": str(run.id)}) + output_directory = str(Path(project.working_directory) / "artifex_runs" / experiment.experiment_id) + command = self.profile.training_command(recipe_configuration, output_directory) if recipe_configuration.get("training_data") and hasattr(self.profile, "training_command") else [] + run.command = command + run.save(update_fields=["command", "updated_at"]) + outcome = self.backend.launch(command=command, working_directory=project.working_directory, timeout_seconds=experiment.maximum_runtime_seconds) + self._apply_training_outcome(run, outcome) + return run + + def _apply_training_outcome(self, run: TrainingRun, outcome: BackendResult) -> None: + experiment = run.experiment + project = experiment.training_project + if outcome.status != "SUCCEEDED": + run.status = getattr(TrainingRunStatus, outcome.status, TrainingRunStatus.FAILED) + run.failure_category = outcome.failure_category or outcome.status + run.failure_details = outcome.failure_details + run.completed_at = timezone.now() + run.save(update_fields=["status", "failure_category", "failure_details", "completed_at", "updated_at"]) + experiment.status = ExperimentStatus.FAILED + experiment.conclusion = Conclusion.EXECUTION_FAILED + experiment.result_summary = run.failure_details + experiment.save(update_fields=["status", "conclusion", "result_summary", "updated_at"]) + self._event("TRAINING_FAILED", project, {"experiment": experiment.experiment_id, "failure": run.failure_category}) + return + checkpoint = ModelCheckpoint.objects.create(training_project=project, name=f"{experiment.experiment_id}-challenger", checkpoint_type=CheckpointType.CHALLENGER, reference=outcome.checkpoint_reference, content_hash=outcome.checkpoint_hash, base_checkpoint=run.input_checkpoint, training_run=run, recipe=run.recipe, validity_status=CheckpointValidityStatus.VALID if self.backend.validate_checkpoint(outcome.checkpoint_reference) else CheckpointValidityStatus.CORRUPT, load_verified=self.backend.validate_checkpoint(outcome.checkpoint_reference)) + run.output_checkpoint = checkpoint + run.status = TrainingRunStatus.SUCCEEDED + run.completed_at = timezone.now() + run.save(update_fields=["output_checkpoint", "status", "completed_at", "updated_at"]) + experiment.status = ExperimentStatus.EVALUATING + experiment.save(update_fields=["status", "updated_at"]) + self._event("TRAINING_COMPLETED", project, {"experiment": experiment.experiment_id, "checkpoint": str(checkpoint.id)}) + + def decide_promotion(self, experiment: TrainingExperiment, evaluation: EvaluationRun, policy: ModelPromotionPolicy) -> ModelPromotionDecision: + project = experiment.training_project + champion = project.current_champion + candidate = evaluation.checkpoint + if champion is None or candidate.training_run_id is None: + raise ValueError("Promotion requires a Challenger and starting Champion.") + baseline = project.baseline_evaluation + if baseline is None or baseline.suite_version_id != evaluation.suite_version_id: + raise ValueError("Champion and Challenger must use the same evaluation version.") + primary = policy.criteria.get("primary_metric", "primary") + minimum_delta = float(policy.criteria.get("minimum_delta", 0)) + candidate_value = float(evaluation.summary.get(primary, 0)) + baseline_value = float(baseline.summary.get(primary, 0)) + regression_ok = all(float(evaluation.summary.get(metric, 0)) >= float(baseline.summary.get(metric, 0)) - float(limit) for metric, limit in policy.criteria.get("max_regression", {}).items()) + eligible = candidate.load_verified and evaluation.status == EvaluationRunStatus.SUCCEEDED and candidate_value - baseline_value >= minimum_delta and regression_ok + decision = PromotionDecision.PROMOTE if eligible else PromotionDecision.REJECT + rationale = "Objective promotion criteria satisfied." if eligible else "Objective promotion criteria not satisfied." + record = ModelPromotionDecision.objects.create(training_project=project, from_champion=champion, candidate=candidate, experiment=experiment, decision=decision, policy=policy, evaluation_evidence={"baseline": baseline.summary, "candidate": evaluation.summary, "delta": candidate_value - baseline_value, "regression_ok": regression_ok}, judge_result={"actor": "MODEL_JUDGE", "deterministic_gate": eligible}, reason=rationale) + if eligible: + champion.checkpoint_type = CheckpointType.IMPORTED + champion.save(update_fields=["checkpoint_type", "updated_at"]) + candidate.checkpoint_type = CheckpointType.CHAMPION + candidate.save(update_fields=["checkpoint_type", "updated_at"]) + project.current_champion = candidate + project.save(update_fields=["current_champion", "updated_at"]) + experiment.status, experiment.conclusion = ExperimentStatus.PROMOTED, Conclusion.SUPPORTED + self._event("CHALLENGER_PROMOTED", project, {"experiment": experiment.experiment_id, "checkpoint": str(candidate.id)}) + else: + experiment.status, experiment.conclusion = ExperimentStatus.REJECTED, Conclusion.REFUTED + experiment.result_summary = rationale + experiment.save(update_fields=["status", "conclusion", "result_summary", "updated_at"]) + return record + + def create_program(self, training_project: TrainingProject, policy: ModelPromotionPolicy, *, wall_seconds: int = 8 * 3600, max_runs: int = 8, max_failed: int = 3, max_single_run: int = 150 * 60, evaluation_reserve: int = 90 * 60) -> OvernightTrainingProgram: + if training_project.current_champion is None: + raise ValueError("A verified Champion is required before starting an overnight program.") + now = timezone.now() + return OvernightTrainingProgram.objects.create(training_project=training_project, starting_champion=training_project.current_champion, deadline=now + timedelta(seconds=wall_seconds), maximum_wall_seconds=wall_seconds, maximum_training_runs=max_runs, maximum_failed_runs=max_failed, maximum_single_run_seconds=max_single_run, evaluation_reserve_seconds=evaluation_reserve, allowed_experiment_types=["DATASET_MIXTURE", "RECIPE", "CHECKPOINT"], promotion_policy=policy) + + def can_start(self, program: OvernightTrainingProgram, experiment: TrainingExperiment, estimated_evaluation_seconds: int) -> tuple[bool, str]: + remaining = max(0, int((program.deadline - timezone.now()).total_seconds())) + runs = TrainingRun.objects.filter(experiment__training_project=program.training_project).count() + failures = TrainingRun.objects.filter(experiment__training_project=program.training_project, status__in=[TrainingRunStatus.FAILED, TrainingRunStatus.OOM, TrainingRunStatus.TIMEOUT]).count() + needed = experiment.estimated_runtime_seconds + estimated_evaluation_seconds + program.evaluation_reserve_seconds + if runs >= program.maximum_training_runs: + return False, "RUN_LIMIT" + if failures >= program.maximum_failed_runs: + return False, "FAILURE_LIMIT" + if needed > remaining: + return False, "FINAL_EVALUATION_RESERVE" + return True, "READY" + + def morning_report(self, program: OvernightTrainingProgram) -> OvernightResearchReport: + project = program.training_project + experiments = list(project.experiments.order_by("created_at")) + payload = {"program_id": str(program.id), "starting_champion": str(program.starting_champion_id), "ending_champion": str(project.current_champion_id), "status": "NO_CHAMPION_CHANGE" if project.current_champion_id == program.starting_champion_id else "CHAMPION_CHANGED", "experiments": [{"id": item.experiment_id, "hypothesis": item.hypothesis, "status": item.status, "conclusion": item.conclusion, "learning": item.result_summary} for item in experiments], "provenance": {"repository": project.repository_path, "baseline_evaluation": str(project.baseline_evaluation_id or "")}} + markdown = "# GUARD OVERNIGHT RESEARCH REPORT\n\n" + json.dumps(payload, indent=2, default=str) + program.ending_champion = project.current_champion + program.status = ProgramStatus.COMPLETED + program.completed_at = timezone.now() + program.termination_reason = program.termination_reason or "FINALIZED" + program.save(update_fields=["ending_champion", "status", "completed_at", "termination_reason", "updated_at"]) + report, _ = OvernightResearchReport.objects.update_or_create(program=program, defaults={"markdown": markdown, "payload": payload}) + self._artifact(project, "OVERNIGHT_RESEARCH_REPORT", "Morning research report", payload, markdown) + self._event("OVERNIGHT_COMPLETED", project, {"program": str(program.id), "status": payload["status"]}) + return report + + def _artifact(self, training_project: TrainingProject, artifact_type: str, name: str, content: dict[str, Any], readable: str = "") -> ModelStudioArtifact: + return ModelStudioArtifact.objects.create(training_project=training_project, artifact_type=artifact_type, name=name, content=content, readable=readable) + + def _event(self, event_type: str, training_project: TrainingProject, payload: dict[str, Any]) -> None: + self.bus.publish(event_type, project=training_project.project, actor="MODEL_STUDIO", payload={"training_project": str(training_project.id), **payload}) + + @staticmethod + def _hash_text(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + @staticmethod + def _read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + + @staticmethod + def _fingerprint(value: dict[str, Any]) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest() + + @staticmethod + def _experiment_value(contract: dict[str, Any]) -> float: + return round(float(contract.get("expected_improvement", 0)) * float(contract.get("confidence", 1)) * float(contract.get("expected_information_gain", 1)) * float(contract.get("novelty", 1)) / max(1.0, float(contract.get("estimated_compute_cost", 1))), 4) diff --git a/control_plane/model_studio/views.py b/control_plane/model_studio/views.py new file mode 100644 index 0000000..b7f7f91 --- /dev/null +++ b/control_plane/model_studio/views.py @@ -0,0 +1,12 @@ +from django.shortcuts import get_object_or_404, render + +from control_plane.model_studio.models import TrainingProject + + +def model_studio(request): + return render(request, "control_plane/model_studio.html", {"projects": TrainingProject.objects.select_related("current_champion", "baseline_evaluation").order_by("-updated_at")}) + + +def model_studio_project(request, project_id): + training_project = get_object_or_404(TrainingProject.objects.select_related("current_champion", "baseline_evaluation"), id=project_id) + return render(request, "control_plane/model_studio_project.html", {"training_project": training_project, "program": training_project.overnight_programs.order_by("-created_at").first(), "experiments": training_project.experiments.order_by("-created_at")[:25], "clusters": training_project.failure_clusters.order_by("-priority")[:25]}) diff --git a/docs/handoff_current_state_20260817.md b/docs/handoff_current_state_20260817.md new file mode 100644 index 0000000..120a435 --- /dev/null +++ b/docs/handoff_current_state_20260817.md @@ -0,0 +1,166 @@ +# Artifex Handoff - Current State - 2026-08-17 + +## Current Objective + +We have been repairing and dogfooding the `CRYPTO / PROTOCOL VENTURE COHORT` pipeline, specifically the V0.3.1 cohort and the V0.3.2B token-judge evidence repair. + +Hard constraint: **NO NEW IDEAS** for the repair work. Only reprocess the existing six former V0.3.1 survivors. + +Safety constraints remain active: + +- Do not issue or sell native tokens. +- Do not sell NFTs or memberships. +- Do not launch mainnet. +- Do not create liquidity or market make. +- Do not contact investors, users, or customers. +- Do not fundraise. +- Do not spend real money. + +## Environment + +- Workspace: `S:\PycharmProjects\Artifex` +- Repo remote: `http://192.168.1.162:3000/daniel/Artifex.git` +- Database: `sqlite:///db.sqlite3` +- SearXNG: `http://192.168.1.162:8080` +- Qwen: `http://192.168.1.162:8002` +- Current date in environment: `2026-08-17` +- Known harmless push warning: `git: 'credential-manager-core' is not a git command. See 'git --help'.` + +Spark remote validation was previously blocked by SSH auth: + +- `daniel@192.168.1.162: Permission denied (publickey,password).` + +## Current Cohort + +- Cohort ID: `CPV031-20260816215626-66d636b2` +- GraphRun: `10` +- Existing six former V0.3.1 survivors only: + - `Agent Passport Clearinghouse` + - `PatchBond Network` + - `ProofGrid Compute Attestation Network` + - `ProofBond` + - `Compute Clearance Network` + - `ProofGrid Compute Escrow` + +## Latest Implemented Work + +### V0.3.2B Token Judge Evidence Repair + +Implemented strict Sol Token Utility Judge evidence validation. + +Required non-empty fields now enforced: + +- `argument_for_native_token` +- `argument_against_native_token` +- `external_collateral_counterfactual` +- `native_token_removed_breaks` +- `native_token_removed_explanation` +- `material_improvements_over_USDC_ETH` +- `native_asset_costs_and_risks` +- `final_rationale` + +Behavior now: + +- Rejects incomplete Sol judge response. +- Retries incomplete judge response up to 3 attempts. +- Raises `ValueError` after exhausted retries. +- Does **not** fall back to deterministic/heuristic classification when Sol produced an incomplete response. +- Normalizes Sol response when `external_collateral_counterfactual` appears nested under `counterfactual_analysis`. +- Uppercases `utility_scores` keys during normalization. +- Persists `final_rationale` in token metadata as well as `TokenUtilityAssessment.rationale`. +- `crypto_score_row()` now reuses persisted `token_utility_assessment` instead of calling the token judge again. +- Final Sol IC prompt was compacted so it returns usable JSON. + +Rationale invariants now checked: + +- Every `TOKEN_OPTIONAL` rationale must explain why ETH/USDC/external collateral is preferable despite claimed degradation. +- Every `TOKEN_ESSENTIAL` / `TOKEN_STRONGLY_JUSTIFIED` rationale must explain what protocol-specific property cannot be reproduced economically enough with ETH/USDC/external collateral. +- The code does not automatically promote due to material degradation. + +## Latest Six-Proposal Results + +Before/after classifications from V0.3.2B: + +| Company | Before | After | Score | Final IC Decision | Final IC Score | Required Field Gaps | +|---|---:|---:|---:|---:|---:|---:| +| Agent Passport Clearinghouse | `TOKEN_OPTIONAL` | `TOKEN_UNNECESSARY` | `18.0` | `REJECT_TOKEN_NOT_NEEDED` | `62.3` | `[]` | +| PatchBond Network | `TOKEN_OPTIONAL` | `TOKEN_OPTIONAL` | `48.0` | `REJECT_TOKEN_NOT_NEEDED` | `66.4` | `[]` | +| ProofGrid Compute Attestation Network | `TOKEN_OPTIONAL` | `TOKEN_OPTIONAL` | `42.0` | `REJECT_TOKEN_UTILITY_WEAK` | `63.6` | `[]` | +| ProofBond | `TOKEN_OPTIONAL` | `TOKEN_OPTIONAL` | `54.0` | `REJECT_TOKEN_UTILITY_WEAK` | `64.3` | `[]` | +| Compute Clearance Network | `TOKEN_OPTIONAL` | `TOKEN_STRONGLY_JUSTIFIED` | `78.0` | `PROTOCOL_VALIDATE` | `79.7` | `[]` | +| ProofGrid Compute Escrow | `TOKEN_OPTIONAL` | `TOKEN_OPTIONAL` | `58.0` | `REVISE_TOKEN_MODEL` | `67.9` | `[]` | + +Current corrected cohort state: + +- Token unnecessary: `1` +- Token optional/routed SaaS: `4` +- Token strongly justified: `1` +- Token essential: `0` +- Serious crypto survivors: `0` +- Crypto survivors: `0` +- Autonomous crypto survivors: `0` +- Assisted high-potential: `0` +- Finalists: `0` +- V0.3.2B required field gaps: `0` +- Token red-team failures: + - `TOKEN_NOT_REQUIRED: 5` + - `YIELD_DEPENDENCY: 1` + - `SLASHING_DEPENDS_ON_HUMAN_JUDGMENT: 1` + +Important nuance: `Compute Clearance Network` became `TOKEN_STRONGLY_JUSTIFIED`, but final cohort finalists remain `0`. It is not automatically promoted to finalist. + +## Latest Files Changed + +- `agents/crypto_venture.py` + - Strict token judge evidence validation/retry/fail-closed behavior. + - Token judge response normalization. + - Persisted required fields. + - Reused persisted token assessment in `crypto_score_row()`. + - Compact final Sol IC prompt. +- `tests/test_crypto_venture_cohort.py` + - Added coverage for complete required judge fields. + - Added retry-success and retry-exhaustion/fail-closed tests. +- `control_plane/ventures/management/commands/export_crypto_venture_cohort.py` + - Added `V0.3.2B Before/After` markdown section. +- `docs/crypto_venture_cohort_v031_20260816.md` + - Updated exported V0.3.2B report. +- `docs/crypto_venture_cohort_v031_20260816.json` + - Updated exported V0.3.2B report data. + +## Latest Commits Pushed + +Recent relevant commits: + +- `35d3346 Repair crypto token judge evidence export` +- `6e89ea8 Require complete crypto token judge evidence` +- `052d1fc Add corrected crypto cohort V0.3.1 export` +- `94b9ab6 Repair crypto token utility scoring` +- `54ffe92 Add crypto protocol cohort V0.3.1 export` + +The most recent push succeeded to `main` with the known harmless credential-manager warning. + +## Verification + +Latest verification passed: + +- `python manage.py check`: passed +- `python -m pytest -p no:django -q tests/test_crypto_venture_cohort.py`: `14 passed` +- Full suite: `157 passed, 1 skipped` + +Warnings are existing Python/Django/LangGraph deprecation warnings and pytest collection warnings for `TestRun`; no failing tests. + +## Current Git State At Time Of Handoff + +After the latest commit/push, `git status --short` was clean. + +This handoff file itself is newly added after that push and should be committed/pushed if the user wants the note persisted remotely. + +## If Continuing + +Recommended next steps only if the user asks to continue: + +1. Commit and push this handoff note if desired. +2. If further V0.3.2B analysis is requested, inspect `docs/crypto_venture_cohort_v031_20260816.md` first. +3. Do not regenerate ideas unless explicitly instructed. +4. Do not rerun broad research unless explicitly requested or needed for a new judge repair. +5. Keep all token/mainnet/fundraising/user-contact/no-spend stop conditions active. diff --git a/templates/control_plane/base.html b/templates/control_plane/base.html index 738f2e5..3baa7d5 100644 --- a/templates/control_plane/base.html +++ b/templates/control_plane/base.html @@ -46,6 +46,7 @@

ARTIFEX

Dashboard Projects + Model Studio Agents Progeny Steward diff --git a/templates/control_plane/model_studio.html b/templates/control_plane/model_studio.html new file mode 100644 index 0000000..b70719c --- /dev/null +++ b/templates/control_plane/model_studio.html @@ -0,0 +1,6 @@ +{% extends "control_plane/base.html" %} +{% block title %}Model Studio | Artifex{% endblock %} +{% block content %} +

Model Studio

Archaeology, reproducible experiments, and bounded overnight research.

+{% for item in projects %}{% empty %}{% endfor %}
Training ProjectStatusChampionProfile
{{ item.name }}{{ item.status }}{{ item.current_champion.name|default:"Not established" }}{{ item.default_profile }}
No Model Studio projects imported.
+{% endblock %} diff --git a/templates/control_plane/model_studio_project.html b/templates/control_plane/model_studio_project.html new file mode 100644 index 0000000..24fe239 --- /dev/null +++ b/templates/control_plane/model_studio_project.html @@ -0,0 +1,8 @@ +{% extends "control_plane/base.html" %} +{% block title %}{{ training_project.name }} | Model Studio{% endblock %} +{% block content %} +

{{ training_project.name }}

{{ training_project.capability_target }}

{{ training_project.status }}
+

Current Champion

{{ training_project.current_champion.name|default:"Not established" }}

Baseline

{{ training_project.baseline_evaluation.status|default:"Not run" }}

Overnight

{{ program.status|default:"Not scheduled" }}

+

Experiments

{% for item in experiments %}{% empty %}{% endfor %}
IDHypothesisStatusValue
{{ item.experiment_id }}{{ item.hypothesis }}{{ item.status }}{{ item.experiment_value_score }}
No experiments proposed.
+

Failure Clusters

{% for item in clusters %}

{{ item.name }}: {{ item.description }}

{% empty %}

Baseline failure mining has not produced clusters yet.

{% endfor %}
+{% endblock %} diff --git a/tests/test_model_studio_v01.py b/tests/test_model_studio_v01.py new file mode 100644 index 0000000..f0322df --- /dev/null +++ b/tests/test_model_studio_v01.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from datetime import timedelta +import json + +import pytest +from django.utils import timezone + +from control_plane.model_studio.backends import FakeTrainingBackend +from control_plane.model_studio.models import ( + CheckpointType, CheckpointValidityStatus, Dataset, DatasetVersion, DatasetValidationStatus, + EvaluationSuite, EvaluationSuiteVersion, ExperimentStatus, ModelCheckpoint, ModelPromotionPolicy, + PromotionDecision, TrainingProject, TrainingProjectStatus, +) +from control_plane.model_studio.services import ModelStudioService +from control_plane.projects.models import Project + + +def studio(outcomes=None): + return ModelStudioService(backend=FakeTrainingBackend(outcomes)) + + +def ready_project(): + project = Project.objects.create(name="Guard", project_type="MODEL", goal="test") + training_project = TrainingProject.objects.create(project=project, name="Guard", slug=f"guard-{project.id.hex[:8]}", goal="test", status=TrainingProjectStatus.READY) + dataset = Dataset.objects.create(training_project=training_project, name="train") + version = DatasetVersion.objects.create(dataset=dataset, version="v1", manifest_reference="fake://train", content_hash="a" * 64, record_count=2, validation_status=DatasetValidationStatus.VALID) + checkpoint = ModelCheckpoint.objects.create(training_project=training_project, name="champion", checkpoint_type=CheckpointType.CHAMPION, reference="fake://champion", content_hash="b" * 64, validity_status=CheckpointValidityStatus.VALID, load_verified=True) + suite = EvaluationSuite.objects.create(training_project=training_project, name="suite") + suite_version = EvaluationSuiteVersion.objects.create(suite=suite, version="v1", reference="fake://suite", content_hash="c" * 64, integrity_status=DatasetValidationStatus.VALID, immutable=True) + training_project.current_champion = checkpoint + training_project.save(update_fields=["current_champion", "updated_at"]) + service = studio() + baseline = service.evaluate(training_project, checkpoint, suite_version, metrics={"primary": 0.5, "critical": 0.5}, synthetic=True) + training_project.baseline_evaluation = baseline + training_project.save(update_fields=["baseline_evaluation", "updated_at"]) + policy = ModelPromotionPolicy.objects.create(training_project=training_project, name="test", version="v1", criteria={"primary_metric": "primary", "minimum_delta": 0.1, "max_regression": {"critical": 0.02}}) + return service, training_project, version, suite_version, policy + + +def contract(**overrides): + payload = {"hypothesis": "Targeted replay improves primary metric.", "reasoning": "A baseline failure cluster supports this.", "intervention": {"learning_rate": 0.0001}, "controls": {"same_champion": True}, "expected_result": "primary +0.1", "primary_success_metric": "primary", "success_threshold": {"minimum_delta": 0.1}, "regression_constraints": {"critical": 0.02}, "rejection_condition": "No meaningful improvement", "ambiguity_policy": "replicate", "maximum_runtime_seconds": 120, "compute_budget": {"gpu_seconds": 120}, "expected_improvement": 0.2, "expected_information_gain": 0.8, "estimated_compute_cost": 1, "novelty": 1, "confidence": 0.8, "estimated_runtime_seconds": 60} + return {**payload, **overrides} + + +def test_recipe_and_completed_dataset_version_are_immutable(): + service, training_project, dataset_version, _, _ = ready_project() + experiment = service.propose_experiment(training_project, contract()) + run = service.run_experiment(experiment, {"dataset_references": ["fake://train"]}) + assert run.recipe.immutable is True + dataset_version.refresh_from_db() + assert dataset_version.immutable is True + run.recipe.configuration = {"changed": True} + with pytest.raises(ValueError, match="TrainingRecipe is immutable"): + run.recipe.save() + dataset_version.record_count = 3 + with pytest.raises(ValueError, match="DatasetVersion is immutable"): + dataset_version.save() + + +def test_incomplete_scientific_contract_and_duplicate_experiment_are_rejected(): + service, training_project, _, _, _ = ready_project() + with pytest.raises(ValueError, match="Incomplete scientific contract"): + service.propose_experiment(training_project, {"hypothesis": "thin"}) + service.propose_experiment(training_project, contract()) + with pytest.raises(ValueError, match="Duplicate experiment"): + service.propose_experiment(training_project, contract()) + + +def test_failed_execution_is_not_hypothesis_rejection(): + service, training_project, _, _, _ = ready_project() + service.backend = FakeTrainingBackend([{"status": "OOM", "failure_category": "OOM", "failure_details": "simulated"}]) + experiment = service.propose_experiment(training_project, contract()) + run = service.run_experiment(experiment, {}) + experiment.refresh_from_db() + assert run.status == "OOM" + assert experiment.status == ExperimentStatus.FAILED + assert experiment.conclusion == "EXECUTION_FAILED" + + +def test_challenger_promotion_requires_same_evaluation_version_and_objective_gate(): + service, training_project, _, suite_version, policy = ready_project() + service.backend = FakeTrainingBackend([{"status": "SUCCEEDED", "checkpoint_reference": "fake://challenger"}]) + experiment = service.propose_experiment(training_project, contract()) + run = service.run_experiment(experiment, {}) + evaluation = service.evaluate(training_project, run.output_checkpoint, suite_version, metrics={"primary": 0.7, "critical": 0.5}, synthetic=True) + decision = service.decide_promotion(experiment, evaluation, policy) + training_project.refresh_from_db() + assert decision.decision == PromotionDecision.PROMOTE + assert training_project.current_champion_id == run.output_checkpoint_id + + +def test_regression_prevents_promotion_and_deadline_reserve_blocks_start(): + service, training_project, _, suite_version, policy = ready_project() + service.backend = FakeTrainingBackend([{"status": "SUCCEEDED", "checkpoint_reference": "fake://challenger"}]) + experiment = service.propose_experiment(training_project, contract()) + run = service.run_experiment(experiment, {}) + evaluation = service.evaluate(training_project, run.output_checkpoint, suite_version, metrics={"primary": 0.8, "critical": 0.4}, synthetic=True) + decision = service.decide_promotion(experiment, evaluation, policy) + assert decision.decision == PromotionDecision.REJECT + program = service.create_program(training_project, policy, wall_seconds=60, evaluation_reserve=30) + program.deadline = timezone.now() + timedelta(seconds=60) + program.save(update_fields=["deadline", "updated_at"]) + allowed, reason = service.can_start(program, experiment, estimated_evaluation_seconds=30) + assert allowed is False + assert reason == "FINAL_EVALUATION_RESERVE" + + +def test_dataset_curation_respects_explicit_no_evmbench_claim(tmp_path): + service, training_project, _, _, _ = ready_project() + manifest = tmp_path / "verified_pilot.json" + manifest.write_text(json.dumps([{"input": "pragma solidity ^0.8.0;", "output": "{}"}]), encoding="utf-8") + summary = tmp_path / "verified_pilot_summary.json" + summary.write_text(json.dumps({"evmbench_source_included": False, "weak_label_data_included": False}), encoding="utf-8") + dataset = Dataset.objects.create(training_project=training_project, name="verified-pilot") + version = DatasetVersion.objects.create(dataset=dataset, version="pilot", manifest_reference=str(manifest), content_hash="d" * 64) + + report = service.curate_guard_datasets(training_project) + + version.refresh_from_db() + assert version.validation_status == DatasetValidationStatus.WARNING + assert version.contamination_status == DatasetValidationStatus.WARNING + assert report["valid"] == 0