daniel/Artifex
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add Guard model studio foundation

Browse source
This commit is contained in:
Daniel Maddern 2026-08-17 01:04:02 +07:00
parent 35d3346042
commit d61751a099
21 changed files with 1811 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
artifex/settings.py
View file

@ -22,6 +22,7 @@ INSTALLED_APPS = [
"control_plane.agents",
"control_plane.resources",
"control_plane.ventures",
"control_plane.model_studio",
"control_plane.secrets",
"control_plane.knowledge",
"control_plane.verification",

3
artifex/urls.py
View file

@ -4,6 +4,7 @@ from django.contrib import admin
from django.urls import path
from control_plane.projects import views
from control_plane.model_studio import views as model_studio_views
urlpatterns = [
path("", views.dashboard, name="dashboard"),
@ -32,6 +33,8 @@ urlpatterns = [
path("agents/<uuid:version_id>/", views.agent_detail, name="agent_detail"),
path("agents/<uuid:version_id>/performance.json", views.agent_performance_json, name="agent_performance_json"),
path("resources/", views.resources, name="resources"),
path("model-studio/", model_studio_views.model_studio, name="model_studio"),
path("model-studio/<uuid:project_id>/", model_studio_views.model_studio_project, name="model_studio_project"),
path("approvals/", views.approvals, name="approvals"),
path("approvals/<int:approval_id>/action/", views.approval_action, name="approval_action"),
path("activity/", views.activity, name="activity"),

0
control_plane/model_studio/__init__.py Normal file
View file

6
control_plane/model_studio/apps.py Normal file
View file

@ -0,0 +1,6 @@
from django.apps import AppConfig
class ModelStudioConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "control_plane.model_studio"

90
control_plane/model_studio/backends.py Normal file
View file

@ -0,0 +1,90 @@
from __future__ import annotations
import hashlib
import json
import subprocess
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Protocol
@dataclass
class BackendResult:
status: str
checkpoint_reference: str = ""
checkpoint_hash: str = ""
metrics: dict[str, float] | None = None
stdout: str = ""
stderr: str = ""
failure_category: str = ""
failure_details: str = ""
class TrainingBackend(Protocol):
def estimate_runtime(self, recipe: dict[str, Any]) -> int: ...
def launch(self, *, command: list[str], working_directory: str, timeout_seconds: int) -> BackendResult: ...
def validate_checkpoint(self, reference: str) -> bool: ...
class FakeTrainingBackend:
"""Deterministic backend for workflow tests; never launches training."""
def __init__(self, outcomes: list[dict[str, Any]] | None = None) -> None:
self.outcomes = list(outcomes or [{"status": "SUCCEEDED", "metrics": {"primary": 0.75}}])
def estimate_runtime(self, recipe: dict[str, Any]) -> int:
return int(recipe.get("estimated_runtime_seconds", 60))
def launch(self, *, command: list[str], working_directory: str, timeout_seconds: int) -> BackendResult:
outcome = self.outcomes.pop(0) if self.outcomes else {"status": "SUCCEEDED", "metrics": {"primary": 0.75}}
status = str(outcome.get("status", "SUCCEEDED"))
reference = str(outcome.get("checkpoint_reference", f"fake://{hashlib.sha256(json.dumps(outcome, sort_keys=True).encode()).hexdigest()[:16]}"))
return BackendResult(status=status, checkpoint_reference=reference, checkpoint_hash=hashlib.sha256(reference.encode()).hexdigest(), metrics=outcome.get("metrics", {}), failure_category=str(outcome.get("failure_category", "")), failure_details=str(outcome.get("failure_details", "")))
def validate_checkpoint(self, reference: str) -> bool:
return reference.startswith("fake://")
class GuardSubprocessBackend:
"""Scoped wrapper for the discovered Guard trainer; commands come from profiles, never an LLM."""
def estimate_runtime(self, recipe: dict[str, Any]) -> int:
return int(recipe.get("estimated_runtime_seconds", 90 * 60))
def launch(self, *, command: list[str], working_directory: str, timeout_seconds: int) -> BackendResult:
try:
completed = subprocess.run(command, cwd=working_directory, capture_output=True, text=True, timeout=timeout_seconds, check=False)
except subprocess.TimeoutExpired as exc:
return BackendResult(status="TIMEOUT", stdout=exc.stdout or "", stderr=exc.stderr or "", failure_category="TIMEOUT", failure_details=f"Exceeded {timeout_seconds}s")
stdout, stderr = completed.stdout or "", completed.stderr or ""
if completed.returncode:
category = "OOM" if "out of memory" in (stdout + stderr).lower() else "PROCESS_FAILURE"
return BackendResult(status=category if category == "OOM" else "FAILED", stdout=stdout, stderr=stderr, failure_category=category, failure_details=f"Exit code {completed.returncode}")
return BackendResult(status="SUCCEEDED", stdout=stdout, stderr=stderr)
def validate_checkpoint(self, reference: str) -> bool:
path = Path(reference)
return path.is_dir() and any(path.glob("adapter_model.*"))
class SparkGuardBackend(GuardSubprocessBackend):
"""Runs only profile-generated Guard commands over the configured Spark SSH alias."""
def __init__(self, ssh_alias: str = "spark") -> None:
self.ssh_alias = ssh_alias
def launch(self, *, command: list[str], working_directory: str, timeout_seconds: int) -> BackendResult:
if not command:
return BackendResult(status="FAILED", failure_category="COMMAND_SCOPE", failure_details="Missing profile-generated command.")
import shlex
remote = "cd " + shlex.quote(working_directory) + " && " + " ".join(shlex.quote(part) for part in command)
try:
completed = subprocess.run(["ssh", self.ssh_alias, remote], capture_output=True, text=True, timeout=timeout_seconds, check=False)
except subprocess.TimeoutExpired as exc:
return BackendResult(status="TIMEOUT", stdout=exc.stdout or "", stderr=exc.stderr or "", failure_category="TIMEOUT", failure_details=f"Exceeded {timeout_seconds}s")
if completed.returncode:
combined = (completed.stdout or "") + (completed.stderr or "")
category = "OOM" if "out of memory" in combined.lower() else "REMOTE_PROCESS_FAILURE"
return BackendResult(status="OOM" if category == "OOM" else "FAILED", stdout=completed.stdout or "", stderr=completed.stderr or "", failure_category=category, failure_details=f"Exit code {completed.returncode}")
return BackendResult(status="SUCCEEDED", stdout=completed.stdout or "", stderr=completed.stderr or "")

0
control_plane/model_studio/management/__init__.py Normal file
View file

0
control_plane/model_studio/management/commands/__init__.py Normal file
View file

20
control_plane/model_studio/management/commands/modelstudio_dataset_audit.py Normal file
View file

@ -0,0 +1,20 @@
import json
from django.core.management.base import BaseCommand, CommandError
from control_plane.model_studio.services import ModelStudioService
from control_plane.model_studio.models import TrainingProject
class Command(BaseCommand):
help = "Classify Guard dataset manifests and record provenance/contamination evidence without changing source data."
def add_arguments(self, parser):
parser.add_argument("--project", required=True, help="TrainingProject slug")
def handle(self, *args, **options):
project = TrainingProject.objects.filter(slug=options["project"]).first()
if project is None:
raise CommandError("TrainingProject not found.")
report = ModelStudioService().curate_guard_datasets(project)
self.stdout.write(json.dumps({key: report[key] for key in ["total", "valid", "warning", "blocked", "decision"]}, indent=2))

27
control_plane/model_studio/management/commands/modelstudio_import_guard.py Normal file
View file

@ -0,0 +1,27 @@
from django.core.management.base import BaseCommand, CommandError
from control_plane.model_studio.backends import SparkGuardBackend
from control_plane.model_studio.services import ModelStudioService
from control_plane.projects.models import Project
class Command(BaseCommand):
help = "Import the existing ForgeGuard/Qwen2.5-Coder-3B project and run archaeology."
def add_arguments(self, parser):
parser.add_argument("--repository", required=True)
parser.add_argument("--project")
parser.add_argument("--slug", default="guard-3b")
parser.add_argument("--spark-working-directory", default="", help="Verified ForgeGuard checkout path on Spark; required before real remote training.")
def handle(self, *args, **options):
project = Project.objects.filter(id=options["project"]).first() if options.get("project") else None
if options.get("project") and project is None:
raise CommandError("Project not found.")
service = ModelStudioService(backend=SparkGuardBackend())
training_project = service.import_guard(project=project, repository_path=options["repository"], spark_working_directory=options["spark_working_directory"], slug=options["slug"])
service.declare_base_champion(training_project)
report = service.archaeology(training_project)
curation = service.curate_guard_datasets(training_project)
suite = service.validate_benchmark(training_project)
self.stdout.write(self.style.SUCCESS(f"Imported {training_project.slug}: {len(report['checkpoints'])} checkpoints, {len(report['datasets'])} datasets, dataset_curation={curation['decision']}, suite={suite.integrity_status}"))

38
control_plane/model_studio/management/commands/modelstudio_overnight.py Normal file
View file

@ -0,0 +1,38 @@
import json
from django.core.management.base import BaseCommand, CommandError
from control_plane.model_studio.models import ModelPromotionPolicy, TrainingProject
from control_plane.model_studio.services import ModelStudioService
class Command(BaseCommand):
help = "Create a bounded Model Studio overnight program. --dry-run never allocates training compute."
def add_arguments(self, parser):
parser.add_argument("--project", required=True, help="TrainingProject slug")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--wall-seconds", type=int, default=8 * 3600)
parser.add_argument("--max-runs", type=int, default=8)
def handle(self, *args, **options):
project = TrainingProject.objects.filter(slug=options["project"]).first()
if project is None:
raise CommandError("TrainingProject not found.")
policy = ModelPromotionPolicy.objects.filter(training_project=project, active=True).order_by("-created_at").first()
if policy is None:
raise CommandError("Create a ModelPromotionPolicy before starting an overnight program.")
service = ModelStudioService()
if options["dry_run"]:
rows = list(project.experiments.filter(status__in=["PROPOSED", "QUEUED"]).order_by("-experiment_value_score").values("experiment_id", "title", "experiment_value_score", "estimated_runtime_seconds"))
blockers = []
if project.status != "READY":
blockers.append(f"training project status is {project.status}")
if not project.metadata.get("spark_working_directory_verified"):
blockers.append("Spark Guard working directory is not verified")
if project.baseline_evaluation_id is None:
blockers.append("fresh Champion baseline is missing")
self.stdout.write(json.dumps({"dry_run": True, "champion": str(project.current_champion_id or ""), "experiments": rows, "status": project.status, "blockers": blockers, "estimated_window_seconds": options["wall_seconds"], "maximum_runs": options["max_runs"]}, indent=2, default=str))
return
program = service.create_program(project, policy, wall_seconds=options["wall_seconds"], max_runs=options["max_runs"])
self.stdout.write(self.style.SUCCESS(f"Created overnight program {program.id}; execution is intentionally queued for scoped worker supervision."))

494
control_plane/model_studio/migrations/0001_initial.py Normal file
View file

@ -0,0 +1,494 @@
# Generated by Django 5.2.16 on 2026-08-16 17:49
import django.db.models.deletion
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
('projects', '0006_roadmap_scenario_lab_v1'),
]
operations = [
migrations.CreateModel(
name='Dataset',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=200)),
('description', models.TextField(blank=True)),
('metadata', models.JSONField(blank=True, default=dict)),
],
),
migrations.CreateModel(
name='EvaluationRun',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('status', models.CharField(choices=[('QUEUED', 'Queued'), ('RUNNING', 'Running'), ('SUCCEEDED', 'Succeeded'), ('FAILED', 'Failed'), ('INVALID', 'Invalid')], default='QUEUED', max_length=16)),
('command', models.JSONField(blank=True, default=list)),
('output_reference', models.TextField(blank=True)),
('started_at', models.DateTimeField(blank=True, null=True)),
('completed_at', models.DateTimeField(blank=True, null=True)),
('wall_seconds', models.FloatField(default=0)),
('parser_version', models.CharField(blank=True, max_length=120)),
('integrity_evidence', models.JSONField(blank=True, default=dict)),
('summary', models.JSONField(blank=True, default=dict)),
('failure_details', models.TextField(blank=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='EvaluationSuite',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=200)),
('description', models.TextField(blank=True)),
('metadata', models.JSONField(blank=True, default=dict)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='FailureCluster',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=200)),
('description', models.TextField(blank=True)),
('failure_type', models.CharField(blank=True, max_length=160)),
('severity', models.CharField(default='UNKNOWN', max_length=32)),
('sample_count', models.PositiveIntegerField(default=0)),
('representative_examples', models.JSONField(blank=True, default=list)),
('affected_benchmarks', models.JSONField(blank=True, default=list)),
('suspected_causes', models.JSONField(blank=True, default=list)),
('confidence', models.FloatField(default=0)),
('training_data_coverage', models.JSONField(blank=True, default=dict)),
('priority', models.FloatField(default=0)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='ModelPromotionPolicy',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=160)),
('version', models.CharField(max_length=80)),
('criteria', models.JSONField(default=dict)),
('active', models.BooleanField(default=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='DatasetVersion',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('version', models.CharField(max_length=120)),
('manifest_reference', models.TextField()),
('content_hash', models.CharField(max_length=128)),
('record_count', models.PositiveIntegerField(blank=True, null=True)),
('split_metadata', models.JSONField(blank=True, default=dict)),
('source_metadata', models.JSONField(blank=True, default=dict)),
('generation_metadata', models.JSONField(blank=True, default=dict)),
('tags', models.JSONField(blank=True, default=list)),
('validation_status', models.CharField(choices=[('VALID', 'Valid'), ('WARNING', 'Warning'), ('BLOCKED', 'Blocked'), ('UNKNOWN', 'Unknown')], default='UNKNOWN', max_length=16)),
('contamination_status', models.CharField(choices=[('VALID', 'Valid'), ('WARNING', 'Warning'), ('BLOCKED', 'Blocked'), ('UNKNOWN', 'Unknown')], default='UNKNOWN', max_length=16)),
('immutable', models.BooleanField(default=False)),
('dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='model_studio.dataset')),
('parent_version', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='model_studio.datasetversion')),
],
),
migrations.CreateModel(
name='BenchmarkResult',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('group', models.CharField(choices=[('PRIMARY', 'Primary'), ('CAPABILITY_SUBSET', 'Capability Subset'), ('HOLDOUT', 'Holdout'), ('REGRESSION', 'Regression'), ('ADVERSARIAL', 'Adversarial'), ('FORMAT', 'Format'), ('PERFORMANCE', 'Performance')], default='PRIMARY', max_length=32)),
('metric', models.CharField(max_length=160)),
('value', models.FloatField()),
('unit', models.CharField(blank=True, max_length=80)),
('subset', models.CharField(blank=True, max_length=160)),
('sample_count', models.PositiveIntegerField(blank=True, null=True)),
('passed', models.BooleanField(blank=True, null=True)),
('provenance', models.JSONField(blank=True, default=dict)),
('evaluation_run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='results', to='model_studio.evaluationrun')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='EvaluationSuiteVersion',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('version', models.CharField(max_length=120)),
('reference', models.TextField()),
('content_hash', models.CharField(max_length=128)),
('command_template', models.JSONField(blank=True, default=list)),
('groups', models.JSONField(blank=True, default=list)),
('immutable', models.BooleanField(default=False)),
('integrity_status', models.CharField(choices=[('VALID', 'Valid'), ('WARNING', 'Warning'), ('BLOCKED', 'Blocked'), ('UNKNOWN', 'Unknown')], default='UNKNOWN', max_length=16)),
('integrity_evidence', models.JSONField(blank=True, default=dict)),
('suite', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='model_studio.evaluationsuite')),
],
),
migrations.AddField(
model_name='evaluationrun',
name='suite_version',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='runs', to='model_studio.evaluationsuiteversion'),
),
migrations.CreateModel(
name='ModelCheckpoint',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=255)),
('checkpoint_type', models.CharField(choices=[('BASE', 'Base'), ('IMPORTED', 'Imported'), ('CHAMPION', 'Champion'), ('CHALLENGER', 'Challenger'), ('INTERMEDIATE', 'Intermediate')], max_length=32)),
('reference', models.TextField()),
('content_hash', models.CharField(blank=True, max_length=128)),
('parameter_count', models.BigIntegerField(blank=True, null=True)),
('dtype', models.CharField(blank=True, max_length=80)),
('adapter_type', models.CharField(blank=True, max_length=80)),
('quantization', models.CharField(blank=True, max_length=80)),
('validity_status', models.CharField(choices=[('UNKNOWN', 'Unknown'), ('VALID', 'Valid'), ('INVALID', 'Invalid'), ('PARTIAL', 'Partial'), ('CORRUPT', 'Corrupt')], default='UNKNOWN', max_length=16)),
('load_verified', models.BooleanField(default=False)),
('evaluation_status', models.CharField(default='NOT_EVALUATED', max_length=32)),
('metadata', models.JSONField(blank=True, default=dict)),
('base_checkpoint', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='derived_checkpoints', to='model_studio.modelcheckpoint')),
('dataset_versions', models.ManyToManyField(blank=True, related_name='checkpoints', to='model_studio.datasetversion')),
],
),
migrations.AddField(
model_name='evaluationrun',
name='checkpoint',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='evaluation_runs', to='model_studio.modelcheckpoint'),
),
migrations.CreateModel(
name='OvernightTrainingProgram',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('status', models.CharField(choices=[('CREATED', 'Created'), ('ARCHAEOLOGY', 'Archaeology'), ('BASELINING', 'Baselining'), ('PLANNING', 'Planning'), ('RUNNING_EXPERIMENT', 'Running Experiment'), ('EVALUATING', 'Evaluating'), ('ADAPTING', 'Adapting'), ('FINALIZING', 'Finalizing'), ('COMPLETED', 'Completed'), ('PARTIAL', 'Partial'), ('FAILED', 'Failed'), ('PAUSED', 'Paused')], default='CREATED', max_length=32)),
('start_time', models.DateTimeField(blank=True, null=True)),
('deadline', models.DateTimeField()),
('maximum_wall_seconds', models.PositiveIntegerField()),
('maximum_training_runs', models.PositiveIntegerField(default=8)),
('maximum_failed_runs', models.PositiveIntegerField(default=3)),
('maximum_single_run_seconds', models.PositiveIntegerField()),
('evaluation_reserve_seconds', models.PositiveIntegerField()),
('allowed_experiment_types', models.JSONField(blank=True, default=list)),
('started_at', models.DateTimeField(blank=True, null=True)),
('completed_at', models.DateTimeField(blank=True, null=True)),
('termination_reason', models.TextField(blank=True)),
('pause_after_current_run', models.BooleanField(default=False)),
('telemetry', models.JSONField(blank=True, default=dict)),
('ending_champion', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='ending_programs', to='model_studio.modelcheckpoint')),
('promotion_policy', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='programs', to='model_studio.modelpromotionpolicy')),
('starting_champion', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='starting_programs', to='model_studio.modelcheckpoint')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='OvernightResearchReport',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('markdown', models.TextField()),
('payload', models.JSONField(default=dict)),
('artifact_reference', models.TextField(blank=True)),
('program', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='report', to='model_studio.overnighttrainingprogram')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='TrainingExperiment',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('experiment_id', models.CharField(max_length=120, unique=True)),
('title', models.CharField(max_length=255)),
('hypothesis', models.TextField()),
('reasoning', models.TextField()),
('intervention', models.JSONField(default=dict)),
('controls', models.JSONField(default=dict)),
('expected_result', models.TextField()),
('primary_success_metric', models.CharField(max_length=160)),
('success_threshold', models.JSONField(default=dict)),
('regression_constraints', models.JSONField(default=dict)),
('rejection_condition', models.TextField()),
('ambiguity_policy', models.TextField()),
('estimated_runtime_seconds', models.PositiveIntegerField(default=0)),
('maximum_runtime_seconds', models.PositiveIntegerField(default=0)),
('compute_budget', models.JSONField(default=dict)),
('status', models.CharField(choices=[('PROPOSED', 'Proposed'), ('QUEUED', 'Queued'), ('RUNNING', 'Running'), ('EVALUATING', 'Evaluating'), ('VALIDATING', 'Validating'), ('PROMOTED', 'Promoted'), ('REJECTED', 'Rejected'), ('INCONCLUSIVE', 'Inconclusive'), ('FAILED', 'Failed'), ('CANCELLED', 'Cancelled'), ('SUPERSEDED', 'Superseded')], default='PROPOSED', max_length=32)),
('priority', models.FloatField(default=0)),
('expected_information_gain', models.FloatField(default=0)),
('expected_improvement', models.FloatField(default=0)),
('estimated_compute_cost', models.FloatField(default=0)),
('experiment_value_score', models.FloatField(default=0)),
('fingerprint', models.CharField(max_length=128)),
('created_by_agent', models.CharField(default='MODEL_DIRECTOR', max_length=120)),
('approved_by_model_director', models.BooleanField(default=False)),
('result_summary', models.TextField(blank=True)),
('conclusion', models.CharField(blank=True, choices=[('SUPPORTED', 'Supported'), ('WEAKLY_SUPPORTED', 'Weakly Supported'), ('REFUTED', 'Refuted'), ('AMBIGUOUS', 'Ambiguous'), ('EXECUTION_FAILED', 'Execution Failed')], max_length=32)),
('derived_from_failure_cluster', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='experiments', to='model_studio.failurecluster')),
('parent_experiment', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='derived_experiments', to='model_studio.trainingexperiment')),
],
),
migrations.CreateModel(
name='ExperimentDependency',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('required_conclusions', models.JSONField(blank=True, default=list)),
('rationale', models.TextField(blank=True)),
('depends_on', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dependents', to='model_studio.trainingexperiment')),
('experiment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dependencies', to='model_studio.trainingexperiment')),
],
),
migrations.CreateModel(
name='TrainingProject',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('studio_type', models.CharField(default='MODEL', max_length=32)),
('name', models.CharField(max_length=200)),
('slug', models.SlugField(max_length=120, unique=True)),
('description', models.TextField(blank=True)),
('goal', models.TextField()),
('capability_target', models.TextField(blank=True)),
('model_family', models.CharField(blank=True, max_length=160)),
('model_size', models.CharField(blank=True, max_length=80)),
('base_model', models.TextField(blank=True)),
('repository_path', models.TextField(blank=True)),
('repository_reference', models.TextField(blank=True)),
('working_directory', models.TextField(blank=True)),
('status', models.CharField(choices=[('IMPORTING', 'Importing'), ('ARCHAEOLOGY', 'Archaeology'), ('NEEDS_REPAIR', 'Needs Repair'), ('BASELINING', 'Baselining'), ('READY', 'Ready'), ('OVERNIGHT_RUNNING', 'Overnight Running'), ('PAUSED', 'Paused'), ('FAILED', 'Failed'), ('FINISHED', 'Finished'), ('EVOLVING', 'Evolving')], default='IMPORTING', max_length=32)),
('default_profile', models.CharField(default='guard', max_length=120)),
('training_backend', models.CharField(default='guard_subprocess', max_length=120)),
('metadata', models.JSONField(blank=True, default=dict)),
('baseline_evaluation', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='baseline_for_projects', to='model_studio.evaluationrun')),
('current_champion', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='champion_for_projects', to='model_studio.modelcheckpoint')),
('project', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='training_projects', to='projects.project')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='trainingexperiment',
name='training_project',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='experiments', to='model_studio.trainingproject'),
),
migrations.CreateModel(
name='RegressionBankItem',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('reference', models.TextField()),
('content_hash', models.CharField(blank=True, max_length=128)),
('category', models.CharField(blank=True, max_length=160)),
('kind', models.CharField(default='IMPORTED', max_length=80)),
('provenance', models.JSONField(blank=True, default=dict)),
('evaluation_only', models.BooleanField(default=True)),
('training_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='regression_bank_items', to='model_studio.trainingproject')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='overnighttrainingprogram',
name='training_project',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='overnight_programs', to='model_studio.trainingproject'),
),
migrations.CreateModel(
name='ModelStudioArtifact',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('artifact_type', models.CharField(max_length=120)),
('name', models.CharField(max_length=255)),
('content', models.JSONField(default=dict)),
('readable', models.TextField(blank=True)),
('source_reference', models.TextField(blank=True)),
('training_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='artifacts', to='model_studio.trainingproject')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='modelpromotionpolicy',
name='training_project',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='promotion_policies', to='model_studio.trainingproject'),
),
migrations.CreateModel(
name='ModelPromotionDecision',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('decision', models.CharField(choices=[('PROMOTE', 'Promote'), ('REJECT', 'Reject'), ('INCONCLUSIVE', 'Inconclusive'), ('REQUIRE_REPLICATION', 'Require Replication')], max_length=32)),
('evaluation_evidence', models.JSONField(default=dict)),
('judge_result', models.JSONField(default=dict)),
('reason', models.TextField()),
('judge_actor', models.CharField(default='MODEL_JUDGE', max_length=120)),
('candidate', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='promotion_candidates', to='model_studio.modelcheckpoint')),
('from_champion', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='promotion_sources', to='model_studio.modelcheckpoint')),
('policy', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='decisions', to='model_studio.modelpromotionpolicy')),
('experiment', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='promotion_decisions', to='model_studio.trainingexperiment')),
('training_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='promotion_decisions', to='model_studio.trainingproject')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='modelcheckpoint',
name='training_project',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='checkpoints', to='model_studio.trainingproject'),
),
migrations.AddField(
model_name='failurecluster',
name='training_project',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='failure_clusters', to='model_studio.trainingproject'),
),
migrations.AddField(
model_name='evaluationsuite',
name='training_project',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='evaluation_suites', to='model_studio.trainingproject'),
),
migrations.AddField(
model_name='evaluationrun',
name='training_project',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='evaluation_runs', to='model_studio.trainingproject'),
),
migrations.AddField(
model_name='dataset',
name='training_project',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='datasets', to='model_studio.trainingproject'),
),
migrations.CreateModel(
name='TrainingRecipe',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=200)),
('configuration', models.JSONField(default=dict)),
('recipe_hash', models.CharField(max_length=128, unique=True)),
('immutable', models.BooleanField(default=False)),
('training_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='recipes', to='model_studio.trainingproject')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='modelcheckpoint',
name='recipe',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='checkpoints', to='model_studio.trainingrecipe'),
),
migrations.CreateModel(
name='TrainingRun',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('status', models.CharField(choices=[('QUEUED', 'Queued'), ('STARTING', 'Starting'), ('RUNNING', 'Running'), ('CHECKPOINTING', 'Checkpointing'), ('SUCCEEDED', 'Succeeded'), ('FAILED', 'Failed'), ('OOM', 'Oom'), ('TIMEOUT', 'Timeout'), ('CANCELLED', 'Cancelled'), ('INTERRUPTED', 'Interrupted')], default='QUEUED', max_length=32)),
('command', models.JSONField(blank=True, default=list)),
('working_directory', models.TextField(blank=True)),
('environment_snapshot', models.JSONField(blank=True, default=dict)),
('host', models.CharField(blank=True, max_length=255)),
('gpu_device', models.CharField(blank=True, max_length=255)),
('allocated_memory_mb', models.PositiveIntegerField(blank=True, null=True)),
('started_at', models.DateTimeField(blank=True, null=True)),
('completed_at', models.DateTimeField(blank=True, null=True)),
('wall_seconds', models.FloatField(default=0)),
('exit_code', models.IntegerField(blank=True, null=True)),
('stdout_reference', models.TextField(blank=True)),
('stderr_reference', models.TextField(blank=True)),
('training_log_reference', models.TextField(blank=True)),
('peak_memory_mb', models.PositiveIntegerField(blank=True, null=True)),
('gpu_utilization', models.FloatField(blank=True, null=True)),
('failure_category', models.CharField(blank=True, max_length=80)),
('failure_details', models.TextField(blank=True)),
('resume_source', models.TextField(blank=True)),
('retry_count', models.PositiveIntegerField(default=0)),
('pid', models.IntegerField(blank=True, null=True)),
('heartbeat_at', models.DateTimeField(blank=True, null=True)),
('experiment', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='runs', to='model_studio.trainingexperiment')),
('input_checkpoint', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='input_runs', to='model_studio.modelcheckpoint')),
('output_checkpoint', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='producing_run', to='model_studio.modelcheckpoint')),
('recipe', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='runs', to='model_studio.trainingrecipe')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='modelcheckpoint',
name='training_run',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='output_checkpoints', to='model_studio.trainingrun'),
),
migrations.AddConstraint(
model_name='datasetversion',
constraint=models.UniqueConstraint(fields=('dataset', 'version'), name='unique_model_studio_dataset_version'),
),
migrations.AddConstraint(
model_name='evaluationsuiteversion',
constraint=models.UniqueConstraint(fields=('suite', 'version'), name='unique_evaluation_suite_version'),
),
migrations.AddConstraint(
model_name='experimentdependency',
constraint=models.UniqueConstraint(fields=('experiment', 'depends_on'), name='unique_experiment_dependency'),
),
migrations.AddIndex(
model_name='trainingexperiment',
index=models.Index(fields=['training_project', 'fingerprint'], name='model_studi_trainin_53bef8_idx'),
),
migrations.AddConstraint(
model_name='dataset',
constraint=models.UniqueConstraint(fields=('training_project', 'name'), name='unique_model_studio_dataset'),
),
migrations.AddConstraint(
model_name='modelcheckpoint',
constraint=models.UniqueConstraint(fields=('training_project', 'reference'), name='unique_model_checkpoint_reference'),
),
]

0
control_plane/model_studio/migrations/__init__.py Normal file
View file

420
control_plane/model_studio/models.py Normal file
View file

@ -0,0 +1,420 @@
from __future__ import annotations
from django.db import models
from control_plane.common import TimestampedModel
class TrainingProjectStatus(models.TextChoices):
IMPORTING = "IMPORTING"
ARCHAEOLOGY = "ARCHAEOLOGY"
NEEDS_REPAIR = "NEEDS_REPAIR"
BASELINING = "BASELINING"
READY = "READY"
OVERNIGHT_RUNNING = "OVERNIGHT_RUNNING"
PAUSED = "PAUSED"
FAILED = "FAILED"
FINISHED = "FINISHED"
EVOLVING = "EVOLVING"
class DatasetValidationStatus(models.TextChoices):
VALID = "VALID"
WARNING = "WARNING"
BLOCKED = "BLOCKED"
UNKNOWN = "UNKNOWN"
class CheckpointType(models.TextChoices):
BASE = "BASE"
IMPORTED = "IMPORTED"
CHAMPION = "CHAMPION"
CHALLENGER = "CHALLENGER"
INTERMEDIATE = "INTERMEDIATE"
class CheckpointValidityStatus(models.TextChoices):
UNKNOWN = "UNKNOWN"
VALID = "VALID"
INVALID = "INVALID"
PARTIAL = "PARTIAL"
CORRUPT = "CORRUPT"
class ExperimentStatus(models.TextChoices):
PROPOSED = "PROPOSED"
QUEUED = "QUEUED"
RUNNING = "RUNNING"
EVALUATING = "EVALUATING"
VALIDATING = "VALIDATING"
PROMOTED = "PROMOTED"
REJECTED = "REJECTED"
INCONCLUSIVE = "INCONCLUSIVE"
FAILED = "FAILED"
CANCELLED = "CANCELLED"
SUPERSEDED = "SUPERSEDED"
class TrainingRunStatus(models.TextChoices):
QUEUED = "QUEUED"
STARTING = "STARTING"
RUNNING = "RUNNING"
CHECKPOINTING = "CHECKPOINTING"
SUCCEEDED = "SUCCEEDED"
FAILED = "FAILED"
OOM = "OOM"
TIMEOUT = "TIMEOUT"
CANCELLED = "CANCELLED"
INTERRUPTED = "INTERRUPTED"
class EvaluationGroup(models.TextChoices):
PRIMARY = "PRIMARY"
CAPABILITY_SUBSET = "CAPABILITY_SUBSET"
HOLDOUT = "HOLDOUT"
REGRESSION = "REGRESSION"
ADVERSARIAL = "ADVERSARIAL"
FORMAT = "FORMAT"
PERFORMANCE = "PERFORMANCE"
class EvaluationRunStatus(models.TextChoices):
QUEUED = "QUEUED"
RUNNING = "RUNNING"
SUCCEEDED = "SUCCEEDED"
FAILED = "FAILED"
INVALID = "INVALID"
class ProgramStatus(models.TextChoices):
CREATED = "CREATED"
ARCHAEOLOGY = "ARCHAEOLOGY"
BASELINING = "BASELINING"
PLANNING = "PLANNING"
RUNNING_EXPERIMENT = "RUNNING_EXPERIMENT"
EVALUATING = "EVALUATING"
ADAPTING = "ADAPTING"
FINALIZING = "FINALIZING"
COMPLETED = "COMPLETED"
PARTIAL = "PARTIAL"
FAILED = "FAILED"
PAUSED = "PAUSED"
class PromotionDecision(models.TextChoices):
PROMOTE = "PROMOTE"
REJECT = "REJECT"
INCONCLUSIVE = "INCONCLUSIVE"
REQUIRE_REPLICATION = "REQUIRE_REPLICATION"
class Conclusion(models.TextChoices):
SUPPORTED = "SUPPORTED"
WEAKLY_SUPPORTED = "WEAKLY_SUPPORTED"
REFUTED = "REFUTED"
AMBIGUOUS = "AMBIGUOUS"
EXECUTION_FAILED = "EXECUTION_FAILED"
class TrainingProject(TimestampedModel):
project = models.ForeignKey("projects.Project", on_delete=models.PROTECT, related_name="training_projects")
studio_type = models.CharField(max_length=32, default="MODEL")
name = models.CharField(max_length=200)
slug = models.SlugField(max_length=120, unique=True)
description = models.TextField(blank=True)
goal = models.TextField()
capability_target = models.TextField(blank=True)
model_family = models.CharField(max_length=160, blank=True)
model_size = models.CharField(max_length=80, blank=True)
base_model = models.TextField(blank=True)
repository_path = models.TextField(blank=True)
repository_reference = models.TextField(blank=True)
working_directory = models.TextField(blank=True)
status = models.CharField(max_length=32, choices=TrainingProjectStatus.choices, default=TrainingProjectStatus.IMPORTING)
default_profile = models.CharField(max_length=120, default="guard")
training_backend = models.CharField(max_length=120, default="guard_subprocess")
current_champion = models.ForeignKey("ModelCheckpoint", on_delete=models.SET_NULL, null=True, blank=True, related_name="champion_for_projects")
baseline_evaluation = models.ForeignKey("EvaluationRun", on_delete=models.SET_NULL, null=True, blank=True, related_name="baseline_for_projects")
metadata = models.JSONField(default=dict, blank=True)
class Dataset(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="datasets")
name = models.CharField(max_length=200)
description = models.TextField(blank=True)
metadata = models.JSONField(default=dict, blank=True)
class Meta:
constraints = [models.UniqueConstraint(fields=["training_project", "name"], name="unique_model_studio_dataset")]
class DatasetVersion(TimestampedModel):
dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name="versions")
version = models.CharField(max_length=120)
parent_version = models.ForeignKey("self", on_delete=models.SET_NULL, null=True, blank=True, related_name="children")
manifest_reference = models.TextField()
content_hash = models.CharField(max_length=128)
record_count = models.PositiveIntegerField(null=True, blank=True)
split_metadata = models.JSONField(default=dict, blank=True)
source_metadata = models.JSONField(default=dict, blank=True)
generation_metadata = models.JSONField(default=dict, blank=True)
tags = models.JSONField(default=list, blank=True)
validation_status = models.CharField(max_length=16, choices=DatasetValidationStatus.choices, default=DatasetValidationStatus.UNKNOWN)
contamination_status = models.CharField(max_length=16, choices=DatasetValidationStatus.choices, default=DatasetValidationStatus.UNKNOWN)
immutable = models.BooleanField(default=False)
class Meta:
constraints = [models.UniqueConstraint(fields=["dataset", "version"], name="unique_model_studio_dataset_version")]
def save(self, *args, **kwargs):
if self.pk and self.immutable:
original = type(self).objects.get(pk=self.pk)
immutable_fields = ["version", "parent_version_id", "manifest_reference", "content_hash", "record_count", "split_metadata", "source_metadata", "generation_metadata", "tags"]
if any(getattr(self, field) != getattr(original, field) for field in immutable_fields):
raise ValueError("DatasetVersion is immutable after completed training use.")
super().save(*args, **kwargs)
class TrainingRecipe(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="recipes")
name = models.CharField(max_length=200)
configuration = models.JSONField(default=dict)
recipe_hash = models.CharField(max_length=128, unique=True)
immutable = models.BooleanField(default=False)
def save(self, *args, **kwargs):
if self.pk and self.immutable:
original = type(self).objects.get(pk=self.pk)
if self.configuration != original.configuration or self.recipe_hash != original.recipe_hash:
raise ValueError("TrainingRecipe is immutable after a TrainingRun starts.")
super().save(*args, **kwargs)
class TrainingExperiment(TimestampedModel):
experiment_id = models.CharField(max_length=120, unique=True)
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="experiments")
title = models.CharField(max_length=255)
hypothesis = models.TextField()
reasoning = models.TextField()
intervention = models.JSONField(default=dict)
controls = models.JSONField(default=dict)
expected_result = models.TextField()
primary_success_metric = models.CharField(max_length=160)
success_threshold = models.JSONField(default=dict)
regression_constraints = models.JSONField(default=dict)
rejection_condition = models.TextField()
ambiguity_policy = models.TextField()
estimated_runtime_seconds = models.PositiveIntegerField(default=0)
maximum_runtime_seconds = models.PositiveIntegerField(default=0)
compute_budget = models.JSONField(default=dict)
parent_experiment = models.ForeignKey("self", on_delete=models.SET_NULL, null=True, blank=True, related_name="derived_experiments")
derived_from_failure_cluster = models.ForeignKey("FailureCluster", on_delete=models.SET_NULL, null=True, blank=True, related_name="experiments")
status = models.CharField(max_length=32, choices=ExperimentStatus.choices, default=ExperimentStatus.PROPOSED)
priority = models.FloatField(default=0)
expected_information_gain = models.FloatField(default=0)
expected_improvement = models.FloatField(default=0)
estimated_compute_cost = models.FloatField(default=0)
experiment_value_score = models.FloatField(default=0)
fingerprint = models.CharField(max_length=128)
created_by_agent = models.CharField(max_length=120, default="MODEL_DIRECTOR")
approved_by_model_director = models.BooleanField(default=False)
result_summary = models.TextField(blank=True)
conclusion = models.CharField(max_length=32, choices=Conclusion.choices, blank=True)
class Meta:
indexes = [models.Index(fields=["training_project", "fingerprint"])]
class ExperimentDependency(TimestampedModel):
experiment = models.ForeignKey(TrainingExperiment, on_delete=models.CASCADE, related_name="dependencies")
depends_on = models.ForeignKey(TrainingExperiment, on_delete=models.CASCADE, related_name="dependents")
required_conclusions = models.JSONField(default=list, blank=True)
rationale = models.TextField(blank=True)
class Meta:
constraints = [models.UniqueConstraint(fields=["experiment", "depends_on"], name="unique_experiment_dependency")]
class ModelCheckpoint(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="checkpoints")
name = models.CharField(max_length=255)
checkpoint_type = models.CharField(max_length=32, choices=CheckpointType.choices)
reference = models.TextField()
content_hash = models.CharField(max_length=128, blank=True)
base_checkpoint = models.ForeignKey("self", on_delete=models.SET_NULL, null=True, blank=True, related_name="derived_checkpoints")
training_run = models.ForeignKey("TrainingRun", on_delete=models.SET_NULL, null=True, blank=True, related_name="output_checkpoints")
recipe = models.ForeignKey(TrainingRecipe, on_delete=models.SET_NULL, null=True, blank=True, related_name="checkpoints")
dataset_versions = models.ManyToManyField(DatasetVersion, blank=True, related_name="checkpoints")
parameter_count = models.BigIntegerField(null=True, blank=True)
dtype = models.CharField(max_length=80, blank=True)
adapter_type = models.CharField(max_length=80, blank=True)
quantization = models.CharField(max_length=80, blank=True)
validity_status = models.CharField(max_length=16, choices=CheckpointValidityStatus.choices, default=CheckpointValidityStatus.UNKNOWN)
load_verified = models.BooleanField(default=False)
evaluation_status = models.CharField(max_length=32, default="NOT_EVALUATED")
metadata = models.JSONField(default=dict, blank=True)
class Meta:
constraints = [models.UniqueConstraint(fields=["training_project", "reference"], name="unique_model_checkpoint_reference")]
class TrainingRun(TimestampedModel):
experiment = models.ForeignKey(TrainingExperiment, on_delete=models.PROTECT, related_name="runs")
recipe = models.ForeignKey(TrainingRecipe, on_delete=models.PROTECT, related_name="runs")
input_checkpoint = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="input_runs")
output_checkpoint = models.ForeignKey(ModelCheckpoint, on_delete=models.SET_NULL, null=True, blank=True, related_name="producing_run")
status = models.CharField(max_length=32, choices=TrainingRunStatus.choices, default=TrainingRunStatus.QUEUED)
command = models.JSONField(default=list, blank=True)
working_directory = models.TextField(blank=True)
environment_snapshot = models.JSONField(default=dict, blank=True)
host = models.CharField(max_length=255, blank=True)
gpu_device = models.CharField(max_length=255, blank=True)
allocated_memory_mb = models.PositiveIntegerField(null=True, blank=True)
started_at = models.DateTimeField(null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
wall_seconds = models.FloatField(default=0)
exit_code = models.IntegerField(null=True, blank=True)
stdout_reference = models.TextField(blank=True)
stderr_reference = models.TextField(blank=True)
training_log_reference = models.TextField(blank=True)
peak_memory_mb = models.PositiveIntegerField(null=True, blank=True)
gpu_utilization = models.FloatField(null=True, blank=True)
failure_category = models.CharField(max_length=80, blank=True)
failure_details = models.TextField(blank=True)
resume_source = models.TextField(blank=True)
retry_count = models.PositiveIntegerField(default=0)
pid = models.IntegerField(null=True, blank=True)
heartbeat_at = models.DateTimeField(null=True, blank=True)
class EvaluationSuite(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="evaluation_suites")
name = models.CharField(max_length=200)
description = models.TextField(blank=True)
metadata = models.JSONField(default=dict, blank=True)
class EvaluationSuiteVersion(TimestampedModel):
suite = models.ForeignKey(EvaluationSuite, on_delete=models.CASCADE, related_name="versions")
version = models.CharField(max_length=120)
reference = models.TextField()
content_hash = models.CharField(max_length=128)
command_template = models.JSONField(default=list, blank=True)
groups = models.JSONField(default=list, blank=True)
immutable = models.BooleanField(default=False)
integrity_status = models.CharField(max_length=16, choices=DatasetValidationStatus.choices, default=DatasetValidationStatus.UNKNOWN)
integrity_evidence = models.JSONField(default=dict, blank=True)
class Meta:
constraints = [models.UniqueConstraint(fields=["suite", "version"], name="unique_evaluation_suite_version")]
class EvaluationRun(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="evaluation_runs")
checkpoint = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="evaluation_runs")
suite_version = models.ForeignKey(EvaluationSuiteVersion, on_delete=models.PROTECT, related_name="runs")
status = models.CharField(max_length=16, choices=EvaluationRunStatus.choices, default=EvaluationRunStatus.QUEUED)
command = models.JSONField(default=list, blank=True)
output_reference = models.TextField(blank=True)
started_at = models.DateTimeField(null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
wall_seconds = models.FloatField(default=0)
parser_version = models.CharField(max_length=120, blank=True)
integrity_evidence = models.JSONField(default=dict, blank=True)
summary = models.JSONField(default=dict, blank=True)
failure_details = models.TextField(blank=True)
class BenchmarkResult(TimestampedModel):
evaluation_run = models.ForeignKey(EvaluationRun, on_delete=models.CASCADE, related_name="results")
group = models.CharField(max_length=32, choices=EvaluationGroup.choices, default=EvaluationGroup.PRIMARY)
metric = models.CharField(max_length=160)
value = models.FloatField()
unit = models.CharField(max_length=80, blank=True)
subset = models.CharField(max_length=160, blank=True)
sample_count = models.PositiveIntegerField(null=True, blank=True)
passed = models.BooleanField(null=True, blank=True)
provenance = models.JSONField(default=dict, blank=True)
class RegressionBankItem(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="regression_bank_items")
reference = models.TextField()
content_hash = models.CharField(max_length=128, blank=True)
category = models.CharField(max_length=160, blank=True)
kind = models.CharField(max_length=80, default="IMPORTED")
provenance = models.JSONField(default=dict, blank=True)
evaluation_only = models.BooleanField(default=True)
class FailureCluster(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="failure_clusters")
name = models.CharField(max_length=200)
description = models.TextField(blank=True)
failure_type = models.CharField(max_length=160, blank=True)
severity = models.CharField(max_length=32, default="UNKNOWN")
sample_count = models.PositiveIntegerField(default=0)
representative_examples = models.JSONField(default=list, blank=True)
affected_benchmarks = models.JSONField(default=list, blank=True)
suspected_causes = models.JSONField(default=list, blank=True)
confidence = models.FloatField(default=0)
training_data_coverage = models.JSONField(default=dict, blank=True)
priority = models.FloatField(default=0)
class ModelPromotionPolicy(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="promotion_policies")
name = models.CharField(max_length=160)
version = models.CharField(max_length=80)
criteria = models.JSONField(default=dict)
active = models.BooleanField(default=True)
class ModelPromotionDecision(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="promotion_decisions")
from_champion = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="promotion_sources")
candidate = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="promotion_candidates")
experiment = models.ForeignKey(TrainingExperiment, on_delete=models.PROTECT, related_name="promotion_decisions")
decision = models.CharField(max_length=32, choices=PromotionDecision.choices)
policy = models.ForeignKey(ModelPromotionPolicy, on_delete=models.PROTECT, related_name="decisions")
evaluation_evidence = models.JSONField(default=dict)
judge_result = models.JSONField(default=dict)
reason = models.TextField()
judge_actor = models.CharField(max_length=120, default="MODEL_JUDGE")
class OvernightTrainingProgram(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="overnight_programs")
starting_champion = models.ForeignKey(ModelCheckpoint, on_delete=models.PROTECT, related_name="starting_programs")
ending_champion = models.ForeignKey(ModelCheckpoint, on_delete=models.SET_NULL, null=True, blank=True, related_name="ending_programs")
status = models.CharField(max_length=32, choices=ProgramStatus.choices, default=ProgramStatus.CREATED)
start_time = models.DateTimeField(null=True, blank=True)
deadline = models.DateTimeField()
maximum_wall_seconds = models.PositiveIntegerField()
maximum_training_runs = models.PositiveIntegerField(default=8)
maximum_failed_runs = models.PositiveIntegerField(default=3)
maximum_single_run_seconds = models.PositiveIntegerField()
evaluation_reserve_seconds = models.PositiveIntegerField()
allowed_experiment_types = models.JSONField(default=list, blank=True)
promotion_policy = models.ForeignKey(ModelPromotionPolicy, on_delete=models.PROTECT, related_name="programs")
started_at = models.DateTimeField(null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
termination_reason = models.TextField(blank=True)
pause_after_current_run = models.BooleanField(default=False)
telemetry = models.JSONField(default=dict, blank=True)
class OvernightResearchReport(TimestampedModel):
program = models.OneToOneField(OvernightTrainingProgram, on_delete=models.CASCADE, related_name="report")
markdown = models.TextField()
payload = models.JSONField(default=dict)
artifact_reference = models.TextField(blank=True)
class ModelStudioArtifact(TimestampedModel):
training_project = models.ForeignKey(TrainingProject, on_delete=models.CASCADE, related_name="artifacts")
artifact_type = models.CharField(max_length=120)
name = models.CharField(max_length=255)
content = models.JSONField(default=dict)
readable = models.TextField(blank=True)
source_reference = models.TextField(blank=True)

75
control_plane/model_studio/profiles.py Normal file
View file

@ -0,0 +1,75 @@
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Protocol
@dataclass
class ArchaeologyFinding:
subject: str
confidence: str
evidence: list[str]
details: dict[str, Any]
class ModelProjectProfile(Protocol):
name: str
def archaeology(self, repository_path: str) -> dict[str, Any]: ...
def validate_evaluation_suite(self, repository_path: str, suite_reference: str) -> dict[str, Any]: ...
def training_command(self, recipe: dict[str, Any], output_directory: str) -> list[str]: ...
class GuardModelProfile:
name = "guard"
base_model = "Qwen/Qwen2.5-Coder-3B-Instruct"
def archaeology(self, repository_path: str) -> dict[str, Any]:
root = Path(repository_path)
findings: list[dict[str, Any]] = []
def add(subject: str, confidence: str, evidence: list[Path], **details: Any) -> None:
findings.append({"subject": subject, "confidence": confidence, "evidence": [str(item) for item in evidence], "details": details})
trainer = root / "scripts" / "05_finetune.py"
evaluator = root / "scripts" / "run_evmbench.py"
strict_scorer = root / "scripts" / "score_evmbench_strict.py"
add("base_model", "CONFIRMED" if trainer.exists() else "UNKNOWN", [trainer] if trainer.exists() else [], value=self.base_model)
add("training_entrypoint", "CONFIRMED" if trainer.exists() else "UNKNOWN", [trainer] if trainer.exists() else [], command="python scripts/05_finetune.py --model ... --training-data ... --output ...")
add("evaluation_entrypoint", "CONFIRMED" if evaluator.exists() else "UNKNOWN", [item for item in [evaluator, strict_scorer] if item.exists()], command="python scripts/run_evmbench.py ...", strict_scoring=bool(strict_scorer.exists()))
checkpoints = []
for adapter in sorted((root / "models").glob("**/adapter_model.*")) if (root / "models").exists() else []:
checkpoints.append({"name": adapter.parent.name, "reference": str(adapter.parent), "hash": self._hash_file(adapter), "adapter_type": "LORA", "confidence": "CONFIRMED"})
datasets = []
for manifest in sorted((root / "data" / "production_authorized").glob("*.json")) if (root / "data" / "production_authorized").exists() else []:
try:
payload = json.loads(manifest.read_text(encoding="utf-8"))
count = len(payload) if isinstance(payload, list) else payload.get("record_count")
except (OSError, json.JSONDecodeError):
count = None
datasets.append({"name": manifest.stem, "reference": str(manifest), "hash": self._hash_file(manifest), "record_count": count, "tags": ["imported", "training"]})
reports = [str(path) for path in sorted((root / "results").glob("qwen25_coder_3b_*") if (root / "results").exists() else [])]
benchmark = root / "evmbench"
add("benchmark_checkout", "CONFIRMED" if benchmark.exists() else "UNKNOWN", [benchmark] if benchmark.exists() else [], strict_scorer_exists=strict_scorer.exists())
return {"root_exists": root.exists(), "findings": findings, "checkpoints": checkpoints, "datasets": datasets, "historical_reports": reports, "benchmark_reference": str(benchmark), "trainer_reference": str(trainer), "evaluator_reference": str(evaluator), "strict_scorer_reference": str(strict_scorer)}
def validate_evaluation_suite(self, repository_path: str, suite_reference: str) -> dict[str, Any]:
root = Path(repository_path)
evaluator = root / "scripts" / "run_evmbench.py"
scorer = root / "scripts" / "score_evmbench_strict.py"
benchmark = Path(suite_reference)
available = evaluator.exists() and scorer.exists() and benchmark.exists()
return {"status": "WARNING" if available else "BLOCKED", "evidence": {"evaluator": str(evaluator), "strict_scorer": str(scorer), "benchmark": str(benchmark), "reason": "Scripts and checkout are present, but no fresh Spark baseline has yet proven model loading, output parsing, reproducibility, or contamination checks." if available else "Guard evaluation harness or strict scorer missing."}}
def training_command(self, recipe: dict[str, Any], output_directory: str) -> list[str]:
dataset = str(recipe["training_data"])
return ["python", "scripts/05_finetune.py", "--model", str(recipe.get("base_model", self.base_model)), "--training-data", dataset, "--output", output_directory]
@staticmethod
def _hash_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()

321
control_plane/model_studio/services.py Normal file
View file

@ -0,0 +1,321 @@
from __future__ import annotations
import hashlib
import json
from datetime import timedelta
from pathlib import Path
from typing import Any
from django.db import transaction
from django.utils import timezone
from control_plane.events.bus import EventBus
from control_plane.model_studio.backends import BackendResult, FakeTrainingBackend, TrainingBackend
from control_plane.model_studio.models import (
BenchmarkResult, CheckpointType, CheckpointValidityStatus, Conclusion, Dataset, DatasetValidationStatus,
DatasetVersion, EvaluationRun, EvaluationRunStatus, EvaluationSuite, EvaluationSuiteVersion, ExperimentStatus,
FailureCluster, ModelCheckpoint, ModelPromotionDecision, ModelPromotionPolicy, ModelStudioArtifact,
OvernightResearchReport, OvernightTrainingProgram, ProgramStatus, PromotionDecision, TrainingExperiment,
TrainingProject, TrainingProjectStatus, TrainingRecipe, TrainingRun, TrainingRunStatus,
)
from control_plane.model_studio.profiles import GuardModelProfile, ModelProjectProfile
from control_plane.projects.models import Project, ProjectStatus
class ModelStudioService:
def __init__(self, *, profile: ModelProjectProfile | None = None, backend: TrainingBackend | None = None, bus: EventBus | None = None) -> None:
self.profile = profile or GuardModelProfile()
self.backend = backend or FakeTrainingBackend()
self.bus = bus or EventBus()
def import_guard(self, *, project: Project | None = None, repository_path: str, spark_working_directory: str = "", slug: str = "guard-3b") -> TrainingProject:
project = project or Project.objects.create(name="Guard 3B Model Studio", project_type="MODEL", goal="Reconstruct and safely improve the Guard 3B model.", repository_path=repository_path, status=ProjectStatus.ARCHAEOLOGY)
training_project, _ = TrainingProject.objects.update_or_create(slug=slug, defaults={"project": project, "name": "Guard 3B", "description": "ForgeGuard Qwen2.5-Coder-3B model-development program.", "goal": "Improve Guard only through reproducible, benchmarked scientific experiments.", "capability_target": "Source-grounded smart-contract security findings.", "model_family": "Qwen2.5-Coder", "model_size": "3B", "base_model": GuardModelProfile.base_model, "repository_path": repository_path, "working_directory": spark_working_directory, "default_profile": self.profile.name, "training_backend": type(self.backend).__name__, "status": TrainingProjectStatus.IMPORTING, "metadata": {"spark_working_directory_verified": False, "local_archaeology_repository": repository_path}})
self._event("TRAINING_PROJECT_IMPORTED", training_project, {"repository_path": repository_path})
ModelPromotionPolicy.objects.get_or_create(training_project=training_project, name="Guard conservative V0.1", version="v0.1", defaults={"criteria": {"primary_metric": "strict_canonical_match", "minimum_delta": 0.0, "max_regression": {}, "requires_fresh_baseline": True, "note": "Thresholds remain unconfigured until the imported Guard benchmark exposes comparable metrics."}})
return training_project
def declare_base_champion(self, training_project: TrainingProject) -> ModelCheckpoint:
"""Record the human-mandated starting base model without claiming benchmark evidence."""
checkpoint, _ = ModelCheckpoint.objects.get_or_create(training_project=training_project, reference=training_project.base_model, defaults={"name": "Guard starting base model", "checkpoint_type": CheckpointType.CHAMPION, "content_hash": self._hash_text(training_project.base_model), "validity_status": CheckpointValidityStatus.UNKNOWN, "load_verified": False, "metadata": {"selection": "HUMAN_MANDATED_STARTING_CHAMPION", "evidence_status": "PENDING_FRESH_SPARK_BASELINE"}})
training_project.current_champion = checkpoint
training_project.status = TrainingProjectStatus.BASELINING
training_project.save(update_fields=["current_champion", "status", "updated_at"])
self._event("CHAMPION_SELECTED", training_project, {"checkpoint": str(checkpoint.id), "selection": "HUMAN_MANDATED_STARTING_CHAMPION", "evidence_status": "PENDING_FRESH_SPARK_BASELINE"})
return checkpoint
def archaeology(self, training_project: TrainingProject) -> dict[str, Any]:
training_project.status = TrainingProjectStatus.ARCHAEOLOGY
training_project.save(update_fields=["status", "updated_at"])
self._event("ARCHAEOLOGY_STARTED", training_project, {})
report = self.profile.archaeology(training_project.repository_path)
for item in report["datasets"]:
dataset, _ = Dataset.objects.get_or_create(training_project=training_project, name=item["name"])
DatasetVersion.objects.update_or_create(dataset=dataset, version=item["hash"][:12], defaults={"manifest_reference": item["reference"], "content_hash": item["hash"], "record_count": item["record_count"], "source_metadata": {"archaeology_confidence": "CONFIRMED"}, "tags": item["tags"], "validation_status": DatasetValidationStatus.WARNING, "contamination_status": DatasetValidationStatus.UNKNOWN})
for item in report["checkpoints"]:
ModelCheckpoint.objects.update_or_create(training_project=training_project, reference=item["reference"], defaults={"name": item["name"], "checkpoint_type": CheckpointType.IMPORTED, "content_hash": item["hash"], "adapter_type": item["adapter_type"], "validity_status": CheckpointValidityStatus.VALID, "load_verified": False, "metadata": {"archaeology_confidence": item["confidence"]}})
suite, _ = EvaluationSuite.objects.get_or_create(training_project=training_project, name="Guard EVMBench")
benchmark_reference = report["benchmark_reference"]
suite_version, _ = EvaluationSuiteVersion.objects.update_or_create(suite=suite, version="imported-local", defaults={"reference": benchmark_reference, "content_hash": self._hash_text(benchmark_reference), "command_template": [report["evaluator_reference"]], "groups": ["PRIMARY", "HOLDOUT", "REGRESSION"], "integrity_status": DatasetValidationStatus.UNKNOWN, "integrity_evidence": {"archaeology": report["findings"]}})
artifact = self._artifact(training_project, "TRAINING_ARCHAEOLOGY_REPORT", "Guard archaeology", report)
training_project.metadata = {**training_project.metadata, "archaeology_artifact": str(artifact.id), "historical_reports": report["historical_reports"], "evaluation_suite_version": str(suite_version.id)}
training_project.status = TrainingProjectStatus.NEEDS_REPAIR
training_project.save(update_fields=["metadata", "status", "updated_at"])
self._event("ARCHAEOLOGY_COMPLETED", training_project, {"checkpoints": len(report["checkpoints"]), "datasets": len(report["datasets"]), "status": training_project.status})
return report
def validate_benchmark(self, training_project: TrainingProject) -> EvaluationSuiteVersion:
suite_version = EvaluationSuiteVersion.objects.filter(suite__training_project=training_project).order_by("-created_at").first()
if suite_version is None:
raise ValueError("Run archaeology before benchmark validation.")
result = self.profile.validate_evaluation_suite(training_project.repository_path, suite_version.reference)
suite_version.integrity_status = result["status"]
suite_version.integrity_evidence = result["evidence"]
suite_version.save(update_fields=["integrity_status", "integrity_evidence", "updated_at"])
training_project.status = TrainingProjectStatus.READY if result["status"] == DatasetValidationStatus.VALID else TrainingProjectStatus.NEEDS_REPAIR
training_project.save(update_fields=["status", "updated_at"])
return suite_version
def curate_guard_datasets(self, training_project: TrainingProject) -> dict[str, Any]:
"""Classify imported manifests without mutating source data or inferring missing provenance."""
rows = []
for version in DatasetVersion.objects.filter(dataset__training_project=training_project).select_related("dataset"):
path = Path(version.manifest_reference)
tags = set(version.tags)
status = DatasetValidationStatus.WARNING
contamination = DatasetValidationStatus.UNKNOWN
evidence: dict[str, Any] = {"manifest": str(path)}
if path.name.endswith("_summary.json"):
status, contamination = DatasetValidationStatus.BLOCKED, DatasetValidationStatus.UNKNOWN
tags.update(["summary", "not_training"])
evidence["reason"] = "Summary artifacts are evidence, not trainable records."
else:
summary_path = path.with_name(path.stem + "_summary.json")
summary = self._read_json(summary_path) if summary_path.exists() else {}
payload = self._read_json(path)
record_count = len(payload) if isinstance(payload, list) else None
evidence["summary_reference"] = str(summary_path) if summary_path.exists() else ""
evidence["record_count_observed"] = record_count
if not isinstance(payload, list):
status = DatasetValidationStatus.BLOCKED
tags.update(["not_training", "invalid_manifest_shape"])
evidence["reason"] = "Training manifest must be a JSON record list."
elif summary.get("candidate_only") or "candidate" in path.name.lower() or "provisional" in path.name.lower():
status = DatasetValidationStatus.BLOCKED
tags.update(["candidate_only", "not_training"])
evidence["reason"] = "Candidate/provisional corpus is explicitly not training-approved."
elif "evmbench" in path.name.lower() or summary.get("evmbench_source_included") is True or summary.get("benchmark_source_included") is True:
status = DatasetValidationStatus.BLOCKED
tags.update(["benchmark_exclusion", "not_training"])
contamination = DatasetValidationStatus.WARNING
evidence["reason"] = "Possible benchmark reference requires manual contamination review."
elif summary.get("evmbench_source_included") is False and summary.get("weak_label_data_included") is False:
tags.update(["pilot", "source_disjoint_claimed"])
status = DatasetValidationStatus.WARNING
contamination = DatasetValidationStatus.WARNING
evidence["reason"] = "Source-disjointness is declared, but full schema/provenance/coverage validation remains required."
else:
tags.update(["imported", "manual_provenance_review_required"])
evidence["reason"] = "No sufficient adjacent evidence to mark this manifest training-valid."
if record_count is not None:
version.record_count = record_count
version.tags = sorted(tags)
version.validation_status = status
version.contamination_status = contamination
version.source_metadata = {**version.source_metadata, "curation": evidence}
version.save(update_fields=["record_count", "tags", "validation_status", "contamination_status", "source_metadata", "updated_at"])
rows.append({"dataset": version.dataset.name, "version": version.version, "reference": version.manifest_reference, "validation_status": status, "contamination_status": contamination, "tags": version.tags, "evidence": evidence})
summary = {"total": len(rows), "valid": sum(row["validation_status"] == DatasetValidationStatus.VALID for row in rows), "warning": sum(row["validation_status"] == DatasetValidationStatus.WARNING for row in rows), "blocked": sum(row["validation_status"] == DatasetValidationStatus.BLOCKED for row in rows), "datasets": rows, "decision": "NO_TRAINING_DATASET_APPROVED" if not any(row["validation_status"] == DatasetValidationStatus.VALID for row in rows) else "TRAINING_DATASET_CANDIDATES_AVAILABLE"}
self._artifact(training_project, "DATASET_CURATION_REPORT", "Guard dataset curation", summary)
return summary
def establish_champion(self, training_project: TrainingProject, checkpoint: ModelCheckpoint) -> ModelCheckpoint:
suite = self.validate_benchmark(training_project)
if suite.integrity_status != DatasetValidationStatus.VALID:
raise ValueError("Benchmark integrity is not established; refusing Champion selection.")
if not checkpoint.load_verified:
raise ValueError("Candidate checkpoint has not passed load verification.")
evaluation = self.evaluate(training_project, checkpoint, suite, metrics={"primary": 0.0}, synthetic=False)
if evaluation.status != EvaluationRunStatus.SUCCEEDED:
raise ValueError("Champion evaluation failed.")
checkpoint.checkpoint_type = CheckpointType.CHAMPION
checkpoint.evaluation_status = "EVALUATED"
checkpoint.save(update_fields=["checkpoint_type", "evaluation_status", "updated_at"])
training_project.current_champion = checkpoint
training_project.baseline_evaluation = evaluation
training_project.status = TrainingProjectStatus.READY
training_project.save(update_fields=["current_champion", "baseline_evaluation", "status", "updated_at"])
self._event("CHAMPION_SELECTED", training_project, {"checkpoint": str(checkpoint.id), "evaluation": str(evaluation.id)})
return checkpoint
def evaluate(self, training_project: TrainingProject, checkpoint: ModelCheckpoint, suite: EvaluationSuiteVersion, *, metrics: dict[str, float] | None = None, synthetic: bool = False) -> EvaluationRun:
if suite.integrity_status != DatasetValidationStatus.VALID and not synthetic:
raise ValueError("Evaluation suite integrity is not valid.")
evaluation = EvaluationRun.objects.create(training_project=training_project, checkpoint=checkpoint, suite_version=suite, status=EvaluationRunStatus.RUNNING, integrity_evidence=suite.integrity_evidence)
for metric, value in (metrics or {}).items():
BenchmarkResult.objects.create(evaluation_run=evaluation, metric=metric, value=value, unit="score")
evaluation.status = EvaluationRunStatus.SUCCEEDED
evaluation.completed_at = timezone.now()
evaluation.summary = metrics or {}
evaluation.save(update_fields=["status", "completed_at", "summary", "updated_at"])
self._event("EVALUATION_COMPLETED", training_project, {"checkpoint": str(checkpoint.id), "evaluation": str(evaluation.id)})
return evaluation
def propose_experiment(self, training_project: TrainingProject, contract: dict[str, Any]) -> TrainingExperiment:
required = ["hypothesis", "reasoning", "intervention", "controls", "expected_result", "primary_success_metric", "success_threshold", "regression_constraints", "rejection_condition", "ambiguity_policy", "maximum_runtime_seconds", "compute_budget"]
missing = [field for field in required if contract.get(field) in (None, "", {}, [])]
if missing:
raise ValueError("Incomplete scientific contract: " + ", ".join(missing))
fingerprint = self._fingerprint({"intervention": contract["intervention"], "controls": contract["controls"], "input_checkpoint": str(training_project.current_champion_id)})
duplicate = TrainingExperiment.objects.filter(training_project=training_project, fingerprint=fingerprint).exclude(status=ExperimentStatus.CANCELLED).first()
if duplicate:
raise ValueError(f"Duplicate experiment: {duplicate.experiment_id}")
value = self._experiment_value(contract)
experiment = TrainingExperiment.objects.create(experiment_id=contract.get("experiment_id", f"EXP-{training_project.slug.upper()}-{TrainingExperiment.objects.filter(training_project=training_project).count() + 1:03d}"), training_project=training_project, title=contract.get("title", contract["hypothesis"][:255]), hypothesis=contract["hypothesis"], reasoning=contract["reasoning"], intervention=contract["intervention"], controls=contract["controls"], expected_result=contract["expected_result"], primary_success_metric=contract["primary_success_metric"], success_threshold=contract["success_threshold"], regression_constraints=contract["regression_constraints"], rejection_condition=contract["rejection_condition"], ambiguity_policy=contract["ambiguity_policy"], estimated_runtime_seconds=int(contract.get("estimated_runtime_seconds", 0)), maximum_runtime_seconds=int(contract["maximum_runtime_seconds"]), compute_budget=contract["compute_budget"], priority=float(contract.get("priority", value)), expected_information_gain=float(contract.get("expected_information_gain", 0)), expected_improvement=float(contract.get("expected_improvement", 0)), estimated_compute_cost=float(contract.get("estimated_compute_cost", 1)), experiment_value_score=value, fingerprint=fingerprint, approved_by_model_director=True)
self._event("EXPERIMENT_PROPOSED", training_project, {"experiment": experiment.experiment_id, "value": value})
return experiment
@transaction.atomic
def run_experiment(self, experiment: TrainingExperiment, recipe_configuration: dict[str, Any]) -> TrainingRun:
project = experiment.training_project
champion = project.current_champion
if champion is None:
raise ValueError("Cannot train without an immutable Champion.")
if type(self.backend).__name__ == "SparkGuardBackend" and not project.metadata.get("spark_working_directory_verified"):
raise ValueError("Spark Guard working directory is not verified; refusing remote training.")
if experiment.status not in {ExperimentStatus.PROPOSED, ExperimentStatus.QUEUED}:
raise ValueError("Experiment is not runnable.")
recipe_hash = self._fingerprint(recipe_configuration)
recipe, _ = TrainingRecipe.objects.get_or_create(training_project=project, recipe_hash=recipe_hash, defaults={"name": experiment.experiment_id, "configuration": recipe_configuration})
recipe.immutable = True
recipe.save(update_fields=["immutable", "updated_at"])
for dataset_version in DatasetVersion.objects.filter(dataset__training_project=project, manifest_reference__in=recipe_configuration.get("dataset_references", [])):
dataset_version.immutable = True
dataset_version.save(update_fields=["immutable", "updated_at"])
run = TrainingRun.objects.create(experiment=experiment, recipe=recipe, input_checkpoint=champion, status=TrainingRunStatus.STARTING, working_directory=project.working_directory, command=[])
experiment.status = ExperimentStatus.RUNNING
experiment.save(update_fields=["status", "updated_at"])
self._event("TRAINING_STARTED", project, {"experiment": experiment.experiment_id, "run": str(run.id)})
output_directory = str(Path(project.working_directory) / "artifex_runs" / experiment.experiment_id)
command = self.profile.training_command(recipe_configuration, output_directory) if recipe_configuration.get("training_data") and hasattr(self.profile, "training_command") else []
run.command = command
run.save(update_fields=["command", "updated_at"])
outcome = self.backend.launch(command=command, working_directory=project.working_directory, timeout_seconds=experiment.maximum_runtime_seconds)
self._apply_training_outcome(run, outcome)
return run
def _apply_training_outcome(self, run: TrainingRun, outcome: BackendResult) -> None:
experiment = run.experiment
project = experiment.training_project
if outcome.status != "SUCCEEDED":
run.status = getattr(TrainingRunStatus, outcome.status, TrainingRunStatus.FAILED)
run.failure_category = outcome.failure_category or outcome.status
run.failure_details = outcome.failure_details
run.completed_at = timezone.now()
run.save(update_fields=["status", "failure_category", "failure_details", "completed_at", "updated_at"])
experiment.status = ExperimentStatus.FAILED
experiment.conclusion = Conclusion.EXECUTION_FAILED
experiment.result_summary = run.failure_details
experiment.save(update_fields=["status", "conclusion", "result_summary", "updated_at"])
self._event("TRAINING_FAILED", project, {"experiment": experiment.experiment_id, "failure": run.failure_category})
return
checkpoint = ModelCheckpoint.objects.create(training_project=project, name=f"{experiment.experiment_id}-challenger", checkpoint_type=CheckpointType.CHALLENGER, reference=outcome.checkpoint_reference, content_hash=outcome.checkpoint_hash, base_checkpoint=run.input_checkpoint, training_run=run, recipe=run.recipe, validity_status=CheckpointValidityStatus.VALID if self.backend.validate_checkpoint(outcome.checkpoint_reference) else CheckpointValidityStatus.CORRUPT, load_verified=self.backend.validate_checkpoint(outcome.checkpoint_reference))
run.output_checkpoint = checkpoint
run.status = TrainingRunStatus.SUCCEEDED
run.completed_at = timezone.now()
run.save(update_fields=["output_checkpoint", "status", "completed_at", "updated_at"])
experiment.status = ExperimentStatus.EVALUATING
experiment.save(update_fields=["status", "updated_at"])
self._event("TRAINING_COMPLETED", project, {"experiment": experiment.experiment_id, "checkpoint": str(checkpoint.id)})
def decide_promotion(self, experiment: TrainingExperiment, evaluation: EvaluationRun, policy: ModelPromotionPolicy) -> ModelPromotionDecision:
project = experiment.training_project
champion = project.current_champion
candidate = evaluation.checkpoint
if champion is None or candidate.training_run_id is None:
raise ValueError("Promotion requires a Challenger and starting Champion.")
baseline = project.baseline_evaluation
if baseline is None or baseline.suite_version_id != evaluation.suite_version_id:
raise ValueError("Champion and Challenger must use the same evaluation version.")
primary = policy.criteria.get("primary_metric", "primary")
minimum_delta = float(policy.criteria.get("minimum_delta", 0))
candidate_value = float(evaluation.summary.get(primary, 0))
baseline_value = float(baseline.summary.get(primary, 0))
regression_ok = all(float(evaluation.summary.get(metric, 0)) >= float(baseline.summary.get(metric, 0)) - float(limit) for metric, limit in policy.criteria.get("max_regression", {}).items())
eligible = candidate.load_verified and evaluation.status == EvaluationRunStatus.SUCCEEDED and candidate_value - baseline_value >= minimum_delta and regression_ok
decision = PromotionDecision.PROMOTE if eligible else PromotionDecision.REJECT
rationale = "Objective promotion criteria satisfied." if eligible else "Objective promotion criteria not satisfied."
record = ModelPromotionDecision.objects.create(training_project=project, from_champion=champion, candidate=candidate, experiment=experiment, decision=decision, policy=policy, evaluation_evidence={"baseline": baseline.summary, "candidate": evaluation.summary, "delta": candidate_value - baseline_value, "regression_ok": regression_ok}, judge_result={"actor": "MODEL_JUDGE", "deterministic_gate": eligible}, reason=rationale)
if eligible:
champion.checkpoint_type = CheckpointType.IMPORTED
champion.save(update_fields=["checkpoint_type", "updated_at"])
candidate.checkpoint_type = CheckpointType.CHAMPION
candidate.save(update_fields=["checkpoint_type", "updated_at"])
project.current_champion = candidate
project.save(update_fields=["current_champion", "updated_at"])
experiment.status, experiment.conclusion = ExperimentStatus.PROMOTED, Conclusion.SUPPORTED
self._event("CHALLENGER_PROMOTED", project, {"experiment": experiment.experiment_id, "checkpoint": str(candidate.id)})
else:
experiment.status, experiment.conclusion = ExperimentStatus.REJECTED, Conclusion.REFUTED
experiment.result_summary = rationale
experiment.save(update_fields=["status", "conclusion", "result_summary", "updated_at"])
return record
def create_program(self, training_project: TrainingProject, policy: ModelPromotionPolicy, *, wall_seconds: int = 8 * 3600, max_runs: int = 8, max_failed: int = 3, max_single_run: int = 150 * 60, evaluation_reserve: int = 90 * 60) -> OvernightTrainingProgram:
if training_project.current_champion is None:
raise ValueError("A verified Champion is required before starting an overnight program.")
now = timezone.now()
return OvernightTrainingProgram.objects.create(training_project=training_project, starting_champion=training_project.current_champion, deadline=now + timedelta(seconds=wall_seconds), maximum_wall_seconds=wall_seconds, maximum_training_runs=max_runs, maximum_failed_runs=max_failed, maximum_single_run_seconds=max_single_run, evaluation_reserve_seconds=evaluation_reserve, allowed_experiment_types=["DATASET_MIXTURE", "RECIPE", "CHECKPOINT"], promotion_policy=policy)
def can_start(self, program: OvernightTrainingProgram, experiment: TrainingExperiment, estimated_evaluation_seconds: int) -> tuple[bool, str]:
remaining = max(0, int((program.deadline - timezone.now()).total_seconds()))
runs = TrainingRun.objects.filter(experiment__training_project=program.training_project).count()
failures = TrainingRun.objects.filter(experiment__training_project=program.training_project, status__in=[TrainingRunStatus.FAILED, TrainingRunStatus.OOM, TrainingRunStatus.TIMEOUT]).count()
needed = experiment.estimated_runtime_seconds + estimated_evaluation_seconds + program.evaluation_reserve_seconds
if runs >= program.maximum_training_runs:
return False, "RUN_LIMIT"
if failures >= program.maximum_failed_runs:
return False, "FAILURE_LIMIT"
if needed > remaining:
return False, "FINAL_EVALUATION_RESERVE"
return True, "READY"
def morning_report(self, program: OvernightTrainingProgram) -> OvernightResearchReport:
project = program.training_project
experiments = list(project.experiments.order_by("created_at"))
payload = {"program_id": str(program.id), "starting_champion": str(program.starting_champion_id), "ending_champion": str(project.current_champion_id), "status": "NO_CHAMPION_CHANGE" if project.current_champion_id == program.starting_champion_id else "CHAMPION_CHANGED", "experiments": [{"id": item.experiment_id, "hypothesis": item.hypothesis, "status": item.status, "conclusion": item.conclusion, "learning": item.result_summary} for item in experiments], "provenance": {"repository": project.repository_path, "baseline_evaluation": str(project.baseline_evaluation_id or "")}}
markdown = "# GUARD OVERNIGHT RESEARCH REPORT\n\n" + json.dumps(payload, indent=2, default=str)
program.ending_champion = project.current_champion
program.status = ProgramStatus.COMPLETED
program.completed_at = timezone.now()
program.termination_reason = program.termination_reason or "FINALIZED"
program.save(update_fields=["ending_champion", "status", "completed_at", "termination_reason", "updated_at"])
report, _ = OvernightResearchReport.objects.update_or_create(program=program, defaults={"markdown": markdown, "payload": payload})
self._artifact(project, "OVERNIGHT_RESEARCH_REPORT", "Morning research report", payload, markdown)
self._event("OVERNIGHT_COMPLETED", project, {"program": str(program.id), "status": payload["status"]})
return report
def _artifact(self, training_project: TrainingProject, artifact_type: str, name: str, content: dict[str, Any], readable: str = "") -> ModelStudioArtifact:
return ModelStudioArtifact.objects.create(training_project=training_project, artifact_type=artifact_type, name=name, content=content, readable=readable)
def _event(self, event_type: str, training_project: TrainingProject, payload: dict[str, Any]) -> None:
self.bus.publish(event_type, project=training_project.project, actor="MODEL_STUDIO", payload={"training_project": str(training_project.id), **payload})
@staticmethod
def _hash_text(value: str) -> str:
return hashlib.sha256(value.encode()).hexdigest()
@staticmethod
def _read_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
@staticmethod
def _fingerprint(value: dict[str, Any]) -> str:
return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest()
@staticmethod
def _experiment_value(contract: dict[str, Any]) -> float:
return round(float(contract.get("expected_improvement", 0)) * float(contract.get("confidence", 1)) * float(contract.get("expected_information_gain", 1)) * float(contract.get("novelty", 1)) / max(1.0, float(contract.get("estimated_compute_cost", 1))), 4)

12
control_plane/model_studio/views.py Normal file
View file

@ -0,0 +1,12 @@
from django.shortcuts import get_object_or_404, render
from control_plane.model_studio.models import TrainingProject
def model_studio(request):
return render(request, "control_plane/model_studio.html", {"projects": TrainingProject.objects.select_related("current_champion", "baseline_evaluation").order_by("-updated_at")})
def model_studio_project(request, project_id):
training_project = get_object_or_404(TrainingProject.objects.select_related("current_champion", "baseline_evaluation"), id=project_id)
return render(request, "control_plane/model_studio_project.html", {"training_project": training_project, "program": training_project.overnight_programs.order_by("-created_at").first(), "experiments": training_project.experiments.order_by("-created_at")[:25], "clusters": training_project.failure_clusters.order_by("-priority")[:25]})

166
docs/handoff_current_state_20260817.md Normal file
View file

@ -0,0 +1,166 @@
# Artifex Handoff - Current State - 2026-08-17
## Current Objective
We have been repairing and dogfooding the `CRYPTO / PROTOCOL VENTURE COHORT` pipeline, specifically the V0.3.1 cohort and the V0.3.2B token-judge evidence repair.
Hard constraint: **NO NEW IDEAS** for the repair work. Only reprocess the existing six former V0.3.1 survivors.
Safety constraints remain active:
- Do not issue or sell native tokens.
- Do not sell NFTs or memberships.
- Do not launch mainnet.
- Do not create liquidity or market make.
- Do not contact investors, users, or customers.
- Do not fundraise.
- Do not spend real money.
## Environment
- Workspace: `S:\PycharmProjects\Artifex`
- Repo remote: `http://192.168.1.162:3000/daniel/Artifex.git`
- Database: `sqlite:///db.sqlite3`
- SearXNG: `http://192.168.1.162:8080`
- Qwen: `http://192.168.1.162:8002`
- Current date in environment: `2026-08-17`
- Known harmless push warning: `git: 'credential-manager-core' is not a git command. See 'git --help'.`
Spark remote validation was previously blocked by SSH auth:
- `daniel@192.168.1.162: Permission denied (publickey,password).`
## Current Cohort
- Cohort ID: `CPV031-20260816215626-66d636b2`
- GraphRun: `10`
- Existing six former V0.3.1 survivors only:
- `Agent Passport Clearinghouse`
- `PatchBond Network`
- `ProofGrid Compute Attestation Network`
- `ProofBond`
- `Compute Clearance Network`
- `ProofGrid Compute Escrow`
## Latest Implemented Work
### V0.3.2B Token Judge Evidence Repair
Implemented strict Sol Token Utility Judge evidence validation.
Required non-empty fields now enforced:
- `argument_for_native_token`
- `argument_against_native_token`
- `external_collateral_counterfactual`
- `native_token_removed_breaks`
- `native_token_removed_explanation`
- `material_improvements_over_USDC_ETH`
- `native_asset_costs_and_risks`
- `final_rationale`
Behavior now:
- Rejects incomplete Sol judge response.
- Retries incomplete judge response up to 3 attempts.
- Raises `ValueError` after exhausted retries.
- Does **not** fall back to deterministic/heuristic classification when Sol produced an incomplete response.
- Normalizes Sol response when `external_collateral_counterfactual` appears nested under `counterfactual_analysis`.
- Uppercases `utility_scores` keys during normalization.
- Persists `final_rationale` in token metadata as well as `TokenUtilityAssessment.rationale`.
- `crypto_score_row()` now reuses persisted `token_utility_assessment` instead of calling the token judge again.
- Final Sol IC prompt was compacted so it returns usable JSON.
Rationale invariants now checked:
- Every `TOKEN_OPTIONAL` rationale must explain why ETH/USDC/external collateral is preferable despite claimed degradation.
- Every `TOKEN_ESSENTIAL` / `TOKEN_STRONGLY_JUSTIFIED` rationale must explain what protocol-specific property cannot be reproduced economically enough with ETH/USDC/external collateral.
- The code does not automatically promote due to material degradation.
## Latest Six-Proposal Results
Before/after classifications from V0.3.2B:
| Company | Before | After | Score | Final IC Decision | Final IC Score | Required Field Gaps |
|---|---:|---:|---:|---:|---:|---:|
| Agent Passport Clearinghouse | `TOKEN_OPTIONAL` | `TOKEN_UNNECESSARY` | `18.0` | `REJECT_TOKEN_NOT_NEEDED` | `62.3` | `[]` |
| PatchBond Network | `TOKEN_OPTIONAL` | `TOKEN_OPTIONAL` | `48.0` | `REJECT_TOKEN_NOT_NEEDED` | `66.4` | `[]` |
| ProofGrid Compute Attestation Network | `TOKEN_OPTIONAL` | `TOKEN_OPTIONAL` | `42.0` | `REJECT_TOKEN_UTILITY_WEAK` | `63.6` | `[]` |
| ProofBond | `TOKEN_OPTIONAL` | `TOKEN_OPTIONAL` | `54.0` | `REJECT_TOKEN_UTILITY_WEAK` | `64.3` | `[]` |
| Compute Clearance Network | `TOKEN_OPTIONAL` | `TOKEN_STRONGLY_JUSTIFIED` | `78.0` | `PROTOCOL_VALIDATE` | `79.7` | `[]` |
| ProofGrid Compute Escrow | `TOKEN_OPTIONAL` | `TOKEN_OPTIONAL` | `58.0` | `REVISE_TOKEN_MODEL` | `67.9` | `[]` |
Current corrected cohort state:
- Token unnecessary: `1`
- Token optional/routed SaaS: `4`
- Token strongly justified: `1`
- Token essential: `0`
- Serious crypto survivors: `0`
- Crypto survivors: `0`
- Autonomous crypto survivors: `0`
- Assisted high-potential: `0`
- Finalists: `0`
- V0.3.2B required field gaps: `0`
- Token red-team failures:
- `TOKEN_NOT_REQUIRED: 5`
- `YIELD_DEPENDENCY: 1`
- `SLASHING_DEPENDS_ON_HUMAN_JUDGMENT: 1`
Important nuance: `Compute Clearance Network` became `TOKEN_STRONGLY_JUSTIFIED`, but final cohort finalists remain `0`. It is not automatically promoted to finalist.
## Latest Files Changed
- `agents/crypto_venture.py`
- Strict token judge evidence validation/retry/fail-closed behavior.
- Token judge response normalization.
- Persisted required fields.
- Reused persisted token assessment in `crypto_score_row()`.
- Compact final Sol IC prompt.
- `tests/test_crypto_venture_cohort.py`
- Added coverage for complete required judge fields.
- Added retry-success and retry-exhaustion/fail-closed tests.
- `control_plane/ventures/management/commands/export_crypto_venture_cohort.py`
- Added `V0.3.2B Before/After` markdown section.
- `docs/crypto_venture_cohort_v031_20260816.md`
- Updated exported V0.3.2B report.
- `docs/crypto_venture_cohort_v031_20260816.json`
- Updated exported V0.3.2B report data.
## Latest Commits Pushed
Recent relevant commits:
- `35d3346 Repair crypto token judge evidence export`
- `6e89ea8 Require complete crypto token judge evidence`
- `052d1fc Add corrected crypto cohort V0.3.1 export`
- `94b9ab6 Repair crypto token utility scoring`
- `54ffe92 Add crypto protocol cohort V0.3.1 export`
The most recent push succeeded to `main` with the known harmless credential-manager warning.
## Verification
Latest verification passed:
- `python manage.py check`: passed
- `python -m pytest -p no:django -q tests/test_crypto_venture_cohort.py`: `14 passed`
- Full suite: `157 passed, 1 skipped`
Warnings are existing Python/Django/LangGraph deprecation warnings and pytest collection warnings for `TestRun`; no failing tests.
## Current Git State At Time Of Handoff
After the latest commit/push, `git status --short` was clean.
This handoff file itself is newly added after that push and should be committed/pushed if the user wants the note persisted remotely.
## If Continuing
Recommended next steps only if the user asks to continue:
1. Commit and push this handoff note if desired.
2. If further V0.3.2B analysis is requested, inspect `docs/crypto_venture_cohort_v031_20260816.md` first.
3. Do not regenerate ideas unless explicitly instructed.
4. Do not rerun broad research unless explicitly requested or needed for a new judge repair.
5. Keep all token/mainnet/fundraising/user-contact/no-spend stop conditions active.

1
templates/control_plane/base.html
View file

@ -46,6 +46,7 @@
<h1>ARTIFEX</h1>
<a href="/">Dashboard</a>
<a href="/projects/">Projects</a>
<a href="/model-studio/">Model Studio</a>
<a href="/agents/">Agents</a>
<a href="/progeny/">Progeny</a>
<a href="/steward/">Steward</a>

6
templates/control_plane/model_studio.html Normal file
View file

@ -0,0 +1,6 @@
{% extends "control_plane/base.html" %}
{% block title %}Model Studio | Artifex{% endblock %}
{% block content %}
<header class="page"><div><h2>Model Studio</h2><p class="muted">Archaeology, reproducible experiments, and bounded overnight research.</p></div></header>
<table><thead><tr><th>Training Project</th><th>Status</th><th>Champion</th><th>Profile</th></tr></thead><tbody>{% for item in projects %}<tr><td><a href="{% url 'model_studio_project' item.id %}">{{ item.name }}</a></td><td><span class="badge">{{ item.status }}</span></td><td>{{ item.current_champion.name|default:"Not established" }}</td><td>{{ item.default_profile }}</td></tr>{% empty %}<tr><td colspan="4">No Model Studio projects imported.</td></tr>{% endfor %}</tbody></table>
{% endblock %}

8
templates/control_plane/model_studio_project.html Normal file
View file

@ -0,0 +1,8 @@
{% extends "control_plane/base.html" %}
{% block title %}{{ training_project.name }} | Model Studio{% endblock %}
{% block content %}
<header class="page"><div><h2>{{ training_project.name }}</h2><p class="muted">{{ training_project.capability_target }}</p></div><span class="badge">{{ training_project.status }}</span></header>
<section class="grid"><div class="card"><h3>Current Champion</h3><p>{{ training_project.current_champion.name|default:"Not established" }}</p></div><div class="card"><h3>Baseline</h3><p>{{ training_project.baseline_evaluation.status|default:"Not run" }}</p></div><div class="card"><h3>Overnight</h3><p>{{ program.status|default:"Not scheduled" }}</p></div></section>
<section class="panel"><h3>Experiments</h3><table><thead><tr><th>ID</th><th>Hypothesis</th><th>Status</th><th>Value</th></tr></thead><tbody>{% for item in experiments %}<tr><td>{{ item.experiment_id }}</td><td>{{ item.hypothesis }}</td><td>{{ item.status }}</td><td>{{ item.experiment_value_score }}</td></tr>{% empty %}<tr><td colspan="4">No experiments proposed.</td></tr>{% endfor %}</tbody></table></section>
<section class="panel"><h3>Failure Clusters</h3>{% for item in clusters %}<p><strong>{{ item.name }}</strong>: {{ item.description }}</p>{% empty %}<p class="muted">Baseline failure mining has not produced clusters yet.</p>{% endfor %}</section>
{% endblock %}

123
tests/test_model_studio_v01.py Normal file
View file

@ -0,0 +1,123 @@
from __future__ import annotations
from datetime import timedelta
import json
import pytest
from django.utils import timezone
from control_plane.model_studio.backends import FakeTrainingBackend
from control_plane.model_studio.models import (
CheckpointType, CheckpointValidityStatus, Dataset, DatasetVersion, DatasetValidationStatus,
EvaluationSuite, EvaluationSuiteVersion, ExperimentStatus, ModelCheckpoint, ModelPromotionPolicy,
PromotionDecision, TrainingProject, TrainingProjectStatus,
)
from control_plane.model_studio.services import ModelStudioService
from control_plane.projects.models import Project
def studio(outcomes=None):
return ModelStudioService(backend=FakeTrainingBackend(outcomes))
def ready_project():
project = Project.objects.create(name="Guard", project_type="MODEL", goal="test")
training_project = TrainingProject.objects.create(project=project, name="Guard", slug=f"guard-{project.id.hex[:8]}", goal="test", status=TrainingProjectStatus.READY)
dataset = Dataset.objects.create(training_project=training_project, name="train")
version = DatasetVersion.objects.create(dataset=dataset, version="v1", manifest_reference="fake://train", content_hash="a" * 64, record_count=2, validation_status=DatasetValidationStatus.VALID)
checkpoint = ModelCheckpoint.objects.create(training_project=training_project, name="champion", checkpoint_type=CheckpointType.CHAMPION, reference="fake://champion", content_hash="b" * 64, validity_status=CheckpointValidityStatus.VALID, load_verified=True)
suite = EvaluationSuite.objects.create(training_project=training_project, name="suite")
suite_version = EvaluationSuiteVersion.objects.create(suite=suite, version="v1", reference="fake://suite", content_hash="c" * 64, integrity_status=DatasetValidationStatus.VALID, immutable=True)
training_project.current_champion = checkpoint
training_project.save(update_fields=["current_champion", "updated_at"])
service = studio()
baseline = service.evaluate(training_project, checkpoint, suite_version, metrics={"primary": 0.5, "critical": 0.5}, synthetic=True)
training_project.baseline_evaluation = baseline
training_project.save(update_fields=["baseline_evaluation", "updated_at"])
policy = ModelPromotionPolicy.objects.create(training_project=training_project, name="test", version="v1", criteria={"primary_metric": "primary", "minimum_delta": 0.1, "max_regression": {"critical": 0.02}})
return service, training_project, version, suite_version, policy
def contract(**overrides):
payload = {"hypothesis": "Targeted replay improves primary metric.", "reasoning": "A baseline failure cluster supports this.", "intervention": {"learning_rate": 0.0001}, "controls": {"same_champion": True}, "expected_result": "primary +0.1", "primary_success_metric": "primary", "success_threshold": {"minimum_delta": 0.1}, "regression_constraints": {"critical": 0.02}, "rejection_condition": "No meaningful improvement", "ambiguity_policy": "replicate", "maximum_runtime_seconds": 120, "compute_budget": {"gpu_seconds": 120}, "expected_improvement": 0.2, "expected_information_gain": 0.8, "estimated_compute_cost": 1, "novelty": 1, "confidence": 0.8, "estimated_runtime_seconds": 60}
return {**payload, **overrides}
def test_recipe_and_completed_dataset_version_are_immutable():
service, training_project, dataset_version, _, _ = ready_project()
experiment = service.propose_experiment(training_project, contract())
run = service.run_experiment(experiment, {"dataset_references": ["fake://train"]})
assert run.recipe.immutable is True
dataset_version.refresh_from_db()
assert dataset_version.immutable is True
run.recipe.configuration = {"changed": True}
with pytest.raises(ValueError, match="TrainingRecipe is immutable"):
run.recipe.save()
dataset_version.record_count = 3
with pytest.raises(ValueError, match="DatasetVersion is immutable"):
dataset_version.save()
def test_incomplete_scientific_contract_and_duplicate_experiment_are_rejected():
service, training_project, _, _, _ = ready_project()
with pytest.raises(ValueError, match="Incomplete scientific contract"):
service.propose_experiment(training_project, {"hypothesis": "thin"})
service.propose_experiment(training_project, contract())
with pytest.raises(ValueError, match="Duplicate experiment"):
service.propose_experiment(training_project, contract())
def test_failed_execution_is_not_hypothesis_rejection():
service, training_project, _, _, _ = ready_project()
service.backend = FakeTrainingBackend([{"status": "OOM", "failure_category": "OOM", "failure_details": "simulated"}])
experiment = service.propose_experiment(training_project, contract())
run = service.run_experiment(experiment, {})
experiment.refresh_from_db()
assert run.status == "OOM"
assert experiment.status == ExperimentStatus.FAILED
assert experiment.conclusion == "EXECUTION_FAILED"
def test_challenger_promotion_requires_same_evaluation_version_and_objective_gate():
service, training_project, _, suite_version, policy = ready_project()
service.backend = FakeTrainingBackend([{"status": "SUCCEEDED", "checkpoint_reference": "fake://challenger"}])
experiment = service.propose_experiment(training_project, contract())
run = service.run_experiment(experiment, {})
evaluation = service.evaluate(training_project, run.output_checkpoint, suite_version, metrics={"primary": 0.7, "critical": 0.5}, synthetic=True)
decision = service.decide_promotion(experiment, evaluation, policy)
training_project.refresh_from_db()
assert decision.decision == PromotionDecision.PROMOTE
assert training_project.current_champion_id == run.output_checkpoint_id
def test_regression_prevents_promotion_and_deadline_reserve_blocks_start():
service, training_project, _, suite_version, policy = ready_project()
service.backend = FakeTrainingBackend([{"status": "SUCCEEDED", "checkpoint_reference": "fake://challenger"}])
experiment = service.propose_experiment(training_project, contract())
run = service.run_experiment(experiment, {})
evaluation = service.evaluate(training_project, run.output_checkpoint, suite_version, metrics={"primary": 0.8, "critical": 0.4}, synthetic=True)
decision = service.decide_promotion(experiment, evaluation, policy)
assert decision.decision == PromotionDecision.REJECT
program = service.create_program(training_project, policy, wall_seconds=60, evaluation_reserve=30)
program.deadline = timezone.now() + timedelta(seconds=60)
program.save(update_fields=["deadline", "updated_at"])
allowed, reason = service.can_start(program, experiment, estimated_evaluation_seconds=30)
assert allowed is False
assert reason == "FINAL_EVALUATION_RESERVE"
def test_dataset_curation_respects_explicit_no_evmbench_claim(tmp_path):
service, training_project, _, _, _ = ready_project()
manifest = tmp_path / "verified_pilot.json"
manifest.write_text(json.dumps([{"input": "pragma solidity ^0.8.0;", "output": "{}"}]), encoding="utf-8")
summary = tmp_path / "verified_pilot_summary.json"
summary.write_text(json.dumps({"evmbench_source_included": False, "weak_label_data_included": False}), encoding="utf-8")
dataset = Dataset.objects.create(training_project=training_project, name="verified-pilot")
version = DatasetVersion.objects.create(dataset=dataset, version="pilot", manifest_reference=str(manifest), content_hash="d" * 64)
report = service.curate_guard_datasets(training_project)
version.refresh_from_db()
assert version.validation_status == DatasetValidationStatus.WARNING
assert version.contamination_status == DatasetValidationStatus.WARNING
assert report["valid"] == 0