From bba15a632bf2aa93922ff757a6d2437362a90f49 Mon Sep 17 00:00:00 2001
From: Daniel Maddern <daniel.maddern@illuvium.io>
Date: Mon, 17 Aug 2026 02:16:25 +0700
Subject: [PATCH] Add Spark Guard overnight controller

---
 scripts/run_guard3b_overnight_spark.sh | 75 ++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100755 scripts/run_guard3b_overnight_spark.sh

diff --git a/scripts/run_guard3b_overnight_spark.sh b/scripts/run_guard3b_overnight_spark.sh
new file mode 100755
index 0000000..77c9ff0
--- /dev/null
+++ b/scripts/run_guard3b_overnight_spark.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Bounded Spark-only Guard 3B program. Commands are fixed; Qwen selects only a safe next recipe.
+set -euo pipefail
+
+ROOT=/home/daniel/forgeguard
+PROJECT="$ROOT/project"
+DATA="$ROOT/artifex_dataset_versions/guard_curated_v03/training.json"
+RUN_ROOT="$PROJECT/artifex_overnight/guard3b_v03_overnight"
+CANARY=artifex-guard3b-v03-canary
+IMAGE=forgeguard-qwen36-unsloth:latest
+MODEL=unsloth/qwen2.5-coder-3b-instruct-bnb-4bit
+MAX_TRAINING_RUNS=${MAX_TRAINING_RUNS:-3}
+DEADLINE_EPOCH=${DEADLINE_EPOCH:-$(( $(date +%s) + 28800 ))}
+mkdir -p "$RUN_ROOT"
+
+docker_run() {
+  docker run --rm --gpus all --shm-size=16g \
+    -v "$PROJECT:/workspace" \
+    -v "$ROOT/artifex_dataset_versions:/workspace/artifex_dataset_versions" \
+    -v /home/daniel/.cache/huggingface:/root/.cache/huggingface \
+    "$IMAGE" "$@"
+}
+
+evaluate() {
+  local name=$1 adapter=${2:-}
+  local result="$RUN_ROOT/${name}_evmbench.json" score="$RUN_ROOT/${name}_strict.json"
+  local args=(python /workspace/scripts/run_evmbench.py --model "$MODEL" --unsloth --save "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json")
+  if [[ -n "$adapter" ]]; then args+=(--adapter "/workspace/artifex_overnight/$adapter"); fi
+  docker_run "${args[@]}" > "$RUN_ROOT/${name}_evaluation.log" 2>&1
+  docker_run python /workspace/scripts/score_evmbench_strict.py "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json" > "$score" 2>&1
+  [[ -s "$result" && -s "$score" ]]
+}
+
+qwen_choice() {
+  local summary
+  summary=$(tail -c 10000 "$RUN_ROOT/base_strict.json" 2>/dev/null; tail -c 10000 "$RUN_ROOT/current_strict.json" 2>/dev/null || true)
+  local body
+  body=$(jq -n --arg prompt "You are MODEL_DIRECTOR for a bounded Guard 3B overnight run. Same-harness strict-score evidence follows: $summary Choose one JSON decision only: {\"action\":\"CONTINUE\"|\"REPLICATE\"|\"STOP\",\"rationale\":\"...\"}. CONTINUE means resume the current adapter with lower LR; REPLICATE means fresh seed from the immutable base; STOP means evidence/remaining time does not justify another run." '{model:"qwen38",messages:[{role:"user",content:$prompt}],temperature:0,max_tokens:200}')
+  curl -fsS --max-time 180 http://127.0.0.1:8002/v1/chat/completions -H 'Content-Type: application/json' -d "$body" | jq -r '.choices[0].message.content' > "$RUN_ROOT/qwen_plan.json" || true
+  jq -r '.action // empty' "$RUN_ROOT/qwen_plan.json" 2>/dev/null || echo CONTINUE
+}
+
+train() {
+  local name=$1 resume=$2 seed=$3 lr=$4 output="$PROJECT/artifex_overnight/$name"
+  local args=(python /workspace/finetune_qwen36_unsloth.py --model "$MODEL" --training-data /workspace/artifex_dataset_versions/guard_curated_v03/training.json --output "/workspace/artifex_overnight/$name" --steps 100 --batch-size 1 --learning-rate "$lr" --max-seq-length 2048 --max-output-tokens 512 --validation-max-examples 64 --seed "$seed")
+  if [[ -n "$resume" ]]; then args+=(--resume-adapter "/workspace/artifex_overnight/$resume"); fi
+  docker_run "${args[@]}" > "$RUN_ROOT/${name}_training.log" 2>&1
+  test -f "$output/adapter_model.safetensors"
+}
+
+docker wait "$CANARY" > "$RUN_ROOT/canary_exit_code.txt"
+docker logs "$CANARY" > "$RUN_ROOT/canary_training.log" 2>&1
+cp "$PROJECT/artifex_overnight/guard3b_v03_canary/pilot_metrics.json" "$RUN_ROOT/canary_metrics.json"
+
+# Base and the initial Challenger use the identical fixed harness/version.
+evaluate base
+cp "$RUN_ROOT/base_strict.json" "$RUN_ROOT/current_strict.json"
+CURRENT=guard3b_v03_canary
+evaluate canary "$CURRENT"
+cp "$RUN_ROOT/canary_strict.json" "$RUN_ROOT/current_strict.json"
+
+for number in $(seq 2 "$MAX_TRAINING_RUNS"); do
+  if (( $(date +%s) + 7200 > DEADLINE_EPOCH )); then echo 'FINAL_EVALUATION_RESERVE' > "$RUN_ROOT/termination_reason.txt"; break; fi
+  action=$(qwen_choice)
+  echo "$action" > "$RUN_ROOT/decision_${number}.txt"
+  [[ "$action" == STOP ]] && break
+  name="guard3b_v03_exp${number}"
+  if [[ "$action" == REPLICATE ]]; then train "$name" "" "$((3407 + number))" 0.00005; else train "$name" "$CURRENT" 3407 0.00005; fi
+  evaluate "exp${number}" "$name"
+  # Qwen may recommend the next recipe, but Champion promotion remains a later Model Judge import decision.
+  CURRENT="$name"
+  cp "$RUN_ROOT/exp${number}_strict.json" "$RUN_ROOT/current_strict.json"
+done
+
+printf '{"finished_at":"%s","last_adapter":"%s"}\n' "$(date -Iseconds)" "$CURRENT" > "$RUN_ROOT/program_state.json"