Artifex/scripts/run_guard3b_overnight_spark.sh

#!/usr/bin/env bash
# Bounded Spark-only Guard 3B program. Commands are fixed; Qwen selects only a safe next recipe.
set -euo pipefail

ROOT=/home/daniel/forgeguard
PROJECT="$ROOT/project"
DATA="$ROOT/artifex_dataset_versions/guard_curated_v03/training.json"
RUN_ROOT="$PROJECT/artifex_overnight/guard3b_v03_overnight"
CANARY=artifex-guard3b-v03-canary
IMAGE=forgeguard-qwen36-unsloth:latest
MODEL=unsloth/qwen2.5-coder-3b-instruct-bnb-4bit
MAX_TRAINING_RUNS=${MAX_TRAINING_RUNS:-3}
DEADLINE_EPOCH=${DEADLINE_EPOCH:-$(( $(date +%s) + 28800 ))}
mkdir -p "$RUN_ROOT"

docker_run() {
  docker run --rm --gpus all --shm-size=16g \
    -v "$PROJECT:/workspace" \
    -v "$ROOT/artifex_dataset_versions:/workspace/artifex_dataset_versions" \
    -v /home/daniel/.cache/huggingface:/root/.cache/huggingface \
    "$IMAGE" "$@"
}

evaluate() {
  local name=$1 adapter=${2:-}
  local result="$RUN_ROOT/${name}_evmbench.json" score="$RUN_ROOT/${name}_strict.json"
  local args=(python /workspace/scripts/run_evmbench.py --model "$MODEL" --unsloth --save "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json")
  if [[ -n "$adapter" ]]; then args+=(--adapter "/workspace/artifex_overnight/$adapter"); fi
  docker_run "${args[@]}" > "$RUN_ROOT/${name}_evaluation.log" 2>&1
  docker_run python /workspace/scripts/score_evmbench_strict.py "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json" > "$score" 2>&1
  [[ -s "$result" && -s "$score" ]]
}

qwen_choice() {
  local summary
  summary=$(tail -c 10000 "$RUN_ROOT/base_strict.json" 2>/dev/null; tail -c 10000 "$RUN_ROOT/current_strict.json" 2>/dev/null || true)
  local body
  body=$(jq -n --arg prompt "You are MODEL_DIRECTOR for a bounded Guard 3B overnight run. Same-harness strict-score evidence follows: $summary Choose one JSON decision only: {\"action\":\"CONTINUE\"|\"REPLICATE\"|\"STOP\",\"rationale\":\"...\"}. CONTINUE means resume the current adapter with lower LR; REPLICATE means fresh seed from the immutable base; STOP means evidence/remaining time does not justify another run." '{model:"qwen38",messages:[{role:"user",content:$prompt}],temperature:0,max_tokens:200}')
  curl -fsS --max-time 180 http://127.0.0.1:8002/v1/chat/completions -H 'Content-Type: application/json' -d "$body" | jq -r '.choices[0].message.content' > "$RUN_ROOT/qwen_plan.json" || true
  jq -r '.action // empty' "$RUN_ROOT/qwen_plan.json" 2>/dev/null || echo CONTINUE
}

train() {
  local name=$1 resume=$2 seed=$3 lr=$4 output="$PROJECT/artifex_overnight/$name"
  local args=(python /workspace/finetune_qwen36_unsloth.py --model "$MODEL" --training-data /workspace/artifex_dataset_versions/guard_curated_v03/training.json --output "/workspace/artifex_overnight/$name" --steps 100 --batch-size 1 --learning-rate "$lr" --max-seq-length 2048 --max-output-tokens 512 --validation-max-examples 64 --seed "$seed")
  if [[ -n "$resume" ]]; then args+=(--resume-adapter "/workspace/artifex_overnight/$resume"); fi
  docker_run "${args[@]}" > "$RUN_ROOT/${name}_training.log" 2>&1
  test -f "$output/adapter_model.safetensors"
}

docker wait "$CANARY" > "$RUN_ROOT/canary_exit_code.txt"
docker logs "$CANARY" > "$RUN_ROOT/canary_training.log" 2>&1
if [[ "$(cat "$RUN_ROOT/canary_exit_code.txt")" != "0" ]]; then
  printf '{"finished_at":"%s","status":"CANARY_FAILED"}\n' "$(date -Iseconds)" > "$RUN_ROOT/program_state.json"
  exit 1
fi
cp "$PROJECT/artifex_overnight/guard3b_v03_canary/pilot_metrics.json" "$RUN_ROOT/canary_metrics.json"

# Base and the initial Challenger use the identical fixed harness/version.
evaluate base
cp "$RUN_ROOT/base_strict.json" "$RUN_ROOT/current_strict.json"
CURRENT=guard3b_v03_canary
evaluate canary "$CURRENT"
cp "$RUN_ROOT/canary_strict.json" "$RUN_ROOT/current_strict.json"

for number in $(seq 2 "$MAX_TRAINING_RUNS"); do
  if (( $(date +%s) + 7200 > DEADLINE_EPOCH )); then echo 'FINAL_EVALUATION_RESERVE' > "$RUN_ROOT/termination_reason.txt"; break; fi
  action=$(qwen_choice)
  echo "$action" > "$RUN_ROOT/decision_${number}.txt"
  [[ "$action" == STOP ]] && break
  name="guard3b_v03_exp${number}"
  if [[ "$action" == REPLICATE ]]; then train "$name" "" "$((3407 + number))" 0.00005; else train "$name" "$CURRENT" 3407 0.00005; fi
  evaluate "exp${number}" "$name"
  # Qwen may recommend the next recipe, but Champion promotion remains a later Model Judge import decision.
  CURRENT="$name"
  cp "$RUN_ROOT/exp${number}_strict.json" "$RUN_ROOT/current_strict.json"
done

printf '{"finished_at":"%s","last_adapter":"%s"}\n' "$(date -Iseconds)" "$CURRENT" > "$RUN_ROOT/program_state.json"