Artifex/scripts/run_guard3b_overnight_spark.sh

#!/usr/bin/env bash
# Bounded Spark-only Guard 3B program. Commands are fixed; Qwen selects only a safe next recipe.
set -euo pipefail

ROOT=/home/daniel/forgeguard
PROJECT="$ROOT/project"
ARTIFEX_ROOT=/home/daniel/Artifex
DATA="$ROOT/artifex_dataset_versions/guard_curated_v03/training.json"
RUN_ROOT="$PROJECT/artifex_overnight/guard3b_v03_overnight"
CANARY=artifex-guard3b-v03-canary
IMAGE=forgeguard-qwen36-unsloth:latest
MODEL=unsloth/qwen2.5-coder-3b-instruct-bnb-4bit
MAX_TRAINING_RUNS=${MAX_TRAINING_RUNS:-3}
DEADLINE_EPOCH=${DEADLINE_EPOCH:-$(( $(date +%s) + 28800 ))}
mkdir -p "$RUN_ROOT"

docker_run() {
  docker run --rm --gpus all --shm-size=16g \
    -v "$PROJECT:/workspace" \
    -v "$ARTIFEX_ROOT/scripts/score_evmbench_type_source_file.py:/workspace/artifex/score_evmbench_type_source_file.py:ro" \
    -v "$ROOT/artifex_dataset_versions:/workspace/artifex_dataset_versions" \
    -v /home/daniel/.cache/huggingface:/root/.cache/huggingface \
    "$IMAGE" "$@"
}

install_evaluator_contract() {
  if ! grep -q 'finding_with_source\["source_path"\]' "$PROJECT/scripts/run_evmbench.py"; then
    patch --batch --forward -d "$PROJECT" -p1 < "$ARTIFEX_ROOT/scripts/patches/forgeguard_evmbench_source_path.patch"
  fi
}

evaluate() {
  local name=$1 adapter=${2:-}
  local result="$RUN_ROOT/${name}_evmbench.json" score="$RUN_ROOT/${name}_strict.json"
  local args=(python /workspace/scripts/run_evmbench.py --model "$MODEL" --unsloth --save "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json")
  if [[ -n "$adapter" ]]; then args+=(--adapter "/workspace/artifex_overnight/$adapter"); fi
  docker_run "${args[@]}" > "$RUN_ROOT/${name}_evaluation.log" 2>&1
  docker_run python /workspace/artifex/score_evmbench_type_source_file.py \
    "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json" \
    --output "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_strict.json" \
    > "$RUN_ROOT/${name}_scoring.log" 2>&1
  [[ -s "$result" && -s "$score" ]]
}

strict_count() {
  jq -er '.summary.canonical_type_and_target_source_file_match.count | if type == "number" then . else error("missing strict score") end' "$1"
}

qwen_choice() {
  local summary
  summary=$(jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/base_strict.json"; jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/current_strict.json")
  local body
  body=$(jq -n --arg prompt "You are MODEL_DIRECTOR for a bounded Guard 3B overnight run. Same-harness strict-score evidence follows: $summary Choose one JSON decision only: {\"action\":\"CONTINUE\"|\"REPLICATE\"|\"STOP\",\"rationale\":\"...\"}. CONTINUE means resume the current adapter with lower LR; REPLICATE means fresh seed from the immutable base; STOP means evidence/remaining time does not justify another run." '{model:"qwen38",messages:[{role:"user",content:$prompt}],temperature:0,max_tokens:200}')
  curl -fsS --max-time 180 http://127.0.0.1:8002/v1/chat/completions -H 'Content-Type: application/json' -d "$body" | jq -r '.choices[0].message.content' > "$RUN_ROOT/qwen_plan.json" || true
  jq -er 'if .action == "CONTINUE" or .action == "REPLICATE" or .action == "STOP" then .action else error("invalid action") end' "$RUN_ROOT/qwen_plan.json" 2>/dev/null || echo STOP
}

train() {
  local name=$1 resume=$2 seed=$3 lr=$4 output="$PROJECT/artifex_overnight/$name"
  local args=(python /workspace/finetune_qwen36_unsloth.py --model "$MODEL" --training-data /workspace/artifex_dataset_versions/guard_curated_v03/training.json --output "/workspace/artifex_overnight/$name" --steps 100 --batch-size 1 --learning-rate "$lr" --max-seq-length 2048 --max-output-tokens 512 --validation-max-examples 64 --seed "$seed")
  if [[ -n "$resume" ]]; then args+=(--resume-adapter "/workspace/artifex_overnight/$resume"); fi
  docker_run "${args[@]}" > "$RUN_ROOT/${name}_training.log" 2>&1
  test -f "$output/adapter_model.safetensors"
}

docker wait "$CANARY" > "$RUN_ROOT/canary_exit_code.txt"
docker logs "$CANARY" > "$RUN_ROOT/canary_training.log" 2>&1
if [[ "$(cat "$RUN_ROOT/canary_exit_code.txt")" != "0" ]]; then
  printf '{"finished_at":"%s","status":"CANARY_FAILED"}\n' "$(date -Iseconds)" > "$RUN_ROOT/program_state.json"
  exit 1
fi
cp "$PROJECT/artifex_overnight/guard3b_v03_canary/pilot_metrics.json" "$RUN_ROOT/canary_metrics.json"
install_evaluator_contract

# Base and the initial Challenger use the identical fixed harness/version.
# Reuse a completed base result when resuming after an infrastructure repair.
if [[ ! -s "$RUN_ROOT/base_evmbench.json" ]]; then evaluate base; else docker_run python /workspace/artifex/score_evmbench_type_source_file.py "/workspace/artifex_overnight/guard3b_v03_overnight/base_evmbench.json" --output "/workspace/artifex_overnight/guard3b_v03_overnight/base_strict.json" > "$RUN_ROOT/base_scoring.log" 2>&1; fi
cp "$RUN_ROOT/base_strict.json" "$RUN_ROOT/current_strict.json"
CURRENT=guard3b_v03_canary
evaluate canary "$CURRENT"
cp "$RUN_ROOT/canary_strict.json" "$RUN_ROOT/current_strict.json"
BASE_STRICT=$(strict_count "$RUN_ROOT/base_strict.json")
CURRENT_STRICT=$(strict_count "$RUN_ROOT/current_strict.json")

for number in $(seq 2 "$MAX_TRAINING_RUNS"); do
  if (( CURRENT_STRICT <= BASE_STRICT )); then
    printf 'NO_STRICT_IMPROVEMENT current=%s base=%s\n' "$CURRENT_STRICT" "$BASE_STRICT" > "$RUN_ROOT/termination_reason.txt"
    break
  fi
  if (( $(date +%s) + 7200 > DEADLINE_EPOCH )); then echo 'FINAL_EVALUATION_RESERVE' > "$RUN_ROOT/termination_reason.txt"; break; fi
  action=$(qwen_choice)
  echo "$action" > "$RUN_ROOT/decision_${number}.txt"
  [[ "$action" == STOP ]] && break
  name="guard3b_v03_exp${number}"
  if [[ "$action" == REPLICATE ]]; then train "$name" "" "$((3407 + number))" 0.00005; else train "$name" "$CURRENT" 3407 0.00005; fi
  evaluate "exp${number}" "$name"
  # Qwen may recommend the next recipe, but Champion promotion remains a later Model Judge import decision.
  CURRENT="$name"
  cp "$RUN_ROOT/exp${number}_strict.json" "$RUN_ROOT/current_strict.json"
  CURRENT_STRICT=$(strict_count "$RUN_ROOT/current_strict.json")
done

printf '{"finished_at":"%s","last_adapter":"%s"}\n' "$(date -Iseconds)" "$CURRENT" > "$RUN_ROOT/program_state.json"