92 lines
5.6 KiB
Bash
Executable file
92 lines
5.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Bounded Spark-only Guard 3B program. Commands are fixed; Qwen selects only a safe next recipe.
|
|
set -euo pipefail
|
|
|
|
ROOT=/home/daniel/forgeguard
|
|
PROJECT="$ROOT/project"
|
|
ARTIFEX_ROOT=/home/daniel/Artifex
|
|
DATA="$ROOT/artifex_dataset_versions/guard_curated_v03/training.json"
|
|
RUN_ROOT="$PROJECT/artifex_overnight/guard3b_v03_overnight"
|
|
CANARY=artifex-guard3b-v03-canary
|
|
IMAGE=forgeguard-qwen36-unsloth:latest
|
|
MODEL=unsloth/qwen2.5-coder-3b-instruct-bnb-4bit
|
|
MAX_TRAINING_RUNS=${MAX_TRAINING_RUNS:-3}
|
|
DEADLINE_EPOCH=${DEADLINE_EPOCH:-$(( $(date +%s) + 28800 ))}
|
|
mkdir -p "$RUN_ROOT"
|
|
|
|
docker_run() {
|
|
docker run --rm --gpus all --shm-size=16g \
|
|
-v "$PROJECT:/workspace" \
|
|
-v "$ARTIFEX_ROOT/scripts/score_evmbench_type_source_file.py:/workspace/artifex/score_evmbench_type_source_file.py:ro" \
|
|
-v "$ROOT/artifex_dataset_versions:/workspace/artifex_dataset_versions" \
|
|
-v /home/daniel/.cache/huggingface:/root/.cache/huggingface \
|
|
"$IMAGE" "$@"
|
|
}
|
|
|
|
install_evaluator_contract() {
|
|
if ! grep -q 'finding_with_source\["source_path"\]' "$PROJECT/scripts/run_evmbench.py"; then
|
|
patch --batch --forward -d "$PROJECT" -p1 < "$ARTIFEX_ROOT/scripts/patches/forgeguard_evmbench_source_path.patch"
|
|
fi
|
|
}
|
|
|
|
evaluate() {
|
|
local name=$1 adapter=${2:-}
|
|
local result="$RUN_ROOT/${name}_evmbench.json" score="$RUN_ROOT/${name}_strict.json"
|
|
local args=(python /workspace/scripts/run_evmbench.py --model "$MODEL" --unsloth --save "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json")
|
|
if [[ -n "$adapter" ]]; then args+=(--adapter "/workspace/artifex_overnight/$adapter"); fi
|
|
docker_run "${args[@]}" > "$RUN_ROOT/${name}_evaluation.log" 2>&1
|
|
docker_run python /workspace/artifex/score_evmbench_type_source_file.py \
|
|
"/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json" \
|
|
--output "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_strict.json" \
|
|
> "$RUN_ROOT/${name}_scoring.log" 2>&1
|
|
[[ -s "$result" && -s "$score" ]]
|
|
}
|
|
|
|
qwen_choice() {
|
|
local summary
|
|
summary=$(jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/base_strict.json"; jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/current_strict.json")
|
|
local body
|
|
body=$(jq -n --arg prompt "You are MODEL_DIRECTOR for a bounded Guard 3B overnight run. Same-harness strict-score evidence follows: $summary Choose one JSON decision only: {\"action\":\"CONTINUE\"|\"REPLICATE\"|\"STOP\",\"rationale\":\"...\"}. CONTINUE means resume the current adapter with lower LR; REPLICATE means fresh seed from the immutable base; STOP means evidence/remaining time does not justify another run." '{model:"qwen38",messages:[{role:"user",content:$prompt}],temperature:0,max_tokens:200}')
|
|
curl -fsS --max-time 180 http://127.0.0.1:8002/v1/chat/completions -H 'Content-Type: application/json' -d "$body" | jq -r '.choices[0].message.content' > "$RUN_ROOT/qwen_plan.json" || true
|
|
jq -r '.action // empty' "$RUN_ROOT/qwen_plan.json" 2>/dev/null || echo CONTINUE
|
|
}
|
|
|
|
train() {
|
|
local name=$1 resume=$2 seed=$3 lr=$4 output="$PROJECT/artifex_overnight/$name"
|
|
local args=(python /workspace/finetune_qwen36_unsloth.py --model "$MODEL" --training-data /workspace/artifex_dataset_versions/guard_curated_v03/training.json --output "/workspace/artifex_overnight/$name" --steps 100 --batch-size 1 --learning-rate "$lr" --max-seq-length 2048 --max-output-tokens 512 --validation-max-examples 64 --seed "$seed")
|
|
if [[ -n "$resume" ]]; then args+=(--resume-adapter "/workspace/artifex_overnight/$resume"); fi
|
|
docker_run "${args[@]}" > "$RUN_ROOT/${name}_training.log" 2>&1
|
|
test -f "$output/adapter_model.safetensors"
|
|
}
|
|
|
|
docker wait "$CANARY" > "$RUN_ROOT/canary_exit_code.txt"
|
|
docker logs "$CANARY" > "$RUN_ROOT/canary_training.log" 2>&1
|
|
if [[ "$(cat "$RUN_ROOT/canary_exit_code.txt")" != "0" ]]; then
|
|
printf '{"finished_at":"%s","status":"CANARY_FAILED"}\n' "$(date -Iseconds)" > "$RUN_ROOT/program_state.json"
|
|
exit 1
|
|
fi
|
|
cp "$PROJECT/artifex_overnight/guard3b_v03_canary/pilot_metrics.json" "$RUN_ROOT/canary_metrics.json"
|
|
install_evaluator_contract
|
|
|
|
# Base and the initial Challenger use the identical fixed harness/version.
|
|
# Reuse a completed base result when resuming after an infrastructure repair.
|
|
if [[ ! -s "$RUN_ROOT/base_evmbench.json" ]]; then evaluate base; else docker_run python /workspace/artifex/score_evmbench_type_source_file.py "/workspace/artifex_overnight/guard3b_v03_overnight/base_evmbench.json" --output "/workspace/artifex_overnight/guard3b_v03_overnight/base_strict.json" > "$RUN_ROOT/base_scoring.log" 2>&1; fi
|
|
cp "$RUN_ROOT/base_strict.json" "$RUN_ROOT/current_strict.json"
|
|
CURRENT=guard3b_v03_canary
|
|
evaluate canary "$CURRENT"
|
|
cp "$RUN_ROOT/canary_strict.json" "$RUN_ROOT/current_strict.json"
|
|
|
|
for number in $(seq 2 "$MAX_TRAINING_RUNS"); do
|
|
if (( $(date +%s) + 7200 > DEADLINE_EPOCH )); then echo 'FINAL_EVALUATION_RESERVE' > "$RUN_ROOT/termination_reason.txt"; break; fi
|
|
action=$(qwen_choice)
|
|
echo "$action" > "$RUN_ROOT/decision_${number}.txt"
|
|
[[ "$action" == STOP ]] && break
|
|
name="guard3b_v03_exp${number}"
|
|
if [[ "$action" == REPLICATE ]]; then train "$name" "" "$((3407 + number))" 0.00005; else train "$name" "$CURRENT" 3407 0.00005; fi
|
|
evaluate "exp${number}" "$name"
|
|
# Qwen may recommend the next recipe, but Champion promotion remains a later Model Judge import decision.
|
|
CURRENT="$name"
|
|
cp "$RUN_ROOT/exp${number}_strict.json" "$RUN_ROOT/current_strict.json"
|
|
done
|
|
|
|
printf '{"finished_at":"%s","last_adapter":"%s"}\n' "$(date -Iseconds)" "$CURRENT" > "$RUN_ROOT/program_state.json"
|