daniel/Artifex
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fail closed on strict evaluation decisions

Browse source
This commit is contained in:
Daniel Maddern 2026-08-17 13:30:17 +07:00
parent c35d4e345d
commit 0c716660cb
1 changed files with 12 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
scripts/run_guard3b_overnight_spark.sh
View file

@ -42,13 +42,17 @@ evaluate() {
[[ -s "$result" && -s "$score" ]]
}
strict_count() {
jq -er '.summary.canonical_type_and_target_source_file_match.count | if type == "number" then . else error("missing strict score") end' "$1"
}
qwen_choice() {
local summary
summary=$(jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/base_strict.json"; jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/current_strict.json")
local body
body=$(jq -n --arg prompt "You are MODEL_DIRECTOR for a bounded Guard 3B overnight run. Same-harness strict-score evidence follows: $summary Choose one JSON decision only: {\"action\":\"CONTINUE\"|\"REPLICATE\"|\"STOP\",\"rationale\":\"...\"}. CONTINUE means resume the current adapter with lower LR; REPLICATE means fresh seed from the immutable base; STOP means evidence/remaining time does not justify another run." '{model:"qwen38",messages:[{role:"user",content:$prompt}],temperature:0,max_tokens:200}')
curl -fsS --max-time 180 http://127.0.0.1:8002/v1/chat/completions -H 'Content-Type: application/json' -d "$body" | jq -r '.choices[0].message.content' > "$RUN_ROOT/qwen_plan.json" || true
jq -r '.action // empty' "$RUN_ROOT/qwen_plan.json" 2>/dev/null || echo CONTINUE
jq -er 'if .action == "CONTINUE" or .action == "REPLICATE" or .action == "STOP" then .action else error("invalid action") end' "$RUN_ROOT/qwen_plan.json" 2>/dev/null || echo STOP
}
train() {
@ -75,8 +79,14 @@ cp "$RUN_ROOT/base_strict.json" "$RUN_ROOT/current_strict.json"
CURRENT=guard3b_v03_canary
evaluate canary "$CURRENT"
cp "$RUN_ROOT/canary_strict.json" "$RUN_ROOT/current_strict.json"
BASE_STRICT=$(strict_count "$RUN_ROOT/base_strict.json")
CURRENT_STRICT=$(strict_count "$RUN_ROOT/current_strict.json")
for number in $(seq 2 "$MAX_TRAINING_RUNS"); do
if (( CURRENT_STRICT <= BASE_STRICT )); then
printf 'NO_STRICT_IMPROVEMENT current=%s base=%s\n' "$CURRENT_STRICT" "$BASE_STRICT" > "$RUN_ROOT/termination_reason.txt"
break
fi
if (( $(date +%s) + 7200 > DEADLINE_EPOCH )); then echo 'FINAL_EVALUATION_RESERVE' > "$RUN_ROOT/termination_reason.txt"; break; fi
action=$(qwen_choice)
echo "$action" > "$RUN_ROOT/decision_${number}.txt"
@ -87,6 +97,7 @@ for number in $(seq 2 "$MAX_TRAINING_RUNS"); do
# Qwen may recommend the next recipe, but Champion promotion remains a later Model Judge import decision.
CURRENT="$name"
cp "$RUN_ROOT/exp${number}_strict.json" "$RUN_ROOT/current_strict.json"
CURRENT_STRICT=$(strict_count "$RUN_ROOT/current_strict.json")
done
printf '{"finished_at":"%s","last_adapter":"%s"}\n' "$(date -Iseconds)" "$CURRENT" > "$RUN_ROOT/program_state.json"