From 0c716660cbe7df3816b703576d2bcecd12f5d1b6 Mon Sep 17 00:00:00 2001 From: Daniel Maddern Date: Mon, 17 Aug 2026 13:30:17 +0700 Subject: [PATCH] Fail closed on strict evaluation decisions --- scripts/run_guard3b_overnight_spark.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/run_guard3b_overnight_spark.sh b/scripts/run_guard3b_overnight_spark.sh index d36ebf5..15c214a 100755 --- a/scripts/run_guard3b_overnight_spark.sh +++ b/scripts/run_guard3b_overnight_spark.sh @@ -42,13 +42,17 @@ evaluate() { [[ -s "$result" && -s "$score" ]] } +strict_count() { + jq -er '.summary.canonical_type_and_target_source_file_match.count | if type == "number" then . else error("missing strict score") end' "$1" +} + qwen_choice() { local summary summary=$(jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/base_strict.json"; jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/current_strict.json") local body body=$(jq -n --arg prompt "You are MODEL_DIRECTOR for a bounded Guard 3B overnight run. Same-harness strict-score evidence follows: $summary Choose one JSON decision only: {\"action\":\"CONTINUE\"|\"REPLICATE\"|\"STOP\",\"rationale\":\"...\"}. CONTINUE means resume the current adapter with lower LR; REPLICATE means fresh seed from the immutable base; STOP means evidence/remaining time does not justify another run." '{model:"qwen38",messages:[{role:"user",content:$prompt}],temperature:0,max_tokens:200}') curl -fsS --max-time 180 http://127.0.0.1:8002/v1/chat/completions -H 'Content-Type: application/json' -d "$body" | jq -r '.choices[0].message.content' > "$RUN_ROOT/qwen_plan.json" || true - jq -r '.action // empty' "$RUN_ROOT/qwen_plan.json" 2>/dev/null || echo CONTINUE + jq -er 'if .action == "CONTINUE" or .action == "REPLICATE" or .action == "STOP" then .action else error("invalid action") end' "$RUN_ROOT/qwen_plan.json" 2>/dev/null || echo STOP } train() { @@ -75,8 +79,14 @@ cp "$RUN_ROOT/base_strict.json" "$RUN_ROOT/current_strict.json" CURRENT=guard3b_v03_canary evaluate canary "$CURRENT" cp "$RUN_ROOT/canary_strict.json" "$RUN_ROOT/current_strict.json" +BASE_STRICT=$(strict_count "$RUN_ROOT/base_strict.json") +CURRENT_STRICT=$(strict_count "$RUN_ROOT/current_strict.json") for number in $(seq 2 "$MAX_TRAINING_RUNS"); do + if (( CURRENT_STRICT <= BASE_STRICT )); then + printf 'NO_STRICT_IMPROVEMENT current=%s base=%s\n' "$CURRENT_STRICT" "$BASE_STRICT" > "$RUN_ROOT/termination_reason.txt" + break + fi if (( $(date +%s) + 7200 > DEADLINE_EPOCH )); then echo 'FINAL_EVALUATION_RESERVE' > "$RUN_ROOT/termination_reason.txt"; break; fi action=$(qwen_choice) echo "$action" > "$RUN_ROOT/decision_${number}.txt" @@ -87,6 +97,7 @@ for number in $(seq 2 "$MAX_TRAINING_RUNS"); do # Qwen may recommend the next recipe, but Champion promotion remains a later Model Judge import decision. CURRENT="$name" cp "$RUN_ROOT/exp${number}_strict.json" "$RUN_ROOT/current_strict.json" + CURRENT_STRICT=$(strict_count "$RUN_ROOT/current_strict.json") done printf '{"finished_at":"%s","last_adapter":"%s"}\n' "$(date -Iseconds)" "$CURRENT" > "$RUN_ROOT/program_state.json"