Guard overnight on failed canary
This commit is contained in:
parent
bba15a632b
commit
38dfb059d7
1 changed files with 4 additions and 0 deletions
|
|
@ -50,6 +50,10 @@ train() {
|
|||
|
||||
docker wait "$CANARY" > "$RUN_ROOT/canary_exit_code.txt"
|
||||
docker logs "$CANARY" > "$RUN_ROOT/canary_training.log" 2>&1
|
||||
if [[ "$(cat "$RUN_ROOT/canary_exit_code.txt")" != "0" ]]; then
|
||||
printf '{"finished_at":"%s","status":"CANARY_FAILED"}\n' "$(date -Iseconds)" > "$RUN_ROOT/program_state.json"
|
||||
exit 1
|
||||
fi
|
||||
cp "$PROJECT/artifex_overnight/guard3b_v03_canary/pilot_metrics.json" "$RUN_ROOT/canary_metrics.json"
|
||||
|
||||
# Base and the initial Challenger use the identical fixed harness/version.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue