Guard overnight on failed canary
This commit is contained in:
parent
bba15a632b
commit
38dfb059d7
1 changed files with 4 additions and 0 deletions
|
|
@ -50,6 +50,10 @@ train() {
|
||||||
|
|
||||||
docker wait "$CANARY" > "$RUN_ROOT/canary_exit_code.txt"
|
docker wait "$CANARY" > "$RUN_ROOT/canary_exit_code.txt"
|
||||||
docker logs "$CANARY" > "$RUN_ROOT/canary_training.log" 2>&1
|
docker logs "$CANARY" > "$RUN_ROOT/canary_training.log" 2>&1
|
||||||
|
if [[ "$(cat "$RUN_ROOT/canary_exit_code.txt")" != "0" ]]; then
|
||||||
|
printf '{"finished_at":"%s","status":"CANARY_FAILED"}\n' "$(date -Iseconds)" > "$RUN_ROOT/program_state.json"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
cp "$PROJECT/artifex_overnight/guard3b_v03_canary/pilot_metrics.json" "$RUN_ROOT/canary_metrics.json"
|
cp "$PROJECT/artifex_overnight/guard3b_v03_canary/pilot_metrics.json" "$RUN_ROOT/canary_metrics.json"
|
||||||
|
|
||||||
# Base and the initial Challenger use the identical fixed harness/version.
|
# Base and the initial Challenger use the identical fixed harness/version.
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue