feat(oncall): warn on a window skip after failure

docker/swarm/stacks/oncall/scripts/dispatch.sh

@@ -18,11 +18,35 @@ for var in GITHUB_DISPATCH_TOKEN DISPATCH_REPO DISPATCH_WORKFLOW; do
 done
 
 now=${DISPATCH_NOW:-$(date -u +%s)}
+ref=${DISPATCH_REF:-main}
+api=${GITHUB_API_URL:-https://api.github.com}
+body=$(mktemp)
+trap 'rm -f "$body"' EXIT
+
+warn_if_last_run_bad() {
+  code=$(curl -s -o "$body" -w '%{http_code}' \
+    -H "Authorization: Bearer $GITHUB_DISPATCH_TOKEN" \
+    -H 'Accept: application/vnd.github+json' \
+    -H 'X-GitHub-Api-Version: 2022-11-28' \
+    --max-time 30 \
+    "$api/repos/$DISPATCH_REPO/actions/workflows/$DISPATCH_WORKFLOW/runs?branch=${ref#refs/heads/}&status=completed&per_page=1" ||
+    true)
+  [ "$code" = 200 ] || return 0
+  last=$(grep -o '"conclusion": *"[a-z_]*"' "$body" | head -n 1 | sed 's/.*"\([a-z_]*\)"$/\1/')
+  case "$last" in
+  failure | cancelled | timed_out | startup_failure) ;;
+  *) return 0 ;;
+  esac
+  link=$(grep -o '"html_url": *"[^"]*"' "$body" | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/')
+  chat_notify warn dispatch "$target" \
+    "skipped: inside the maintenance window, the last run is $last" "${link:-$runs}" "last run"
+}
 
 case "${WINDOW_POLICY:-none}" in
 avoid)
   if in_window "$now"; then
     printf 'dispatch: skip %s, inside the maintenance window\n' "$target"
+    in_window "$now" 3 && warn_if_last_run_bad
     exit 0
   fi
   ;;
@@ -39,15 +63,12 @@ none) ;;
 *) config_error "WINDOW_POLICY must be avoid, require or none, got '${WINDOW_POLICY}'" ;;
 esac
 
-api=${GITHUB_API_URL:-https://api.github.com}
-body=$(mktemp)
-trap 'rm -f "$body"' EXIT
 code=$(curl -s -o "$body" -w '%{http_code}' -X POST \
   -H "Authorization: Bearer $GITHUB_DISPATCH_TOKEN" \
   -H 'Accept: application/vnd.github+json' \
   -H 'X-GitHub-Api-Version: 2022-11-28' \
   --max-time 30 \
-  -d "{\"ref\":$(json_string "${DISPATCH_REF:-main}")}" \
+  -d "{\"ref\":$(json_string "$ref")}" \
   "$api/repos/$DISPATCH_REPO/actions/workflows/$DISPATCH_WORKFLOW/dispatches" || true)
 
 case "$code" in

docker/swarm/stacks/oncall/scripts/test.sh

@@ -72,6 +72,7 @@ check() {
 }
 
 api_calls() { grep -c '^API ' "$log"; }
+dispatch_calls() { grep -c '^API .*/dispatches$' "$log"; }
 chat_calls() { grep -c '^CHAT ' "$log"; }
 
 json_probe='import json, sys
@@ -89,24 +90,53 @@ chat_lines() {
   grep '^CHATDATA ' "$log" | sed 's/^CHATDATA //' | python3 -c "$lines_probe" 2>&1
 }
 
-run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0100
+LAST_OK='{"workflow_runs":[{"id":1,"status":"completed","conclusion":"success","html_url":"https://github.com/freeCodeCamp/news/actions/runs/1"}]}'
+LAST_FAILED='{"workflow_runs":[{"id":2,"status":"completed","conclusion":"failure","html_url":"https://github.com/freeCodeCamp/news/actions/runs/2"}]}'
+
+run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0100 FAKE_CODE=200 FAKE_BODY="$LAST_OK"
 check "avoid inside window exits 0" "$status" 0
-check "avoid inside window makes no API call" "$(api_calls)" 0
+check "avoid inside window starts nothing" "$(dispatch_calls)" 0
+check "avoid inside window reads the last main run" "$(grep '^API ' "$log")" \
+  "API http://api.test/repos/freeCodeCamp/news/actions/workflows/deploy-eng.yml/runs?branch=main&status=completed&per_page=1"
+check "avoid skip after a good run posts no chat" "$(chat_calls)" 0
+
+run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0100 FAKE_CODE=200 FAKE_BODY="$LAST_FAILED"
+check "avoid skip after a failed run exits 0" "$status" 0
+check "avoid skip after a failed run starts nothing" "$(dispatch_calls)" 0
+check "avoid skip after a failed run is a standard warn line" \
+  "$(chat_text_has '⚠️ *dispatch* · freeCodeCamp/news deploy-eng.yml · skipped: inside the maintenance window, the last run is failure')" 1
+check "avoid skip after a failed run links that run" \
+  "$(chat_text_has '<https://github.com/freeCodeCamp/news/actions/runs/2|last run>')" 1
+check "avoid skip warn is one line" "$(chat_lines)" 1
+
+run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$((WED_0100 + 2 * 3600 + 5 * 60)) FAKE_CODE=200 FAKE_BODY="$LAST_FAILED"
+check "second window skip after a failed run posts no chat" "$(chat_calls)" 0
+
+run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0100 DISPATCH_REF=refs/heads/main FAKE_CODE=200 FAKE_BODY="$LAST_FAILED"
+check "full ref reads runs by branch name" "$(grep -c 'runs?branch=main&' "$log")" 1
+
+run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0100 FAKE_CODE=200 \
+  FAKE_BODY='{"workflow_runs":[{"conclusion":"cancelled","html_url":"https://x.test/3"}]}'
+check "avoid skip after a cancelled run posts chat" "$(chat_text_has 'the last run is cancelled')" 1
+
+run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0100 FAKE_CODE=503 FAKE_BODY=down
+check "avoid skip with an unreadable API exits 0" "$status" 0
+check "avoid skip with an unreadable API posts no chat" "$(chat_calls)" 0
 
 run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0559
-check "window includes Wed 05:59" "$(api_calls)" 0
+check "window includes Wed 05:59" "$(dispatch_calls)" 0
 
 run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$SAT_0000
-check "window includes Sat 00:00" "$(api_calls)" 0
+check "window includes Sat 00:00" "$(dispatch_calls)" 0
 
 run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0600
-check "window excludes Wed 06:00" "$(api_calls)" 1
+check "window excludes Wed 06:00" "$(dispatch_calls)" 1
 
 run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$TUE_2359
-check "window excludes Tue 23:59" "$(api_calls)" 1
+check "window excludes Tue 23:59" "$(dispatch_calls)" 1
 
 run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$WED_0905
-check "window excludes Wed 09:05 (leading-zero hour)" "$(api_calls)" 1
+check "window excludes Wed 09:05 (leading-zero hour)" "$(dispatch_calls)" 1
 
 run_dispatch WINDOW_POLICY=avoid DISPATCH_NOW=$THU_0605
 check "avoid outside window exits 0" "$status" 0

docker/swarm/stacks/oncall/stack-oncall.yml

@@ -226,7 +226,7 @@ configs:
     name: oncall_scripts_lib_v1
     file: ./scripts/lib.sh
   scripts_dispatch:
-    name: oncall_scripts_dispatch_v1
+    name: oncall_scripts_dispatch_v2
     file: ./scripts/dispatch.sh
   scripts_watchdog:
     name: oncall_scripts_watchdog_v1

docs/infra-guides/docker-swarm-oncall.md

@@ -80,7 +80,7 @@ GitHub `schedule` triggers start late or do not start at all (observed since 202
 
 The maintenance window is Wed/Sat 00:00–06:00 UTC. `scripts/lib.sh` defines it once.
 
-- `avoid`: inside the window, the dispatcher does nothing.
+- `avoid`: inside the window, the dispatcher starts nothing. If the last completed run on the branch failed or was cancelled, the first skip of the window (00:00–03:00 UTC) posts a warning to Google Chat.
 - `require`: outside the window, the dispatcher refuses and posts to Google Chat.
 
 The dispatcher posts to Google Chat when the API refuses the request or when a variable is missing. The housekeeping workflow also checks its own start time. It refuses to run outside the window unless you start it with `window_check=false`.