Stop one blocked address from reading as an internet outage

bin/omarchy-network-status

@@ -5,7 +5,18 @@
 # omarchy:args=[--verbose]
 
 verbose=false
-internet_probe=1.1.1.1
+
+# Only used for `ip route get`, to find the interface carrying default traffic.
+# Reachability does not matter here: the lookup is answered from the local
+# routing table, so a blocked address still resolves the right route.
+route_probe=1.1.1.1
+
+# Reachability probes used when the system has no public resolver configured.
+# More than one, because any single address can be blocked on a given network.
+fallback_probes=(1.1.1.1 8.8.8.8 9.9.9.9)
+
+# Upper bound on addresses probed per sample, to keep a sample cheap.
+max_probes=3
 
 case "${1:-}" in
   "")
@@ -22,7 +33,7 @@ esac
 print_status() {
   local device nm state ssid signal freq
 
-  device=$(ip route get "$internet_probe" 2>/dev/null | awk '{ for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit } }')
+  device=$(ip route get "$route_probe" 2>/dev/null | awk '{ for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit } }')
 
   if [[ -z $device ]]; then
     printf 'disconnected\t\t\t\n'
@@ -48,12 +59,97 @@ print_status() {
   printf 'wifi\t%s\t%s\t%s\n' "${ssid:-$device}" "$signal" "$freq"
 }
 
+# A resolver inside the LAN says nothing about whether the internet is
+# reachable, and the systemd-resolved stub listens on loopback, so only public
+# unicast IPv4 addresses make usable reachability probes.
+is_public_ipv4() {
+  local ip=$1 a b c d
+
+  [[ $ip =~ ^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]] || return 1
+
+  a=$((10#${BASH_REMATCH[1]}))
+  b=$((10#${BASH_REMATCH[2]}))
+  c=$((10#${BASH_REMATCH[3]}))
+  d=$((10#${BASH_REMATCH[4]}))
+
+  if (( a > 255 || b > 255 || c > 255 || d > 255 )); then
+    return 1
+  fi
+
+  if (( a == 0 || a == 10 || a == 127 || a >= 224 )) ||
+    (( a == 169 && b == 254 )) ||
+    (( a == 172 && b >= 16 && b <= 31 )) ||
+    (( a == 192 && b == 168 )) ||
+    (( a == 198 && (b == 18 || b == 19) )) ||
+    (( a == 100 && b >= 64 && b <= 127 )); then
+    return 1
+  fi
+
+  return 0
+}
+
+# Resolvers this system is configured to use. `resolvectl` reports the global
+# and per-link servers; /etc/resolv.conf covers hosts not running
+# systemd-resolved, where it lists real servers rather than the local stub.
+configured_dns_servers() {
+  resolvectl dns 2>/dev/null | sed 's/^[^:]*://'
+  awk '$1 == "nameserver" { print $2 }' /etc/resolv.conf 2>/dev/null
+}
+
+# Probe targets for the internet sample: configured public resolvers first,
+# since those are addresses this system already depends on reaching, then the
+# fallbacks. Deduplicated and capped.
+internet_probes() {
+  local ip seen="" probes=()
+
+  for ip in $(configured_dns_servers) "${fallback_probes[@]}"; do
+    # resolvectl prints servers as addr[:port][%ifname][#name], and `omarchy dns`
+    # writes its DNS-over-TLS servers with the #name. IPv6 is rejected either way.
+    ip=${ip%%[:%#]*}
+    is_public_ipv4 "$ip" || continue
+    [[ $seen == *" $ip "* ]] && continue
+
+    seen+=" $ip "
+    probes+=("$ip")
+
+    if (( ${#probes[@]} >= max_probes )); then
+      break
+    fi
+  done
+
+  if (( ${#probes[@]} > 0 )); then
+    printf '%s\n' "${probes[@]}"
+  fi
+}
+
 ping_latency_ms() {
   local host=$1
 
   LC_ALL=C ping -n -c 1 -W 1 "$host" 2>/dev/null | awk -F'time[=<]' '/time[=<]/ { split($2, parts, " "); print parts[1]; exit }'
 }
 
+# Probe the candidates together and keep the best answer, so one unreachable
+# address no longer reads as a total outage. Networks that blackhole a single
+# well-known resolver are common enough to plan for, and probing concurrently
+# keeps a sample within the same timeout as a single ping.
+ping_internet_ms() {
+  local tmpdir=$1
+  local host index=0 pids=()
+
+  for host in $(internet_probes); do
+    ping_latency_ms "$host" >"$tmpdir/internet.$index" &
+    pids+=($!)
+    index=$((index + 1))
+  done
+
+  if (( ${#pids[@]} > 0 )); then
+    wait "${pids[@]}" 2>/dev/null
+  fi
+
+  cat "$tmpdir"/internet.* 2>/dev/null |
+    awk 'NF { if (best == "" || $1 + 0 < best + 0) best = $1 } END { if (best != "") print best }'
+}
+
 print_ping_samples() {
   local gateway=$1
   local tmpdir router_file internet_file
@@ -69,7 +165,7 @@ print_ping_samples() {
     router_pid=$!
   fi
 
-  ping_latency_ms "$internet_probe" >"$internet_file" &
+  ping_internet_ms "$tmpdir" >"$internet_file" &
   internet_pid=$!
 
   if [[ -n $router_pid ]]; then
@@ -85,7 +181,7 @@ print_ping_samples() {
 print_verbose() {
   local route_json iface gw src prefix link
 
-  route_json=$(ip -j route get "$internet_probe" 2>/dev/null)
+  route_json=$(ip -j route get "$route_probe" 2>/dev/null)
   [[ -z $route_json ]] && return
 
   iface=$(jq -r '.[0].dev // ""' <<<"$route_json" 2>/dev/null)

test/shell.d/network-status-probe-test.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"
+
+STATUS="$ROOT/bin/omarchy-network-status"
+
+# Load the address classifier and probe selection on their own, so the
+# reachability rules can be checked without any real network.
+eval "$(sed -n '/^is_public_ipv4()/,/^}$/p' "$STATUS")"
+eval "$(sed -n '/^configured_dns_servers()/,/^}$/p' "$STATUS")"
+eval "$(sed -n '/^internet_probes()/,/^}$/p' "$STATUS")"
+
+fallback_probes=(1.1.1.1 8.8.8.8 9.9.9.9)
+max_probes=3
+
+for ip in 8.8.8.8 1.0.0.1 208.67.222.222 172.15.0.1 172.32.0.1 192.169.0.1 100.63.0.1 198.17.0.1 198.20.0.1; do
+  is_public_ipv4 "$ip" || fail "public address is usable as a probe: $ip"
+done
+pass "public addresses are usable as probes"
+
+# A resolver on the LAN, on loopback, behind a fake-IP proxy, or outside
+# unicast IPv4 proves nothing about internet reachability.
+for ip in 127.0.0.53 192.168.1.4 10.0.0.1 172.16.5.5 172.31.255.1 169.254.1.1 \
+  100.64.0.1 198.18.0.2 198.19.255.1 0.0.0.0 224.0.0.1 2001:4860:4860::8888 999.1.1.1 8.8.8 abc ""; do
+  ! is_public_ipv4 "$ip" || fail "address is rejected as a probe: ${ip:-<empty>}"
+done
+pass "non-public and malformed addresses are rejected as probes"
+
+probes_for() {
+  local dns_output=$1 stub probes
+
+  stub=$(mktemp -d)
+  cat >"$stub/resolvectl" <<EOF
+#!/bin/bash
+[[ \$1 == "dns" ]] && cat <<'DNS'
+$dns_output
+DNS
+EOF
+  chmod +x "$stub/resolvectl"
+  probes=$(PATH="$stub:$PATH" bash -c "$(declare -f is_public_ipv4 configured_dns_servers internet_probes)
+    fallback_probes=(1.1.1.1 8.8.8.8 9.9.9.9)
+    max_probes=3
+    internet_probes" | tr '\n' ' ')
+  rm -rf "$stub"
+  printf '%s' "${probes% }"
+}
+
+[[ $(probes_for 'Global: 8.8.8.8 8.8.4.4 9.9.9.9') == "8.8.8.8 8.8.4.4 9.9.9.9" ]] ||
+  fail "configured public resolvers are probed" "got: $(probes_for 'Global: 8.8.8.8 8.8.4.4 9.9.9.9')"
+pass "configured public resolvers are probed"
+
+# `omarchy dns Cloudflare` configures DNS-over-TLS servers, which resolvectl
+# prints with their server name attached, as it does a port or an interface.
+probes=$(probes_for 'Global: 1.1.1.1#cloudflare-dns.com 1.0.0.1#cloudflare-dns.com 2606:4700:4700::1111#cloudflare-dns.com')
+[[ $probes == "1.1.1.1 1.0.0.1 8.8.8.8" ]] || fail "configured DNS-over-TLS resolvers are probed" "got: $probes"
+probes=$(probes_for 'Link 2 (eth0): 8.8.4.4:9953 9.9.9.9%eth0 149.112.112.112:853#dns.quad9.net')
+[[ $probes == "8.8.4.4 9.9.9.9 149.112.112.112" ]] || fail "resolvers with a port or interface are probed" "got: $probes"
+pass "configured resolvers are probed whatever resolvectl appends to them"
+
+# `omarchy dns DHCP` typically yields a resolver inside the LAN. Probing it
+# would report the internet as reachable whenever the router is up.
+[[ $(probes_for 'Link 2 (wlp3s0): 192.168.1.4') == "1.1.1.1 8.8.8.8 9.9.9.9" ]] ||
+  fail "a LAN-only resolver falls back to public probes" "got: $(probes_for 'Link 2 (wlp3s0): 192.168.1.4')"
+pass "a LAN-only resolver falls back to public probes"
+
+[[ $(probes_for 'Global: 2001:4860:4860::8888') == "1.1.1.1 8.8.8.8 9.9.9.9" ]] ||
+  fail "IPv6-only resolvers fall back to public probes"
+pass "IPv6-only resolvers fall back to public probes"
+
+# The whole point: more than one address, so a network that blocks a single
+# well-known resolver does not read as an outage.
+probes=$(probes_for 'Global: 1.1.1.1 1.0.0.1')
+(( $(wc -w <<<"$probes") > 1 )) || fail "more than one address is probed" "got: $probes"
+pass "more than one address is probed"
+
+# The router lookup must stay on a fixed public address: pointing it at a
+# configured resolver would resolve an on-link route instead of the default one.
+grep -q '^route_probe=' "$STATUS" || fail "route lookup uses a dedicated probe address"
+grep -q 'ip route get "$route_probe"' "$STATUS" || fail "route lookup uses the dedicated probe address"
+grep -q 'ip -j route get "$route_probe"' "$STATUS" || fail "verbose route lookup uses the dedicated probe address"
+pass "route lookup uses a dedicated public address"
+
+# An unreachable internet must still report as unreachable.
+stub=$(mktemp -d)
+cat >"$stub/ping" <<'PING'
+#!/bin/bash
+host=${!#}
+if [[ $host == 192.168.1.1 ]]; then
+  echo "64 bytes from $host: icmp_seq=1 ttl=64 time=1.23 ms"
+  exit 0
+fi
+exit 1
+PING
+chmod +x "$stub/ping"
+eval "$(sed -n '/^ping_latency_ms()/,/^}$/p' "$STATUS")"
+eval "$(sed -n '/^ping_internet_ms()/,/^}$/p' "$STATUS")"
+workdir=$(mktemp -d)
+result=$(PATH="$stub:$PATH" ping_internet_ms "$workdir")
+rm -rf "$stub" "$workdir"
+[[ -z $result ]] || fail "an unreachable internet reports no latency" "got: $result"
+pass "an unreachable internet reports no latency"