Report why a server call failed everywhere, not just at task completion

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog

@@ -6,7 +6,15 @@ if [[ $boottype == usb && ! -z $web ]]; then
     sysuuid=${sysuuid,,}
     mac=$(getMACAddresses)
     base64mac=$(echo $mac | base64)
-    token=$(curl -Lks --data "mac=$base64mac" "${web}status/hostgetkey.php")
+    callServer "${web}status/hostgetkey.php" "mac=$base64mac"
+    token="$serverBody"
+    # Said nothing at all when the key could not be fetched. The next call then
+    # posts an empty hosttoken, and whatever comes back is written to
+    # /tmp/hinfo.txt and SOURCED -- so a failure here surfaced much later, as
+    # missing kernel arguments rather than as a failed request.
+    [[ -z $token ]] && echo " * Could not get this host's key: ${serverReason:-the server returned no key}"
+    # Left as a raw curl deliberately: -o writes a file rather than returning a
+    # body, which is not what callServer is for.
     curl -Lks -o /tmp/hinfo.txt --data "sysuuid=${sysuuid}&mac=$mac&hosttoken=${token}" "${web}service/hostinfo.php" -A ''
     [[ -f /tmp/hinfo.txt ]] && . /tmp/hinfo.txt
 fi

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.auto.reg

@@ -73,7 +73,8 @@ if [[ -f /sys/firmware/acpi/tables/MSDM ]]; then
     productKey=$(tail -c+57 /sys/firmware/acpi/tables/MSDM | base64)
 fi
 while [[ -z $res ]]; do
-    res=$(curl -Lks --data "sysserial=${sysserial}&sysuuid=${sysuuid}&mac=$mac&productKey=${productKey}" ${web}service/auto.register.php 2>/dev/null)
+    callServer "${web}service/auto.register.php" "sysserial=${sysserial}&sysuuid=${sysuuid}&mac=$mac&productKey=${productKey}"
+    res="$serverBody"
     case $count in
         [0-8])
             let count+=1
@@ -82,7 +83,9 @@ while [[ -z $res ]]; do
         9)
             echo "Failed"
             debugPause
-            handleError "Cannot register host. ($0)\n   Args Passed: $*"
+            # Was "Cannot register host." and nothing else, for every cause:
+            # server down, server refusing, server answering with nothing.
+            handleError "Cannot register host: ${serverReason:-no reason reported} ($0)\n   Args Passed: $*"
             ;;
     esac
 done

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.av

@@ -169,6 +169,8 @@ for disk in $disks; do
             let cnt+=1
             echo " * File id: $cnt"
             dots "Sending file id"
+            # Fire and forget: nothing reads a reply, so there is no reply to
+            # misreport and callServer would add nothing.
             curl -Lks --data "sysuuid=${sysuuid}&mac=$mac&string=$b64&mode=$avmode" ${web}service/av.php 2>/dev/null
             case $? in
                 0)

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.capone

@@ -23,12 +23,16 @@ dots "Looking for images"
 count=0
 imgret=""
 while [[ -z $imgret ]]; do
-    imgret=$(curl -Lks --data "action=imagelookup&key=${dmi64}" ${web}service/capone.php 2>/dev/null)
+    callServer "${web}service/capone.php" "action=imagelookup&key=${dmi64}"
+    imgret="$serverBody"
     [[ -n $imgret ]] && continue
     if [[ $count -ge 10 ]]; then
         echo "Failed"
         debugPause
-        handleError "Count not find an Image definition ($0)\n   Args Passed: $*"
+        # "Count not" was a typo for "Cannot", and the message blamed a
+        # missing image definition for what is just as often a server that
+        # could not be reached -- the reply is empty either way.
+        handleError "Cannot find an Image definition: ${serverReason:-the server returned no image for this machine} ($0)\n   Args Passed: $*"
     fi
     let count+=1
     usleep 5000000

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.checkin

@@ -54,12 +54,18 @@ checkin() {
     local count=0
     local res=""
     local waittime=0
+    # No bail-out, and that is deliberate: a host waiting its turn in a queue
+    # must keep waiting however long that takes. What was NOT deliberate is that
+    # a dead server looked identical to a queue position -- $res was empty, so
+    # this printed " *  (In line for 5s)" forever with nothing to diagnose.
+    # serverReason distinguishes them; the waiting behaviour is unchanged.
     while [[ $res != "##@GO" ]]; do
-        res=$(curl -Lks --data "$poststring" ${web}service/$php_post 2>/dev/null)
+        callServer "${web}service/$php_post" "$poststring"
+        res="$serverBody"
         if [[ $res != "##@GO" ]]; then
             echo "Failed"
             debugPause
-            echo -n " * $res (In line for "
+            echo -n " * ${serverReason:-$res} (In line for "
             sec2string "$waittime"
             echo ")"
             let waittime+=5

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.checkmount

@@ -5,7 +5,11 @@ dots "Checking Mounted File System"
 if [[ ! -f /images/.mntcheck ]]; then
     count=0
     while [[ $blame != '##' ]]; do
-        blame=$(curl -Lks --data "sysuuid=${sysuuid}&mac=$mac&type=$type" ${web}service/blame.php 2>/dev/null)
+        # $blame was reported verbatim in the handleError below, so a server
+        # that answered with nothing produced "Error during failure
+        # notification: " -- an error message about an error, carrying neither.
+        callServer "${web}service/blame.php" "sysuuid=${sysuuid}&mac=$mac&type=$type"
+        blame="${serverBody:-$serverReason}"
         case $count in
             [0-8])
                 let count+=1
@@ -14,7 +18,7 @@ if [[ ! -f /images/.mntcheck ]]; then
             9)
                 echo "Failed"
                 debugPause
-                handleError "Error during failure notification: $blame ($0)\n   Args Passed: $*"
+                handleError "Error during failure notification: ${serverReason:-$blame} ($0)\n   Args Passed: $*"
                 ;;
         esac
     done

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.imgcomplete

@@ -23,7 +23,7 @@ while [[ $res != "##" ]]; do
     # a subshell would discard both. This used to print $res as the error, so an
     # empty reply was reported as an empty error and there was nothing to
     # diagnose from here -- see fogproject GH-1380.
-    postToServer "${web}service/$php_post" "$poststring"
+    callServer "${web}service/$php_post" "$poststring"
     res="$serverBody"
     [[ $res == "##" ]] && break
     echo "Failed"

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.inventory

@@ -97,19 +97,35 @@ fi
 poststring="mac=${mac}&sysman=${sysman64}&sysproduct=${sysproduct64}&sysversion=${sysversion64}&sysserial=${sysserial64}&sysuuid=${sysuuid64}&systype=${systype64}&biosversion=${biosversion64}&biosvendor=${biosvendor64}&biosdate=${biosdate64}&mbman=${mbman64}&mbproductname=${mbproductname64}&mbversion=${mbversion64}&mbserial=${mbserial64}&mbasset=${mbasset64}&cpuman=${cpuman64}&cpuversion=${cpuversion64}&cpucurrent=${cpucurrent64}&cpumax=${cpumax64}&mem=${mem64}&hdinfo=${hdinfo64}&caseman=${caseman64}&casever=${casever64}&caseserial=${caseserial64}&caseasset=${caseasset64}&gpuvendors=${inventory_graphics_vendor64}&gpuproducts=${inventory_graphics_product64}"
 count=0
 res=""
+sent=0
 while [[ -z $res ]]; do
     dots "Attempting to send inventory"
-    res=$(curl -Lks --data "$poststring" ${web}service/inventory.php 2>/dev/null)
+    callServer "${web}service/inventory.php" "$poststring"
+    res="$serverBody"
     if [[ $count -ge 10 ]]; then
         echo "Failed"
         debugPause
+        # Says why it gave up. It used to break in silence, and the
+        # unconditional "Done" below then reported the inventory as sent when
+        # eleven attempts had just failed.
+        echo " * Inventory was not recorded: ${serverReason:-no reply from the server}"
+        debugPause
         break
     fi
     if [[ -z $res ]]; then
         let count+=1
         usleep 2000000
     fi
 done
-echo "Done"
+# Gated on the operation, not printed regardless. Inventory is not fatal -- a
+# deploy carries on without it, which is why this breaks rather than
+# handleError-ing -- but reporting success for something that did not happen is
+# how a missing inventory goes unnoticed until someone looks for the host.
+[[ -n $res ]] && sent=1
+if [[ $sent -eq 1 ]]; then
+    echo "Done"
+else
+    echo " * Continuing without inventory"
+fi
 debugPause
 [[ $deployed -eq 1 ]] && . /bin/fog.nonimgcomplete $mac

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.man.reg

@@ -61,7 +61,8 @@ mac=$(getMACAddresses | base64)
 sysuuid=$(dmidecode -s system-uuid)
 sysuuid=${sysuuid,,}
 sysuuid=$(echo $sysuuid | base64)
-exists=$(curl -Lks --data "sysuuid=${sysuuid}&mac=$mac" ${web}service/man.hostexists.php 2>/dev/null)
+callServer "${web}service/man.hostexists.php" "sysuuid=${sysuuid}&mac=$mac"
+exists="$serverBody"
 checkAndSet() {
     local testvar="$1"
     local onlynum="$2"
@@ -92,7 +93,15 @@ setIDs() {
             [?])
                 url="${web}service/${str}listing.php"
                 clearScreen
-                res=$(echo -e $(curl -ks $url 2>/dev/null))
+                # Unquoted $serverBody, as the original's unquoted $( ) was:
+                # the word splitting is what the loop below iterates on.
+                # NOTE: this call now follows redirects (callServer uses -L,
+                # this site used bare -ks). That is a deliberate improvement --
+                # a server that redirects HTTP to HTTPS previously returned the
+                # redirect page here and the listing came out as noise.
+                callServer "$url"
+                res=$(echo -e $serverBody)
+                [[ -z $res ]] && res="${serverReason:-no entries returned}"
                 i=0
                 IFS=$'\n'
                 for line in $res; do
@@ -166,12 +175,22 @@ while [[ $res != "#!ok" ]]; do
         usleep 2000000
     fi
     host=$(echo $host | base64)
-    res=$(curl -Lks --data "host=$host" ${web}service/hostnameloop.php 2>/dev/null)
-    [[ $res != "#!ok" ]] && echo "$res"
+    callServer "${web}service/hostnameloop.php" "host=$host"
+    res="$serverBody"
+    # ${serverReason:-$res}: a failed call printed a blank line and asked for the
+    # hostname again, forever, with no hint that the server was the problem.
+    [[ $res != "#!ok" ]] && echo "${serverReason:-$res}"
 done
 imageid=""
 setIDs "imageid" "image" "" "" 20
-if [[ $(curl -Lks ${web}service/locationcheck.php 2>/dev/null) == "##" ]]; then
+# The prompt is offered only when the server says "##". A failed call is not
+# "##" either, so this used to skip the question in silence and register the
+# host with no location. Return 2 (answered, empty body) is a legitimate "no
+# locations configured" here and stays quiet; only a real failure is reported,
+# and the default is unchanged so the flow is not blocked by a warning.
+callServer "${web}service/locationcheck.php"
+[[ $? -eq 1 ]] && echo " * Could not ask the server about locations: ${serverReason}"
+if [[ $serverBody == "##" ]]; then
     while [[ -z $askme ]]; do
         echo -n "    Would you like to assign a location for this host? (y/N) "
         read askme
@@ -191,7 +210,9 @@ if [[ $(curl -Lks ${web}service/locationcheck.php 2>/dev/null) == "##" ]]; then
     done
 fi
 askme=""
-if [[ $(curl -Lks ${web}service/oucheck.php 2>/dev/null) == "##" ]]; then
+callServer "${web}service/oucheck.php"
+[[ $? -eq 1 ]] && echo " * Could not ask the server about OUs: ${serverReason}"
+if [[ $serverBody == "##" ]]; then
     while [[ -z $askme ]]; do
         echo -n "    Would you like to assign an ou for this host? (y/N) "
         read askme
@@ -313,7 +334,8 @@ while [[ -z $askme ]]; do
                 read -s password
                 user64=$(echo $username | tr -d '\012' | base64)
                 pass64=$(echo $password | tr -d '\012' | base64)
-                ret=$(curl -Lks --data "sysuuid=${sysuuid}&mac=$mac&username=$user64&password=$pass64" ${web}service/checkcredentials.php 2>/dev/null)
+                callServer "${web}service/checkcredentials.php" "sysuuid=${sysuuid}&mac=$mac&username=$user64&password=$pass64"
+                ret="${serverBody:-$serverReason}"
                 case $ret in
                     '#!ok')
                         echo
@@ -355,9 +377,14 @@ while [[ -z $askme ]]; do
 done
 dots "Attempting to register host"
 res=""
+# Unbounded on purpose -- registration is the whole point of this screen and
+# there is a human present to abort -- but it used to echo an empty line every
+# two seconds when the server was unreachable, so the operator watched a blank
+# console with no idea anything was wrong.
 while [[ -z $res ]]; do
-    res=$(curl -Lks --data "sysuuid=${sysuuid}&mac=$mac&advanced=$(echo -n 1 | base64)&host=$host&imageid=$imageid&primaryuser=$primaryuser&other1=$other1&other2=$other2&doimage=$realdoimage&doad=$blDoAD&location=$location64&ou=$ou64&username=$user64&password=$pass64&groupid=$group64&snapinid=$snapin64&productKey=$productKey" ${web}service/auto.register.php 2>/dev/null)
-    echo "$res"
+    callServer "${web}service/auto.register.php" "sysuuid=${sysuuid}&mac=$mac&advanced=$(echo -n 1 | base64)&host=$host&imageid=$imageid&primaryuser=$primaryuser&other1=$other1&other2=$other2&doimage=$realdoimage&doad=$blDoAD&location=$location64&ou=$ou64&username=$user64&password=$pass64&groupid=$group64&snapinid=$snapin64&productKey=$productKey"
+    res="$serverBody"
+    echo "${serverReason:-$res}"
     usleep 2000000
 done
 . /bin/fog.inventory

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.nonimgcomplete

@@ -12,7 +12,7 @@ while [[ $res != "##" ]]; do
     # not even the empty "Error returned:" its sibling managed -- so eleven
     # attempts could go by with no clue why. Same reporting as fog.imgcomplete
     # now; see fogproject GH-1380.
-    postToServer "${web}service/Post_Wipe.php" "sysuuid=${sysuuid}&mac=$mactosend"
+    callServer "${web}service/Post_Wipe.php" "sysuuid=${sysuuid}&mac=$mactosend"
     res="$serverBody"
     [[ $res == "##" ]] && break
     echo "Failed"

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.statusreporter

@@ -12,6 +12,11 @@ while :; do
     cat /dev/null > $tmpfile 2>/dev/null
     [[ -z $mac ]] && continue
     status=$(echo $status | base64)
+    # Deliberately NOT callServer, and $res is deliberately never read. This is
+    # a best-effort progress ping every three seconds for the life of the task;
+    # reporting each failure would paint the console over the top of the imaging
+    # output it exists to report on, and a missed progress update costs nothing.
+    # The completion scripts are where a failure actually matters.
     res=$(curl -Lks --data "sysuuid=${sysuuid}&mac=$mac&status=$status" ${web}service/progress.php 2>/dev/null)
     usleep 3000000
 done

Buildroot/board/FOG/FOS/rootfs_overlay/bin/fog.surfacetest

@@ -20,11 +20,12 @@ count=0
 sysuuid=$(dmidecode -s system-uuid)
 sysuuid=${sysuuid,,}
 while [[ $res != "##@GO" ]]; do
-    res=$(curl -Lks --data "sysuuid=${sysuuid}&mac=$mac" ${web}service/Pre_Stage1.php 2>/dev/null)
+    callServer "${web}service/Pre_Stage1.php" "sysuuid=${sysuuid}&mac=$mac"
+    res="$serverBody"
     if [[ $count -ge 10 ]]; then
         echo "Failed"
         debugPause
-        handleError "Could not checkin. ($0)\n   Args Passed: $*"
+        handleError "Could not checkin: ${serverReason:-last reply was \"$res\"} ($0)\n   Args Passed: $*"
     fi
     let count+=1
     usleep 5000000