Re-order stack.sh 3: logging and error traps

Part 3 of a series

Re-order the setup and check bits in the top portion of stack.sh to
have a logical flow with similar things done together.

No behaviour changes are intended aside from the order of execution.
Any such changes are bugs.

* Move logging and error configuration earlier to cover initial project setup.

Change-Id: Ib16bbe20f224b1cf5e86c7a2fda0d9472c108873
diff --git a/lib/rpc_backend b/lib/rpc_backend
index 38da50c..d527c6b 100644
--- a/lib/rpc_backend
+++ b/lib/rpc_backend
@@ -43,7 +43,7 @@
     local rpc_backend_cnt=0
     for svc in qpid zeromq rabbit; do
         is_service_enabled $svc &&
-        ((rpc_backend_cnt++))
+        (( rpc_backend_cnt++ )) || true
     done
     if [ "$rpc_backend_cnt" -gt 1 ]; then
         echo "ERROR: only one rpc backend may be enabled,"
diff --git a/stack.sh b/stack.sh
index 68eac7c..f42e50a 100755
--- a/stack.sh
+++ b/stack.sh
@@ -177,9 +177,6 @@
     exit 1
 fi
 
-# Set up logging level
-VERBOSE=$(trueorfalse True $VERBOSE)
-
 # Configure sudo
 # --------------
 
@@ -285,6 +282,182 @@
 fi
 
 
+# Configure Logging
+# -----------------
+
+# Set up logging level
+VERBOSE=$(trueorfalse True $VERBOSE)
+
+# Draw a spinner so the user knows something is happening
+function spinner {
+    local delay=0.75
+    local spinstr='/-\|'
+    printf "..." >&3
+    while [ true ]; do
+        local temp=${spinstr#?}
+        printf "[%c]" "$spinstr" >&3
+        local spinstr=$temp${spinstr%"$temp"}
+        sleep $delay
+        printf "\b\b\b" >&3
+    done
+}
+
+function kill_spinner {
+    if [ ! -z "$LAST_SPINNER_PID" ]; then
+        kill >/dev/null 2>&1 $LAST_SPINNER_PID
+        printf "\b\b\bdone\n" >&3
+    fi
+}
+
+# Echo text to the log file, summary log file and stdout
+# echo_summary "something to say"
+function echo_summary {
+    if [[ -t 3 && "$VERBOSE" != "True" ]]; then
+        kill_spinner
+        echo -n -e $@ >&6
+        spinner &
+        LAST_SPINNER_PID=$!
+    else
+        echo -e $@ >&6
+    fi
+}
+
+# Echo text only to stdout, no log files
+# echo_nolog "something not for the logs"
+function echo_nolog {
+    echo $@ >&3
+}
+
+if [[ is_fedora && $DISTRO == "rhel6" ]]; then
+    # poor old python2.6 doesn't have argparse by default, which
+    # outfilter.py uses
+    is_package_installed python-argparse || install_package python-argparse
+fi
+
+# Set up logging for ``stack.sh``
+# Set ``LOGFILE`` to turn on logging
+# Append '.xxxxxxxx' to the given name to maintain history
+# where 'xxxxxxxx' is a representation of the date the file was created
+TIMESTAMP_FORMAT=${TIMESTAMP_FORMAT:-"%F-%H%M%S"}
+if [[ -n "$LOGFILE" || -n "$SCREEN_LOGDIR" ]]; then
+    LOGDAYS=${LOGDAYS:-7}
+    CURRENT_LOG_TIME=$(date "+$TIMESTAMP_FORMAT")
+fi
+
+if [[ -n "$LOGFILE" ]]; then
+    # First clean up old log files.  Use the user-specified ``LOGFILE``
+    # as the template to search for, appending '.*' to match the date
+    # we added on earlier runs.
+    LOGDIR=$(dirname "$LOGFILE")
+    LOGFILENAME=$(basename "$LOGFILE")
+    mkdir -p $LOGDIR
+    find $LOGDIR -maxdepth 1 -name $LOGFILENAME.\* -mtime +$LOGDAYS -exec rm {} \;
+    LOGFILE=$LOGFILE.${CURRENT_LOG_TIME}
+    SUMFILE=$LOGFILE.${CURRENT_LOG_TIME}.summary
+
+    # Redirect output according to config
+
+    # Set fd 3 to a copy of stdout. So we can set fd 1 without losing
+    # stdout later.
+    exec 3>&1
+    if [[ "$VERBOSE" == "True" ]]; then
+        # Set fd 1 and 2 to write the log file
+        exec 1> >( $TOP_DIR/tools/outfilter.py -v -o "${LOGFILE}" ) 2>&1
+        # Set fd 6 to summary log file
+        exec 6> >( $TOP_DIR/tools/outfilter.py -o "${SUMFILE}" )
+    else
+        # Set fd 1 and 2 to primary logfile
+        exec 1> >( $TOP_DIR/tools/outfilter.py -o "${LOGFILE}" ) 2>&1
+        # Set fd 6 to summary logfile and stdout
+        exec 6> >( $TOP_DIR/tools/outfilter.py -v -o "${SUMFILE}" >&3 )
+    fi
+
+    echo_summary "stack.sh log $LOGFILE"
+    # Specified logfile name always links to the most recent log
+    ln -sf $LOGFILE $LOGDIR/$LOGFILENAME
+    ln -sf $SUMFILE $LOGDIR/$LOGFILENAME.summary
+else
+    # Set up output redirection without log files
+    # Set fd 3 to a copy of stdout. So we can set fd 1 without losing
+    # stdout later.
+    exec 3>&1
+    if [[ "$VERBOSE" != "True" ]]; then
+        # Throw away stdout and stderr
+        exec 1>/dev/null 2>&1
+    fi
+    # Always send summary fd to original stdout
+    exec 6> >( $TOP_DIR/tools/outfilter.py -v >&3 )
+fi
+
+# Set up logging of screen windows
+# Set ``SCREEN_LOGDIR`` to turn on logging of screen windows to the
+# directory specified in ``SCREEN_LOGDIR``, we will log to the the file
+# ``screen-$SERVICE_NAME-$TIMESTAMP.log`` in that dir and have a link
+# ``screen-$SERVICE_NAME.log`` to the latest log file.
+# Logs are kept for as long specified in ``LOGDAYS``.
+if [[ -n "$SCREEN_LOGDIR" ]]; then
+
+    # We make sure the directory is created.
+    if [[ -d "$SCREEN_LOGDIR" ]]; then
+        # We cleanup the old logs
+        find $SCREEN_LOGDIR -maxdepth 1 -name screen-\*.log -mtime +$LOGDAYS -exec rm {} \;
+    else
+        mkdir -p $SCREEN_LOGDIR
+    fi
+fi
+
+
+# Configure Error Traps
+# ---------------------
+
+# Kill background processes on exit
+trap exit_trap EXIT
+function exit_trap {
+    local r=$?
+    jobs=$(jobs -p)
+    # Only do the kill when we're logging through a process substitution,
+    # which currently is only to verbose logfile
+    if [[ -n $jobs && -n "$LOGFILE" && "$VERBOSE" == "True" ]]; then
+        echo "exit_trap: cleaning up child processes"
+        kill 2>&1 $jobs
+    fi
+
+    # Kill the last spinner process
+    kill_spinner
+
+    if [[ $r -ne 0 ]]; then
+        echo "Error on exit"
+        if [[ -z $LOGDIR ]]; then
+            $TOP_DIR/tools/worlddump.py
+        else
+            $TOP_DIR/tools/worlddump.py -d $LOGDIR
+        fi
+    fi
+
+    exit $r
+}
+
+# Exit on any errors so that errors don't compound
+trap err_trap ERR
+function err_trap {
+    local r=$?
+    set +o xtrace
+    if [[ -n "$LOGFILE" ]]; then
+        echo "${0##*/} failed: full log in $LOGFILE"
+    else
+        echo "${0##*/} failed"
+    fi
+    exit $r
+}
+
+# Begin trapping error exit codes
+set -o errexit
+
+# Print the commands being run so that we can see the command that triggers
+# an error.  It is also useful for following along as the install occurs.
+set -o xtrace
+
+
 # Common Configuration
 # --------------------
 
@@ -494,179 +667,6 @@
 fi
 
 
-# Configure logging
-# -----------------
-
-# Draw a spinner so the user knows something is happening
-function spinner {
-    local delay=0.75
-    local spinstr='/-\|'
-    printf "..." >&3
-    while [ true ]; do
-        local temp=${spinstr#?}
-        printf "[%c]" "$spinstr" >&3
-        local spinstr=$temp${spinstr%"$temp"}
-        sleep $delay
-        printf "\b\b\b" >&3
-    done
-}
-
-function kill_spinner {
-    if [ ! -z "$LAST_SPINNER_PID" ]; then
-        kill >/dev/null 2>&1 $LAST_SPINNER_PID
-        printf "\b\b\bdone\n" >&3
-    fi
-}
-
-# Echo text to the log file, summary log file and stdout
-# echo_summary "something to say"
-function echo_summary {
-    if [[ -t 3 && "$VERBOSE" != "True" ]]; then
-        kill_spinner
-        echo -n -e $@ >&6
-        spinner &
-        LAST_SPINNER_PID=$!
-    else
-        echo -e $@ >&6
-    fi
-}
-
-# Echo text only to stdout, no log files
-# echo_nolog "something not for the logs"
-function echo_nolog {
-    echo $@ >&3
-}
-
-if [[ is_fedora && $DISTRO == "rhel6" ]]; then
-    # poor old python2.6 doesn't have argparse by default, which
-    # outfilter.py uses
-    is_package_installed python-argparse || install_package python-argparse
-fi
-
-# Set up logging for ``stack.sh``
-# Set ``LOGFILE`` to turn on logging
-# Append '.xxxxxxxx' to the given name to maintain history
-# where 'xxxxxxxx' is a representation of the date the file was created
-TIMESTAMP_FORMAT=${TIMESTAMP_FORMAT:-"%F-%H%M%S"}
-if [[ -n "$LOGFILE" || -n "$SCREEN_LOGDIR" ]]; then
-    LOGDAYS=${LOGDAYS:-7}
-    CURRENT_LOG_TIME=$(date "+$TIMESTAMP_FORMAT")
-fi
-
-if [[ -n "$LOGFILE" ]]; then
-    # First clean up old log files.  Use the user-specified ``LOGFILE``
-    # as the template to search for, appending '.*' to match the date
-    # we added on earlier runs.
-    LOGDIR=$(dirname "$LOGFILE")
-    LOGFILENAME=$(basename "$LOGFILE")
-    mkdir -p $LOGDIR
-    find $LOGDIR -maxdepth 1 -name $LOGFILENAME.\* -mtime +$LOGDAYS -exec rm {} \;
-    LOGFILE=$LOGFILE.${CURRENT_LOG_TIME}
-    SUMFILE=$LOGFILE.${CURRENT_LOG_TIME}.summary
-
-    # Redirect output according to config
-
-    # Set fd 3 to a copy of stdout. So we can set fd 1 without losing
-    # stdout later.
-    exec 3>&1
-    if [[ "$VERBOSE" == "True" ]]; then
-        # Set fd 1 and 2 to write the log file
-        exec 1> >( $TOP_DIR/tools/outfilter.py -v -o "${LOGFILE}" ) 2>&1
-        # Set fd 6 to summary log file
-        exec 6> >( $TOP_DIR/tools/outfilter.py -o "${SUMFILE}" )
-    else
-        # Set fd 1 and 2 to primary logfile
-        exec 1> >( $TOP_DIR/tools/outfilter.py -o "${LOGFILE}" ) 2>&1
-        # Set fd 6 to summary logfile and stdout
-        exec 6> >( $TOP_DIR/tools/outfilter.py -v -o "${SUMFILE}" >&3 )
-    fi
-
-    echo_summary "stack.sh log $LOGFILE"
-    # Specified logfile name always links to the most recent log
-    ln -sf $LOGFILE $LOGDIR/$LOGFILENAME
-    ln -sf $SUMFILE $LOGDIR/$LOGFILENAME.summary
-else
-    # Set up output redirection without log files
-    # Set fd 3 to a copy of stdout. So we can set fd 1 without losing
-    # stdout later.
-    exec 3>&1
-    if [[ "$VERBOSE" != "True" ]]; then
-        # Throw away stdout and stderr
-        exec 1>/dev/null 2>&1
-    fi
-    # Always send summary fd to original stdout
-    exec 6> >( $TOP_DIR/tools/outfilter.py -v >&3 )
-fi
-
-# Set up logging of screen windows
-# Set ``SCREEN_LOGDIR`` to turn on logging of screen windows to the
-# directory specified in ``SCREEN_LOGDIR``, we will log to the the file
-# ``screen-$SERVICE_NAME-$TIMESTAMP.log`` in that dir and have a link
-# ``screen-$SERVICE_NAME.log`` to the latest log file.
-# Logs are kept for as long specified in ``LOGDAYS``.
-if [[ -n "$SCREEN_LOGDIR" ]]; then
-
-    # We make sure the directory is created.
-    if [[ -d "$SCREEN_LOGDIR" ]]; then
-        # We cleanup the old logs
-        find $SCREEN_LOGDIR -maxdepth 1 -name screen-\*.log -mtime +$LOGDAYS -exec rm {} \;
-    else
-        mkdir -p $SCREEN_LOGDIR
-    fi
-fi
-
-
-# Set Up Script Execution
-# -----------------------
-
-# Kill background processes on exit
-trap exit_trap EXIT
-function exit_trap {
-    local r=$?
-    jobs=$(jobs -p)
-    # Only do the kill when we're logging through a process substitution,
-    # which currently is only to verbose logfile
-    if [[ -n $jobs && -n "$LOGFILE" && "$VERBOSE" == "True" ]]; then
-        echo "exit_trap: cleaning up child processes"
-        kill 2>&1 $jobs
-    fi
-
-    # Kill the last spinner process
-    kill_spinner
-
-    if [[ $r -ne 0 ]]; then
-        echo "Error on exit"
-        if [[ -z $LOGDIR ]]; then
-            $TOP_DIR/tools/worlddump.py
-        else
-            $TOP_DIR/tools/worlddump.py -d $LOGDIR
-        fi
-    fi
-
-    exit $r
-}
-
-# Exit on any errors so that errors don't compound
-trap err_trap ERR
-function err_trap {
-    local r=$?
-    set +o xtrace
-    if [[ -n "$LOGFILE" ]]; then
-        echo "${0##*/} failed: full log in $LOGFILE"
-    else
-        echo "${0##*/} failed"
-    fi
-    exit $r
-}
-
-
-set -o errexit
-
-# Print the commands being run so that we can see the command that triggers
-# an error.  It is also useful for following along as the install occurs.
-set -o xtrace
-
-
 # Install Packages
 # ================