Improve exercise robustness

* Test returns and exit codes on most command invocations
* Add start and end banners to make output easier to find in
  long log files
* Adds die_if_error(), die_if_not_set() and is_set() to functions
* Add some function tests

Fixes bug 944593

Change-Id: I55e2962c5fec9aad237b674732b1e922ad37a62e
diff --git a/exercises/euca.sh b/exercises/euca.sh
index 86cd673..b766bab 100755
--- a/exercises/euca.sh
+++ b/exercises/euca.sh
@@ -2,7 +2,10 @@
 
 # we will use the ``euca2ools`` cli tool that wraps the python boto
 # library to test ec2 compatibility
-#
+
+echo "**************************************************"
+echo "Begin DevStack Exercise: $0"
+echo "**************************************************"
 
 # This script exits on an error so that errors don't compound and you see
 # only the first error that occured.
@@ -16,9 +19,14 @@
 # ========
 
 # Use openrc + stackrc + localrc for settings
-pushd $(cd $(dirname "$0")/.. && pwd)
+pushd $(cd $(dirname "$0")/.. && pwd) >/dev/null
+
+# Import common functions
+source ./functions
+
+# Import configuration
 source ./openrc
-popd
+popd >/dev/null
 
 # Max time to wait while vm goes from build to active state
 ACTIVE_TIMEOUT=${ACTIVE_TIMEOUT:-30}
@@ -49,6 +57,7 @@
 
 # Launch it
 INSTANCE=`euca-run-instances -g $SECGROUP -t $DEFAULT_INSTANCE_TYPE $IMAGE | grep INSTANCE | cut -f2`
+die_if_not_set INSTANCE "Failure launching instance"
 
 # Assure it has booted within a reasonable time
 if ! timeout $RUNNING_TIMEOUT sh -c "while ! euca-describe-instances $INSTANCE | grep -q running; do sleep 1; done"; then
@@ -58,12 +67,15 @@
 
 # Allocate floating address
 FLOATING_IP=`euca-allocate-address | cut -f2`
+die_if_not_set FLOATING_IP "Failure allocating floating IP"
 
 # Associate floating address
 euca-associate-address -i $INSTANCE $FLOATING_IP
+die_if_error "Failure associating address $FLOATING_IP to $INSTANCE"
 
 # Authorize pinging
 euca-authorize -P icmp -s 0.0.0.0/0 -t -1:-1 $SECGROUP
+die_if_error "Failure authorizing rule in $SECGROUP"
 
 # Test we can ping our floating ip within ASSOCIATE_TIMEOUT seconds
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while ! ping -c1 -w1 $FLOATING_IP; do sleep 1; done"; then
@@ -73,9 +85,11 @@
 
 # Revoke pinging
 euca-revoke -P icmp -s 0.0.0.0/0 -t -1:-1 $SECGROUP
+die_if_error "Failure revoking rule in $SECGROUP"
 
 # Release floating address
 euca-disassociate-address $FLOATING_IP
+die_if_error "Failure disassociating address $FLOATING_IP"
 
 # Wait just a tick for everything above to complete so release doesn't fail
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while euca-describe-addresses | grep $INSTANCE | grep -q $FLOATING_IP; do sleep 1; done"; then
@@ -85,6 +99,7 @@
 
 # Release floating address
 euca-release-address $FLOATING_IP
+die_if_error "Failure releasing address $FLOATING_IP"
 
 # Wait just a tick for everything above to complete so terminate doesn't fail
 if ! timeout $ASSOCIATE_TIMEOUT sh -c "while euca-describe-addresses | grep -q $FLOATING_IP; do sleep 1; done"; then
@@ -94,6 +109,7 @@
 
 # Terminate instance
 euca-terminate-instances $INSTANCE
+die_if_error "Failure terminating instance $INSTANCE"
 
 # Assure it has terminated within a reasonable time
 if ! timeout $TERMINATE_TIMEOUT sh -c "while euca-describe-instances $INSTANCE | grep -q running; do sleep 1; done"; then
@@ -103,3 +119,9 @@
 
 # Delete group
 euca-delete-group $SECGROUP
+die_if_error "Failure deleting security group $SECGROUP"
+
+set +o xtrace
+echo "**************************************************"
+echo "End DevStack Exercise: $0"
+echo "**************************************************"