Merge "Fix noVNC git repo"
diff --git a/exercises/aggregates.sh b/exercises/aggregates.sh
index 96241f9..1b1ac06 100755
--- a/exercises/aggregates.sh
+++ b/exercises/aggregates.sh
@@ -67,7 +67,10 @@
 exit_if_aggregate_present $AGGREGATE_NAME
 
 AGGREGATE_ID=$(nova aggregate-create $AGGREGATE_NAME $AGGREGATE_A_ZONE | grep " $AGGREGATE_NAME " | get_field 1)
+die_if_not_set $LINENO AGGREGATE_ID "Failure creating AGGREGATE_ID for $AGGREGATE_NAME $AGGREGATE_A_ZONE"
+
 AGGREGATE2_ID=$(nova aggregate-create $AGGREGATE2_NAME $AGGREGATE_A_ZONE | grep " $AGGREGATE2_NAME " | get_field 1)
+die_if_not_set $LINENO AGGREGATE2_ID "Fail creating AGGREGATE2_ID for $AGGREGATE2_NAME $AGGREGATE_A_ZONE"
 
 # check aggregate created
 nova aggregate-list | grep -q " $AGGREGATE_NAME " || die $LINENO "Aggregate $AGGREGATE_NAME not created"
diff --git a/exercises/floating_ips.sh b/exercises/floating_ips.sh
index 1a1608c..4d71d49 100755
--- a/exercises/floating_ips.sh
+++ b/exercises/floating_ips.sh
@@ -114,6 +114,7 @@
 if [[ -z "$INSTANCE_TYPE" ]]; then
     # grab the first flavor in the list to launch if default doesn't exist
     INSTANCE_TYPE=$(nova flavor-list | head -n 4 | tail -n 1 | get_field 1)
+    die_if_not_set $LINENO INSTANCE_TYPE "Failure retrieving INSTANCE_TYPE"
 fi
 
 # Clean-up from previous runs
diff --git a/exercises/neutron-adv-test.sh b/exercises/neutron-adv-test.sh
index 7dfa5dc..28e0a3d 100755
--- a/exercises/neutron-adv-test.sh
+++ b/exercises/neutron-adv-test.sh
@@ -139,24 +139,28 @@
 
 function get_image_id {
     local IMAGE_ID=$(glance image-list | egrep " $DEFAULT_IMAGE_NAME " | get_field 1)
+    die_if_not_set $LINENO IMAGE_ID "Failure retrieving IMAGE_ID"
     echo "$IMAGE_ID"
 }
 
 function get_tenant_id {
     local TENANT_NAME=$1
     local TENANT_ID=`keystone tenant-list | grep " $TENANT_NAME " | head -n 1 | get_field 1`
+    die_if_not_set $LINENO TENANT_ID "Failure retrieving TENANT_ID for $TENANT_NAME"
     echo "$TENANT_ID"
 }
 
 function get_user_id {
     local USER_NAME=$1
     local USER_ID=`keystone user-list | grep $USER_NAME | awk '{print $2}'`
+    die_if_not_set $LINENO USER_ID "Failure retrieving USER_ID for $USER_NAME"
     echo "$USER_ID"
 }
 
 function get_role_id {
     local ROLE_NAME=$1
     local ROLE_ID=`keystone role-list | grep $ROLE_NAME | awk '{print $2}'`
+    die_if_not_set $LINENO ROLE_ID "Failure retrieving ROLE_ID for $ROLE_NAME"
     echo "$ROLE_ID"
 }
 
@@ -169,6 +173,7 @@
 function get_flavor_id {
     local INSTANCE_TYPE=$1
     local FLAVOR_ID=`nova flavor-list | grep $INSTANCE_TYPE | awk '{print $2}'`
+    die_if_not_set $LINENO FLAVOR_ID "Failure retrieving FLAVOR_ID for $INSTANCE_TYPE"
     echo "$FLAVOR_ID"
 }
 
@@ -234,6 +239,7 @@
     local TENANT_ID=$(get_tenant_id $TENANT)
     source $TOP_DIR/openrc $TENANT $TENANT
     local NET_ID=$(neutron net-create --tenant_id $TENANT_ID $NET_NAME $EXTRA| grep ' id ' | awk '{print $4}' )
+    die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $TENANT_ID $NET_NAME $EXTRA"
     neutron subnet-create --ip_version 4 --tenant_id $TENANT_ID --gateway $GATEWAY $NET_ID $CIDR
     neutron-debug probe-create --device-owner compute $NET_ID
     source $TOP_DIR/openrc demo demo
diff --git a/exercises/sec_groups.sh b/exercises/sec_groups.sh
index 7d80570..eb32cc7 100755
--- a/exercises/sec_groups.sh
+++ b/exercises/sec_groups.sh
@@ -56,6 +56,7 @@
 
 # Check to make sure rules were added
 SEC_GROUP_RULES=( $(nova secgroup-list-rules $SEC_GROUP_NAME | grep -v \- | grep -v 'Source Group' | cut -d '|' -f3 | tr -d ' ') )
+die_if_not_set $LINENO SEC_GROUP_RULES "Failure retrieving SEC_GROUP_RULES for $SEC_GROUP_NAME"
 for i in "${RULES_TO_ADD[@]}"; do
     skip=
     for j in "${SEC_GROUP_RULES[@]}"; do
diff --git a/exercises/volumes.sh b/exercises/volumes.sh
index 9ee9fa9..77fa4eb 100755
--- a/exercises/volumes.sh
+++ b/exercises/volumes.sh
@@ -118,6 +118,7 @@
 if [[ -z "$INSTANCE_TYPE" ]]; then
     # grab the first flavor in the list to launch if default doesn't exist
     INSTANCE_TYPE=$(nova flavor-list | head -n 4 | tail -n 1 | get_field 1)
+    die_if_not_set $LINENO INSTANCE_TYPE "Failure retrieving INSTANCE_TYPE"
 fi
 
 # Clean-up from previous runs
diff --git a/functions b/functions
index 6137aaf..4d5b4b5 100644
--- a/functions
+++ b/functions
@@ -1351,10 +1351,9 @@
 
     # Create a directory for the downloaded image tarballs.
     mkdir -p $FILES/images
-
+    IMAGE_FNAME=`basename "$image_url"`
     if [[ $image_url != file* ]]; then
         # Downloads the image (uec ami+aki style), then extracts it.
-        IMAGE_FNAME=`basename "$image_url"`
         if [[ ! -f $FILES/$IMAGE_FNAME || "$(stat -c "%s" $FILES/$IMAGE_FNAME)" = "0" ]]; then
              wget -c $image_url -O $FILES/$IMAGE_FNAME
              if [[ $? -ne 0 ]]; then
@@ -1410,13 +1409,92 @@
         vmdk_create_type="$(head -25 $IMAGE | grep -a -F -m 1 'createType=' $IMAGE)"
         vmdk_create_type="${vmdk_create_type#*\"}"
         vmdk_create_type="${vmdk_create_type%?}"
+
+        descriptor_data_pair_msg="Monolithic flat and VMFS disks "`
+                                 `"should use a descriptor-data pair."
         if [[ "$vmdk_create_type" = "monolithicSparse" ]]; then
             vmdk_disktype="sparse"
-        elif [[ "$vmdk_create_type" = "monolithicFlat" ]]; then
-            die $LINENO "Monolithic flat disks should use a descriptor-data pair." \
-            "Please provide the disk and not the descriptor."
+        elif [[ "$vmdk_create_type" = "monolithicFlat" || \
+        "$vmdk_create_type" = "vmfs" ]]; then
+            # Attempt to retrieve the *-flat.vmdk
+            flat_fname="$(head -25 $IMAGE | grep -G 'RW\|RDONLY [0-9]+ FLAT\|VMFS' $IMAGE)"
+            flat_fname="${flat_fname#*\"}"
+            flat_fname="${flat_fname%?}"
+            if [[ -z "$flat_name" ]]; then
+                flat_fname="$IMAGE_NAME-flat.vmdk"
+            fi
+            path_len=`expr ${#image_url} - ${#IMAGE_FNAME}`
+            flat_url="${image_url:0:$path_len}$flat_fname"
+            warn $LINENO "$descriptor_data_pair_msg"`
+                         `" Attempt to retrieve the *-flat.vmdk: $flat_url"
+            if [[ $flat_url != file* ]]; then
+                if [[ ! -f $FILES/$flat_fname || \
+                "$(stat -c "%s" $FILES/$flat_fname)" = "0" ]]; then
+                    wget -c $flat_url -O $FILES/$flat_fname
+                    if [[ $? -ne 0 ]]; then
+                        echo "Flat disk not found: $flat_url"
+                        flat_found=false
+                    fi
+                fi
+                if $flat_found; then
+                    IMAGE="$FILES/${flat_fname}"
+                fi
+            else
+                IMAGE=$(echo $flat_url | sed "s/^file:\/\///g")
+                if [[ ! -f $IMAGE || "$(stat -c "%s" $IMAGE)" == "0" ]]; then
+                    echo "Flat disk not found: $flat_url"
+                    flat_found=false
+                fi
+                if ! $flat_found; then
+                    IMAGE=$(echo $image_url | sed "s/^file:\/\///g")
+                fi
+            fi
+            if $flat_found; then
+                IMAGE_NAME="${flat_fname}"
+            fi
+            vmdk_disktype="preallocated"
+        elif [[ -z "$vmdk_create_type" ]]; then
+            # *-flat.vmdk provided: attempt to retrieve the descriptor (*.vmdk)
+            # to retrieve appropriate metadata
+            if [[ ${IMAGE_NAME: -5} != "-flat" ]]; then
+                warn $LINENO "Expected filename suffix: '-flat'."`
+                            `" Filename provided: ${IMAGE_NAME}"
+            else
+                descriptor_fname="${IMAGE_NAME:0:${#IMAGE_NAME} - 5}.vmdk"
+                path_len=`expr ${#image_url} - ${#IMAGE_FNAME}`
+                flat_path="${image_url:0:$path_len}"
+                descriptor_url=$flat_path$descriptor_fname
+                warn $LINENO "$descriptor_data_pair_msg"`
+                             `" Attempt to retrieve the descriptor *.vmdk: $descriptor_url"
+                if [[ $flat_path != file* ]]; then
+                    if [[ ! -f $FILES/$descriptor_fname || \
+                    "$(stat -c "%s" $FILES/$descriptor_fname)" = "0" ]]; then
+                        wget -c $descriptor_url -O $FILES/$descriptor_fname
+                        if [[ $? -ne 0 ]]; then
+                            warn $LINENO "Descriptor not found $descriptor_url"
+                            descriptor_found=false
+                        fi
+                    fi
+                    descriptor_url="$FILES/$descriptor_fname"
+                else
+                    descriptor_url=$(echo $descriptor_url | sed "s/^file:\/\///g")
+                    if [[ ! -f $descriptor_url || \
+                    "$(stat -c "%s" $descriptor_url)" == "0" ]]; then
+                         warn $LINENO "Descriptor not found $descriptor_url"
+                         descriptor_found=false
+                    fi
+                fi
+                if $descriptor_found; then
+                    vmdk_adapter_type="$(head -25 $descriptor_url |"`
+                    `"grep -a -F -m 1 'ddb.adapterType =' $descriptor_url)"
+                    vmdk_adapter_type="${vmdk_adapter_type#*\"}"
+                    vmdk_adapter_type="${vmdk_adapter_type%?}"
+                 fi
+             fi
+             #TODO(alegendre): handle streamOptimized once supported by the VMware driver.
+             vmdk_disktype="preallocated"
         else
-            #TODO(alegendre): handle streamOptimized once supported by VMware driver.
+            #TODO(alegendre): handle streamOptimized once supported by the VMware driver.
             vmdk_disktype="preallocated"
         fi
 
diff --git a/lib/neutron b/lib/neutron
index 7f1a9d8..7376772 100644
--- a/lib/neutron
+++ b/lib/neutron
@@ -346,6 +346,7 @@
 
 function create_neutron_initial_network() {
     TENANT_ID=$(keystone tenant-list | grep " demo " | get_field 1)
+    die_if_not_set $LINENO TENANT_ID "Failure retrieving TENANT_ID for demo"
 
     # Create a small network
     # Since neutron command is executed in admin context at this point,
@@ -360,12 +361,16 @@
             sudo ip addr add $IP dev $OVS_PHYSICAL_BRIDGE
         done
         NET_ID=$(neutron net-create $PHYSICAL_NETWORK --tenant_id $TENANT_ID --provider:network_type flat --provider:physical_network "$PHYSICAL_NETWORK" | grep ' id ' | get_field 2)
+        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK $TENANT_ID"
         SUBNET_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} --gateway $NETWORK_GATEWAY --name $PRIVATE_SUBNET_NAME $NET_ID $FIXED_RANGE | grep ' id ' | get_field 2)
+        die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $TENANT_ID"
         sudo ifconfig $OVS_PHYSICAL_BRIDGE up
         sudo route add default gw $NETWORK_GATEWAY dev $OVS_PHYSICAL_BRIDGE
     else
         NET_ID=$(neutron net-create --tenant_id $TENANT_ID "$PRIVATE_NETWORK_NAME" | grep ' id ' | get_field 2)
+        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK $TENANT_ID"
         SUBNET_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 4 --gateway $NETWORK_GATEWAY --name $PRIVATE_SUBNET_NAME $NET_ID $FIXED_RANGE | grep ' id ' | get_field 2)
+        die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $TENANT_ID"
     fi
 
     if [[ "$Q_L3_ENABLED" == "True" ]]; then
@@ -373,14 +378,18 @@
         if [[ "$Q_L3_ROUTER_PER_TENANT" == "True" ]]; then
             # create a tenant-owned router.
             ROUTER_ID=$(neutron router-create --tenant_id $TENANT_ID $Q_ROUTER_NAME | grep ' id ' | get_field 2)
+            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $TENANT_ID $Q_ROUTER_NAME"
         else
             # Plugin only supports creating a single router, which should be admin owned.
             ROUTER_ID=$(neutron router-create $Q_ROUTER_NAME | grep ' id ' | get_field 2)
+            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $Q_ROUTER_NAME"
         fi
         neutron router-interface-add $ROUTER_ID $SUBNET_ID
         # Create an external network, and a subnet. Configure the external network as router gw
         EXT_NET_ID=$(neutron net-create "$PUBLIC_NETWORK_NAME" -- --router:external=True | grep ' id ' | get_field 2)
+        die_if_not_set $LINENO EXT_NET_ID "Failure creating EXT_NET_ID for $PUBLIC_NETWORK_NAME"
         EXT_GW_IP=$(neutron subnet-create --ip_version 4 ${Q_FLOATING_ALLOCATION_POOL:+--allocation-pool $Q_FLOATING_ALLOCATION_POOL} --gateway $PUBLIC_NETWORK_GATEWAY --name $PUBLIC_SUBNET_NAME $EXT_NET_ID $FLOATING_RANGE -- --enable_dhcp=False | grep 'gateway_ip' | get_field 2)
+        die_if_not_set $LINENO EXT_GW_IP "Failure creating EXT_GW_IP"
         neutron router-gateway-set $ROUTER_ID $EXT_NET_ID
 
         if is_service_enabled q-l3; then
@@ -390,6 +399,7 @@
                 sudo ip addr add $EXT_GW_IP/$CIDR_LEN dev $PUBLIC_BRIDGE
                 sudo ip link set $PUBLIC_BRIDGE up
                 ROUTER_GW_IP=`neutron port-list -c fixed_ips -c device_owner | grep router_gateway | awk -F '"' '{ print $8; }'`
+                die_if_not_set $LINENO ROUTER_GW_IP "Failure retrieving ROUTER_GW_IP"
                 sudo route add -net $FIXED_RANGE gw $ROUTER_GW_IP
             fi
             if [[ "$Q_USE_NAMESPACE" == "False" ]]; then
diff --git a/lib/swift b/lib/swift
index 8a1489b..5ff6055 100644
--- a/lib/swift
+++ b/lib/swift
@@ -526,14 +526,19 @@
     fi
 
     SWIFT_TENANT_TEST1=$(keystone tenant-create --name=swifttenanttest1 | grep " id " | get_field 2)
+    die_if_not_set $LINENO SWIFT_TENANT_TEST1 "Failure creating SWIFT_TENANT_TEST1"
     SWIFT_USER_TEST1=$(keystone user-create --name=swiftusertest1 --pass=testing --email=test@example.com | grep " id " | get_field 2)
+    die_if_not_set $LINENO SWIFT_USER_TEST1 "Failure creating SWIFT_USER_TEST1"
     keystone user-role-add --user-id $SWIFT_USER_TEST1 --role-id $ADMIN_ROLE --tenant-id $SWIFT_TENANT_TEST1
 
     SWIFT_USER_TEST3=$(keystone user-create --name=swiftusertest3 --pass=testing3 --email=test3@example.com | grep " id " | get_field 2)
+    die_if_not_set $LINENO SWIFT_USER_TEST3 "Failure creating SWIFT_USER_TEST3"
     keystone user-role-add --user-id $SWIFT_USER_TEST3 --role-id $ANOTHER_ROLE --tenant-id $SWIFT_TENANT_TEST1
 
     SWIFT_TENANT_TEST2=$(keystone tenant-create --name=swifttenanttest2 | grep " id " | get_field 2)
+    die_if_not_set $LINENO SWIFT_TENANT_TEST2 "Failure creating SWIFT_TENANT_TEST2"
     SWIFT_USER_TEST2=$(keystone user-create --name=swiftusertest2 --pass=testing2 --email=test2@example.com | grep " id " | get_field 2)
+    die_if_not_set $LINENO SWIFT_USER_TEST2 "Failure creating SWIFT_USER_TEST2"
     keystone user-role-add --user-id $SWIFT_USER_TEST2 --role-id $ADMIN_ROLE --tenant-id $SWIFT_TENANT_TEST2
 }
 
diff --git a/stack.sh b/stack.sh
index a2ef679..0f4329a 100755
--- a/stack.sh
+++ b/stack.sh
@@ -1080,7 +1080,9 @@
 # Create an access key and secret key for nova ec2 register image
 if is_service_enabled key && is_service_enabled swift3 && is_service_enabled nova; then
     NOVA_USER_ID=$(keystone user-list | grep ' nova ' | get_field 1)
+    die_if_not_set $LINENO NOVA_USER_ID "Failure retrieving NOVA_USER_ID for nova"
     NOVA_TENANT_ID=$(keystone tenant-list | grep " $SERVICE_TENANT_NAME " | get_field 1)
+    die_if_not_set $LINENO NOVA_TENANT_ID "Failure retrieving NOVA_TENANT_ID for $SERVICE_TENANT_NAME"
     CREDS=$(keystone ec2-credentials-create --user_id $NOVA_USER_ID --tenant_id $NOVA_TENANT_ID)
     ACCESS_KEY=$(echo "$CREDS" | awk '/ access / { print $4 }')
     SECRET_KEY=$(echo "$CREDS" | awk '/ secret / { print $4 }')
diff --git a/tools/jenkins/adapters/euca.sh b/tools/jenkins/adapters/euca.sh
index b49ce9f..a7e635c 100755
--- a/tools/jenkins/adapters/euca.sh
+++ b/tools/jenkins/adapters/euca.sh
@@ -5,4 +5,5 @@
 
 TOP_DIR=$(cd ../../.. && pwd)
 HEAD_IP=`cat $TOP_DIR/addresses | grep HEAD | cut -d "=" -f2`
+die_if_not_set $LINENO HEAD_IP "Failure retrieving HEAD_IP"
 ssh stack@$HEAD_IP 'cd devstack && source openrc && cd exercises &&  ./euca.sh'
diff --git a/tools/jenkins/adapters/floating_ips.sh b/tools/jenkins/adapters/floating_ips.sh
index a97f935..8da1eeb 100755
--- a/tools/jenkins/adapters/floating_ips.sh
+++ b/tools/jenkins/adapters/floating_ips.sh
@@ -5,4 +5,5 @@
 
 TOP_DIR=$(cd ../../.. && pwd)
 HEAD_IP=`cat $TOP_DIR/addresses | grep HEAD | cut -d "=" -f2`
+die_if_not_set $LINENO HEAD_IP "Failure retrieving HEAD_IP"
 ssh stack@$HEAD_IP 'cd devstack && source openrc && cd exercises &&  ./floating_ips.sh'
diff --git a/tools/jenkins/adapters/volumes.sh b/tools/jenkins/adapters/volumes.sh
index ec29209..0a0b6c0 100755
--- a/tools/jenkins/adapters/volumes.sh
+++ b/tools/jenkins/adapters/volumes.sh
@@ -5,4 +5,5 @@
 
 TOP_DIR=$(cd ../../.. && pwd)
 HEAD_IP=`cat $TOP_DIR/addresses | grep HEAD | cut -d "=" -f2`
+die_if_not_set $LINENO HEAD_IP "Failure retrieving HEAD_IP"
 ssh stack@$HEAD_IP 'cd devstack && source openrc && cd exercises &&  ./volumes.sh'