Robustify service shutdown

* Save PID when using screen in screen_it()
* Add screen_stop()
* Call out service stop_*() in unstack.sh functions so screen_stop()
  can do its thing

Closes-bug: 1183449
Change-Id: Iac84231cfda960c4197de5b6e8ba6eb19225169a
diff --git a/functions b/functions
index 6f09685..92b61ed 100644
--- a/functions
+++ b/functions
@@ -1132,10 +1132,39 @@
             sleep 1.5
 
             NL=`echo -ne '\015'`
-            screen -S $SCREEN_NAME -p $1 -X stuff "$2 || echo \"$1 failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/$1.failure\"$NL"
+            # This fun command does the following:
+            # - the passed server command is backgrounded
+            # - the pid of the background process is saved in the usual place
+            # - the server process is brought back to the foreground
+            # - if the server process exits prematurely the fg command errors
+            #   and a message is written to stdout and the service failure file
+            # The pid saved can be used in screen_stop() as a process group
+            # id to kill off all child processes
+            screen -S $SCREEN_NAME -p $1 -X stuff "$2 & echo \$! >$SERVICE_DIR/$SCREEN_NAME/$1.pid; fg || echo \"$1 failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/$1.failure\"$NL"
         else
             # Spawn directly without screen
-            run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$service.pid
+            run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$1.pid
+        fi
+    fi
+}
+
+
+# Stop a service in screen
+# screen_stop service
+function screen_stop() {
+    SCREEN_NAME=${SCREEN_NAME:-stack}
+    SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
+    USE_SCREEN=$(trueorfalse True $USE_SCREEN)
+
+    if is_service_enabled $1; then
+        # Kill via pid if we have one available
+        if [[ -r $SERVICE_DIR/$SCREEN_NAME/$1.pid ]]; then
+            pkill -TERM -P $(cat $SERVICE_DIR/$SCREEN_NAME/$1.pid)
+            rm $SERVICE_DIR/$SCREEN_NAME/$1.pid
+        fi
+        if [[ "$USE_SCREEN" = "True" ]]; then
+            # Clean up the screen window
+            screen -S $SCREEN_NAME -p $1 -X kill
         fi
     fi
 }
diff --git a/lib/ceilometer b/lib/ceilometer
index fac3be1..211303f 100644
--- a/lib/ceilometer
+++ b/lib/ceilometer
@@ -162,7 +162,7 @@
 function stop_ceilometer() {
     # Kill the ceilometer screen windows
     for serv in ceilometer-acompute ceilometer-acentral ceilometer-anotification ceilometer-collector ceilometer-api ceilometer-alarm-notifier ceilometer-alarm-evaluator; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
 }
 
diff --git a/lib/cinder b/lib/cinder
index cbe732e..11414be 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -556,7 +556,7 @@
 function stop_cinder() {
     # Kill the cinder screen windows
     for serv in c-api c-bak c-sch c-vol; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
 
     if is_service_enabled c-vol; then
diff --git a/lib/glance b/lib/glance
index 135136d..80868ae 100644
--- a/lib/glance
+++ b/lib/glance
@@ -206,8 +206,8 @@
 # stop_glance() - Stop running processes
 function stop_glance() {
     # Kill the Glance screen windows
-    screen -S $SCREEN_NAME -p g-api -X kill
-    screen -S $SCREEN_NAME -p g-reg -X kill
+    screen_stop g-api
+    screen_stop g-reg
 }
 
 
diff --git a/lib/heat b/lib/heat
index e44a618..29cd967 100644
--- a/lib/heat
+++ b/lib/heat
@@ -175,7 +175,7 @@
 function stop_heat() {
     # Kill the screen windows
     for serv in h-eng h-api h-api-cfn h-api-cw; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
 }
 
diff --git a/lib/keystone b/lib/keystone
index 29b9604..dc6a730 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -421,7 +421,7 @@
 # stop_keystone() - Stop running processes
 function stop_keystone() {
     # Kill the Keystone screen window
-    screen -S $SCREEN_NAME -p key -X kill
+    screen_stop key
 }
 
 
diff --git a/lib/nova b/lib/nova
index 39685a8..178f8ee 100644
--- a/lib/nova
+++ b/lib/nova
@@ -705,7 +705,7 @@
     # Some services are listed here twice since more than one instance
     # of a service may be running in certain configs.
     for serv in n-api n-cpu n-crt n-net n-sch n-novnc n-xvnc n-cauth n-spice n-cond n-cell n-cell n-api-meta; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
     if is_service_enabled n-cpu && [[ -r $NOVA_PLUGINS/hypervisor-$VIRT_DRIVER ]]; then
         stop_nova_hypervisor
diff --git a/lib/trove b/lib/trove
index f8e3edd..870afbe 100644
--- a/lib/trove
+++ b/lib/trove
@@ -198,7 +198,7 @@
 function stop_trove() {
     # Kill the trove screen windows
     for serv in tr-api tr-tmgr tr-cond; do
-        screen -S $SCREEN_NAME -p $serv -X kill
+        screen_stop $serv
     done
 }
 
diff --git a/stackrc b/stackrc
index 3fdc566..49fb26b 100644
--- a/stackrc
+++ b/stackrc
@@ -9,6 +9,9 @@
 # Destination for working data
 DATA_DIR=${DEST}/data
 
+# Destination for status files
+SERVICE_DIR=${DEST}/status
+
 # Determine stack user
 if [[ $EUID -eq 0 ]]; then
     STACK_USER=stack
diff --git a/unstack.sh b/unstack.sh
index 67c8b7c..77dbe07 100755
--- a/unstack.sh
+++ b/unstack.sh
@@ -36,6 +36,9 @@
 # Get project function libraries
 source $TOP_DIR/lib/baremetal
 source $TOP_DIR/lib/cinder
+source $TOP_DIR/lib/keystone
+source $TOP_DIR/lib/glance
+source $TOP_DIR/lib/nova
 source $TOP_DIR/lib/horizon
 source $TOP_DIR/lib/swift
 source $TOP_DIR/lib/neutron
@@ -75,21 +78,29 @@
     teardown_neutron_debug
 fi
 
-# Shut down devstack's screen to get the bulk of OpenStack services in one shot
-SCREEN=$(which screen)
-if [[ -n "$SCREEN" ]]; then
-    SESSION=$(screen -ls | awk '/[0-9].stack/ { print $1 }')
-    if [[ -n "$SESSION" ]]; then
-        screen -X -S $SESSION quit
-    fi
+# Call service stop
+if is_service_enabled trove; then
+    stop_trove
 fi
 
-# Shut down Nova hypervisor plugins after Nova
-NOVA_PLUGINS=$TOP_DIR/lib/nova_plugins
-if is_service_enabled nova && [[ -r $NOVA_PLUGINS/hypervisor-$VIRT_DRIVER ]]; then
-    # Load plugin
-    source $NOVA_PLUGINS/hypervisor-$VIRT_DRIVER
-    stop_nova_hypervisor
+if is_service_enabled heat; then
+    stop_heat
+fi
+
+if is_service_enabled ceilometer; then
+    stop_ceilometer
+fi
+
+if is_service_enabled nova; then
+    stop_nova
+fi
+
+if is_service_enabled g-api g-reg; then
+    stop_glance
+fi
+
+if is_service_enabled key; then
+    stop_keystone
 fi
 
 # Swift runs daemons
@@ -123,6 +134,7 @@
 
 # Get the iSCSI volumes
 if is_service_enabled cinder; then
+    stop_cinder
     cleanup_cinder
 fi
 
@@ -152,4 +164,13 @@
     cleanup_trove
 fi
 
+# Clean up the remainder of the screen processes
+SCREEN=$(which screen)
+if [[ -n "$SCREEN" ]]; then
+    SESSION=$(screen -ls | awk '/[0-9].stack/ { print $1 }')
+    if [[ -n "$SESSION" ]]; then
+        screen -X -S $SESSION quit
+    fi
+fi
+
 cleanup_tmp