Wait for hypervisor-stats to reflect ironic nodes
When enrolling nodes into Ironic, poll nova's hypervisor-stats until
the hypervisor count reflects the number of enrolled nodes. This
eliminates a race where devstack completes and an instance is spawned
before the first post-enrollment periodic task ticks on the Nova side,
which has recently started popping up in the gate.
Change-Id: Ib3d8005e0094ee8af2d5fcb65aca6cd92736da90
Closes-bug: #1398128
diff --git a/lib/ironic b/lib/ironic
index 622c189..28f8fe8 100644
--- a/lib/ironic
+++ b/lib/ironic
@@ -500,9 +500,23 @@
create_ovs_taps
}
+function wait_for_nova_resources {
+ # After nodes have been enrolled, we need to wait for n-cpu's periodic
+ # task populate the resource tracker with available nodes. Wait for 2
+ # minutes before timing out.
+ local expected_count=$1
+ echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $expected_count Ironic nodes"
+ for i in $(seq 1 120); do
+ if [ $(nova hypervisor-stats | grep " count " | get_field 2) -ge $expected_count ]; then
+ return 0
+ fi
+ sleep 1
+ done
+ die $LINENO "Nova hypervisor-stats did not register at least $expected_count nodes"
+}
+
function enroll_nodes {
local chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2)
- local idx=0
if [[ "$IRONIC_DEPLOY_DRIVER" == "pxe_ssh" ]] ; then
local _IRONIC_DEPLOY_KERNEL_KEY=pxe_deploy_kernel
@@ -536,6 +550,7 @@
fi
fi
+ local total_nodes=0
while read hardware_info; do
if ! is_ironic_hardware; then
local mac_address=$hardware_info
@@ -566,7 +581,7 @@
ironic port-create --address $mac_address --node_uuid $node_id
- idx=$((idx+1))
+ total_nodes=$((total_nodes+1))
done < $ironic_hwinfo_file
# create the nova flavor
@@ -581,6 +596,10 @@
# from the flavor after the completion of
# https://blueprints.launchpad.net/ironic/+spec/add-node-instance-info
nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$IRONIC_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$IRONIC_DEPLOY_RAMDISK_ID"
+
+ if [ "$VIRT_DRIVER" == "ironic" ]; then
+ wait_for_nova_resources $total_nodes
+ fi
}
function configure_iptables {