Merge "Fix devstack with linuxbridge without l3 agent"
diff --git a/doc/source/plugin-registry.rst b/doc/source/plugin-registry.rst
index 6ece997..89f74c3 100644
--- a/doc/source/plugin-registry.rst
+++ b/doc/source/plugin-registry.rst
@@ -76,6 +76,7 @@
 magnum                                 `git://git.openstack.org/openstack/magnum <https://git.openstack.org/cgit/openstack/magnum>`__
 magnum-ui                              `git://git.openstack.org/openstack/magnum-ui <https://git.openstack.org/cgit/openstack/magnum-ui>`__
 manila                                 `git://git.openstack.org/openstack/manila <https://git.openstack.org/cgit/openstack/manila>`__
+manila-ui                              `git://git.openstack.org/openstack/manila-ui <https://git.openstack.org/cgit/openstack/manila-ui>`__
 masakari                               `git://git.openstack.org/openstack/masakari <https://git.openstack.org/cgit/openstack/masakari>`__
 mistral                                `git://git.openstack.org/openstack/mistral <https://git.openstack.org/cgit/openstack/mistral>`__
 mixmatch                               `git://git.openstack.org/openstack/mixmatch <https://git.openstack.org/cgit/openstack/mixmatch>`__
diff --git a/extras.d/80-tempest.sh b/extras.d/80-tempest.sh
index 6a3d121..15ecfe3 100644
--- a/extras.d/80-tempest.sh
+++ b/extras.d/80-tempest.sh
@@ -11,13 +11,16 @@
         # Tempest config must come after layer 2 services are running
         :
     elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
+        # Tempest config must come after all other plugins are run
+        :
+    elif [[ "$1" == "stack" && "$2" == "post-extra" ]]; then
+        # local.conf Tempest option overrides
+        :
+    elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
         echo_summary "Initializing Tempest"
         configure_tempest
         echo_summary "Installing Tempest Plugins"
         install_tempest_plugins
-    elif [[ "$1" == "stack" && "$2" == "post-extra" ]]; then
-        # local.conf Tempest option overrides
-        :
     fi
 
     if [[ "$1" == "unstack" ]]; then
diff --git a/files/apache-keystone.template b/files/apache-keystone.template
index 428544f..84dc273 100644
--- a/files/apache-keystone.template
+++ b/files/apache-keystone.template
@@ -34,6 +34,12 @@
     %SSLKEYFILE%
 </VirtualHost>
 
+%SSLLISTEN%<VirtualHost *:443>
+%SSLLISTEN%    %SSLENGINE%
+%SSLLISTEN%    %SSLCERTFILE%
+%SSLLISTEN%    %SSLKEYFILE%
+%SSLLISTEN%</VirtualHost>
+
 Alias /identity %KEYSTONE_BIN%/keystone-wsgi-public
 <Location /identity>
     SetHandler wsgi-script
diff --git a/lib/cinder b/lib/cinder
index 9ff74e8..f6ad780 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -425,11 +425,7 @@
 }
 
 # init_cinder() - Initialize database and volume group
-# Uses global ``NOVA_ENABLED_APIS``
 function init_cinder {
-    # Force nova volumes off
-    NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/osapi_volume,//")
-
     if is_service_enabled $DATABASE_BACKENDS; then
         # (Re)create cinder database
         recreate_database cinder
diff --git a/lib/cinder_backends/ceph b/lib/cinder_backends/ceph
index ba86ccf..00a0bb3 100644
--- a/lib/cinder_backends/ceph
+++ b/lib/cinder_backends/ceph
@@ -48,7 +48,7 @@
     iniset $CINDER_CONF $be_name rbd_ceph_conf "$CEPH_CONF_FILE"
     iniset $CINDER_CONF $be_name rbd_pool "$CINDER_CEPH_POOL"
     iniset $CINDER_CONF $be_name rbd_user "$CINDER_CEPH_USER"
-    iniset $CINDER_CONF $be_name rbd_uuid "$CINDER_CEPH_UUID"
+    iniset $CINDER_CONF $be_name rbd_secret_uuid "$CINDER_CEPH_UUID"
     iniset $CINDER_CONF $be_name rbd_flatten_volume_from_snapshot False
     iniset $CINDER_CONF $be_name rbd_max_clone_depth 5
     iniset $CINDER_CONF DEFAULT glance_api_version 2
diff --git a/lib/glance b/lib/glance
index 5259174..da9cd43 100644
--- a/lib/glance
+++ b/lib/glance
@@ -235,7 +235,7 @@
     iniset $GLANCE_CACHE_CONF DEFAULT use_syslog $SYSLOG
     iniset $GLANCE_CACHE_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
     iniuncomment $GLANCE_CACHE_CONF DEFAULT auth_url
-    iniset $GLANCE_CACHE_CONF DEFAULT auth_url $KEYSTONE_AUTH_URI/v2.0
+    iniset $GLANCE_CACHE_CONF DEFAULT auth_url $KEYSTONE_AUTH_URI/v3
     iniuncomment $GLANCE_CACHE_CONF DEFAULT auth_tenant_name
     iniset $GLANCE_CACHE_CONF DEFAULT admin_tenant_name $SERVICE_PROJECT_NAME
     iniuncomment $GLANCE_CACHE_CONF DEFAULT auth_user
diff --git a/lib/horizon b/lib/horizon
index c0faed7..830da09 100644
--- a/lib/horizon
+++ b/lib/horizon
@@ -126,9 +126,7 @@
     if is_ubuntu; then
         disable_apache_site 000-default
         sudo touch $horizon_conf
-    elif is_fedora; then
-        sudo sed '/^Listen/s/^.*$/Listen 0.0.0.0:80/' -i /etc/httpd/conf/httpd.conf
-    elif is_suse; then
+    elif is_fedora || is_suse; then
         : # nothing to do
     else
         exit_distro_not_supported "horizon apache configuration"
diff --git a/lib/keystone b/lib/keystone
index 948d5b4..fd1d1d4 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -153,6 +153,7 @@
 function _config_keystone_apache_wsgi {
     local keystone_apache_conf
     keystone_apache_conf=$(apache_site_config_for keystone)
+    keystone_ssl_listen="#"
     local keystone_ssl=""
     local keystone_certfile=""
     local keystone_keyfile=""
@@ -161,6 +162,7 @@
     local venv_path=""
 
     if is_ssl_enabled_service key; then
+        keystone_ssl_listen=""
         keystone_ssl="SSLEngine On"
         keystone_certfile="SSLCertificateFile $KEYSTONE_SSL_CERT"
         keystone_keyfile="SSLCertificateKeyFile $KEYSTONE_SSL_KEY"
@@ -178,6 +180,7 @@
         s|%PUBLICPORT%|$keystone_service_port|g;
         s|%ADMINPORT%|$keystone_auth_port|g;
         s|%APACHE_NAME%|$APACHE_NAME|g;
+        s|%SSLLISTEN%|$keystone_ssl_listen|g;
         s|%SSLENGINE%|$keystone_ssl|g;
         s|%SSLCERTFILE%|$keystone_certfile|g;
         s|%SSLKEYFILE%|$keystone_keyfile|g;
diff --git a/lib/neutron-legacy b/lib/neutron-legacy
index 613e0f1..0b0caf1 100644
--- a/lib/neutron-legacy
+++ b/lib/neutron-legacy
@@ -509,6 +509,10 @@
 
 function stop_mutnauq_l2_agent {
     stop_process q-agt
+
+    if [ "$VIRT_DRIVER" = 'xenserver' ]; then
+        stop_process q-domua
+    fi
 }
 
 # stop_mutnauq_other() - Stop running processes (non-screen)
diff --git a/lib/neutron_plugins/openvswitch_agent b/lib/neutron_plugins/openvswitch_agent
index e27b8a6..f009966 100644
--- a/lib/neutron_plugins/openvswitch_agent
+++ b/lib/neutron_plugins/openvswitch_agent
@@ -81,19 +81,24 @@
         # integration bridge.  This is enabled by using a root wrapper
         # that executes commands on dom0 via a XenAPI plugin.
         # XenAPI does not support daemon rootwrap now, so set root_helper_daemon empty
-        iniset /$Q_PLUGIN_CONF_FILE agent root_helper "$Q_RR_DOM0_COMMAND"
-        iniset /$Q_PLUGIN_CONF_FILE agent root_helper_daemon ""
+        iniset "/$Q_PLUGIN_CONF_FILE.domU" agent root_helper "$Q_RR_DOM0_COMMAND"
+        iniset "/$Q_PLUGIN_CONF_FILE.domU" agent root_helper_daemon ""
 
         # Disable minimize polling, so that it can always detect OVS and Port changes
         # This is a problem of xenserver + neutron, bug has been reported
         # https://bugs.launchpad.net/neutron/+bug/1495423
-        iniset /$Q_PLUGIN_CONF_FILE agent minimize_polling False
+        iniset "/$Q_PLUGIN_CONF_FILE.domU" agent minimize_polling False
 
         # Set "physical" mapping
-        iniset /$Q_PLUGIN_CONF_FILE ovs bridge_mappings "physnet1:$FLAT_NETWORK_BRIDGE"
+        iniset "/$Q_PLUGIN_CONF_FILE.domU" ovs bridge_mappings "physnet1:$FLAT_NETWORK_BRIDGE"
 
         # XEN_INTEGRATION_BRIDGE is the integration bridge in dom0
-        iniset /$Q_PLUGIN_CONF_FILE ovs integration_bridge $XEN_INTEGRATION_BRIDGE
+        iniset "/$Q_PLUGIN_CONF_FILE.domU" ovs integration_bridge $XEN_INTEGRATION_BRIDGE
+
+        # Set OVS native interface for ovs-agent in compute node
+        XEN_DOM0_IP=$(echo "$XENAPI_CONNECTION_URL" | cut -d "/" -f 3)
+        iniset /$Q_PLUGIN_CONF_FILE ovs ovsdb_connection tcp:$XEN_DOM0_IP:6640
+        iniset /$Q_PLUGIN_CONF_FILE ovs of_listen_address $HOST_IP
 
         # Set up domU's L2 agent:
 
@@ -107,11 +112,11 @@
         sudo ovs-vsctl -- --may-exist add-port $PUBLIC_BRIDGE $PUBLIC_INTERFACE
 
         # Set bridge mappings to "physnet1:br-$GUEST_INTERFACE_DEFAULT"
-        iniset "/$Q_PLUGIN_CONF_FILE.domU" ovs bridge_mappings "physnet1:br-$VLAN_INTERFACE,physnet-ex:$PUBLIC_BRIDGE"
+        iniset /$Q_PLUGIN_CONF_FILE ovs bridge_mappings "physnet1:br-$VLAN_INTERFACE,physnet-ex:$PUBLIC_BRIDGE"
         # Set integration bridge to domU's
-        iniset "/$Q_PLUGIN_CONF_FILE.domU" ovs integration_bridge $OVS_BRIDGE
+        iniset /$Q_PLUGIN_CONF_FILE ovs integration_bridge $OVS_BRIDGE
         # Set root wrap
-        iniset "/$Q_PLUGIN_CONF_FILE.domU" agent root_helper "$Q_RR_COMMAND"
+        iniset /$Q_PLUGIN_CONF_FILE agent root_helper "$Q_RR_COMMAND"
     fi
     iniset /$Q_PLUGIN_CONF_FILE agent tunnel_types $Q_TUNNEL_TYPES
     iniset /$Q_PLUGIN_CONF_FILE ovs datapath_type $OVS_DATAPATH_TYPE
diff --git a/lib/neutron_plugins/ovs_base b/lib/neutron_plugins/ovs_base
index baf7d7f..1004325 100644
--- a/lib/neutron_plugins/ovs_base
+++ b/lib/neutron_plugins/ovs_base
@@ -30,7 +30,7 @@
 
 function _neutron_ovs_base_setup_bridge {
     local bridge=$1
-    neutron-ovs-cleanup
+    neutron-ovs-cleanup --config-file $NEUTRON_CONF
     _neutron_ovs_base_add_bridge $bridge
     sudo ovs-vsctl --no-wait br-set-external-id $bridge bridge-id $bridge
 }
@@ -97,7 +97,7 @@
         iniset $Q_L3_CONF_FILE DEFAULT external_network_bridge $PUBLIC_BRIDGE
     fi
 
-    neutron-ovs-cleanup
+    neutron-ovs-cleanup --config-file $NEUTRON_CONF
     if [[ "$Q_USE_PUBLIC_VETH" = "True" ]]; then
         ip link show $Q_PUBLIC_VETH_INT > /dev/null 2>&1 ||
         sudo ip link add $Q_PUBLIC_VETH_INT type veth \
diff --git a/lib/nova b/lib/nova
index ca9a6c7..d5db5ea 100644
--- a/lib/nova
+++ b/lib/nova
@@ -85,9 +85,6 @@
 # NOTE: Set ``FORCE_CONFIG_DRIVE="False"`` to turn OFF config drive
 FORCE_CONFIG_DRIVE=${FORCE_CONFIG_DRIVE:-"False"}
 
-# Option to initialize CellsV2 environment
-NOVA_CONFIGURE_CELLSV2=$(trueorfalse False NOVA_CONFIGURE_CELLSV2)
-
 # Nova supports pluggable schedulers.  The default ``FilterScheduler``
 # should work in most cases.
 SCHEDULER=${SCHEDULER:-filter_scheduler}
@@ -681,9 +678,7 @@
     if is_service_enabled $DATABASE_BACKENDS && is_service_enabled n-api; then
         # (Re)create nova databases
         recreate_database nova
-        if [ "$NOVA_CONFIGURE_CELLSV2" != "False" ]; then
-            recreate_database nova_api_cell0
-        fi
+        recreate_database nova_api_cell0
 
         # Migrate nova database. If "nova-manage cell_v2 simple_cell_setup" has
         # been run this migrates the "nova" and "nova_api_cell0" database.
diff --git a/lib/nova_plugins/hypervisor-libvirt b/lib/nova_plugins/hypervisor-libvirt
index 167ab6f..f3c8add 100644
--- a/lib/nova_plugins/hypervisor-libvirt
+++ b/lib/nova_plugins/hypervisor-libvirt
@@ -105,6 +105,16 @@
     if [[ "$ENABLE_FILE_INJECTION" == "True" ]] ; then
         if is_ubuntu; then
             install_package python-guestfs
+            # NOTE(andreaf) Ubuntu kernel can only be read by root, which breaks libguestfs:
+            # https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725)
+            INSTALLED_KERNELS="$(ls /boot/vmlinuz-*)"
+            for kernel in $INSTALLED_KERNELS; do
+                STAT_OVERRIDE="root root 644 ${kernel}"
+                # unstack won't remove the statoverride, so make this idempotent
+                if [[ ! $(dpkg-statoverride --list | grep "$STAT_OVERRIDE") ]]; then
+                    sudo dpkg-statoverride --add --update $STAT_OVERRIDE
+                fi
+            done
         elif is_fedora || is_suse; then
             install_package python-libguestfs
         fi
diff --git a/lib/nova_plugins/hypervisor-xenserver b/lib/nova_plugins/hypervisor-xenserver
index e5d25da..b053856 100644
--- a/lib/nova_plugins/hypervisor-xenserver
+++ b/lib/nova_plugins/hypervisor-xenserver
@@ -90,6 +90,19 @@
         echo "install_conntrack_tools"
     } | $ssh_dom0
 
+    if is_service_enabled neutron; then
+        # Remove restriction on linux bridge in Dom0 when neutron is enabled
+        $ssh_dom0 "rm -f /etc/modprobe.d/blacklist-bridge*"
+
+        count=`$ssh_dom0 "iptables -t filter -L XenServerDevstack |wc -l"`
+        if [ "$count" = "0" ]; then
+        {
+            echo "iptables -t filter --new XenServerDevstack"
+            echo "iptables -t filter -I INPUT -j XenServerDevstack"
+            echo "iptables -t filter -I XenServerDevstack -p tcp --dport 6640 -j ACCEPT"
+        } | $ssh_dom0
+        fi
+    fi
 }
 
 # install_nova_hypervisor() - Install external components
diff --git a/lib/swift b/lib/swift
index f9ea028..b175f2e 100644
--- a/lib/swift
+++ b/lib/swift
@@ -397,6 +397,9 @@
     iniset ${SWIFT_CONFIG_PROXY_SERVER} app:proxy-server node_timeout 120
     iniset ${SWIFT_CONFIG_PROXY_SERVER} app:proxy-server conn_timeout 20
 
+    # Versioned Writes
+    iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:versioned_writes allow_versioned_writes true
+
     # Configure Ceilometer
     if is_service_enabled ceilometer; then
         iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:ceilometer "set log_level" "WARN"
@@ -489,8 +492,6 @@
         generate_swift_config_services ${swift_node_config} ${node_number} $(( CONTAINER_PORT_BASE + 10 * (node_number - 1) )) container
         iniuncomment ${swift_node_config} DEFAULT bind_ip
         iniset ${swift_node_config} DEFAULT bind_ip ${SWIFT_SERVICE_LISTEN_ADDRESS}
-        iniuncomment ${swift_node_config} app:container-server allow_versions
-        iniset ${swift_node_config} app:container-server allow_versions  "true"
 
         swift_node_config=${SWIFT_CONF_DIR}/account-server/${node_number}.conf
         cp ${SWIFT_DIR}/etc/account-server.conf-sample ${swift_node_config}
diff --git a/lib/tempest b/lib/tempest
index a5dd531..6dc83b5 100644
--- a/lib/tempest
+++ b/lib/tempest
@@ -269,13 +269,15 @@
         iniset $TEMPEST_CONFIG auth admin_project_name $admin_project_name
         iniset $TEMPEST_CONFIG auth admin_domain_name $admin_domain_name
     fi
-    if [ "$ENABLE_IDENTITY_V2" == "False" ]; then
-        # Only Identity v3 is available; then skip Identity API v2 tests
-        iniset $TEMPEST_CONFIG identity-feature-enabled api_v2 False
-        # In addition, use v3 auth tokens for running all Tempest tests
-        iniset $TEMPEST_CONFIG identity auth_version v3
-    else
+    if [ "$ENABLE_IDENTITY_V2" == "True" ]; then
+        # Run Identity API v2 tests ONLY if needed
+        iniset $TEMPEST_CONFIG identity-feature-enabled api_v2 True
         iniset $TEMPEST_CONFIG identity auth_version ${TEMPEST_AUTH_VERSION:-v2}
+    else
+        # Skip Identity API v2 tests by default
+        iniset $TEMPEST_CONFIG identity-feature-enabled api_v2 False
+        # Use v3 auth tokens for running all Tempest tests
+        iniset $TEMPEST_CONFIG identity auth_version v3
     fi
 
     if is_ssl_enabled_service "key" || is_service_enabled tls-proxy; then
@@ -383,11 +385,6 @@
 
     # Orchestration Tests
     if is_service_enabled heat; then
-        # Though this is not needed by heat, some tempest tests explicitly
-        # try to set this role. Removing them from the tempest tests breaks
-        # some non-devstack CIs.
-        get_or_create_role "heat_stack_owner"
-
         if [[ ! -z "$HEAT_CFN_IMAGE_URL" ]]; then
             iniset $TEMPEST_CONFIG orchestration image_ref $(basename "${HEAT_CFN_IMAGE_URL%.*}")
         fi
@@ -401,7 +398,7 @@
             iniset $TEMPEST_CONFIG orchestration instance_type "m1.heat"
         fi
         iniset $TEMPEST_CONFIG orchestration build_timeout 900
-        iniset $TEMPEST_CONFIG orchestration stack_owner_role "heat_stack_owner"
+        iniset $TEMPEST_CONFIG orchestration stack_owner_role Member
     fi
 
     # Scenario
diff --git a/openrc b/openrc
index 8d8ae8b..d1c6129 100644
--- a/openrc
+++ b/openrc
@@ -81,12 +81,12 @@
 KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
 
 # Identity API version
-export OS_IDENTITY_API_VERSION=${IDENTITY_API_VERSION:-2.0}
+export OS_IDENTITY_API_VERSION=${IDENTITY_API_VERSION:-3}
 
 # Authenticating against an OpenStack cloud using Keystone returns a **Token**
 # and **Service Catalog**.  The catalog contains the endpoints for all services
 # the user/project has access to - including nova, glance, keystone, swift, ...
-# We currently recommend using the 2.0 *identity api*.
+# We currently recommend using the version 3 *identity api*.
 #
 export OS_AUTH_URL=$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:5000/v${OS_IDENTITY_API_VERSION}
 
diff --git a/stack.sh b/stack.sh
index 74edb10..05a7666 100755
--- a/stack.sh
+++ b/stack.sh
@@ -1385,8 +1385,16 @@
 # ----------------------
 
 # Do this late because it requires compute hosts to have started
-if is_service_enabled n-api && [ "$NOVA_CONFIGURE_CELLSV2" == "True" ]; then
-    create_cell
+if is_service_enabled n-api; then
+    if is_service_enabled n-cpu; then
+        create_cell
+    else
+        # Some CI systems like Hyper-V build the control plane on
+        # Linux, and join in non Linux Computes after setup. This
+        # allows them to delay the processing until after their whole
+        # environment is up.
+        echo_summary "SKIPPING Cell setup because n-cpu is not enabled. You will have to do this manually before you have a working environment."
+    fi
 fi
 
 # Bash completion
@@ -1411,6 +1419,9 @@
 # Phase: test-config
 run_phase stack test-config
 
+# Apply late configuration from ``local.conf`` if it exists for layer 2 services
+# Phase: test-config
+merge_config_group $TOP_DIR/local.conf test-config
 
 # Fin
 # ===
diff --git a/stackrc b/stackrc
index b5018de..9799d1d 100644
--- a/stackrc
+++ b/stackrc
@@ -148,7 +148,7 @@
 fi
 
 # Configure Identity API version: 2.0, 3
-IDENTITY_API_VERSION=${IDENTITY_API_VERSION:-2.0}
+IDENTITY_API_VERSION=${IDENTITY_API_VERSION:-3}
 
 # Set the option ENABLE_IDENTITY_V2 to True. It defines whether the DevStack
 # deployment will be deploying the Identity v2 pipelines. If this option is set
diff --git a/tools/create_userrc.sh b/tools/create_userrc.sh
index 30d1a01..f4a4edc 100755
--- a/tools/create_userrc.sh
+++ b/tools/create_userrc.sh
@@ -152,7 +152,7 @@
 fi
 
 if [ -z "$OS_AUTH_URL" ]; then
-    export OS_AUTH_URL=http://localhost:5000/v2.0/
+    export OS_AUTH_URL=http://localhost:5000/v3/
 fi
 
 if [ -z "$OS_USER_DOMAIN_ID" -a -z "$OS_USER_DOMAIN_NAME" ]; then
diff --git a/tools/worlddump.py b/tools/worlddump.py
index e1ef544..1ce931e 100755
--- a/tools/worlddump.py
+++ b/tools/worlddump.py
@@ -34,6 +34,7 @@
     'neutron-linuxbridge-agent',
     'neutron-metadata-agent',
     'neutron-openvswitch-agent',
+    'cinder-volume',
 )