lib/neutron

# lib/neutron
# functions - funstions specific to neutron

# Dependencies:
# ``functions`` file
# ``DEST`` must be defined

# ``stack.sh`` calls the entry points in this order:
#
# install_neutron
# install_neutronclient
# install_neutron_agent_packages
# install_neutron_third_party
# configure_neutron
# init_neutron
# configure_neutron_third_party
# init_neutron_third_party
# start_neutron_third_party
# create_nova_conf_neutron
# start_neutron_service_and_check
# create_neutron_initial_network
# setup_neutron_debug
# start_neutron_agents
#
# ``unstack.sh`` calls the entry points in this order:
#
# stop_neutron

# Functions in lib/neutron are classified into the following categories:
#
# - entry points (called from stack.sh or unstack.sh)
# - internal functions
# - neutron exercises
# - 3rd party programs


# Neutron Networking
# ------------------

# Make sure that neutron is enabled in ``ENABLED_SERVICES``.  If you want
# to run Neutron on this host, make sure that q-svc is also in
# ``ENABLED_SERVICES``.
#
# If you're planning to use the Neutron openvswitch plugin, set
# ``Q_PLUGIN`` to "openvswitch" and make sure the q-agt service is enabled
# in ``ENABLED_SERVICES``.  If you're planning to use the Neutron
# linuxbridge plugin, set ``Q_PLUGIN`` to "linuxbridge" and make sure the
# q-agt service is enabled in ``ENABLED_SERVICES``.
#
# See "Neutron Network Configuration" below for additional variables
# that must be set in localrc for connectivity across hosts with
# Neutron.
#
# With Neutron networking the NETWORK_MANAGER variable is ignored.
#
# To enable specific configuration options for either the Open vSwitch or
# LinuxBridge plugin, please see the top level README file under the
# Neutron section.

# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace


# Neutron Network Configuration
# -----------------------------

# Gateway and subnet defaults, in case they are not customized in localrc
NETWORK_GATEWAY=${NETWORK_GATEWAY:-10.0.0.1}
PUBLIC_NETWORK_GATEWAY=${PUBLIC_NETWORK_GATEWAY:-172.24.4.225}
PRIVATE_SUBNET_NAME=${PRIVATE_SUBNET_NAME:-"private-subnet"}
PUBLIC_SUBNET_NAME=${PUBLIC_SUBNET_NAME:-"public-subnet"}

# Set up default directories
NEUTRON_DIR=$DEST/neutron
NEUTRONCLIENT_DIR=$DEST/python-neutronclient
NEUTRON_AUTH_CACHE_DIR=${NEUTRON_AUTH_CACHE_DIR:-/var/cache/neutron}

# Support entry points installation of console scripts
if [[ -d $NEUTRON_DIR/bin/neutron-server ]]; then
    NEUTRON_BIN_DIR=$NEUTRON_DIR/bin
     else
NEUTRON_BIN_DIR=$(get_python_exec_prefix)
fi

NEUTRON_CONF_DIR=/etc/neutron
NEUTRON_CONF=$NEUTRON_CONF_DIR/neutron.conf
export NEUTRON_TEST_CONFIG_FILE=${NEUTRON_TEST_CONFIG_FILE:-"$NEUTRON_CONF_DIR/debug.ini"}

# Default Neutron Plugin
Q_PLUGIN=${Q_PLUGIN:-openvswitch}
# Default Neutron Port
Q_PORT=${Q_PORT:-9696}
# Default Neutron Host
Q_HOST=${Q_HOST:-$SERVICE_HOST}
# Default admin username
Q_ADMIN_USERNAME=${Q_ADMIN_USERNAME:-neutron}
# Default auth strategy
Q_AUTH_STRATEGY=${Q_AUTH_STRATEGY:-keystone}
# Use namespace or not
Q_USE_NAMESPACE=${Q_USE_NAMESPACE:-True}
# RHEL's support for namespaces requires using veths with ovs
Q_OVS_USE_VETH=${Q_OVS_USE_VETH:-False}
Q_USE_ROOTWRAP=${Q_USE_ROOTWRAP:-True}
# Meta data IP
Q_META_DATA_IP=${Q_META_DATA_IP:-$SERVICE_HOST}
# Allow Overlapping IP among subnets
Q_ALLOW_OVERLAPPING_IP=${Q_ALLOW_OVERLAPPING_IP:-True}
# Use neutron-debug command
Q_USE_DEBUG_COMMAND=${Q_USE_DEBUG_COMMAND:-False}
# The name of the default q-l3 router
Q_ROUTER_NAME=${Q_ROUTER_NAME:-router1}
# List of config file names in addition to the main plugin config file
# See _configure_neutron_common() for details about setting it up
declare -a Q_PLUGIN_EXTRA_CONF_FILES

if is_service_enabled neutron; then
    Q_RR_CONF_FILE=$NEUTRON_CONF_DIR/rootwrap.conf
    if [[ "$Q_USE_ROOTWRAP" == "False" ]]; then
        Q_RR_COMMAND="sudo"
    else
        NEUTRON_ROOTWRAP=$(get_rootwrap_location neutron)
        Q_RR_COMMAND="sudo $NEUTRON_ROOTWRAP $Q_RR_CONF_FILE"
    fi

    # Provider Network Configurations
    # --------------------------------

    # The following variables control the Neutron openvswitch and
    # linuxbridge plugins' allocation of tenant networks and
    # availability of provider networks. If these are not configured
    # in ``localrc``, tenant networks will be local to the host (with no
    # remote connectivity), and no physical resources will be
    # available for the allocation of provider networks.

    # To use GRE tunnels for tenant networks, set to True in
    # ``localrc``. GRE tunnels are only supported by the openvswitch
    # plugin, and currently only on Ubuntu.
    ENABLE_TENANT_TUNNELS=${ENABLE_TENANT_TUNNELS:-False}

    # If using GRE tunnels for tenant networks, specify the range of
    # tunnel IDs from which tenant networks are allocated. Can be
    # overriden in ``localrc`` in necesssary.
    TENANT_TUNNEL_RANGES=${TENANT_TUNNEL_RANGE:-1:1000}

    # To use VLANs for tenant networks, set to True in localrc. VLANs
    # are supported by the openvswitch and linuxbridge plugins, each
    # requiring additional configuration described below.
    ENABLE_TENANT_VLANS=${ENABLE_TENANT_VLANS:-False}

    # If using VLANs for tenant networks, set in ``localrc`` to specify
    # the range of VLAN VIDs from which tenant networks are
    # allocated. An external network switch must be configured to
    # trunk these VLANs between hosts for multi-host connectivity.
    #
    # Example: ``TENANT_VLAN_RANGE=1000:1999``
    TENANT_VLAN_RANGE=${TENANT_VLAN_RANGE:-}

    # If using VLANs for tenant networks, or if using flat or VLAN
    # provider networks, set in ``localrc`` to the name of the physical
    # network, and also configure ``OVS_PHYSICAL_BRIDGE`` for the
    # openvswitch agent or ``LB_PHYSICAL_INTERFACE`` for the linuxbridge
    # agent, as described below.
    #
    # Example: ``PHYSICAL_NETWORK=default``
    PHYSICAL_NETWORK=${PHYSICAL_NETWORK:-}

    # With the openvswitch plugin, if using VLANs for tenant networks,
    # or if using flat or VLAN provider networks, set in ``localrc`` to
    # the name of the OVS bridge to use for the physical network. The
    # bridge will be created if it does not already exist, but a
    # physical interface must be manually added to the bridge as a
    # port for external connectivity.
    #
    # Example: ``OVS_PHYSICAL_BRIDGE=br-eth1``
    OVS_PHYSICAL_BRIDGE=${OVS_PHYSICAL_BRIDGE:-}

    # With the linuxbridge plugin, if using VLANs for tenant networks,
    # or if using flat or VLAN provider networks, set in ``localrc`` to
    # the name of the network interface to use for the physical
    # network.
    #
    # Example: ``LB_PHYSICAL_INTERFACE=eth1``
    LB_PHYSICAL_INTERFACE=${LB_PHYSICAL_INTERFACE:-}

    # With the openvswitch plugin, set to True in ``localrc`` to enable
    # provider GRE tunnels when ``ENABLE_TENANT_TUNNELS`` is False.
    #
    # Example: ``OVS_ENABLE_TUNNELING=True``
    OVS_ENABLE_TUNNELING=${OVS_ENABLE_TUNNELING:-$ENABLE_TENANT_TUNNELS}
fi

# Neutron plugin specific functions
# ---------------------------------

# Please refer to ``lib/neutron_plugins/README.md`` for details.
source $TOP_DIR/lib/neutron_plugins/$Q_PLUGIN

# Agent loadbalancer service plugin functions
# -------------------------------------------

# Hardcoding for 1 service plugin for now
source $TOP_DIR/lib/neutron_plugins/services/loadbalancer

# VPN service plugin functions
# -------------------------------------------
# Hardcoding for 1 service plugin for now
source $TOP_DIR/lib/neutron_plugins/services/vpn

# Firewall Service Plugin functions
# --------------------------------
source $TOP_DIR/lib/neutron_plugins/services/firewall

# Use security group or not
if has_neutron_plugin_security_group; then
    Q_USE_SECGROUP=${Q_USE_SECGROUP:-True}
else
    Q_USE_SECGROUP=False
fi

# Functions
# ---------

# configure_neutron()
# Set common config for all neutron server and agents.
function configure_neutron() {
    _configure_neutron_common
    iniset_rpc_backend neutron $NEUTRON_CONF DEFAULT

    # goes before q-svc to init Q_SERVICE_PLUGIN_CLASSES
    if is_service_enabled q-lbaas; then
        _configure_neutron_lbaas
    fi
    if is_service_enabled q-vpn; then
        _configure_neutron_vpn
    fi
    if is_service_enabled q-fwaas; then
        _configure_neutron_fwaas
    fi
    if is_service_enabled q-svc; then
        _configure_neutron_service
    fi
    if is_service_enabled q-agt; then
        _configure_neutron_plugin_agent
    fi
    if is_service_enabled q-dhcp; then
        _configure_neutron_dhcp_agent
    fi
    if is_service_enabled q-l3; then
        _configure_neutron_l3_agent
    fi
    if is_service_enabled q-meta; then
        _configure_neutron_metadata_agent
    fi

    _configure_neutron_debug_command
}

function create_nova_conf_neutron() {
    iniset $NOVA_CONF DEFAULT network_api_class "nova.network.quantumv2.api.API"
    iniset $NOVA_CONF DEFAULT quantum_admin_username "$Q_ADMIN_USERNAME"
    iniset $NOVA_CONF DEFAULT quantum_admin_password "$SERVICE_PASSWORD"
    iniset $NOVA_CONF DEFAULT quantum_admin_auth_url "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_AUTH_PORT/v2.0"
    iniset $NOVA_CONF DEFAULT quantum_auth_strategy "$Q_AUTH_STRATEGY"
    iniset $NOVA_CONF DEFAULT quantum_admin_tenant_name "$SERVICE_TENANT_NAME"
    iniset $NOVA_CONF DEFAULT quantum_region_name "RegionOne"
    iniset $NOVA_CONF DEFAULT quantum_url "http://$Q_HOST:$Q_PORT"

    if [[ "$Q_USE_SECGROUP" == "True" ]]; then
        LIBVIRT_FIREWALL_DRIVER=nova.virt.firewall.NoopFirewallDriver
        iniset $NOVA_CONF DEFAULT security_group_api quantum
    fi

    # set NOVA_VIF_DRIVER and optionally set options in nova_conf
    neutron_plugin_create_nova_conf

    iniset $NOVA_CONF DEFAULT libvirt_vif_driver "$NOVA_VIF_DRIVER"
    iniset $NOVA_CONF DEFAULT linuxnet_interface_driver "$LINUXNET_VIF_DRIVER"
    if is_service_enabled q-meta; then
        iniset $NOVA_CONF DEFAULT service_quantum_metadata_proxy "True"
    fi
}

# create_neutron_accounts() - Set up common required neutron accounts

# Tenant               User       Roles
# ------------------------------------------------------------------
# service              neutron    admin        # if enabled

# Migrated from keystone_data.sh
function create_neutron_accounts() {

    SERVICE_TENANT=$(keystone tenant-list | awk "/ $SERVICE_TENANT_NAME / { print \$2 }")
    ADMIN_ROLE=$(keystone role-list | awk "/ admin / { print \$2 }")

    if [[ "$ENABLED_SERVICES" =~ "q-svc" ]]; then
        NEUTRON_USER=$(keystone user-create \
            --name=neutron \
            --pass="$SERVICE_PASSWORD" \
            --tenant_id $SERVICE_TENANT \
            --email=neutron@example.com \
            | grep " id " | get_field 2)
        keystone user-role-add \
            --tenant_id $SERVICE_TENANT \
            --user_id $NEUTRON_USER \
            --role_id $ADMIN_ROLE
        if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then
            NEUTRON_SERVICE=$(keystone service-create \
                --name=neutron \
                --type=network \
                --description="Neutron Service" \
                | grep " id " | get_field 2)
            keystone endpoint-create \
                --region RegionOne \
                --service_id $NEUTRON_SERVICE \
                --publicurl "http://$SERVICE_HOST:9696/" \
                --adminurl "http://$SERVICE_HOST:9696/" \
                --internalurl "http://$SERVICE_HOST:9696/"
        fi
    fi
}

function create_neutron_initial_network() {
    TENANT_ID=$(keystone tenant-list | grep " demo " | get_field 1)

    # Create a small network
    # Since neutron command is executed in admin context at this point,
    # ``--tenant_id`` needs to be specified.
    if is_baremetal; then
        sudo ovs-vsctl add-port $OVS_PHYSICAL_BRIDGE $PUBLIC_INTERFACE
        for IP in $(ip addr show dev $PUBLIC_INTERFACE | grep ' inet ' | awk '{print $2}'); do
            sudo ip addr del $IP dev $PUBLIC_INTERFACE
            sudo ip addr add $IP dev $OVS_PHYSICAL_BRIDGE
        done
        NET_ID=$(neutron net-create $PHYSICAL_NETWORK --tenant_id $TENANT_ID --provider:network_type flat --provider:physical_network "$PHYSICAL_NETWORK" | grep ' id ' | get_field 2)
        SUBNET_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} --gateway $NETWORK_GATEWAY --name $PRIVATE_SUBNET_NAME $NET_ID $FIXED_RANGE | grep ' id ' | get_field 2)
        sudo ifconfig $OVS_PHYSICAL_BRIDGE up
    else
        NET_ID=$(neutron net-create --tenant_id $TENANT_ID "$PRIVATE_NETWORK_NAME" | grep ' id ' | get_field 2)
        SUBNET_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 4 --gateway $NETWORK_GATEWAY --name $PRIVATE_SUBNET_NAME $NET_ID $FIXED_RANGE | grep ' id ' | get_field 2)
    fi

    if [[ "$Q_L3_ENABLED" == "True" ]]; then
        # Create a router, and add the private subnet as one of its interfaces
        if [[ "$Q_L3_ROUTER_PER_TENANT" == "True" ]]; then
            # create a tenant-owned router.
            ROUTER_ID=$(neutron router-create --tenant_id $TENANT_ID $Q_ROUTER_NAME | grep ' id ' | get_field 2)
        else
            # Plugin only supports creating a single router, which should be admin owned.
            ROUTER_ID=$(neutron router-create $Q_ROUTER_NAME | grep ' id ' | get_field 2)
        fi
        neutron router-interface-add $ROUTER_ID $SUBNET_ID
        # Create an external network, and a subnet. Configure the external network as router gw
        EXT_NET_ID=$(neutron net-create "$PUBLIC_NETWORK_NAME" -- --router:external=True | grep ' id ' | get_field 2)
        EXT_GW_IP=$(neutron subnet-create --ip_version 4 ${Q_FLOATING_ALLOCATION_POOL:+--allocation-pool $Q_FLOATING_ALLOCATION_POOL} --gateway $PUBLIC_NETWORK_GATEWAY --name $PUBLIC_SUBNET_NAME $EXT_NET_ID $FLOATING_RANGE -- --enable_dhcp=False | grep 'gateway_ip' | get_field 2)
        neutron router-gateway-set $ROUTER_ID $EXT_NET_ID

        if is_service_enabled q-l3; then
            # logic is specific to using the l3-agent for l3
            if is_neutron_ovs_base_plugin && [[ "$Q_USE_NAMESPACE" = "True" ]]; then
                CIDR_LEN=${FLOATING_RANGE#*/}
                sudo ip addr add $EXT_GW_IP/$CIDR_LEN dev $PUBLIC_BRIDGE
                sudo ip link set $PUBLIC_BRIDGE up
                ROUTER_GW_IP=`neutron port-list -c fixed_ips -c device_owner | grep router_gateway | awk -F '"' '{ print $8; }'`
                sudo route add -net $FIXED_RANGE gw $ROUTER_GW_IP
            fi
            if [[ "$Q_USE_NAMESPACE" == "False" ]]; then
                # Explicitly set router id in l3 agent configuration
                iniset $Q_L3_CONF_FILE DEFAULT router_id $ROUTER_ID
            fi
        fi
   fi
}

# init_neutron() - Initialize databases, etc.
function init_neutron() {
    recreate_database $Q_DB_NAME utf8
    # Run Neutron db migrations
    $NEUTRON_BIN_DIR/neutron-db-manage --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE upgrade head
}

# install_neutron() - Collect source and prepare
function install_neutron() {
    git_clone $NEUTRON_REPO $NEUTRON_DIR $NEUTRON_BRANCH
    setup_develop $NEUTRON_DIR
}

# install_neutronclient() - Collect source and prepare
function install_neutronclient() {
    git_clone $NEUTRONCLIENT_REPO $NEUTRONCLIENT_DIR $NEUTRONCLIENT_BRANCH
    setup_develop $NEUTRONCLIENT_DIR
}

# install_neutron_agent_packages() - Collect source and prepare
function install_neutron_agent_packages() {
    # install packages that are specific to plugin agent(s)
    if is_service_enabled q-agt q-dhcp q-l3; then
        neutron_plugin_install_agent_packages
    fi

    if is_service_enabled q-lbaas; then
       neutron_agent_lbaas_install_agent_packages
    fi
}

# Start running processes, including screen
function start_neutron_service_and_check() {
    # build config-file options
    local cfg_file
    local CFG_FILE_OPTIONS="--config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE"
    for cfg_file in ${Q_PLUGIN_EXTRA_CONF_FILES[@]}; do
         CFG_FILE_OPTIONS+=" --config-file /$cfg_file"
    done
    # Start the Neutron service
    screen_it q-svc "cd $NEUTRON_DIR && python $NEUTRON_BIN_DIR/neutron-server $CFG_FILE_OPTIONS"
    echo "Waiting for Neutron to start..."
    if ! timeout $SERVICE_TIMEOUT sh -c "while ! http_proxy= wget -q -O- http://$Q_HOST:$Q_PORT; do sleep 1; done"; then
      die $LINENO "Neutron did not start"
    fi
}

# Start running processes, including screen
function start_neutron_agents() {
    # Start up the neutron agents if enabled
    screen_it q-agt "cd $NEUTRON_DIR && python $AGENT_BINARY --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE"
    screen_it q-dhcp "cd $NEUTRON_DIR && python $AGENT_DHCP_BINARY --config-file $NEUTRON_CONF --config-file=$Q_DHCP_CONF_FILE"

    L3_CONF_FILES="--config-file $NEUTRON_CONF --config-file=$Q_L3_CONF_FILE"

    if is_service_enabled q-fwaas; then
        L3_CONF_FILES="$L3_CONF_FILES --config-file $Q_FWAAS_CONF_FILE"
    fi
    if is_service_enabled q-vpn; then
        screen_it q-vpn "cd $NEUTRON_DIR && $AGENT_VPN_BINARY $L3_CONF_FILES"
    else
        screen_it q-l3 "cd $NEUTRON_DIR && python $AGENT_L3_BINARY $L3_CONF_FILES"
    fi

    screen_it q-meta "cd $NEUTRON_DIR && python $AGENT_META_BINARY --config-file $NEUTRON_CONF --config-file=$Q_META_CONF_FILE"

    if [ "$VIRT_DRIVER" = 'xenserver' ]; then
        # For XenServer, start an agent for the domU openvswitch
        screen_it q-domua "cd $NEUTRON_DIR && python $AGENT_BINARY --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE.domU"
    fi

    if is_service_enabled q-lbaas; then
        screen_it q-lbaas "cd $NEUTRON_DIR && python $AGENT_LBAAS_BINARY --config-file $NEUTRON_CONF --config-file=$LBAAS_AGENT_CONF_FILENAME"
    fi
}

# stop_neutron() - Stop running processes (non-screen)
function stop_neutron() {
    if is_service_enabled q-dhcp; then
        pid=$(ps aux | awk '/[d]nsmasq.+interface=(tap|ns-)/ { print $2 }')
        [ ! -z "$pid" ] && sudo kill -9 $pid
    fi
    if is_service_enabled q-meta; then
        pid=$(ps aux | awk '/neutron-ns-metadata-proxy/ { print $2 }')
        [ ! -z "$pid" ] && sudo kill -9 $pid
    fi
}

# cleanup_neutron() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_neutron() {
    if is_neutron_ovs_base_plugin; then
        neutron_ovs_base_cleanup
    fi

    # delete all namespaces created by neutron
    for ns in $(sudo ip netns list | grep -o -e qdhcp-[0-9a-f\-]* -e qrouter-[0-9a-f\-]*); do
        sudo ip netns delete ${ns}
    done
}

# _configure_neutron_common()
# Set common config for all neutron server and agents.
# This MUST be called before other ``_configure_neutron_*`` functions.
function _configure_neutron_common() {
    # Put config files in ``NEUTRON_CONF_DIR`` for everyone to find
    if [[ ! -d $NEUTRON_CONF_DIR ]]; then
        sudo mkdir -p $NEUTRON_CONF_DIR
    fi
    sudo chown $STACK_USER $NEUTRON_CONF_DIR

    cp $NEUTRON_DIR/etc/neutron.conf $NEUTRON_CONF

    # Set plugin-specific variables ``Q_DB_NAME``, ``Q_PLUGIN_CLASS``.
    # For main plugin config file, set ``Q_PLUGIN_CONF_PATH``, ``Q_PLUGIN_CONF_FILENAME``.
    # For addition plugin config files, set ``Q_PLUGIN_EXTRA_CONF_PATH``,
    # ``Q_PLUGIN_EXTRA_CONF_FILES``.  For example:
    #    ``Q_PLUGIN_EXTRA_CONF_FILES=(file1, file2)``
    neutron_plugin_configure_common

    if [[ $Q_PLUGIN_CONF_PATH == '' || $Q_PLUGIN_CONF_FILENAME == '' || $Q_PLUGIN_CLASS == '' ]]; then
        die $LINENO "Neutron plugin not set.. exiting"
    fi

    # If needed, move config file from ``$NEUTRON_DIR/etc/neutron`` to ``NEUTRON_CONF_DIR``
    mkdir -p /$Q_PLUGIN_CONF_PATH
    Q_PLUGIN_CONF_FILE=$Q_PLUGIN_CONF_PATH/$Q_PLUGIN_CONF_FILENAME
    cp $NEUTRON_DIR/$Q_PLUGIN_CONF_FILE /$Q_PLUGIN_CONF_FILE

    iniset /$Q_PLUGIN_CONF_FILE database connection `database_connection_url $Q_DB_NAME`
    iniset $NEUTRON_CONF DEFAULT state_path $DATA_DIR/neutron

    # If addition config files are set, make sure their path name is set as well
    if [[ ${#Q_PLUGIN_EXTRA_CONF_FILES[@]} > 0 && $Q_PLUGIN_EXTRA_CONF_PATH == '' ]]; then
        die $LINENO "Neutron additional plugin config not set.. exiting"
    fi

    # If additional config files exist, copy them over to neutron configuration
    # directory
    if [[ $Q_PLUGIN_EXTRA_CONF_PATH != '' ]]; then
        mkdir -p /$Q_PLUGIN_EXTRA_CONF_PATH
        local f
        for (( f=0; $f < ${#Q_PLUGIN_EXTRA_CONF_FILES[@]}; f+=1 )); do
            Q_PLUGIN_EXTRA_CONF_FILES[$f]=$Q_PLUGIN_EXTRA_CONF_PATH/${Q_PLUGIN_EXTRA_CONF_FILES[$f]}
            cp $NEUTRON_DIR/${Q_PLUGIN_EXTRA_CONF_FILES[$f]} /${Q_PLUGIN_EXTRA_CONF_FILES[$f]}
        done
    fi

    _neutron_setup_rootwrap
}

function _configure_neutron_debug_command() {
    if [[ "$Q_USE_DEBUG_COMMAND" != "True" ]]; then
        return
    fi

    cp $NEUTRON_DIR/etc/l3_agent.ini $NEUTRON_TEST_CONFIG_FILE

    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT verbose False
    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT debug False
    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT use_namespaces $Q_USE_NAMESPACE
    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT root_helper "$Q_RR_COMMAND"
    # Intermediate fix until Neutron patch lands and then line above will
    # be cleaned.
    iniset $NEUTRON_TEST_CONFIG_FILE agent root_helper "$Q_RR_COMMAND"

    _neutron_setup_interface_driver $NEUTRON_TEST_CONFIG_FILE

    neutron_plugin_configure_debug_command
}

function _configure_neutron_dhcp_agent() {
    AGENT_DHCP_BINARY="$NEUTRON_BIN_DIR/neutron-dhcp-agent"
    Q_DHCP_CONF_FILE=$NEUTRON_CONF_DIR/dhcp_agent.ini

    cp $NEUTRON_DIR/etc/dhcp_agent.ini $Q_DHCP_CONF_FILE

    iniset $Q_DHCP_CONF_FILE DEFAULT verbose True
    iniset $Q_DHCP_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_DHCP_CONF_FILE DEFAULT use_namespaces $Q_USE_NAMESPACE
    iniset $Q_DHCP_CONF_FILE DEFAULT root_helper "$Q_RR_COMMAND"

    _neutron_setup_interface_driver $Q_DHCP_CONF_FILE

    neutron_plugin_configure_dhcp_agent
}

function _configure_neutron_l3_agent() {
    Q_L3_ENABLED=True
    # for l3-agent, only use per tenant router if we have namespaces
    Q_L3_ROUTER_PER_TENANT=$Q_USE_NAMESPACE

    AGENT_L3_BINARY=${AGENT_L3_BINARY:-"$NEUTRON_BIN_DIR/neutron-l3-agent"}
    Q_L3_CONF_FILE=$NEUTRON_CONF_DIR/l3_agent.ini

    if is_service_enabled q-fwaas; then
        Q_FWAAS_CONF_FILE=$NEUTRON_CONF_DIR/fwaas_driver.ini
    fi

    cp $NEUTRON_DIR/etc/l3_agent.ini $Q_L3_CONF_FILE

    iniset $Q_L3_CONF_FILE DEFAULT verbose True
    iniset $Q_L3_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_L3_CONF_FILE DEFAULT use_namespaces $Q_USE_NAMESPACE
    iniset $Q_L3_CONF_FILE DEFAULT root_helper "$Q_RR_COMMAND"

    _neutron_setup_interface_driver $Q_L3_CONF_FILE

    neutron_plugin_configure_l3_agent
}

function _configure_neutron_metadata_agent() {
    AGENT_META_BINARY="$NEUTRON_BIN_DIR/neutron-metadata-agent"
    Q_META_CONF_FILE=$NEUTRON_CONF_DIR/metadata_agent.ini

    cp $NEUTRON_DIR/etc/metadata_agent.ini $Q_META_CONF_FILE

    iniset $Q_META_CONF_FILE DEFAULT verbose True
    iniset $Q_META_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_META_CONF_FILE DEFAULT nova_metadata_ip $Q_META_DATA_IP
    iniset $Q_META_CONF_FILE DEFAULT root_helper "$Q_RR_COMMAND"

    _neutron_setup_keystone $Q_META_CONF_FILE DEFAULT True True

}

function _configure_neutron_lbaas() {
    neutron_agent_lbaas_configure_common
    neutron_agent_lbaas_configure_agent
}

function _configure_neutron_fwaas() {
    neutron_fwaas_configure_common
    neutron_fwaas_configure_driver
}

function _configure_neutron_vpn()
{
    neutron_vpn_install_agent_packages
    neutron_vpn_configure_common
}

# _configure_neutron_plugin_agent() - Set config files for neutron plugin agent
# It is called when q-agt is enabled.
function _configure_neutron_plugin_agent() {
    # Specify the default root helper prior to agent configuration to
    # ensure that an agent's configuration can override the default
    iniset /$Q_PLUGIN_CONF_FILE agent root_helper "$Q_RR_COMMAND"
    iniset $NEUTRON_CONF DEFAULT verbose True
    iniset $NEUTRON_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

    # Configure agent for plugin
    neutron_plugin_configure_plugin_agent
}

# _configure_neutron_service() - Set config files for neutron service
# It is called when q-svc is enabled.
function _configure_neutron_service() {
    Q_API_PASTE_FILE=$NEUTRON_CONF_DIR/api-paste.ini
    Q_POLICY_FILE=$NEUTRON_CONF_DIR/policy.json

    cp $NEUTRON_DIR/etc/api-paste.ini $Q_API_PASTE_FILE
    cp $NEUTRON_DIR/etc/policy.json $Q_POLICY_FILE

    # Update either configuration file with plugin
    iniset $NEUTRON_CONF DEFAULT core_plugin $Q_PLUGIN_CLASS

    if [[ $Q_SERVICE_PLUGIN_CLASSES != '' ]]; then
        iniset $NEUTRON_CONF DEFAULT service_plugins $Q_SERVICE_PLUGIN_CLASSES
    fi

    iniset $NEUTRON_CONF DEFAULT verbose True
    iniset $NEUTRON_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $NEUTRON_CONF DEFAULT policy_file $Q_POLICY_FILE
    iniset $NEUTRON_CONF DEFAULT allow_overlapping_ips $Q_ALLOW_OVERLAPPING_IP

    iniset $NEUTRON_CONF DEFAULT auth_strategy $Q_AUTH_STRATEGY
    _neutron_setup_keystone $NEUTRON_CONF keystone_authtoken

    # Define extra "DEFAULT" configuration options when q-svc is configured by
    # defining the array ``Q_SRV_EXTRA_DEFAULT_OPTS``.
    # For Example: ``Q_SRV_EXTRA_DEFAULT_OPTS=(foo=true bar=2)``
    for I in "${Q_SRV_EXTRA_DEFAULT_OPTS[@]}"; do
        # Replace the first '=' with ' ' for iniset syntax
        iniset $NEUTRON_CONF DEFAULT ${I/=/ }
    done

    # Configure plugin
    neutron_plugin_configure_service
}

# Utility Functions
#------------------

# _neutron_setup_rootwrap() - configure Neutron's rootwrap
function _neutron_setup_rootwrap() {
    if [[ "$Q_USE_ROOTWRAP" == "False" ]]; then
        return
    fi
    # Deploy new rootwrap filters files (owned by root).
    # Wipe any existing ``rootwrap.d`` files first
    Q_CONF_ROOTWRAP_D=$NEUTRON_CONF_DIR/rootwrap.d
    if [[ -d $Q_CONF_ROOTWRAP_D ]]; then
        sudo rm -rf $Q_CONF_ROOTWRAP_D
    fi
    # Deploy filters to ``$NEUTRON_CONF_DIR/rootwrap.d``
    mkdir -p -m 755 $Q_CONF_ROOTWRAP_D
    cp -pr $NEUTRON_DIR/etc/neutron/rootwrap.d/* $Q_CONF_ROOTWRAP_D/
    sudo chown -R root:root $Q_CONF_ROOTWRAP_D
    sudo chmod 644 $Q_CONF_ROOTWRAP_D/*
    # Set up ``rootwrap.conf``, pointing to ``$NEUTRON_CONF_DIR/rootwrap.d``
    # location moved in newer versions, prefer new location
    if test -r $NEUTRON_DIR/etc/neutron/rootwrap.conf; then
      sudo cp -p $NEUTRON_DIR/etc/neutron/rootwrap.conf $Q_RR_CONF_FILE
    else
      sudo cp -p $NEUTRON_DIR/etc/rootwrap.conf $Q_RR_CONF_FILE
    fi
    sudo sed -e "s:^filters_path=.*$:filters_path=$Q_CONF_ROOTWRAP_D:" -i $Q_RR_CONF_FILE
    sudo chown root:root $Q_RR_CONF_FILE
    sudo chmod 0644 $Q_RR_CONF_FILE
    # Specify ``rootwrap.conf`` as first parameter to neutron-rootwrap
    ROOTWRAP_SUDOER_CMD="$NEUTRON_ROOTWRAP $Q_RR_CONF_FILE *"

    # Set up the rootwrap sudoers for neutron
    TEMPFILE=`mktemp`
    echo "$USER ALL=(root) NOPASSWD: $ROOTWRAP_SUDOER_CMD" >$TEMPFILE
    chmod 0440 $TEMPFILE
    sudo chown root:root $TEMPFILE
    sudo mv $TEMPFILE /etc/sudoers.d/neutron-rootwrap

    # Update the root_helper
    iniset $NEUTRON_CONF agent root_helper "$Q_RR_COMMAND"
}

# Configures keystone integration for neutron service and agents
function _neutron_setup_keystone() {
    local conf_file=$1
    local section=$2
    local use_auth_url=$3
    local skip_auth_cache=$4
    if [[ -n $use_auth_url ]]; then
        iniset $conf_file $section auth_url "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v2.0"
    else
        iniset $conf_file $section auth_host $KEYSTONE_SERVICE_HOST
        iniset $conf_file $section auth_port $KEYSTONE_AUTH_PORT
        iniset $conf_file $section auth_protocol $KEYSTONE_SERVICE_PROTOCOL
    fi
    iniset $conf_file $section admin_tenant_name $SERVICE_TENANT_NAME
    iniset $conf_file $section admin_user $Q_ADMIN_USERNAME
    iniset $conf_file $section admin_password $SERVICE_PASSWORD
    if [[ -z $skip_auth_cache ]]; then
        iniset $conf_file $section signing_dir $NEUTRON_AUTH_CACHE_DIR
        # Create cache dir
        sudo mkdir -p $NEUTRON_AUTH_CACHE_DIR
        sudo chown $STACK_USER $NEUTRON_AUTH_CACHE_DIR
        rm -f $NEUTRON_AUTH_CACHE_DIR/*
    fi
}

function _neutron_setup_interface_driver() {

    # ovs_use_veth needs to be set before the plugin configuration
    # occurs to allow plugins to override the setting.
    iniset $1 DEFAULT ovs_use_veth $Q_OVS_USE_VETH

    neutron_plugin_setup_interface_driver $1
}

# Functions for Neutron Exercises
#--------------------------------

function delete_probe() {
    local from_net="$1"
    net_id=`_get_net_id $from_net`
    probe_id=`neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-list -c id -c network_id | grep $net_id | awk '{print $2}'`
    neutron-debug --os-tenant-name admin --os-username admin probe-delete $probe_id
}

function setup_neutron_debug() {
    if [[ "$Q_USE_DEBUG_COMMAND" == "True" ]]; then
        public_net_id=`_get_net_id $PUBLIC_NETWORK_NAME`
        neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-create --device-owner compute $public_net_id
        private_net_id=`_get_net_id $PRIVATE_NETWORK_NAME`
        neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-create --device-owner compute $private_net_id
    fi
}

function teardown_neutron_debug() {
    delete_probe $PUBLIC_NETWORK_NAME
    delete_probe $PRIVATE_NETWORK_NAME
}

function _get_net_id() {
    neutron --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD net-list | grep $1 | awk '{print $2}'
}

function _get_probe_cmd_prefix() {
    local from_net="$1"
    net_id=`_get_net_id $from_net`
    probe_id=`neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-list -c id -c network_id | grep $net_id | awk '{print $2}' | head -n 1`
    echo "$Q_RR_COMMAND ip netns exec qprobe-$probe_id"
}

function _ping_check_neutron() {
    local from_net=$1
    local ip=$2
    local timeout_sec=$3
    local expected=${4:-"True"}
    local check_command=""
    probe_cmd=`_get_probe_cmd_prefix $from_net`
    if [[ "$expected" = "True" ]]; then
        check_command="while ! $probe_cmd ping -w 1 -c 1 $ip; do sleep 1; done"
    else
        check_command="while $probe_cmd ping -w 1 -c 1 $ip; do sleep 1; done"
    fi
    if ! timeout $timeout_sec sh -c "$check_command"; then
        if [[ "$expected" = "True" ]]; then
            die $LINENO "[Fail] Couldn't ping server"
        else
            die $LINENO "[Fail] Could ping server"
        fi
    fi
}

# ssh check
function _ssh_check_neutron() {
    local from_net=$1
    local key_file=$2
    local ip=$3
    local user=$4
    local timeout_sec=$5
    local probe_cmd = ""
    probe_cmd=`_get_probe_cmd_prefix $from_net`
    if ! timeout $timeout_sec sh -c "while ! $probe_cmd ssh -o StrictHostKeyChecking=no -i $key_file ${user}@$ip echo success; do sleep 1; done"; then
        die $LINENO "server didn't become ssh-able!"
    fi
}

# Neutron 3rd party programs
#---------------------------

# please refer to ``lib/neutron_thirdparty/README.md`` for details
NEUTRON_THIRD_PARTIES=""
for f in $TOP_DIR/lib/neutron_thirdparty/*; do
     third_party=$(basename $f)
     if is_service_enabled $third_party; then
         source $TOP_DIR/lib/neutron_thirdparty/$third_party
         NEUTRON_THIRD_PARTIES="$NEUTRON_THIRD_PARTIES,$third_party"
     fi
done

function _neutron_third_party_do() {
    for third_party in ${NEUTRON_THIRD_PARTIES//,/ }; do
        ${1}_${third_party}
    done
}

# configure_neutron_third_party() - Set config files, create data dirs, etc
function configure_neutron_third_party() {
    _neutron_third_party_do configure
}

# init_neutron_third_party() - Initialize databases, etc.
function init_neutron_third_party() {
    _neutron_third_party_do init
}

# install_neutron_third_party() - Collect source and prepare
function install_neutron_third_party() {
    _neutron_third_party_do install
}

# start_neutron_third_party() - Start running processes, including screen
function start_neutron_third_party() {
    _neutron_third_party_do start
}

# stop_neutron_third_party - Stop running processes (non-screen)
function stop_neutron_third_party() {
    _neutron_third_party_do stop
}


# Restore xtrace
$XTRACE

# Local variables:
# mode: shell-script
# End: