lib/neutron-legacy

#!/bin/bash
#
# lib/neutron
# functions - functions specific to neutron

# Dependencies:
# ``functions`` file
# ``DEST`` must be defined
# ``STACK_USER`` must be defined

# ``stack.sh`` calls the entry points in this order:
#
# - install_neutron_agent_packages
# - install_neutronclient
# - install_neutron
# - install_neutron_third_party
# - configure_neutron
# - init_neutron
# - configure_neutron_third_party
# - init_neutron_third_party
# - start_neutron_third_party
# - create_nova_conf_neutron
# - start_neutron_service_and_check
# - check_neutron_third_party_integration
# - start_neutron_agents
# - create_neutron_initial_network
# - setup_neutron_debug
#
# ``unstack.sh`` calls the entry points in this order:
#
# - teardown_neutron_debug
# - stop_neutron
# - stop_neutron_third_party
# - cleanup_neutron

# Functions in lib/neutron are classified into the following categories:
#
# - entry points (called from stack.sh or unstack.sh)
# - internal functions
# - neutron exercises
# - 3rd party programs


# Neutron Networking
# ------------------

# Make sure that neutron is enabled in ``ENABLED_SERVICES``.  If you want
# to run Neutron on this host, make sure that q-svc is also in
# ``ENABLED_SERVICES``.
#
# See "Neutron Network Configuration" below for additional variables
# that must be set in localrc for connectivity across hosts with
# Neutron.
#
# With Neutron networking the NETWORK_MANAGER variable is ignored.

# Settings
# --------


# Neutron Network Configuration
# -----------------------------

# Subnet IP version
IP_VERSION=${IP_VERSION:-"4+6"}
# Validate IP_VERSION
if [[ $IP_VERSION != "4" ]] && [[ $IP_VERSION != "6" ]] && [[ $IP_VERSION != "4+6" ]]; then
    die $LINENO "IP_VERSION must be either 4, 6, or 4+6"
fi
# Gateway and subnet defaults, in case they are not customized in localrc
NETWORK_GATEWAY=${NETWORK_GATEWAY:-10.0.0.1}
PUBLIC_NETWORK_GATEWAY=${PUBLIC_NETWORK_GATEWAY:-172.24.4.1}
PRIVATE_SUBNET_NAME=${PRIVATE_SUBNET_NAME:-"private-subnet"}
PUBLIC_SUBNET_NAME=${PUBLIC_SUBNET_NAME:-"public-subnet"}

if is_ssl_enabled_service "neutron" || is_service_enabled tls-proxy; then
    Q_PROTOCOL="https"
fi

# Generate 40-bit IPv6 Global ID to comply with RFC 4193
IPV6_GLOBAL_ID=`uuidgen | sed s/-//g | cut -c 23- | sed -e "s/\(..\)\(....\)\(....\)/\1:\2:\3/"`

# IPv6 gateway and subnet defaults, in case they are not customized in localrc
IPV6_RA_MODE=${IPV6_RA_MODE:-slaac}
IPV6_ADDRESS_MODE=${IPV6_ADDRESS_MODE:-slaac}
IPV6_PUBLIC_SUBNET_NAME=${IPV6_PUBLIC_SUBNET_NAME:-ipv6-public-subnet}
IPV6_PRIVATE_SUBNET_NAME=${IPV6_PRIVATE_SUBNET_NAME:-ipv6-private-subnet}
FIXED_RANGE_V6=${FIXED_RANGE_V6:-fd$IPV6_GLOBAL_ID::/64}
IPV6_PRIVATE_NETWORK_GATEWAY=${IPV6_PRIVATE_NETWORK_GATEWAY:-fd$IPV6_GLOBAL_ID::1}
IPV6_PUBLIC_RANGE=${IPV6_PUBLIC_RANGE:-2001:db8::/64}
IPV6_PUBLIC_NETWORK_GATEWAY=${IPV6_PUBLIC_NETWORK_GATEWAY:-2001:db8::2}
IPV6_ROUTER_GW_IP=${IPV6_ROUTER_GW_IP:-2001:db8::1}

# Set up default directories
GITDIR["python-neutronclient"]=$DEST/python-neutronclient


NEUTRON_DIR=$DEST/neutron
NEUTRON_FWAAS_DIR=$DEST/neutron-fwaas
NEUTRON_LBAAS_DIR=$DEST/neutron-lbaas
NEUTRON_VPNAAS_DIR=$DEST/neutron-vpnaas
NEUTRON_AUTH_CACHE_DIR=${NEUTRON_AUTH_CACHE_DIR:-/var/cache/neutron}

# Support entry points installation of console scripts
if [[ -d $NEUTRON_DIR/bin/neutron-server ]]; then
    NEUTRON_BIN_DIR=$NEUTRON_DIR/bin
else
    NEUTRON_BIN_DIR=$(get_python_exec_prefix)
fi

NEUTRON_CONF_DIR=/etc/neutron
NEUTRON_CONF=$NEUTRON_CONF_DIR/neutron.conf
export NEUTRON_TEST_CONFIG_FILE=${NEUTRON_TEST_CONFIG_FILE:-"$NEUTRON_CONF_DIR/debug.ini"}

# Default provider for load balancer service
DEFAULT_LB_PROVIDER=LOADBALANCER:Haproxy:neutron_lbaas.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default

# Agent binaries.  Note, binary paths for other agents are set in per-service
# scripts in lib/neutron_plugins/services/
AGENT_DHCP_BINARY="$NEUTRON_BIN_DIR/neutron-dhcp-agent"
AGENT_L3_BINARY=${AGENT_L3_BINARY:-"$NEUTRON_BIN_DIR/neutron-l3-agent"}
AGENT_META_BINARY="$NEUTRON_BIN_DIR/neutron-metadata-agent"

# Agent config files. Note, plugin-specific Q_PLUGIN_CONF_FILE is set and
# loaded from per-plugin  scripts in lib/neutron_plugins/
Q_DHCP_CONF_FILE=$NEUTRON_CONF_DIR/dhcp_agent.ini
Q_L3_CONF_FILE=$NEUTRON_CONF_DIR/l3_agent.ini
Q_FWAAS_CONF_FILE=$NEUTRON_CONF_DIR/fwaas_driver.ini
Q_VPN_CONF_FILE=$NEUTRON_CONF_DIR/vpn_agent.ini
Q_META_CONF_FILE=$NEUTRON_CONF_DIR/metadata_agent.ini

# Default name for Neutron database
Q_DB_NAME=${Q_DB_NAME:-neutron}
# Default Neutron Plugin
Q_PLUGIN=${Q_PLUGIN:-ml2}
# Default Neutron Port
Q_PORT=${Q_PORT:-9696}
# Default Neutron Internal Port when using TLS proxy
Q_PORT_INT=${Q_PORT_INT:-19696}
# Default Neutron Host
Q_HOST=${Q_HOST:-$SERVICE_HOST}
# Default protocol
Q_PROTOCOL=${Q_PROTOCOL:-$SERVICE_PROTOCOL}
# Default listen address
Q_LISTEN_ADDRESS=${Q_LISTEN_ADDRESS:-$SERVICE_LISTEN_ADDRESS}
# Default admin username
Q_ADMIN_USERNAME=${Q_ADMIN_USERNAME:-neutron}
# Default auth strategy
Q_AUTH_STRATEGY=${Q_AUTH_STRATEGY:-keystone}
# RHEL's support for namespaces requires using veths with ovs
Q_OVS_USE_VETH=${Q_OVS_USE_VETH:-False}
Q_USE_ROOTWRAP=${Q_USE_ROOTWRAP:-True}
Q_USE_ROOTWRAP_DAEMON=$(trueorfalse True Q_USE_ROOTWRAP_DAEMON)
# Meta data IP
Q_META_DATA_IP=${Q_META_DATA_IP:-$SERVICE_HOST}
# Allow Overlapping IP among subnets
Q_ALLOW_OVERLAPPING_IP=${Q_ALLOW_OVERLAPPING_IP:-True}
# Use neutron-debug command
Q_USE_DEBUG_COMMAND=${Q_USE_DEBUG_COMMAND:-False}
# The name of the default q-l3 router
Q_ROUTER_NAME=${Q_ROUTER_NAME:-router1}
Q_NOTIFY_NOVA_PORT_STATUS_CHANGES=${Q_NOTIFY_NOVA_PORT_STATUS_CHANGES:-True}
Q_NOTIFY_NOVA_PORT_DATA_CHANGES=${Q_NOTIFY_NOVA_PORT_DATA_CHANGES:-True}
VIF_PLUGGING_IS_FATAL=${VIF_PLUGGING_IS_FATAL:-True}
VIF_PLUGGING_TIMEOUT=${VIF_PLUGGING_TIMEOUT:-300}
# Specify if the initial private and external networks should be created
NEUTRON_CREATE_INITIAL_NETWORKS=${NEUTRON_CREATE_INITIAL_NETWORKS:-True}

## Provider Network Information
PROVIDER_SUBNET_NAME=${PROVIDER_SUBNET_NAME:-"provider_net"}

# Define the public bridge that will transmit traffic from VMs to the
# physical network - used by both the OVS and Linux Bridge drivers.
PUBLIC_BRIDGE=${PUBLIC_BRIDGE:-br-ex}

# Use flat providernet for public network
#
# If Q_USE_PROVIDERNET_FOR_PUBLIC=True, use a flat provider network
# for external interface of neutron l3-agent.  In that case,
# PUBLIC_PHYSICAL_NETWORK specifies provider:physical_network value
# used for the network.  In case of ofagent, you should add the
# corresponding entry to your OFAGENT_PHYSICAL_INTERFACE_MAPPINGS.
# For openvswitch agent, you should add the corresponding entry to
# your OVS_BRIDGE_MAPPINGS.
#
# eg.  (ofagent)
#    Q_USE_PROVIDERNET_FOR_PUBLIC=True
#    Q_USE_PUBLIC_VETH=True
#    PUBLIC_PHYSICAL_NETWORK=public
#    OFAGENT_PHYSICAL_INTERFACE_MAPPINGS=public:veth-pub-int
#
# eg.  (openvswitch agent)
#    Q_USE_PROVIDERNET_FOR_PUBLIC=True
#    PUBLIC_PHYSICAL_NETWORK=public
#    OVS_BRIDGE_MAPPINGS=public:br-ex
Q_USE_PROVIDERNET_FOR_PUBLIC=${Q_USE_PROVIDERNET_FOR_PUBLIC:-False}
PUBLIC_PHYSICAL_NETWORK=${PUBLIC_PHYSICAL_NETWORK:-public}

# If Q_USE_PUBLIC_VETH=True, create and use a veth pair instead of
# PUBLIC_BRIDGE.  This is intended to be used with
# Q_USE_PROVIDERNET_FOR_PUBLIC=True.
Q_USE_PUBLIC_VETH=${Q_USE_PUBLIC_VETH:-False}
Q_PUBLIC_VETH_EX=${Q_PUBLIC_VETH_EX:-veth-pub-ex}
Q_PUBLIC_VETH_INT=${Q_PUBLIC_VETH_INT:-veth-pub-int}

# The next two variables are configured by plugin
# e.g.  _configure_neutron_l3_agent or lib/neutron_plugins/*
#
# The plugin supports L3.
Q_L3_ENABLED=${Q_L3_ENABLED:-False}
# L3 routers exist per tenant
Q_L3_ROUTER_PER_TENANT=${Q_L3_ROUTER_PER_TENANT:-True}

# List of config file names in addition to the main plugin config file
# See _configure_neutron_common() for details about setting it up
declare -a Q_PLUGIN_EXTRA_CONF_FILES

# List of (optional) config files for VPN device drivers to use with
# the neutron-q-vpn agent
declare -a Q_VPN_EXTRA_CONF_FILES


Q_RR_CONF_FILE=$NEUTRON_CONF_DIR/rootwrap.conf
if [[ "$Q_USE_ROOTWRAP" == "False" ]]; then
    Q_RR_COMMAND="sudo"
else
    NEUTRON_ROOTWRAP=$(get_rootwrap_location neutron)
    Q_RR_COMMAND="sudo $NEUTRON_ROOTWRAP $Q_RR_CONF_FILE"
    if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then
        Q_RR_DAEMON_COMMAND="sudo $NEUTRON_ROOTWRAP-daemon $Q_RR_CONF_FILE"
    fi
fi


# Distributed Virtual Router (DVR) configuration
# Can be:
# - ``legacy``   - No DVR functionality
# - ``dvr_snat`` - Controller or single node DVR
# - ``dvr``      - Compute node in multi-node DVR
#
Q_DVR_MODE=${Q_DVR_MODE:-legacy}
if [[ "$Q_DVR_MODE" != "legacy" ]]; then
    Q_ML2_PLUGIN_MECHANISM_DRIVERS=openvswitch,linuxbridge,l2population
fi

# Provider Network Configurations
# --------------------------------

# The following variables control the Neutron ML2 plugins' allocation
# of tenant networks and availability of provider networks. If these
# are not configured in ``localrc``, tenant networks will be local to
# the host (with no remote connectivity), and no physical resources
# will be available for the allocation of provider networks.

# To disable tunnels (GRE or VXLAN) for tenant networks,
# set to False in ``local.conf``.
# GRE tunnels are only supported by the openvswitch.
ENABLE_TENANT_TUNNELS=${ENABLE_TENANT_TUNNELS:-True}

# If using GRE tunnels for tenant networks, specify the range of
# tunnel IDs from which tenant networks are allocated. Can be
# overridden in ``localrc`` in necessary.
TENANT_TUNNEL_RANGES=${TENANT_TUNNEL_RANGES:-1:1000}

# To use VLANs for tenant networks, set to True in localrc. VLANs
# are supported by the ML2 plugins, requiring additional configuration
# described below.
ENABLE_TENANT_VLANS=${ENABLE_TENANT_VLANS:-False}

# If using VLANs for tenant networks, set in ``localrc`` to specify
# the range of VLAN VIDs from which tenant networks are
# allocated. An external network switch must be configured to
# trunk these VLANs between hosts for multi-host connectivity.
#
# Example: ``TENANT_VLAN_RANGE=1000:1999``
TENANT_VLAN_RANGE=${TENANT_VLAN_RANGE:-}

# If using VLANs for tenant networks, or if using flat or VLAN
# provider networks, set in ``localrc`` to the name of the physical
# network, and also configure ``OVS_PHYSICAL_BRIDGE`` for the
# openvswitch agent or ``LB_PHYSICAL_INTERFACE`` for the linuxbridge
# agent, as described below.
#
# Example: ``PHYSICAL_NETWORK=default``
PHYSICAL_NETWORK=${PHYSICAL_NETWORK:-}

# With the openvswitch agent, if using VLANs for tenant networks,
# or if using flat or VLAN provider networks, set in ``localrc`` to
# the name of the OVS bridge to use for the physical network. The
# bridge will be created if it does not already exist, but a
# physical interface must be manually added to the bridge as a
# port for external connectivity.
#
# Example: ``OVS_PHYSICAL_BRIDGE=br-eth1``
OVS_PHYSICAL_BRIDGE=${OVS_PHYSICAL_BRIDGE:-}

# With the linuxbridge agent, if using VLANs for tenant networks,
# or if using flat or VLAN provider networks, set in ``localrc`` to
# the name of the network interface to use for the physical
# network.
#
# Example: ``LB_PHYSICAL_INTERFACE=eth1``
LB_PHYSICAL_INTERFACE=${LB_PHYSICAL_INTERFACE:-}

# When Neutron tunnels are enabled it is needed to specify the
# IP address of the end point in the local server. This IP is set
# by default to the same IP address that the HOST IP.
# This variable can be used to specify a different end point IP address
# Example: ``TUNNEL_ENDPOINT_IP=1.1.1.1``
TUNNEL_ENDPOINT_IP=${TUNNEL_ENDPOINT_IP:-$HOST_IP}

# With the openvswitch plugin, set to True in ``localrc`` to enable
# provider GRE tunnels when ``ENABLE_TENANT_TUNNELS`` is False.
#
# Example: ``OVS_ENABLE_TUNNELING=True``
OVS_ENABLE_TUNNELING=${OVS_ENABLE_TUNNELING:-$ENABLE_TENANT_TUNNELS}

# Use DHCP agent for providing metadata service in the case of
# without L3 agent (No Route Agent), set to True in localrc.
ENABLE_ISOLATED_METADATA=${ENABLE_ISOLATED_METADATA:-False}

# Add a static route as dhcp option, so the request to 169.254.169.254
# will be able to reach through a route(DHCP agent)
# This option require ENABLE_ISOLATED_METADATA = True
ENABLE_METADATA_NETWORK=${ENABLE_METADATA_NETWORK:-False}
# Neutron plugin specific functions
# ---------------------------------

# Please refer to ``lib/neutron_plugins/README.md`` for details.
if [ -f $TOP_DIR/lib/neutron_plugins/$Q_PLUGIN ]; then
    source $TOP_DIR/lib/neutron_plugins/$Q_PLUGIN
fi

# Agent loadbalancer service plugin functions
# -------------------------------------------

# Hardcoding for 1 service plugin for now
source $TOP_DIR/lib/neutron_plugins/services/loadbalancer

# Agent metering service plugin functions
# -------------------------------------------

# Hardcoding for 1 service plugin for now
source $TOP_DIR/lib/neutron_plugins/services/metering

# VPN service plugin functions
# -------------------------------------------
# Hardcoding for 1 service plugin for now
source $TOP_DIR/lib/neutron_plugins/services/vpn

# Firewall Service Plugin functions
# ---------------------------------
source $TOP_DIR/lib/neutron_plugins/services/firewall

# Use security group or not
if has_neutron_plugin_security_group; then
    Q_USE_SECGROUP=${Q_USE_SECGROUP:-True}
else
    Q_USE_SECGROUP=False
fi

# Save trace setting
_XTRACE_NEUTRON=$(set +o | grep xtrace)
set +o xtrace


# Functions
# ---------

function _determine_config_server {
    local cfg_file
    local opts="--config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE"
    for cfg_file in ${Q_PLUGIN_EXTRA_CONF_FILES[@]}; do
        opts+=" --config-file /$cfg_file"
    done
    echo "$opts"
}

function _determine_config_vpn {
    local cfg_file
    local opts="--config-file $NEUTRON_CONF --config-file=$Q_L3_CONF_FILE --config-file=$Q_VPN_CONF_FILE"
    if is_service_enabled q-fwaas; then
        opts+=" --config-file $Q_FWAAS_CONF_FILE"
    fi
    for cfg_file in ${Q_VPN_EXTRA_CONF_FILES[@]}; do
        opts+=" --config-file $cfg_file"
    done
    echo "$opts"

}

function _determine_config_l3 {
    local opts="--config-file $NEUTRON_CONF --config-file=$Q_L3_CONF_FILE"
    if is_service_enabled q-fwaas; then
        opts+=" --config-file $Q_FWAAS_CONF_FILE"
    fi
    echo "$opts"
}

# For services and agents that require it, dynamically construct a list of
# --config-file arguments that are passed to the binary.
function determine_config_files {
    local opts=""
    case "$1" in
        "neutron-server") opts="$(_determine_config_server)" ;;
        "neutron-vpn-agent") opts="$(_determine_config_vpn)" ;;
        "neutron-l3-agent") opts="$(_determine_config_l3)" ;;
    esac
    if [ -z "$opts" ] ; then
        die $LINENO "Could not determine config files for $1."
    fi
    echo "$opts"
}

# Test if any Neutron services are enabled
# is_neutron_enabled
function is_neutron_enabled {
    [[ ,${ENABLED_SERVICES} =~ ,"q-" ]] && return 0
    return 1
}

# configure_neutron()
# Set common config for all neutron server and agents.
function configure_neutron {
    _configure_neutron_common
    iniset_rpc_backend neutron $NEUTRON_CONF

    # goes before q-svc to init Q_SERVICE_PLUGIN_CLASSES
    if is_service_enabled q-lbaas; then
        _configure_neutron_lbaas
    fi
    if is_service_enabled q-metering; then
        _configure_neutron_metering
    fi
    if is_service_enabled q-vpn; then
        _configure_neutron_vpn
    fi
    if is_service_enabled q-fwaas; then
        _configure_neutron_fwaas
    fi
    if is_service_enabled q-agt q-svc; then
        _configure_neutron_service
    fi
    if is_service_enabled q-agt; then
        _configure_neutron_plugin_agent
    fi
    if is_service_enabled q-dhcp; then
        _configure_neutron_dhcp_agent
    fi
    if is_service_enabled q-l3; then
        _configure_neutron_l3_agent
    fi
    if is_service_enabled q-meta; then
        _configure_neutron_metadata_agent
    fi

    if [[ "$Q_DVR_MODE" != "legacy" ]]; then
        _configure_dvr
    fi
    if is_service_enabled ceilometer; then
        _configure_neutron_ceilometer_notifications
    fi

    _configure_neutron_debug_command

    iniset $NEUTRON_CONF DEFAULT api_workers "$API_WORKERS"
}

function create_nova_conf_neutron {
    iniset $NOVA_CONF DEFAULT network_api_class "nova.network.neutronv2.api.API"

    iniset $NOVA_CONF neutron auth_plugin "v3password"
    iniset $NOVA_CONF neutron auth_url "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_AUTH_PORT/v3"
    iniset $NOVA_CONF neutron username "$Q_ADMIN_USERNAME"
    iniset $NOVA_CONF neutron password "$SERVICE_PASSWORD"
    iniset $NOVA_CONF neutron user_domain_name "Default"
    iniset $NOVA_CONF neutron project_name "$SERVICE_TENANT_NAME"
    iniset $NOVA_CONF neutron project_domain_name "Default"
    iniset $NOVA_CONF neutron auth_strategy "$Q_AUTH_STRATEGY"
    iniset $NOVA_CONF neutron region_name "$REGION_NAME"
    iniset $NOVA_CONF neutron url "${Q_PROTOCOL}://$Q_HOST:$Q_PORT"

    if [[ "$Q_USE_SECGROUP" == "True" ]]; then
        LIBVIRT_FIREWALL_DRIVER=nova.virt.firewall.NoopFirewallDriver
        iniset $NOVA_CONF DEFAULT firewall_driver $LIBVIRT_FIREWALL_DRIVER
        iniset $NOVA_CONF DEFAULT security_group_api neutron
    fi

    # optionally set options in nova_conf
    neutron_plugin_create_nova_conf

    if is_service_enabled q-meta; then
        iniset $NOVA_CONF neutron service_metadata_proxy "True"
    fi

    iniset $NOVA_CONF DEFAULT vif_plugging_is_fatal "$VIF_PLUGGING_IS_FATAL"
    iniset $NOVA_CONF DEFAULT vif_plugging_timeout "$VIF_PLUGGING_TIMEOUT"
}

# create_neutron_cache_dir() - Part of the _neutron_setup_keystone() process
function create_neutron_cache_dir {
    # Create cache dir
    sudo install -d -o $STACK_USER $NEUTRON_AUTH_CACHE_DIR
    rm -f $NEUTRON_AUTH_CACHE_DIR/*
}

# create_neutron_accounts() - Set up common required neutron accounts

# Tenant               User       Roles
# ------------------------------------------------------------------
# service              neutron    admin        # if enabled

# Migrated from keystone_data.sh
function create_neutron_accounts {
    if [[ "$ENABLED_SERVICES" =~ "q-svc" ]]; then

        create_service_user "neutron"

        if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then

            get_or_create_service "neutron" "network" "Neutron Service"
            get_or_create_endpoint "network" \
                "$REGION_NAME" \
                "$Q_PROTOCOL://$SERVICE_HOST:$Q_PORT/" \
                "$Q_PROTOCOL://$SERVICE_HOST:$Q_PORT/" \
                "$Q_PROTOCOL://$SERVICE_HOST:$Q_PORT/"
        fi
    fi
}

function create_neutron_initial_network {
    TENANT_ID=$(openstack project list | grep " demo " | get_field 1)
    die_if_not_set $LINENO TENANT_ID "Failure retrieving TENANT_ID for demo"

    # Allow drivers that need to create an initial network to do so here
    if type -p neutron_plugin_create_initial_network_profile > /dev/null; then
        neutron_plugin_create_initial_network_profile $PHYSICAL_NETWORK
    fi

    if is_provider_network; then
        die_if_not_set $LINENO PHYSICAL_NETWORK "You must specify the PHYSICAL_NETWORK"
        die_if_not_set $LINENO PROVIDER_NETWORK_TYPE "You must specify the PROVIDER_NETWORK_TYPE"
        NET_ID=$(neutron net-create $PHYSICAL_NETWORK --tenant_id $TENANT_ID --provider:network_type $PROVIDER_NETWORK_TYPE --provider:physical_network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider:segmentation_id $SEGMENTATION_ID} --shared | grep ' id ' | get_field 2)
        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK $TENANT_ID"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            SUBNET_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} --name $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY --subnetpool None $NET_ID $FIXED_RANGE | grep ' id ' | get_field 2)
            die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME $TENANT_ID"
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
            SUBNET_V6_ID=$(neutron subnet-create --tenant_id $TENANT_ID --ip_version 6 --ipv6-address-mode $IPV6_ADDRESS_MODE --gateway $V6_NETWORK_GATEWAY --name $PROVIDER_SUBNET_NAME_V6 --subnetpool_id None $NET_ID $FIXED_RANGE_V6 | grep 'id' | get_field 2)
            die_if_not_set $LINENO SUBNET_V6_ID "Failure creating SUBNET_V6_ID for $PROVIDER_SUBNET_NAME_V6 $TENANT_ID"
        fi

        if [[ $Q_AGENT == "openvswitch" ]]; then
            sudo ip link set $OVS_PHYSICAL_BRIDGE up
            sudo ip link set br-int up
            sudo ip link set $PUBLIC_INTERFACE up
        fi
    else
        NET_ID=$(neutron net-create --tenant-id $TENANT_ID "$PRIVATE_NETWORK_NAME" | grep ' id ' | get_field 2)
        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PRIVATE_NETWORK_NAME $TENANT_ID"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            # Create IPv4 private subnet
            SUBNET_ID=$(_neutron_create_private_subnet_v4)
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
            # Create IPv6 private subnet
            IPV6_SUBNET_ID=$(_neutron_create_private_subnet_v6)
        fi
    fi

    if [[ "$Q_L3_ENABLED" == "True" ]]; then
        # Create a router, and add the private subnet as one of its interfaces
        if [[ "$Q_L3_ROUTER_PER_TENANT" == "True" ]]; then
            # create a tenant-owned router.
            ROUTER_ID=$(neutron router-create --tenant-id $TENANT_ID $Q_ROUTER_NAME | grep ' id ' | get_field 2)
            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $TENANT_ID $Q_ROUTER_NAME"
        else
            # Plugin only supports creating a single router, which should be admin owned.
            ROUTER_ID=$(neutron router-create $Q_ROUTER_NAME | grep ' id ' | get_field 2)
            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $Q_ROUTER_NAME"
        fi

        # Create an external network, and a subnet. Configure the external network as router gw
        if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" = "True" ]; then
            EXT_NET_ID=$(neutron net-create "$PUBLIC_NETWORK_NAME" -- --router:external=True --provider:network_type=flat --provider:physical_network=${PUBLIC_PHYSICAL_NETWORK} | grep ' id ' | get_field 2)
        else
            EXT_NET_ID=$(neutron net-create "$PUBLIC_NETWORK_NAME" -- --router:external=True | grep ' id ' | get_field 2)
        fi
        die_if_not_set $LINENO EXT_NET_ID "Failure creating EXT_NET_ID for $PUBLIC_NETWORK_NAME"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            # Configure router for IPv4 public access
            _neutron_configure_router_v4
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
            # Configure router for IPv6 public access
            _neutron_configure_router_v6
        fi
    fi
}

# init_neutron() - Initialize databases, etc.
function init_neutron {
    recreate_database $Q_DB_NAME
    # Run Neutron db migrations
    $NEUTRON_BIN_DIR/neutron-db-manage --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE upgrade head
}

# install_neutron() - Collect source and prepare
function install_neutron {
    git_clone $NEUTRON_REPO $NEUTRON_DIR $NEUTRON_BRANCH
    setup_develop $NEUTRON_DIR
    if is_service_enabled q-fwaas; then
        git_clone $NEUTRON_FWAAS_REPO $NEUTRON_FWAAS_DIR $NEUTRON_FWAAS_BRANCH
        setup_develop $NEUTRON_FWAAS_DIR
    fi
    if is_service_enabled q-lbaas; then
        git_clone $NEUTRON_LBAAS_REPO $NEUTRON_LBAAS_DIR $NEUTRON_LBAAS_BRANCH
        setup_develop $NEUTRON_LBAAS_DIR
    fi
    if is_service_enabled q-vpn; then
        git_clone $NEUTRON_VPNAAS_REPO $NEUTRON_VPNAAS_DIR $NEUTRON_VPNAAS_BRANCH
        setup_develop $NEUTRON_VPNAAS_DIR
    fi

    if [ "$VIRT_DRIVER" == 'xenserver' ]; then
        local dom0_ip
        dom0_ip=$(echo "$XENAPI_CONNECTION_URL" | cut -d "/" -f 3-)

        local ssh_dom0
        ssh_dom0="sudo -u $DOMZERO_USER ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$dom0_ip"

        # Find where the plugins should go in dom0
        local xen_functions
        xen_functions=$(cat $TOP_DIR/tools/xen/functions)
        local plugin_dir
        plugin_dir=$($ssh_dom0 "$xen_functions; set -eux; xapi_plugin_location")

        # install neutron plugins to dom0
        tar -czf - -C $NEUTRON_DIR/neutron/plugins/ml2/drivers/openvswitch/agent/xenapi/etc/xapi.d/plugins/ ./ |
            $ssh_dom0 "tar -xzf - -C $plugin_dir && chmod a+x $plugin_dir/*"
    fi
}

# install_neutronclient() - Collect source and prepare
function install_neutronclient {
    if use_library_from_git "python-neutronclient"; then
        git_clone_by_name "python-neutronclient"
        setup_dev_lib "python-neutronclient"
        sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-neutronclient"]}/tools/,/etc/bash_completion.d/}neutron.bash_completion
    fi
}

# install_neutron_agent_packages() - Collect source and prepare
function install_neutron_agent_packages {
    # radvd doesn't come with the OS. Install it if the l3 service is enabled.
    if is_service_enabled q-l3; then
        install_package radvd
    fi
    # install packages that are specific to plugin agent(s)
    if is_service_enabled q-agt q-dhcp q-l3; then
        neutron_plugin_install_agent_packages
    fi

    if is_service_enabled q-lbaas; then
        neutron_agent_lbaas_install_agent_packages
    fi
}

# Start running processes, including screen
function start_neutron_service_and_check {
    local cfg_file_options="$(determine_config_files neutron-server)"
    local service_port=$Q_PORT
    local service_protocol=$Q_PROTOCOL
    if is_service_enabled tls-proxy; then
        service_port=$Q_PORT_INT
        service_protocol="http"
    fi
    # Start the Neutron service
    run_process q-svc "$NEUTRON_BIN_DIR/neutron-server $cfg_file_options"
    echo "Waiting for Neutron to start..."
    if is_ssl_enabled_service "neutron"; then
        ssl_ca="--ca-certificate=${SSL_BUNDLE_FILE}"
    fi

    local testcmd="wget ${ssl_ca} --no-proxy -q -O- $service_protocol://$Q_HOST:$service_port"
    test_with_retry "$testcmd" "Neutron did not start" $SERVICE_TIMEOUT

    # Start proxy if enabled
    if is_service_enabled tls-proxy; then
        start_tls_proxy '*' $Q_PORT $Q_HOST $Q_PORT_INT &
    fi
}

# Control of the l2 agent is separated out to make it easier to test partial
# upgrades (everything upgraded except the L2 agent)
function start_neutron_l2_agent {
    run_process q-agt "$AGENT_BINARY --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE"

    if is_provider_network && [[ $Q_AGENT == "openvswitch" ]]; then
        sudo ovs-vsctl --no-wait -- --may-exist add-port $OVS_PHYSICAL_BRIDGE $PUBLIC_INTERFACE
        sudo ip link set $OVS_PHYSICAL_BRIDGE up
        sudo ip link set br-int up
        sudo ip link set $PUBLIC_INTERFACE up
        if is_ironic_hardware; then
            for IP in $(ip addr show dev $PUBLIC_INTERFACE | grep ' inet ' | awk '{print $2}'); do
                sudo ip addr del $IP dev $PUBLIC_INTERFACE
                sudo ip addr add $IP dev $OVS_PHYSICAL_BRIDGE
            done
            sudo ip route replace $FIXED_RANGE via $NETWORK_GATEWAY dev $OVS_PHYSICAL_BRIDGE
        fi
    fi
}

function start_neutron_other_agents {
    run_process q-dhcp "$AGENT_DHCP_BINARY --config-file $NEUTRON_CONF --config-file=$Q_DHCP_CONF_FILE"

    if is_service_enabled neutron-vpnaas; then
        :  # Started by plugin
    elif is_service_enabled q-vpn; then
        run_process q-vpn "$AGENT_VPN_BINARY $(determine_config_files neutron-vpn-agent)"
    else
        run_process q-l3 "$AGENT_L3_BINARY $(determine_config_files neutron-l3-agent)"
    fi

    run_process q-meta "$AGENT_META_BINARY --config-file $NEUTRON_CONF --config-file=$Q_META_CONF_FILE"
    run_process q-lbaas "$AGENT_LBAAS_BINARY --config-file $NEUTRON_CONF --config-file=$LBAAS_AGENT_CONF_FILENAME"
    run_process q-metering "$AGENT_METERING_BINARY --config-file $NEUTRON_CONF --config-file $METERING_AGENT_CONF_FILENAME"

    if [ "$VIRT_DRIVER" = 'xenserver' ]; then
        # For XenServer, start an agent for the domU openvswitch
        run_process q-domua "$AGENT_BINARY --config-file $NEUTRON_CONF --config-file /$Q_PLUGIN_CONF_FILE.domU"
    fi
}

# Start running processes, including screen
function start_neutron_agents {
    # Start up the neutron agents if enabled
    start_neutron_l2_agent
    start_neutron_other_agents
}

function stop_neutron_l2_agent {
    stop_process q-agt
}

function stop_neutron_other {
    if is_service_enabled q-dhcp; then
        stop_process q-dhcp
        pid=$(ps aux | awk '/[d]nsmasq.+interface=(tap|ns-)/ { print $2 }')
        [ ! -z "$pid" ] && sudo kill -9 $pid
    fi

    stop_process q-svc
    stop_process q-l3

    if is_service_enabled q-meta; then
        sudo pkill -9 -f neutron-ns-metadata-proxy || :
        stop_process q-meta
    fi

    if is_service_enabled q-lbaas; then
        neutron_lbaas_stop
    fi
    if is_service_enabled q-fwaas; then
        neutron_fwaas_stop
    fi
    if is_service_enabled q-vpn; then
        neutron_vpn_stop
    fi
    if is_service_enabled q-metering; then
        neutron_metering_stop
    fi
}

# stop_neutron() - Stop running processes (non-screen)
function stop_neutron {
    stop_neutron_other
    stop_neutron_l2_agent
}

# _move_neutron_addresses_route() - Move the primary IP to the OVS bridge
# on startup, or back to the public interface on cleanup. If no IP is
# configured on the interface, just add it as a port to the OVS bridge.
function _move_neutron_addresses_route {
    local from_intf=$1
    local to_intf=$2
    local add_ovs_port=$3
    local af=$4

    if [[ -n "$from_intf" && -n "$to_intf" ]]; then
        # Remove the primary IP address from $from_intf and add it to $to_intf,
        # along with the default route, if it exists.  Also, when called
        # on configure we will also add $from_intf as a port on $to_intf,
        # assuming it is an OVS bridge.

        local IP_ADD=""
        local IP_DEL=""
        local DEFAULT_ROUTE_GW
        DEFAULT_ROUTE_GW=$(ip -f $af r | awk "/default.+$from_intf/ { print \$3; exit }")
        local ADD_OVS_PORT=""

        IP_BRD=$(ip -f $af a s dev $from_intf scope global primary | grep inet | awk '{ print $2, $3, $4; exit }')

        if [ "$DEFAULT_ROUTE_GW" != "" ]; then
            ADD_DEFAULT_ROUTE="sudo ip -f $af r replace default via $DEFAULT_ROUTE_GW dev $to_intf"
        fi

        if [[ "$add_ovs_port" == "True" ]]; then
            ADD_OVS_PORT="sudo ovs-vsctl --may-exist add-port $to_intf $from_intf"
        fi

        if [[ "$IP_BRD" != "" ]]; then
            IP_DEL="sudo ip addr del $IP_BRD dev $from_intf"
            IP_ADD="sudo ip addr add $IP_BRD dev $to_intf"
        fi

        $IP_DEL; $IP_ADD; $ADD_OVS_PORT; $ADD_DEFAULT_ROUTE
    fi
}

# cleanup_neutron() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_neutron {

    if [[ -n "$OVS_PHYSICAL_BRIDGE" ]]; then
        _move_neutron_addresses_route "$OVS_PHYSICAL_BRIDGE" "$PUBLIC_INTERFACE" False "inet"

        if [[ $(ip -f inet6 a s dev "$OVS_PHYSICAL_BRIDGE" | grep -c 'global') != 0 ]]; then
            # ip(8) wants the prefix length when deleting
            local v6_gateway
            v6_gateway=$(ip -6 a s dev $OVS_PHYSICAL_BRIDGE | grep $IPV6_PUBLIC_NETWORK_GATEWAY | awk '{ print $2 }')
            sudo ip -6 addr del $v6_gateway dev $OVS_PHYSICAL_BRIDGE
            _move_neutron_addresses_route "$OVS_PHYSICAL_BRIDGE" "$PUBLIC_INTERFACE" False "inet6"
        fi

        if is_provider_network && is_ironic_hardware; then
            for IP in $(ip addr show dev $OVS_PHYSICAL_BRIDGE | grep ' inet ' | awk '{print $2}'); do
                sudo ip addr del $IP dev $OVS_PHYSICAL_BRIDGE
                sudo ip addr add $IP dev $PUBLIC_INTERFACE
            done
            sudo route del -net $FIXED_RANGE gw $NETWORK_GATEWAY dev $OVS_PHYSICAL_BRIDGE
        fi
    fi

    if is_neutron_ovs_base_plugin; then
        neutron_ovs_base_cleanup
    fi

    if [[ $Q_AGENT == "linuxbridge" ]]; then
        neutron_lb_cleanup
    fi

    # delete all namespaces created by neutron
    for ns in $(sudo ip netns list | grep -o -E '(qdhcp|qrouter|qlbaas|fip|snat)-[0-9a-f-]*'); do
        sudo ip netns delete ${ns}
    done
}


function _create_neutron_conf_dir {
    # Put config files in ``NEUTRON_CONF_DIR`` for everyone to find
    sudo install -d -o $STACK_USER $NEUTRON_CONF_DIR
}

# _configure_neutron_common()
# Set common config for all neutron server and agents.
# This MUST be called before other ``_configure_neutron_*`` functions.
function _configure_neutron_common {
    _create_neutron_conf_dir

    # Uses oslo config generator to generate core sample configuration files
    (cd $NEUTRON_DIR && exec ./tools/generate_config_file_samples.sh)

    cp $NEUTRON_DIR/etc/neutron.conf.sample $NEUTRON_CONF

    Q_POLICY_FILE=$NEUTRON_CONF_DIR/policy.json
    cp $NEUTRON_DIR/etc/policy.json $Q_POLICY_FILE

    # allow neutron user to administer neutron to match neutron account
    sed -i 's/"context_is_admin":  "role:admin"/"context_is_admin":  "role:admin or user_name:neutron"/g' $Q_POLICY_FILE

    # Set plugin-specific variables ``Q_DB_NAME``, ``Q_PLUGIN_CLASS``.
    # For main plugin config file, set ``Q_PLUGIN_CONF_PATH``, ``Q_PLUGIN_CONF_FILENAME``.
    # For addition plugin config files, set ``Q_PLUGIN_EXTRA_CONF_PATH``,
    # ``Q_PLUGIN_EXTRA_CONF_FILES``.  For example:
    #
    #    ``Q_PLUGIN_EXTRA_CONF_FILES=(file1, file2)``
    neutron_plugin_configure_common

    if [[ "$Q_PLUGIN_CONF_PATH" == '' || "$Q_PLUGIN_CONF_FILENAME" == '' || "$Q_PLUGIN_CLASS" == '' ]]; then
        die $LINENO "Neutron plugin not set.. exiting"
    fi

    # If needed, move config file from ``$NEUTRON_DIR/etc/neutron`` to ``NEUTRON_CONF_DIR``
    mkdir -p /$Q_PLUGIN_CONF_PATH
    Q_PLUGIN_CONF_FILE=$Q_PLUGIN_CONF_PATH/$Q_PLUGIN_CONF_FILENAME
    # NOTE(hichihara): Some neutron vendor plugins were already decomposed and
    # there is no config file in Neutron tree. They should prepare the file in each plugin.
    if [ -f "$NEUTRON_DIR/$Q_PLUGIN_CONF_FILE.sample" ]; then
        cp "$NEUTRON_DIR/$Q_PLUGIN_CONF_FILE.sample" /$Q_PLUGIN_CONF_FILE
    elif [ -f $NEUTRON_DIR/$Q_PLUGIN_CONF_FILE ]; then
        cp $NEUTRON_DIR/$Q_PLUGIN_CONF_FILE /$Q_PLUGIN_CONF_FILE
    fi

    iniset $NEUTRON_CONF database connection `database_connection_url $Q_DB_NAME`
    iniset $NEUTRON_CONF DEFAULT state_path $DATA_DIR/neutron
    iniset $NEUTRON_CONF DEFAULT use_syslog $SYSLOG
    iniset $NEUTRON_CONF DEFAULT bind_host $Q_LISTEN_ADDRESS
    iniset $NEUTRON_CONF oslo_concurrency lock_path $DATA_DIR/neutron/lock

    # If addition config files are set, make sure their path name is set as well
    if [[ ${#Q_PLUGIN_EXTRA_CONF_FILES[@]} > 0 && $Q_PLUGIN_EXTRA_CONF_PATH == '' ]]; then
        die $LINENO "Neutron additional plugin config not set.. exiting"
    fi

    # If additional config files exist, copy them over to neutron configuration
    # directory
    if [[ $Q_PLUGIN_EXTRA_CONF_PATH != '' ]]; then
        local f
        for (( f=0; $f < ${#Q_PLUGIN_EXTRA_CONF_FILES[@]}; f+=1 )); do
            Q_PLUGIN_EXTRA_CONF_FILES[$f]=$Q_PLUGIN_EXTRA_CONF_PATH/${Q_PLUGIN_EXTRA_CONF_FILES[$f]}
        done
    fi

    if [ "$VIRT_DRIVER" = 'fake' ]; then
        # Disable arbitrary limits
        iniset $NEUTRON_CONF quotas quota_network -1
        iniset $NEUTRON_CONF quotas quota_subnet -1
        iniset $NEUTRON_CONF quotas quota_port -1
        iniset $NEUTRON_CONF quotas quota_security_group -1
        iniset $NEUTRON_CONF quotas quota_security_group_rule -1
    fi

    # Format logging
    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
        setup_colorized_logging $NEUTRON_CONF DEFAULT project_id
    else
        # Show user_name and project_name by default like in nova
        iniset $NEUTRON_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s"
    fi

    if is_service_enabled tls-proxy; then
        # Set the service port for a proxy to take the original
        iniset $NEUTRON_CONF DEFAULT bind_port "$Q_PORT_INT"
    fi

    if is_ssl_enabled_service "nova"; then
        iniset $NEUTRON_CONF nova cafile $SSL_BUNDLE_FILE
    fi

    if is_ssl_enabled_service "neutron"; then
        ensure_certificates NEUTRON

        iniset $NEUTRON_CONF DEFAULT use_ssl True
        iniset $NEUTRON_CONF DEFAULT ssl_cert_file "$NEUTRON_SSL_CERT"
        iniset $NEUTRON_CONF DEFAULT ssl_key_file "$NEUTRON_SSL_KEY"
    fi

    _neutron_setup_rootwrap
}

function _configure_neutron_debug_command {
    if [[ "$Q_USE_DEBUG_COMMAND" != "True" ]]; then
        return
    fi

    cp $NEUTRON_DIR/etc/l3_agent.ini.sample $NEUTRON_TEST_CONFIG_FILE

    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT verbose False
    iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT debug False
    iniset $NEUTRON_TEST_CONFIG_FILE AGENT root_helper "$Q_RR_COMMAND"
    if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then
        iniset $NEUTRON_TEST_CONFIG_FILE AGENT root_helper_daemon "$Q_RR_DAEMON_COMMAND"
    fi

    _neutron_setup_interface_driver $NEUTRON_TEST_CONFIG_FILE

    neutron_plugin_configure_debug_command
}

function _configure_neutron_dhcp_agent {

    cp $NEUTRON_DIR/etc/dhcp_agent.ini.sample $Q_DHCP_CONF_FILE

    iniset $Q_DHCP_CONF_FILE DEFAULT verbose True
    iniset $Q_DHCP_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_DHCP_CONF_FILE AGENT root_helper "$Q_RR_COMMAND"
    if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then
        iniset $Q_DHCP_CONF_FILE AGENT root_helper_daemon "$Q_RR_DAEMON_COMMAND"
    fi

    if ! is_service_enabled q-l3; then
        if [[ "$ENABLE_ISOLATED_METADATA" = "True" ]]; then
            iniset $Q_DHCP_CONF_FILE DEFAULT enable_isolated_metadata $ENABLE_ISOLATED_METADATA
            iniset $Q_DHCP_CONF_FILE DEFAULT enable_metadata_network $ENABLE_METADATA_NETWORK
        else
            if [[ "$ENABLE_METADATA_NETWORK" = "True" ]]; then
                die "$LINENO" "Enable isolated metadata is a must for metadata network"
            fi
        fi
    fi

    _neutron_setup_interface_driver $Q_DHCP_CONF_FILE

    neutron_plugin_configure_dhcp_agent
}

function _configure_neutron_l3_agent {
    Q_L3_ENABLED=True

    if is_service_enabled q-vpn; then
        neutron_vpn_configure_agent
    fi

    cp $NEUTRON_DIR/etc/l3_agent.ini.sample $Q_L3_CONF_FILE

    iniset $Q_L3_CONF_FILE DEFAULT verbose True
    iniset $Q_L3_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_L3_CONF_FILE AGENT root_helper "$Q_RR_COMMAND"
    if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then
        iniset $Q_L3_CONF_FILE AGENT root_helper_daemon "$Q_RR_DAEMON_COMMAND"
    fi

    _neutron_setup_interface_driver $Q_L3_CONF_FILE

    neutron_plugin_configure_l3_agent

    _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" True "inet"

    if [[ $(ip -f inet6 a s dev "$PUBLIC_INTERFACE" | grep -c 'global') != 0 ]]; then
        _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" False "inet6"
    fi
}

function _configure_neutron_metadata_agent {
    cp $NEUTRON_DIR/etc/metadata_agent.ini.sample $Q_META_CONF_FILE

    iniset $Q_META_CONF_FILE DEFAULT verbose True
    iniset $Q_META_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $Q_META_CONF_FILE DEFAULT nova_metadata_ip $Q_META_DATA_IP
    iniset $Q_META_CONF_FILE AGENT root_helper "$Q_RR_COMMAND"
    if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then
        iniset $Q_META_CONF_FILE AGENT root_helper_daemon "$Q_RR_DAEMON_COMMAND"
    fi

    # Configures keystone for metadata_agent
    # The third argument "True" sets auth_url needed to communicate with keystone
    _neutron_setup_keystone $Q_META_CONF_FILE DEFAULT True

}

function _configure_neutron_ceilometer_notifications {
    iniset $NEUTRON_CONF DEFAULT notification_driver messaging
}

function _configure_neutron_lbaas {
    # Uses oslo config generator to generate LBaaS sample configuration files
    (cd $NEUTRON_LBAAS_DIR && exec ./tools/generate_config_file_samples.sh)

    if [ -f $NEUTRON_LBAAS_DIR/etc/neutron_lbaas.conf.sample ]; then
        cp $NEUTRON_LBAAS_DIR/etc/neutron_lbaas.conf.sample $NEUTRON_CONF_DIR/neutron_lbaas.conf
        iniset $NEUTRON_CONF_DIR/neutron_lbaas.conf service_providers service_provider $DEFAULT_LB_PROVIDER
    fi
    neutron_agent_lbaas_configure_common
    neutron_agent_lbaas_configure_agent
}

function _configure_neutron_metering {
    neutron_agent_metering_configure_common
    neutron_agent_metering_configure_agent
}

function _configure_neutron_fwaas {
    if [ -f $NEUTRON_FWAAS_DIR/etc/neutron_fwaas.conf ]; then
        cp $NEUTRON_FWAAS_DIR/etc/neutron_fwaas.conf $NEUTRON_CONF_DIR
    fi
    neutron_fwaas_configure_common
    neutron_fwaas_configure_driver
}

function _configure_neutron_vpn {
    if [ -f $NEUTRON_VPNAAS_DIR/etc/neutron_vpnaas.conf ]; then
        cp $NEUTRON_VPNAAS_DIR/etc/neutron_vpnaas.conf $NEUTRON_CONF_DIR
    fi
    neutron_vpn_install_agent_packages
    neutron_vpn_configure_common
}

function _configure_dvr {
    iniset $NEUTRON_CONF DEFAULT router_distributed True
    iniset $Q_L3_CONF_FILE DEFAULT agent_mode $Q_DVR_MODE
}


# _configure_neutron_plugin_agent() - Set config files for neutron plugin agent
# It is called when q-agt is enabled.
function _configure_neutron_plugin_agent {
    # Specify the default root helper prior to agent configuration to
    # ensure that an agent's configuration can override the default
    iniset /$Q_PLUGIN_CONF_FILE agent root_helper "$Q_RR_COMMAND"
    if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then
        iniset /$Q_PLUGIN_CONF_FILE  agent root_helper_daemon "$Q_RR_DAEMON_COMMAND"
    fi
    iniset $NEUTRON_CONF DEFAULT verbose True
    iniset $NEUTRON_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

    # Configure agent for plugin
    neutron_plugin_configure_plugin_agent
}

# _configure_neutron_service() - Set config files for neutron service
# It is called when q-svc is enabled.
function _configure_neutron_service {
    Q_API_PASTE_FILE=$NEUTRON_CONF_DIR/api-paste.ini
    cp $NEUTRON_DIR/etc/api-paste.ini $Q_API_PASTE_FILE

    # Update either configuration file with plugin
    iniset $NEUTRON_CONF DEFAULT core_plugin $Q_PLUGIN_CLASS

    if [[ $Q_SERVICE_PLUGIN_CLASSES != '' ]]; then
        iniset $NEUTRON_CONF DEFAULT service_plugins $Q_SERVICE_PLUGIN_CLASSES
    fi

    iniset $NEUTRON_CONF DEFAULT verbose True
    iniset $NEUTRON_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $NEUTRON_CONF oslo_policy policy_file $Q_POLICY_FILE
    iniset $NEUTRON_CONF DEFAULT allow_overlapping_ips $Q_ALLOW_OVERLAPPING_IP

    iniset $NEUTRON_CONF DEFAULT auth_strategy $Q_AUTH_STRATEGY
    _neutron_setup_keystone $NEUTRON_CONF keystone_authtoken

    # Configuration for neutron notifications to nova.
    iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_status_changes $Q_NOTIFY_NOVA_PORT_STATUS_CHANGES
    iniset $NEUTRON_CONF DEFAULT notify_nova_on_port_data_changes $Q_NOTIFY_NOVA_PORT_DATA_CHANGES

    iniset $NEUTRON_CONF nova auth_plugin password
    iniset $NEUTRON_CONF nova auth_url $KEYSTONE_AUTH_URI
    iniset $NEUTRON_CONF nova username nova
    iniset $NEUTRON_CONF nova password $SERVICE_PASSWORD
    iniset $NEUTRON_CONF nova user_domain_id default
    iniset $NEUTRON_CONF nova project_name $SERVICE_TENANT_NAME
    iniset $NEUTRON_CONF nova project_domain_id default
    iniset $NEUTRON_CONF nova region_name $REGION_NAME

    # Configure plugin
    neutron_plugin_configure_service
}

# Utility Functions
#------------------

# _neutron_service_plugin_class_add() - add service plugin class
function _neutron_service_plugin_class_add {
    local service_plugin_class=$1
    if [[ $Q_SERVICE_PLUGIN_CLASSES == '' ]]; then
        Q_SERVICE_PLUGIN_CLASSES=$service_plugin_class
    elif [[ ! ,${Q_SERVICE_PLUGIN_CLASSES}, =~ ,${service_plugin_class}, ]]; then
        Q_SERVICE_PLUGIN_CLASSES="$Q_SERVICE_PLUGIN_CLASSES,$service_plugin_class"
    fi
}

# _neutron_deploy_rootwrap_filters() - deploy rootwrap filters to $Q_CONF_ROOTWRAP_D (owned by root).
function _neutron_deploy_rootwrap_filters {
    if [[ "$Q_USE_ROOTWRAP" == "False" ]]; then
        return
    fi
    local srcdir=$1
    sudo install -d -o root -m 755 $Q_CONF_ROOTWRAP_D
    sudo install -o root -m 644 $srcdir/etc/neutron/rootwrap.d/* $Q_CONF_ROOTWRAP_D/
}

# _neutron_setup_rootwrap() - configure Neutron's rootwrap
function _neutron_setup_rootwrap {
    if [[ "$Q_USE_ROOTWRAP" == "False" ]]; then
        return
    fi
    # Wipe any existing ``rootwrap.d`` files first
    Q_CONF_ROOTWRAP_D=$NEUTRON_CONF_DIR/rootwrap.d
    if [[ -d $Q_CONF_ROOTWRAP_D ]]; then
        sudo rm -rf $Q_CONF_ROOTWRAP_D
    fi

    _neutron_deploy_rootwrap_filters $NEUTRON_DIR

    # Set up ``rootwrap.conf``, pointing to ``$NEUTRON_CONF_DIR/rootwrap.d``
    # location moved in newer versions, prefer new location
    if test -r $NEUTRON_DIR/etc/neutron/rootwrap.conf; then
        sudo install -o root -g root -m 644 $NEUTRON_DIR/etc/neutron/rootwrap.conf $Q_RR_CONF_FILE
    else
        sudo install -o root -g root -m 644 $NEUTRON_DIR/etc/rootwrap.conf $Q_RR_CONF_FILE
    fi
    sudo sed -e "s:^filters_path=.*$:filters_path=$Q_CONF_ROOTWRAP_D:" -i $Q_RR_CONF_FILE
    sudo sed -e 's:^exec_dirs=\(.*\)$:exec_dirs=\1,/usr/local/bin:' -i $Q_RR_CONF_FILE

    # Specify ``rootwrap.conf`` as first parameter to neutron-rootwrap
    ROOTWRAP_SUDOER_CMD="$NEUTRON_ROOTWRAP $Q_RR_CONF_FILE *"
    ROOTWRAP_DAEMON_SUDOER_CMD="$NEUTRON_ROOTWRAP-daemon $Q_RR_CONF_FILE"

    # Set up the rootwrap sudoers for neutron
    TEMPFILE=`mktemp`
    echo "$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_SUDOER_CMD" >$TEMPFILE
    echo "$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_DAEMON_SUDOER_CMD" >>$TEMPFILE
    chmod 0440 $TEMPFILE
    sudo chown root:root $TEMPFILE
    sudo mv $TEMPFILE /etc/sudoers.d/neutron-rootwrap

    # Update the root_helper
    iniset $NEUTRON_CONF agent root_helper "$Q_RR_COMMAND"
    if [[ "$Q_USE_ROOTWRAP_DAEMON" == "True" ]]; then
        iniset $NEUTRON_CONF agent root_helper_daemon "$Q_RR_DAEMON_COMMAND"
    fi
}

# Configures keystone integration for neutron service and agents
function _neutron_setup_keystone {
    local conf_file=$1
    local section=$2
    local use_auth_url=$3

    # Configures keystone for metadata_agent
    # metadata_agent needs auth_url to communicate with keystone
    if [[ "$use_auth_url" == "True" ]]; then
        iniset $conf_file $section auth_url $KEYSTONE_SERVICE_URI/v2.0
    fi

    create_neutron_cache_dir
    configure_auth_token_middleware $conf_file $Q_ADMIN_USERNAME $NEUTRON_AUTH_CACHE_DIR $section
}

function _neutron_setup_interface_driver {

    # ovs_use_veth needs to be set before the plugin configuration
    # occurs to allow plugins to override the setting.
    iniset $1 DEFAULT ovs_use_veth $Q_OVS_USE_VETH

    neutron_plugin_setup_interface_driver $1
}

# Create private IPv4 subnet
function _neutron_create_private_subnet_v4 {
    local subnet_params="--tenant-id $TENANT_ID "
    subnet_params+="--ip_version 4 "
    subnet_params+="--gateway $NETWORK_GATEWAY "
    subnet_params+="--name $PRIVATE_SUBNET_NAME "
    subnet_params+="--subnetpool None "
    subnet_params+="$NET_ID $FIXED_RANGE"
    local subnet_id
    subnet_id=$(neutron subnet-create $subnet_params | grep ' id ' | get_field 2)
    die_if_not_set $LINENO subnet_id "Failure creating private IPv4 subnet for $TENANT_ID"
    echo $subnet_id
}

# Create private IPv6 subnet
function _neutron_create_private_subnet_v6 {
    die_if_not_set $LINENO IPV6_RA_MODE "IPV6 RA Mode not set"
    die_if_not_set $LINENO IPV6_ADDRESS_MODE "IPV6 Address Mode not set"
    local ipv6_modes="--ipv6-ra-mode $IPV6_RA_MODE --ipv6-address-mode $IPV6_ADDRESS_MODE"
    local subnet_params="--tenant-id $TENANT_ID "
    subnet_params+="--ip_version 6 "
    subnet_params+="--gateway $IPV6_PRIVATE_NETWORK_GATEWAY "
    subnet_params+="--name $IPV6_PRIVATE_SUBNET_NAME "
    subnet_params+="--subnetpool None "
    subnet_params+="$NET_ID $FIXED_RANGE_V6 $ipv6_modes"
    local ipv6_subnet_id
    ipv6_subnet_id=$(neutron subnet-create $subnet_params | grep ' id ' | get_field 2)
    die_if_not_set $LINENO ipv6_subnet_id "Failure creating private IPv6 subnet for $TENANT_ID"
    echo $ipv6_subnet_id
}

# Create public IPv4 subnet
function _neutron_create_public_subnet_v4 {
    local subnet_params+="--ip_version 4 "
    subnet_params+="${Q_FLOATING_ALLOCATION_POOL:+--allocation-pool $Q_FLOATING_ALLOCATION_POOL} "
    subnet_params+="--gateway $PUBLIC_NETWORK_GATEWAY "
    subnet_params+="--name $PUBLIC_SUBNET_NAME "
    subnet_params+="--subnetpool None "
    subnet_params+="$EXT_NET_ID $FLOATING_RANGE "
    subnet_params+="-- --enable_dhcp=False"
    local id_and_ext_gw_ip
    id_and_ext_gw_ip=$(neutron subnet-create $subnet_params | grep -e 'gateway_ip' -e ' id ')
    die_if_not_set $LINENO id_and_ext_gw_ip "Failure creating public IPv4 subnet"
    echo $id_and_ext_gw_ip
}

# Create public IPv6 subnet
function _neutron_create_public_subnet_v6 {
    local subnet_params="--ip_version 6 "
    subnet_params+="--gateway $IPV6_PUBLIC_NETWORK_GATEWAY "
    subnet_params+="--name $IPV6_PUBLIC_SUBNET_NAME "
    subnet_params+="--subnetpool None "
    subnet_params+="$EXT_NET_ID $IPV6_PUBLIC_RANGE "
    subnet_params+="-- --enable_dhcp=False"
    local ipv6_id_and_ext_gw_ip
    ipv6_id_and_ext_gw_ip=$(neutron subnet-create $subnet_params | grep -e 'gateway_ip' -e ' id ')
    die_if_not_set $LINENO ipv6_id_and_ext_gw_ip "Failure creating an IPv6 public subnet"
    echo $ipv6_id_and_ext_gw_ip
}

# Configure neutron router for IPv4 public access
function _neutron_configure_router_v4 {
    neutron router-interface-add $ROUTER_ID $SUBNET_ID
    # Create a public subnet on the external network
    local id_and_ext_gw_ip
    id_and_ext_gw_ip=$(_neutron_create_public_subnet_v4 $EXT_NET_ID)
    local ext_gw_ip
    ext_gw_ip=$(echo $id_and_ext_gw_ip  | get_field 2)
    PUB_SUBNET_ID=$(echo $id_and_ext_gw_ip | get_field 5)
    # Configure the external network as the default router gateway
    neutron router-gateway-set $ROUTER_ID $EXT_NET_ID

    # This logic is specific to using the l3-agent for layer 3
    if is_service_enabled q-l3; then
        # Configure and enable public bridge
        local ext_gw_interface="none"
        if is_neutron_ovs_base_plugin; then
            ext_gw_interface=$(_neutron_get_ext_gw_interface)
        elif [[ "$Q_AGENT" = "linuxbridge" ]]; then
            # Search for the brq device the neutron router and network for $FIXED_RANGE
            # will be using.
            # e.x. brq3592e767-da for NET_ID 3592e767-da66-4bcb-9bec-cdb03cd96102
            ext_gw_interface=brq${EXT_NET_ID:0:11}
        fi
        if [[ "$ext_gw_interface" != "none" ]]; then
            local cidr_len=${FLOATING_RANGE#*/}
            local testcmd="ip -o link | grep -q $ext_gw_interface"
            test_with_retry "$testcmd" "$ext_gw_interface creation failed"
            if [[ $(ip addr show dev $ext_gw_interface | grep -c $ext_gw_ip) == 0 && ( $Q_USE_PROVIDERNET_FOR_PUBLIC == "False" || $Q_USE_PUBLIC_VETH == "True" ) ]]; then
                sudo ip addr add $ext_gw_ip/$cidr_len dev $ext_gw_interface
                sudo ip link set $ext_gw_interface up
            fi
            ROUTER_GW_IP=`neutron port-list -c fixed_ips -c device_owner | grep router_gateway | awk -F '"' -v subnet_id=$PUB_SUBNET_ID '$4 == subnet_id { print $8; }'`
            die_if_not_set $LINENO ROUTER_GW_IP "Failure retrieving ROUTER_GW_IP"
            sudo ip route replace  $FIXED_RANGE via $ROUTER_GW_IP
        fi
        _neutron_set_router_id
    fi
}

# Configure neutron router for IPv6 public access
function _neutron_configure_router_v6 {
    neutron router-interface-add $ROUTER_ID $IPV6_SUBNET_ID
    # Create a public subnet on the external network
    local ipv6_id_and_ext_gw_ip
    ipv6_id_and_ext_gw_ip=$(_neutron_create_public_subnet_v6 $EXT_NET_ID)
    local ipv6_ext_gw_ip
    ipv6_ext_gw_ip=$(echo $ipv6_id_and_ext_gw_ip | get_field 2)
    local ipv6_pub_subnet_id
    ipv6_pub_subnet_id=$(echo $ipv6_id_and_ext_gw_ip | get_field 5)

    # If the external network has not already been set as the default router
    # gateway when configuring an IPv4 public subnet, do so now
    if [[ "$IP_VERSION" == "6" ]]; then
        neutron router-gateway-set $ROUTER_ID $EXT_NET_ID
    fi

    # This logic is specific to using the l3-agent for layer 3
    if is_service_enabled q-l3; then
        # Ensure IPv6 forwarding is enabled on the host
        sudo sysctl -w net.ipv6.conf.all.forwarding=1
        # Configure and enable public bridge
        # Override global IPV6_ROUTER_GW_IP with the true value from neutron
        IPV6_ROUTER_GW_IP=`neutron port-list -c fixed_ips | grep $ipv6_pub_subnet_id | awk -F '"' -v subnet_id=$ipv6_pub_subnet_id '$4 == subnet_id { print $8; }'`
        die_if_not_set $LINENO IPV6_ROUTER_GW_IP "Failure retrieving IPV6_ROUTER_GW_IP"

        if is_neutron_ovs_base_plugin; then
            local ext_gw_interface
            ext_gw_interface=$(_neutron_get_ext_gw_interface)
            local ipv6_cidr_len=${IPV6_PUBLIC_RANGE#*/}

            # Configure interface for public bridge
            sudo ip -6 addr add $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface
            sudo ip -6 route replace $FIXED_RANGE_V6 via $IPV6_ROUTER_GW_IP dev $ext_gw_interface
        fi
        _neutron_set_router_id
    fi
}

# Explicitly set router id in l3 agent configuration
function _neutron_set_router_id {
    if [[ "$Q_L3_ROUTER_PER_TENANT" == "False" ]]; then
        iniset $Q_L3_CONF_FILE DEFAULT router_id $ROUTER_ID
    fi
}

# Get ext_gw_interface depending on value of Q_USE_PUBLIC_VETH
function _neutron_get_ext_gw_interface {
    if [[ "$Q_USE_PUBLIC_VETH" == "True" ]]; then
        echo $Q_PUBLIC_VETH_EX
    else
        # Disable in-band as we are going to use local port
        # to communicate with VMs
        sudo ovs-vsctl set Bridge $PUBLIC_BRIDGE \
            other_config:disable-in-band=true
        echo $PUBLIC_BRIDGE
    fi
}

# Functions for Neutron Exercises
#--------------------------------

function delete_probe {
    local from_net="$1"
    net_id=`_get_net_id $from_net`
    probe_id=`neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-list -c id -c network_id | grep $net_id | awk '{print $2}'`
    neutron-debug --os-tenant-name admin --os-username admin probe-delete $probe_id
}

function setup_neutron_debug {
    if [[ "$Q_USE_DEBUG_COMMAND" == "True" ]]; then
        public_net_id=`_get_net_id $PUBLIC_NETWORK_NAME`
        neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-create --device-owner compute $public_net_id
        private_net_id=`_get_net_id $PRIVATE_NETWORK_NAME`
        neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-create --device-owner compute $private_net_id
    fi
}

function teardown_neutron_debug {
    delete_probe $PUBLIC_NETWORK_NAME
    delete_probe $PRIVATE_NETWORK_NAME
}

function _get_net_id {
    neutron --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD net-list | grep $1 | awk '{print $2}'
}

function _get_probe_cmd_prefix {
    local from_net="$1"
    net_id=`_get_net_id $from_net`
    probe_id=`neutron-debug --os-tenant-name admin --os-username admin --os-password $ADMIN_PASSWORD probe-list -c id -c network_id | grep $net_id | awk '{print $2}' | head -n 1`
    echo "$Q_RR_COMMAND ip netns exec qprobe-$probe_id"
}

# ssh check
function _ssh_check_neutron {
    local from_net=$1
    local key_file=$2
    local ip=$3
    local user=$4
    local timeout_sec=$5
    local probe_cmd = ""
    probe_cmd=`_get_probe_cmd_prefix $from_net`
    local testcmd="$probe_cmd ssh -o StrictHostKeyChecking=no -i $key_file ${user}@$ip echo success"
    test_with_retry "$testcmd" "server $ip didn't become ssh-able" $timeout_sec
}

# Neutron 3rd party programs
#---------------------------

# please refer to ``lib/neutron_thirdparty/README.md`` for details
NEUTRON_THIRD_PARTIES=""
for f in $TOP_DIR/lib/neutron_thirdparty/*; do
    third_party=$(basename $f)
    if is_service_enabled $third_party; then
        source $TOP_DIR/lib/neutron_thirdparty/$third_party
        NEUTRON_THIRD_PARTIES="$NEUTRON_THIRD_PARTIES,$third_party"
    fi
done

function _neutron_third_party_do {
    for third_party in ${NEUTRON_THIRD_PARTIES//,/ }; do
        ${1}_${third_party}
    done
}

# configure_neutron_third_party() - Set config files, create data dirs, etc
function configure_neutron_third_party {
    _neutron_third_party_do configure
}

# init_neutron_third_party() - Initialize databases, etc.
function init_neutron_third_party {
    _neutron_third_party_do init
}

# install_neutron_third_party() - Collect source and prepare
function install_neutron_third_party {
    _neutron_third_party_do install
}

# start_neutron_third_party() - Start running processes, including screen
function start_neutron_third_party {
    _neutron_third_party_do start
}

# stop_neutron_third_party - Stop running processes (non-screen)
function stop_neutron_third_party {
    _neutron_third_party_do stop
}

# check_neutron_third_party_integration() - Check that third party integration is sane
function check_neutron_third_party_integration {
    _neutron_third_party_do check
}

function is_provider_network {
    if [ "$Q_USE_PROVIDER_NETWORKING" == "True" ] && [ "$Q_L3_ENABLED" == "False" ]; then
        return 0
    fi
    return 1
}


# Restore xtrace
$_XTRACE_NEUTRON

# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End: