# lib/cinder
# Install and start **Cinder** volume service

# Dependencies:
#
# - functions
# - DEST, DATA_DIR, STACK_USER must be defined
# - SERVICE_{TENANT_NAME|PASSWORD} must be defined
# - ``KEYSTONE_TOKEN_FORMAT`` must be defined

# stack.sh
# ---------
# - install_cinder
# - configure_cinder
# - init_cinder
# - start_cinder
# - stop_cinder
# - cleanup_cinder

# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace


# Defaults
# --------

# set up default driver
CINDER_DRIVER=${CINDER_DRIVER:-default}
CINDER_PLUGINS=$TOP_DIR/lib/cinder_plugins
CINDER_BACKENDS=$TOP_DIR/lib/cinder_backends

# grab plugin config if specified via cinder_driver
if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
    source $CINDER_PLUGINS/$CINDER_DRIVER
fi

# set up default directories
GITDIR["python-cinderclient"]=$DEST/python-cinderclient

CINDER_DIR=$DEST/cinder
CINDER_STATE_PATH=${CINDER_STATE_PATH:=$DATA_DIR/cinder}
CINDER_AUTH_CACHE_DIR=${CINDER_AUTH_CACHE_DIR:-/var/cache/cinder}

CINDER_CONF_DIR=/etc/cinder
CINDER_CONF=$CINDER_CONF_DIR/cinder.conf
CINDER_API_PASTE_INI=$CINDER_CONF_DIR/api-paste.ini

# Public facing bits
if is_ssl_enabled_service "cinder" || is_service_enabled tls-proxy; then
    CINDER_SERVICE_PROTOCOL="https"
fi
CINDER_SERVICE_HOST=${CINDER_SERVICE_HOST:-$SERVICE_HOST}
CINDER_SERVICE_PORT=${CINDER_SERVICE_PORT:-8776}
CINDER_SERVICE_PORT_INT=${CINDER_SERVICE_PORT_INT:-18776}
CINDER_SERVICE_PROTOCOL=${CINDER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}

# Support entry points installation of console scripts
if [[ -d $CINDER_DIR/bin ]]; then
    CINDER_BIN_DIR=$CINDER_DIR/bin
else
    CINDER_BIN_DIR=$(get_python_exec_prefix)
fi


# Maintain this here for backward-compatibility with the old configuration
# DEPRECATED: Use CINDER_ENABLED_BACKENDS instead
# Support for multi lvm backend configuration (default is no support)
CINDER_MULTI_LVM_BACKEND=$(trueorfalse False $CINDER_MULTI_LVM_BACKEND)

# Default backends
# The backend format is type:name where type is one of the supported backend
# types (lvm, nfs, etc) and name is the identifier used in the Cinder
# configuration and for the volume type name.  Multiple backends are
# comma-separated.
if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then
    CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
else
    CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2}
fi


# Should cinder perform secure deletion of volumes?
# Defaults to true, can be set to False to avoid this bug when testing:
# https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023755
CINDER_SECURE_DELETE=`trueorfalse True $CINDER_SECURE_DELETE`

# Cinder reports allocations back to the scheduler on periodic intervals
# it turns out we can get an "out of space" issue when we run tests too
# quickly just because cinder didn't realize we'd freed up resources.
# Make this configurable so that devstack-gate/tempest can set it to
# less than the 60 second default
# https://bugs.launchpad.net/cinder/+bug/1180976
CINDER_PERIODIC_INTERVAL=${CINDER_PERIODIC_INTERVAL:-60}

# Tell Tempest this project is present
TEMPEST_SERVICES+=,cinder


# Source the enabled backends
if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
    for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
        be_type=${be%%:*}
        be_name=${be##*:}
        if [[ -r $CINDER_BACKENDS/${be_type} ]]; then
            source $CINDER_BACKENDS/${be_type}
        fi
    done
fi


# Functions
# ---------

# Test if any Cinder services are enabled
# is_cinder_enabled
function is_cinder_enabled {
    [[ ,${ENABLED_SERVICES} =~ ,"c-" ]] && return 0
    return 1
}

# cleanup_cinder() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_cinder {
    # ensure the volume group is cleared up because fails might
    # leave dead volumes in the group
    local targets=$(sudo tgtadm --op show --mode target)
    if [ $? -ne 0 ]; then
        # If tgt driver isn't running this won't work obviously
        # So check the response and restart if need be
        echo "tgtd seems to be in a bad state, restarting..."
        if is_ubuntu; then
            restart_service tgt
        else
            restart_service tgtd
        fi
        targets=$(sudo tgtadm --op show --mode target)
    fi

    if [[ -n "$targets" ]]; then
        local iqn_list=( $(grep --no-filename -r iqn $SCSI_PERSIST_DIR | sed 's/<target //' | sed 's/>//') )
        for i in "${iqn_list[@]}"; do
            echo removing iSCSI target: $i
            sudo tgt-admin --delete $i
        done
    fi

    if is_ubuntu; then
        stop_service tgt
    else
        stop_service tgtd
    fi

    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
        local be be_name be_type
        for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
            be_type=${be%%:*}
            be_name=${be##*:}
            if type cleanup_cinder_backend_${be_type} >/dev/null 2>&1; then
                cleanup_cinder_backend_${be_type} ${be_name}
            fi
        done
    fi
}

# configure_cinder_rootwrap() - configure Cinder's rootwrap
function configure_cinder_rootwrap {
    # Set the paths of certain binaries
    local cinder_rootwrap=$(get_rootwrap_location cinder)

    # Deploy new rootwrap filters files (owned by root).
    # Wipe any existing rootwrap.d files first
    if [[ -d $CINDER_CONF_DIR/rootwrap.d ]]; then
        sudo rm -rf $CINDER_CONF_DIR/rootwrap.d
    fi
    # Deploy filters to /etc/cinder/rootwrap.d
    sudo mkdir -m 755 $CINDER_CONF_DIR/rootwrap.d
    sudo cp $CINDER_DIR/etc/cinder/rootwrap.d/*.filters $CINDER_CONF_DIR/rootwrap.d
    sudo chown -R root:root $CINDER_CONF_DIR/rootwrap.d
    sudo chmod 644 $CINDER_CONF_DIR/rootwrap.d/*
    # Set up rootwrap.conf, pointing to /etc/cinder/rootwrap.d
    sudo cp $CINDER_DIR/etc/cinder/rootwrap.conf $CINDER_CONF_DIR/
    sudo sed -e "s:^filters_path=.*$:filters_path=$CINDER_CONF_DIR/rootwrap.d:" -i $CINDER_CONF_DIR/rootwrap.conf
    sudo chown root:root $CINDER_CONF_DIR/rootwrap.conf
    sudo chmod 0644 $CINDER_CONF_DIR/rootwrap.conf
    # Specify rootwrap.conf as first parameter to rootwrap
    ROOTWRAP_CSUDOER_CMD="$cinder_rootwrap $CINDER_CONF_DIR/rootwrap.conf *"

    # Set up the rootwrap sudoers for cinder
    local tempfile=`mktemp`
    echo "$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_CSUDOER_CMD" >$tempfile
    chmod 0440 $tempfile
    sudo chown root:root $tempfile
    sudo mv $tempfile /etc/sudoers.d/cinder-rootwrap
}

# configure_cinder() - Set config files, create data dirs, etc
function configure_cinder {
    if [[ ! -d $CINDER_CONF_DIR ]]; then
        sudo mkdir -p $CINDER_CONF_DIR
    fi
    sudo chown $STACK_USER $CINDER_CONF_DIR

    cp -p $CINDER_DIR/etc/cinder/policy.json $CINDER_CONF_DIR

    configure_cinder_rootwrap

    cp $CINDER_DIR/etc/cinder/api-paste.ini $CINDER_API_PASTE_INI

    inicomment $CINDER_API_PASTE_INI filter:authtoken auth_host
    inicomment $CINDER_API_PASTE_INI filter:authtoken auth_port
    inicomment $CINDER_API_PASTE_INI filter:authtoken auth_protocol
    inicomment $CINDER_API_PASTE_INI filter:authtoken cafile
    inicomment $CINDER_API_PASTE_INI filter:authtoken admin_tenant_name
    inicomment $CINDER_API_PASTE_INI filter:authtoken admin_user
    inicomment $CINDER_API_PASTE_INI filter:authtoken admin_password
    inicomment $CINDER_API_PASTE_INI filter:authtoken signing_dir

    configure_auth_token_middleware $CINDER_CONF cinder $CINDER_AUTH_CACHE_DIR

    iniset $CINDER_CONF DEFAULT auth_strategy keystone
    iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $CINDER_CONF DEFAULT verbose True

    iniset $CINDER_CONF DEFAULT my_ip "$CINDER_SERVICE_HOST"
    iniset $CINDER_CONF DEFAULT iscsi_helper tgtadm
    iniset $CINDER_CONF DEFAULT sql_connection `database_connection_url cinder`
    iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI
    iniset $CINDER_CONF DEFAULT rootwrap_config "$CINDER_CONF_DIR/rootwrap.conf"
    iniset $CINDER_CONF DEFAULT osapi_volume_extension cinder.api.contrib.standard_extensions
    iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH
    iniset $CINDER_CONF DEFAULT lock_path $CINDER_STATE_PATH
    iniset $CINDER_CONF DEFAULT periodic_interval $CINDER_PERIODIC_INTERVAL
    # NOTE(thingee): Cinder V1 API is deprecated and defaults to off as of
    # Juno. Keep it enabled so we can continue testing while it's still
    # supported.
    iniset $CINDER_CONF DEFAULT enable_v1_api true

    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
        local enabled_backends=""
        local default_name=""
        local be be_name be_type
        for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
            be_type=${be%%:*}
            be_name=${be##*:}
            if type configure_cinder_backend_${be_type} >/dev/null 2>&1; then
                configure_cinder_backend_${be_type} ${be_name}
            fi
            if [[ -z "$default_name" ]]; then
                default_name=$be_name
            fi
            enabled_backends+=$be_name,
        done
        iniset $CINDER_CONF DEFAULT enabled_backends ${enabled_backends%,*}
        if [[ -n "$default_name" ]]; then
            iniset $CINDER_CONF DEFAULT default_volume_type ${default_name}
        fi
    fi

    if is_service_enabled swift; then
        iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:8080/v1/AUTH_"
    fi

    if is_service_enabled ceilometer; then
        iniset $CINDER_CONF DEFAULT notification_driver "messaging"
    fi

    if is_service_enabled tls-proxy; then
        # Set the service port for a proxy to take the original
        iniset $CINDER_CONF DEFAULT osapi_volume_listen_port $CINDER_SERVICE_PORT_INT
    fi

    if [ "$SYSLOG" != "False" ]; then
        iniset $CINDER_CONF DEFAULT use_syslog True
    fi

    iniset_rpc_backend cinder $CINDER_CONF DEFAULT

    if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then
        iniset $CINDER_CONF DEFAULT secure_delete False
        iniset $CINDER_CONF DEFAULT volume_clear none
    fi

    # Format logging
    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
        setup_colorized_logging $CINDER_CONF DEFAULT "project_id" "user_id"
    fi

    if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
        configure_cinder_driver
    fi

    if [[ is_fedora && $DISTRO =~ (rhel6) ]]; then
        # Cinder clones are slightly larger due to some extra
        # metadata.  RHEL6 will not allow auto-extending of LV's
        # without this, leading to clones giving hard-to-track disk
        # I/O errors.
        # see https://bugzilla.redhat.com/show_bug.cgi?id=975052
        sudo sed -i~ \
            -e 's/snapshot_autoextend_threshold =.*/snapshot_autoextend_threshold = 80/' \
            -e 's/snapshot_autoextend_percent =.*/snapshot_autoextend_percent = 20/' \
            /etc/lvm/lvm.conf
    fi

    iniset $CINDER_CONF DEFAULT osapi_volume_workers "$API_WORKERS"

    iniset $CINDER_CONF DEFAULT glance_api_servers "${GLANCE_SERVICE_PROTOCOL}://${GLANCE_HOSTPORT}"
    if is_ssl_enabled_service glance || is_service_enabled tls-proxy; then
        iniset $CINDER_CONF DEFAULT glance_protocol https
        iniset $CINDER_CONF DEFAULT glance_ca_certificates_file $SSL_BUNDLE_FILE
    fi

    # Register SSL certificates if provided
    if is_ssl_enabled_service cinder; then
        ensure_certificates CINDER

        iniset $CINDER_CONF DEFAULT ssl_cert_file "$CINDER_SSL_CERT"
        iniset $CINDER_CONF DEFAULT ssl_key_file "$CINDER_SSL_KEY"
    fi

}

# create_cinder_accounts() - Set up common required cinder accounts

# Tenant               User       Roles
# ------------------------------------------------------------------
# service              cinder     admin        # if enabled

# Migrated from keystone_data.sh
function create_cinder_accounts {

    local service_tenant=$(openstack project list | awk "/ $SERVICE_TENANT_NAME / { print \$2 }")
    local admin_role=$(openstack role list | awk "/ admin / { print \$2 }")

    # Cinder
    if [[ "$ENABLED_SERVICES" =~ "c-api" ]]; then

        local cinder_user=$(get_or_create_user "cinder" \
            "$SERVICE_PASSWORD" $service_tenant)
        get_or_add_user_role $admin_role $cinder_user $service_tenant

        if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then

            local cinder_service=$(get_or_create_service "cinder" \
                "volume" "Cinder Volume Service")
            get_or_create_endpoint $cinder_service "$REGION_NAME" \
                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s" \
                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s" \
                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s"

            local cinder_v2_service=$(get_or_create_service "cinderv2" \
                "volumev2" "Cinder Volume Service V2")
            get_or_create_endpoint $cinder_v2_service "$REGION_NAME" \
                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s" \
                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s" \
                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s"
        fi
    fi
}

# create_cinder_cache_dir() - Part of the init_cinder() process
function create_cinder_cache_dir {
    # Create cache dir
    sudo mkdir -p $CINDER_AUTH_CACHE_DIR
    sudo chown $STACK_USER $CINDER_AUTH_CACHE_DIR
    rm -f $CINDER_AUTH_CACHE_DIR/*
}

# init_cinder() - Initialize database and volume group
# Uses global ``NOVA_ENABLED_APIS``
function init_cinder {
    # Force nova volumes off
    NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/osapi_volume,//")

    if is_service_enabled $DATABASE_BACKENDS; then
        # (Re)create cinder database
        recreate_database cinder utf8

        # Migrate cinder database
        $CINDER_BIN_DIR/cinder-manage db sync
    fi

    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
        local be be_name be_type
        for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
            be_type=${be%%:*}
            be_name=${be##*:}
            if type init_cinder_backend_${be_type} >/dev/null 2>&1; then
                init_cinder_backend_${be_type} ${be_name}
            fi
        done
    fi

    mkdir -p $CINDER_STATE_PATH/volumes
    create_cinder_cache_dir
}

# install_cinder() - Collect source and prepare
function install_cinder {
    git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH
    setup_develop $CINDER_DIR
}

# install_cinderclient() - Collect source and prepare
function install_cinderclient {
    if use_library_from_git "python-cinderclient"; then
        git_clone_by_name "python-cinderclient"
        setup_dev_lib "python-cinderclient"
        sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-cinderclient"]}/tools/,/etc/bash_completion.d/}cinder.bash_completion
    fi
}

# apply config.d approach for cinder volumes directory
function _configure_tgt_for_config_d {
    if [[ ! -d /etc/tgt/stack.d/ ]]; then
        sudo ln -sf $CINDER_STATE_PATH/volumes /etc/tgt/stack.d
        echo "include /etc/tgt/stack.d/*" | sudo tee -a /etc/tgt/targets.conf
    fi
}

# start_cinder() - Start running processes, including screen
function start_cinder {
    local service_port=$CINDER_SERVICE_PORT
    local service_protocol=$CINDER_SERVICE_PROTOCOL
    if is_service_enabled tls-proxy; then
        service_port=$CINDER_SERVICE_PORT_INT
        service_protocol="http"
    fi
    if is_service_enabled c-vol; then
        # Delete any old stack.conf
        sudo rm -f /etc/tgt/conf.d/stack.conf
        _configure_tgt_for_config_d
        if is_ubuntu; then
            sudo service tgt restart
        elif is_fedora; then
            if [[ $DISTRO =~ (rhel6) ]]; then
                sudo /sbin/service tgtd restart
            else
                # bypass redirection to systemctl during restart
                sudo /sbin/service --skip-redirect tgtd restart
            fi
        elif is_suse; then
            restart_service tgtd
        else
            # note for other distros: unstack.sh also uses the tgt/tgtd service
            # name, and would need to be adjusted too
            exit_distro_not_supported "restarting tgt"
        fi
        # NOTE(gfidente): ensure tgtd is running in debug mode
        sudo tgtadm --mode system --op update --name debug --value on
    fi

    run_process c-api "$CINDER_BIN_DIR/cinder-api --config-file $CINDER_CONF"
    echo "Waiting for Cinder API to start..."
    if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$CINDER_SERVICE_HOST:$service_port; then
        die $LINENO "c-api did not start"
    fi

    run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"

    # NOTE(jdg): For cinder, startup order matters.  To ensure that repor_capabilities is received
    # by the scheduler start the cinder-volume service last (or restart it) after the scheduler
    # has started.  This is a quick fix for lp bug/1189595

    # Start proxies if enabled
    if is_service_enabled c-api && is_service_enabled tls-proxy; then
        start_tls_proxy '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT &
    fi
}

# stop_cinder() - Stop running processes
function stop_cinder {
    # Kill the cinder screen windows
    local serv
    for serv in c-api c-bak c-sch c-vol; do
        stop_process $serv
    done

    if is_service_enabled c-vol; then
        if is_ubuntu; then
            stop_service tgt
        else
            stop_service tgtd
        fi
    fi
}

# create_volume_types() - Create Cinder's configured volume types
function create_volume_types {
    # Create volume types
    if is_service_enabled c-api && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
        local be be_name be_type
        for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
            be_type=${be%%:*}
            be_name=${be##*:}
            # openstack volume type create --property volume_backend_name="${be_type}" ${be_name}
            cinder type-create ${be_name} && \
                cinder type-key ${be_name} set volume_backend_name="${be_name}"
        done
    fi
}

# Compatibility for Grenade

function create_cinder_volume_group {
    # During a transition period Grenade needs to have this function defined
    # It is effectively a no-op in the Grenade 'target' use case
    :
}


# Restore xtrace
$XTRACE

# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End: