lib/keystone

# lib/keystone
# Functions to control the configuration and operation of **Keystone**

# Dependencies:
#
# - ``functions`` file
# - ``tls`` file
# - ``DEST``, ``STACK_USER``
# - ``IDENTITY_API_VERSION``
# - ``BASE_SQL_CONN``
# - ``SERVICE_HOST``, ``SERVICE_PROTOCOL``
# - ``SERVICE_TOKEN``
# - ``S3_SERVICE_PORT`` (template backend only)

# ``stack.sh`` calls the entry points in this order:
#
# - install_keystone
# - configure_keystone
# - _config_keystone_apache_wsgi
# - init_keystone
# - start_keystone
# - create_keystone_accounts
# - stop_keystone
# - cleanup_keystone
# - _cleanup_keystone_apache_wsgi

# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace

# Defaults
# --------

# Set up default directories
KEYSTONE_DIR=$DEST/keystone
KEYSTONE_CONF_DIR=${KEYSTONE_CONF_DIR:-/etc/keystone}
KEYSTONE_CONF=$KEYSTONE_CONF_DIR/keystone.conf
KEYSTONE_PASTE_INI=${KEYSTONE_PASTE_INI:-$KEYSTONE_CONF_DIR/keystone-paste.ini}
KEYSTONE_AUTH_CACHE_DIR=${KEYSTONE_AUTH_CACHE_DIR:-/var/cache/keystone}
KEYSTONE_WSGI_DIR=${KEYSTONE_WSGI_DIR:-/var/www/keystone}

KEYSTONECLIENT_DIR=$DEST/python-keystoneclient

# Set up additional extensions, such as oauth1, federation
# Example of KEYSTONE_EXTENSIONS=oauth1,federation
KEYSTONE_EXTENSIONS=${KEYSTONE_EXTENSIONS:-}

# Select the backend for Keystone's service catalog
KEYSTONE_CATALOG_BACKEND=${KEYSTONE_CATALOG_BACKEND:-sql}
KEYSTONE_CATALOG=$KEYSTONE_CONF_DIR/default_catalog.templates

# Select the backend for Tokens
KEYSTONE_TOKEN_BACKEND=${KEYSTONE_TOKEN_BACKEND:-sql}

# Select the backend for Identity
KEYSTONE_IDENTITY_BACKEND=${KEYSTONE_IDENTITY_BACKEND:-sql}

# Select the backend for Assignment
KEYSTONE_ASSIGNMENT_BACKEND=${KEYSTONE_ASSIGNMENT_BACKEND:-sql}

# Select Keystone's token format
# Choose from 'UUID', 'PKI', or 'PKIZ'
KEYSTONE_TOKEN_FORMAT=$(echo ${KEYSTONE_TOKEN_FORMAT} | tr '[:upper:]' '[:lower:]')

# Set Keystone interface configuration
KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
KEYSTONE_AUTH_PORT_INT=${KEYSTONE_AUTH_PORT_INT:-35358}
KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}

# Public facing bits
KEYSTONE_SERVICE_HOST=${KEYSTONE_SERVICE_HOST:-$SERVICE_HOST}
KEYSTONE_SERVICE_PORT=${KEYSTONE_SERVICE_PORT:-5000}
KEYSTONE_SERVICE_PORT_INT=${KEYSTONE_SERVICE_PORT_INT:-5001}
KEYSTONE_SERVICE_PROTOCOL=${KEYSTONE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}

# Bind hosts
KEYSTONE_ADMIN_BIND_HOST=${KEYSTONE_ADMIN_BIND_HOST:-$KEYSTONE_SERVICE_HOST}
# Set the tenant for service accounts in Keystone
SERVICE_TENANT_NAME=${SERVICE_TENANT_NAME:-service}

# valid identity backends as per dir keystone/identity/backends
KEYSTONE_VALID_IDENTITY_BACKENDS=kvs,ldap,pam,sql

# valid assignment backends as per dir keystone/identity/backends
KEYSTONE_VALID_ASSIGNMENT_BACKENDS=kvs,ldap,sql

# if we are running with SSL use https protocols
if is_ssl_enabled_service "key"; then
    KEYSTONE_AUTH_PROTOCOL="https"
    KEYSTONE_SERVICE_PROTOCOL="https"
fi

# complete URIs
KEYSTONE_AUTH_URI=${KEYSTONE_AUTH_PROTOCOL}://${KEYSTONE_AUTH_HOST}:${KEYSTONE_AUTH_PORT}
KEYSTONE_SERVICE_URI=${KEYSTONE_SERVICE_PROTOCOL}://${KEYSTONE_SERVICE_HOST}:${KEYSTONE_SERVICE_PORT}

# Functions
# ---------
# cleanup_keystone() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_keystone {
    # kill instances (nova)
    # delete image files (glance)
    # This function intentionally left blank
    :
}

# _cleanup_keystone_apache_wsgi() - Remove wsgi files, disable and remove apache vhost file
function _cleanup_keystone_apache_wsgi {
    sudo rm -f $KEYSTONE_WSGI_DIR/*.wsgi
    disable_apache_site keystone
    sudo rm -f $(apache_site_config_for keystone)
}

# _config_keystone_apache_wsgi() - Set WSGI config files of Keystone
function _config_keystone_apache_wsgi {
    sudo mkdir -p $KEYSTONE_WSGI_DIR

    local keystone_apache_conf=$(apache_site_config_for keystone)

    # copy proxy vhost and wsgi file
    sudo cp $KEYSTONE_DIR/httpd/keystone.py $KEYSTONE_WSGI_DIR/main
    sudo cp $KEYSTONE_DIR/httpd/keystone.py $KEYSTONE_WSGI_DIR/admin

    sudo cp $FILES/apache-keystone.template $keystone_apache_conf
    sudo sed -e "
        s|%PUBLICPORT%|$KEYSTONE_SERVICE_PORT|g;
        s|%ADMINPORT%|$KEYSTONE_AUTH_PORT|g;
        s|%APACHE_NAME%|$APACHE_NAME|g;
        s|%PUBLICWSGI%|$KEYSTONE_WSGI_DIR/main|g;
        s|%ADMINWSGI%|$KEYSTONE_WSGI_DIR/admin|g;
        s|%USER%|$STACK_USER|g
    " -i $keystone_apache_conf
    enable_apache_site keystone
}

# configure_keystone() - Set config files, create data dirs, etc
function configure_keystone {
    if [[ ! -d $KEYSTONE_CONF_DIR ]]; then
        sudo mkdir -p $KEYSTONE_CONF_DIR
    fi
    sudo chown $STACK_USER $KEYSTONE_CONF_DIR

    if [[ "$KEYSTONE_CONF_DIR" != "$KEYSTONE_DIR/etc" ]]; then
        cp -p $KEYSTONE_DIR/etc/keystone.conf.sample $KEYSTONE_CONF
        chmod 600 $KEYSTONE_CONF
        cp -p $KEYSTONE_DIR/etc/policy.json $KEYSTONE_CONF_DIR
        if [[ -f "$KEYSTONE_DIR/etc/keystone-paste.ini" ]]; then
            cp -p "$KEYSTONE_DIR/etc/keystone-paste.ini" "$KEYSTONE_PASTE_INI"
        fi
    fi
    if [[ -f "$KEYSTONE_PASTE_INI" ]]; then
        iniset "$KEYSTONE_CONF" paste_deploy config_file "$KEYSTONE_PASTE_INI"
    else
        # compatibility with mixed cfg and paste.deploy configuration
        KEYSTONE_PASTE_INI="$KEYSTONE_CONF"
    fi

    configure_keystone_extensions

    # Rewrite stock ``keystone.conf``

    if is_service_enabled ldap; then
        #Set all needed ldap values
        iniset $KEYSTONE_CONF ldap password $LDAP_PASSWORD
        iniset $KEYSTONE_CONF ldap user $LDAP_MANAGER_DN
        iniset $KEYSTONE_CONF ldap suffix $LDAP_BASE_DN
        iniset $KEYSTONE_CONF ldap use_dumb_member "True"
        iniset $KEYSTONE_CONF ldap user_attribute_ignore "enabled,email,tenants,default_project_id"
        iniset $KEYSTONE_CONF ldap tenant_attribute_ignore "enabled"
        iniset $KEYSTONE_CONF ldap tenant_domain_id_attribute "businessCategory"
        iniset $KEYSTONE_CONF ldap tenant_desc_attribute "description"
        iniset $KEYSTONE_CONF ldap tenant_tree_dn "ou=Projects,$LDAP_BASE_DN"
        iniset $KEYSTONE_CONF ldap user_domain_id_attribute "businessCategory"
        iniset $KEYSTONE_CONF ldap user_tree_dn "ou=Users,$LDAP_BASE_DN"
        iniset $KEYSTONE_CONF DEFAULT member_role_id "9fe2ff9ee4384b1894a90878d3e92bab"
        iniset $KEYSTONE_CONF DEFAULT member_role_name "_member_"
    fi

    # check if identity backend is valid
    if [[ "$KEYSTONE_VALID_IDENTITY_BACKENDS" =~ "$KEYSTONE_IDENTITY_BACKEND" ]]; then
        iniset $KEYSTONE_CONF identity driver "keystone.identity.backends.$KEYSTONE_IDENTITY_BACKEND.Identity"
    fi

    # check if assignment backend is valid
    if [[ "$KEYSTONE_VALID_ASSIGNMENT_BACKENDS" =~ "$KEYSTONE_ASSIGNMENT_BACKEND" ]]; then
        iniset $KEYSTONE_CONF assignment driver "keystone.assignment.backends.$KEYSTONE_ASSIGNMENT_BACKEND.Assignment"
    fi

    # Set the URL advertised in the ``versions`` structure returned by the '/' route
    iniset $KEYSTONE_CONF DEFAULT public_endpoint "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:%(public_port)s/"
    iniset $KEYSTONE_CONF DEFAULT admin_endpoint "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:%(admin_port)s/"
    iniset $KEYSTONE_CONF DEFAULT admin_bind_host "$KEYSTONE_ADMIN_BIND_HOST"

    # Register SSL certificates if provided
    if is_ssl_enabled_service key; then
        ensure_certificates KEYSTONE

        iniset $KEYSTONE_CONF ssl enable True
        iniset $KEYSTONE_CONF ssl certfile $KEYSTONE_SSL_CERT
        iniset $KEYSTONE_CONF ssl keyfile $KEYSTONE_SSL_KEY
    fi

    if is_service_enabled tls-proxy; then
        # Set the service ports for a proxy to take the originals
        iniset $KEYSTONE_CONF DEFAULT public_port $KEYSTONE_SERVICE_PORT_INT
        iniset $KEYSTONE_CONF DEFAULT admin_port $KEYSTONE_AUTH_PORT_INT
    fi

    iniset $KEYSTONE_CONF DEFAULT admin_token "$SERVICE_TOKEN"

    if [[ "$KEYSTONE_TOKEN_FORMAT" != "" ]]; then
        iniset $KEYSTONE_CONF token provider keystone.token.providers.$KEYSTONE_TOKEN_FORMAT.Provider
    fi

    iniset $KEYSTONE_CONF database connection `database_connection_url keystone`
    iniset $KEYSTONE_CONF ec2 driver "keystone.contrib.ec2.backends.sql.Ec2"

    if [[ "$KEYSTONE_TOKEN_BACKEND" = "sql" ]]; then
        iniset $KEYSTONE_CONF token driver keystone.token.backends.sql.Token
    elif [[ "$KEYSTONE_TOKEN_BACKEND" = "memcache" ]]; then
        iniset $KEYSTONE_CONF token driver keystone.token.backends.memcache.Token
    else
        iniset $KEYSTONE_CONF token driver keystone.token.backends.kvs.Token
    fi

    if [[ "$KEYSTONE_CATALOG_BACKEND" = "sql" ]]; then
        # Configure ``keystone.conf`` to use sql
        iniset $KEYSTONE_CONF catalog driver keystone.catalog.backends.sql.Catalog
        inicomment $KEYSTONE_CONF catalog template_file
    else
        cp -p $FILES/default_catalog.templates $KEYSTONE_CATALOG

        # Add swift endpoints to service catalog if swift is enabled
        if is_service_enabled s-proxy; then
            echo "catalog.RegionOne.object_store.publicURL = http://%SERVICE_HOST%:8080/v1/AUTH_\$(tenant_id)s" >> $KEYSTONE_CATALOG
            echo "catalog.RegionOne.object_store.adminURL = http://%SERVICE_HOST%:8080/" >> $KEYSTONE_CATALOG
            echo "catalog.RegionOne.object_store.internalURL = http://%SERVICE_HOST%:8080/v1/AUTH_\$(tenant_id)s" >> $KEYSTONE_CATALOG
            echo "catalog.RegionOne.object_store.name = Swift Service" >> $KEYSTONE_CATALOG
        fi

        # Add neutron endpoints to service catalog if neutron is enabled
        if is_service_enabled neutron; then
            echo "catalog.RegionOne.network.publicURL = http://%SERVICE_HOST%:$Q_PORT/" >> $KEYSTONE_CATALOG
            echo "catalog.RegionOne.network.adminURL = http://%SERVICE_HOST%:$Q_PORT/" >> $KEYSTONE_CATALOG
            echo "catalog.RegionOne.network.internalURL = http://%SERVICE_HOST%:$Q_PORT/" >> $KEYSTONE_CATALOG
            echo "catalog.RegionOne.network.name = Neutron Service" >> $KEYSTONE_CATALOG
        fi

        sed -e "
            s,%SERVICE_HOST%,$SERVICE_HOST,g;
            s,%S3_SERVICE_PORT%,$S3_SERVICE_PORT,g;
        " -i $KEYSTONE_CATALOG

        # Configure ``keystone.conf`` to use templates
        iniset $KEYSTONE_CONF catalog driver "keystone.catalog.backends.templated.TemplatedCatalog"
        iniset $KEYSTONE_CONF catalog template_file "$KEYSTONE_CATALOG"
    fi

    # Set up logging
    if [ "$SYSLOG" != "False" ]; then
        iniset $KEYSTONE_CONF DEFAULT use_syslog "True"
    fi

    # Format logging
    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ] &&  ! is_apache_enabled_service key ; then
        setup_colorized_logging $KEYSTONE_CONF DEFAULT
    fi

    if is_apache_enabled_service key; then
        iniset $KEYSTONE_CONF DEFAULT debug "True"
        # Eliminate the %(asctime)s.%(msecs)03d from the log format strings
        iniset $KEYSTONE_CONF DEFAULT logging_context_format_string "%(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s"
        iniset $KEYSTONE_CONF DEFAULT logging_default_format_string "%(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s"
        iniset $KEYSTONE_CONF DEFAULT logging_debug_format_suffix "%(funcName)s %(pathname)s:%(lineno)d"
        iniset $KEYSTONE_CONF DEFAULT logging_exception_prefix "%(process)d TRACE %(name)s %(instance)s"
        _config_keystone_apache_wsgi
    fi
}

function configure_keystone_extensions {
    # Add keystone extension into keystone v3 application pipeline
    local extension_value
    local api_v3
    local extension
    local api_v3_extension
    for extension_value in ${KEYSTONE_EXTENSIONS//,/ }; do
        if [[ -z "${extension_value}" ]]; then
            continue
        fi
        api_v3=$(iniget $KEYSTONE_PASTE_INI pipeline:api_v3 pipeline)
        extension=$(echo $api_v3 | sed -ne "/${extension_value}/ p;" )
        if [[ -z $extension ]]; then
            api_v3_extension=$(echo $api_v3 | sed -ne "s/service_v3/${extension_value}_extension service_v3/p;" )
            iniset $KEYSTONE_PASTE_INI pipeline:api_v3 pipeline "$api_v3_extension"
        fi
    done
}

# create_keystone_accounts() - Sets up common required keystone accounts

# Tenant               User       Roles
# ------------------------------------------------------------------
# admin                admin      admin
# service              --         --
# --                   --         service
# --                   --         ResellerAdmin
# --                   --         Member
# demo                 admin      admin
# demo                 demo       Member, anotherrole
# invisible_to_admin   demo       Member

# Migrated from keystone_data.sh
function create_keystone_accounts {

    # admin
    ADMIN_TENANT=$(openstack project create \
        admin \
        | grep " id " | get_field 2)
    ADMIN_USER=$(openstack user create \
        admin \
        --project "$ADMIN_TENANT" \
        --email admin@example.com \
        --password "$ADMIN_PASSWORD" \
        | grep " id " | get_field 2)
    ADMIN_ROLE=$(openstack role create \
        admin \
        | grep " id " | get_field 2)
    openstack role add \
        $ADMIN_ROLE \
        --project $ADMIN_TENANT \
        --user $ADMIN_USER

    # Create service project/role
    openstack project create $SERVICE_TENANT_NAME

    # Service role, so service users do not have to be admins
    openstack role create service

    # The ResellerAdmin role is used by Nova and Ceilometer so we need to keep it.
    # The admin role in swift allows a user to act as an admin for their tenant,
    # but ResellerAdmin is needed for a user to act as any tenant. The name of this
    # role is also configurable in swift-proxy.conf
    openstack role create ResellerAdmin

    # The Member role is used by Horizon and Swift so we need to keep it:
    MEMBER_ROLE=$(openstack role create \
        Member \
        | grep " id " | get_field 2)
    # ANOTHER_ROLE demonstrates that an arbitrary role may be created and used
    # TODO(sleepsonthefloor): show how this can be used for rbac in the future!
    ANOTHER_ROLE=$(openstack role create \
        anotherrole \
        | grep " id " | get_field 2)

    # invisible tenant - admin can't see this one
    INVIS_TENANT=$(openstack project create \
        invisible_to_admin \
        | grep " id " | get_field 2)

    # demo
    DEMO_TENANT=$(openstack project create \
        demo \
        | grep " id " | get_field 2)
    DEMO_USER=$(openstack user create \
        demo \
        --project $DEMO_TENANT \
        --email demo@example.com \
        --password "$ADMIN_PASSWORD" \
        | grep " id " | get_field 2)

    openstack role add --project $DEMO_TENANT --user $DEMO_USER $MEMBER_ROLE
    openstack role add --project $DEMO_TENANT --user $ADMIN_USER $ADMIN_ROLE
    openstack role add --project $DEMO_TENANT --user $DEMO_USER $ANOTHER_ROLE
    openstack role add --project $INVIS_TENANT --user $DEMO_USER $MEMBER_ROLE

    # Keystone
    if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then
        KEYSTONE_SERVICE=$(openstack service create \
            keystone \
            --type identity \
            --description "Keystone Identity Service" \
            | grep " id " | get_field 2)
        openstack endpoint create \
            $KEYSTONE_SERVICE \
            --region RegionOne \
            --publicurl "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_SERVICE_PORT/v$IDENTITY_API_VERSION" \
            --adminurl "$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v$IDENTITY_API_VERSION" \
            --internalurl "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_SERVICE_PORT/v$IDENTITY_API_VERSION"
    fi
}

# Configure the API version for the OpenStack projects.
# configure_API_version conf_file version
function configure_API_version {
    local conf_file=$1
    local api_version=$2
    iniset $conf_file keystone_authtoken auth_uri $KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_SERVICE_PORT/v$api_version
}

# init_keystone() - Initialize databases, etc.
function init_keystone {
    if is_service_enabled ldap; then
        init_ldap
    fi

    # (Re)create keystone database
    recreate_database keystone utf8

    # Initialize keystone database
    $KEYSTONE_DIR/bin/keystone-manage db_sync

    local extension_value
    for extension_value in ${KEYSTONE_EXTENSIONS//,/ }; do
        if [[ -z "${extension_value}" ]]; then
            continue
        fi
        $KEYSTONE_DIR/bin/keystone-manage db_sync --extension "${extension_value}"
    done

    if [[ "$KEYSTONE_TOKEN_FORMAT" != "uuid" ]]; then
        # Set up certificates
        rm -rf $KEYSTONE_CONF_DIR/ssl
        $KEYSTONE_DIR/bin/keystone-manage pki_setup

        # Create cache dir
        sudo mkdir -p $KEYSTONE_AUTH_CACHE_DIR
        sudo chown $STACK_USER $KEYSTONE_AUTH_CACHE_DIR
        rm -f $KEYSTONE_AUTH_CACHE_DIR/*
    fi
}

# install_keystoneclient() - Collect source and prepare
function install_keystoneclient {
    git_clone $KEYSTONECLIENT_REPO $KEYSTONECLIENT_DIR $KEYSTONECLIENT_BRANCH
    setup_develop $KEYSTONECLIENT_DIR
    sudo install -D -m 0644 -o $STACK_USER {$KEYSTONECLIENT_DIR/tools/,/etc/bash_completion.d/}keystone.bash_completion
}

# install_keystone() - Collect source and prepare
function install_keystone {
    # only install ldap if the service has been enabled
    if is_service_enabled ldap; then
        install_ldap
    fi
    if [[ "$KEYSTONE_TOKEN_BACKEND" = "memcache" ]]; then
        # Install memcached and the memcache Python library that keystone uses.
        # Unfortunately the Python library goes by different names in the .deb
        # and .rpm circles.
        install_package memcached
        if is_ubuntu; then
            install_package python-memcache
        else
            install_package python-memcached
        fi
    fi
    git_clone $KEYSTONE_REPO $KEYSTONE_DIR $KEYSTONE_BRANCH
    setup_develop $KEYSTONE_DIR
    if is_apache_enabled_service key; then
        install_apache_wsgi
    fi
}

# start_keystone() - Start running processes, including screen
function start_keystone {
    # Get right service port for testing
    local service_port=$KEYSTONE_SERVICE_PORT
    if is_service_enabled tls-proxy; then
        service_port=$KEYSTONE_SERVICE_PORT_INT
    fi

    if is_apache_enabled_service key; then
        restart_apache_server
        screen_it key "cd $KEYSTONE_DIR && sudo tail -f /var/log/$APACHE_NAME/keystone"
    else
        # Start Keystone in a screen window
        screen_it key "cd $KEYSTONE_DIR && $KEYSTONE_DIR/bin/keystone-all --config-file $KEYSTONE_CONF --debug"
    fi

    echo "Waiting for keystone to start..."
    # Check that the keystone service is running. Even if the tls tunnel
    # should be enabled, make sure the internal port is checked using
    # unencryted traffic at this point.
    if ! timeout $SERVICE_TIMEOUT sh -c "while ! curl --noproxy '*' -k -s http://$KEYSTONE_SERVICE_HOST:$service_port/v$IDENTITY_API_VERSION/ >/dev/null; do sleep 1; done"; then
        die $LINENO "keystone did not start"
    fi

    # Start proxies if enabled
    if is_service_enabled tls-proxy; then
        start_tls_proxy '*' $KEYSTONE_SERVICE_PORT $KEYSTONE_SERVICE_HOST $KEYSTONE_SERVICE_PORT_INT &
        start_tls_proxy '*' $KEYSTONE_AUTH_PORT $KEYSTONE_AUTH_HOST $KEYSTONE_AUTH_PORT_INT &
    fi
}

# stop_keystone() - Stop running processes
function stop_keystone {
    # Kill the Keystone screen window
    screen_stop key
    # Cleanup the WSGI files and VHOST
    _cleanup_keystone_apache_wsgi
}


# Restore xtrace
$XTRACE

# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End: