Merge "Fix neutron log format with colorization"
diff --git a/.gitignore b/.gitignore
index 0c22c6b..a3d5b0d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,5 +13,6 @@
accrc
.stackenv
.prereqs
+devstack-docs-*
docs/
docs-files
diff --git a/driver_certs/cinder_driver_cert.sh b/driver_certs/cinder_driver_cert.sh
new file mode 100755
index 0000000..18bef8b
--- /dev/null
+++ b/driver_certs/cinder_driver_cert.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# **cinder_cert.sh**
+
+CERT_DIR=$(cd $(dirname "$0") && pwd)
+TOP_DIR=$(cd $CERT_DIR/..; pwd)
+
+source $TOP_DIR/functions
+source $TOP_DIR/stackrc
+source $TOP_DIR/openrc
+source $TOP_DIR/lib/tempest
+source $TOP_DIR/lib/cinder
+
+TEMPFILE=`mktemp`
+RECLONE=True
+
+function log_message() {
+ MESSAGE=$1
+ STEP_HEADER=$2
+ if [[ "$STEP_HEADER" = "True" ]]; then
+ echo -e "\n========================================================" | tee -a $TEMPFILE
+ fi
+ echo -e `date +%m/%d/%y/%T:`"${MESSAGE}" | tee -a $TEMPFILE
+ if [[ "$STEP_HEADER" = "True" ]]; then
+ echo -e "========================================================" | tee -a $TEMPFILE
+ fi
+}
+
+if [[ "$OFFLINE" = "True" ]]; then
+ echo "ERROR: Driver cert requires fresh clone/pull from ${CINDER_BRANCH}"
+ echo " Please set OFFLINE=False and retry."
+ exit 1
+fi
+
+log_message "RUNNING CINDER DRIVER CERTIFICATION CHECK", True
+log_message "Output is being logged to: $TEMPFILE"
+
+cd $CINDER_DIR
+log_message "Cloning to ${CINDER_REPO}...", True
+install_cinder
+
+log_message "Pull a fresh Clone of cinder repo...", True
+git status | tee -a $TEMPFILE
+git log --pretty=oneline -n 1 | tee -a $TEMPFILE
+
+log_message "Gathering copy of cinder.conf file (passwords will be scrubbed)...", True
+cat /etc/cinder/cinder.conf | egrep -v "(^#.*|^$)" | tee -a $TEMPFILE
+sed -i "s/\(.*password.*=\).*$/\1 xxx/i" $TEMPFILE
+log_message "End of cinder.conf.", True
+
+cd $TOP_DIR
+# Verify tempest is installed/enabled
+if ! is_service_enabled tempest; then
+ log_message "ERROR!!! Cert requires tempest in enabled_services!", True
+ log_message" Please add tempest to enabled_services and retry."
+ exit 1
+fi
+
+cd $TEMPEST_DIR
+install_tempest
+
+log_message "Verify tempest is current....", True
+git status | tee -a $TEMPFILE
+log_message "Check status and get latest commit..."
+git log --pretty=oneline -n 1 | tee -a $TEMPFILE
+
+
+#stop and restart cinder services
+log_message "Restart Cinder services...", True
+stop_cinder
+sleep 1
+start_cinder
+sleep 5
+
+# run tempest api/volume/test_*
+log_message "Run the actual tempest volume tests (run_tests.sh -N tempest.api.volume.test_*)...", True
+exec 2> >(tee -a $TEMPFILE)
+`./run_tests.sh -N tempest.api.volume.test_*`
+if [[ $? = 0 ]]; then
+ log_message "CONGRATULATIONS!!! Device driver PASSED!", True
+ log_message "Submit output: ($TEMPFILE)"
+ exit 0
+else
+ log_message "SORRY!!! Device driver FAILED!", True
+ log_message "Check output in $TEMPFILE"
+ exit 1
+fi
diff --git a/lib/glance b/lib/glance
index eb727f1..f40b1a7 100644
--- a/lib/glance
+++ b/lib/glance
@@ -172,7 +172,7 @@
recreate_database glance utf8
# Migrate glance database
- $GLANCE_BIN_DIR/glance-manage db_sync
+ $GLANCE_BIN_DIR/glance-manage db sync
create_glance_cache_dir
}
diff --git a/lib/keystone b/lib/keystone
index 4353eba..c1fa0af 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -179,7 +179,6 @@
fi
iniset $KEYSTONE_CONF DEFAULT admin_token "$SERVICE_TOKEN"
- iniset $KEYSTONE_CONF signing token_format "$KEYSTONE_TOKEN_FORMAT"
if [[ "$KEYSTONE_TOKEN_FORMAT" = "UUID" ]]; then
iniset $KEYSTONE_CONF token provider keystone.token.providers.uuid.Provider
@@ -190,6 +189,8 @@
if [[ "$KEYSTONE_TOKEN_BACKEND" = "sql" ]]; then
iniset $KEYSTONE_CONF token driver keystone.token.backends.sql.Token
+ elif [[ "$KEYSTONE_TOKEN_BACKEND" = "memcache" ]]; then
+ iniset $KEYSTONE_CONF token driver keystone.token.backends.memcache.Token
else
iniset $KEYSTONE_CONF token driver keystone.token.backends.kvs.Token
fi
@@ -350,6 +351,17 @@
if is_service_enabled ldap; then
install_ldap
fi
+ if [[ "$KEYSTONE_TOKEN_BACKEND" = "memcache" ]]; then
+ # Install memcached and the memcache Python library that keystone uses.
+ # Unfortunately the Python library goes by different names in the .deb
+ # and .rpm circles.
+ install_package memcached
+ if is_ubuntu; then
+ install_package python-memcache
+ else
+ install_package python-memcached
+ fi
+ fi
git_clone $KEYSTONE_REPO $KEYSTONE_DIR $KEYSTONE_BRANCH
setup_develop $KEYSTONE_DIR
if is_apache_enabled_service key; then
diff --git a/lib/neutron b/lib/neutron
index 1c01d05..7f1a9d8 100644
--- a/lib/neutron
+++ b/lib/neutron
@@ -17,6 +17,7 @@
# - configure_neutron_third_party
# - init_neutron_third_party
# - start_neutron_third_party
+# - create_neutron_cache_dir
# - create_nova_conf_neutron
# - start_neutron_service_and_check
# - create_neutron_initial_network
@@ -296,6 +297,14 @@
fi
}
+# create_neutron_cache_dir() - Part of the _neutron_setup_keystone() process
+function create_neutron_cache_dir() {
+ # Create cache dir
+ sudo mkdir -p $NEUTRON_AUTH_CACHE_DIR
+ sudo chown $STACK_USER $NEUTRON_AUTH_CACHE_DIR
+ rm -f $NEUTRON_AUTH_CACHE_DIR/*
+}
+
# create_neutron_accounts() - Set up common required neutron accounts
# Tenant User Roles
@@ -782,9 +791,7 @@
if [[ -z $skip_auth_cache ]]; then
iniset $conf_file $section signing_dir $NEUTRON_AUTH_CACHE_DIR
# Create cache dir
- sudo mkdir -p $NEUTRON_AUTH_CACHE_DIR
- sudo chown $STACK_USER $NEUTRON_AUTH_CACHE_DIR
- rm -f $NEUTRON_AUTH_CACHE_DIR/*
+ create_neutron_cache_dir
fi
}
diff --git a/lib/nova_plugins/hypervisor-powervm b/lib/nova_plugins/hypervisor-powervm
deleted file mode 100644
index 561dd9f..0000000
--- a/lib/nova_plugins/hypervisor-powervm
+++ /dev/null
@@ -1,76 +0,0 @@
-# lib/nova_plugins/hypervisor-powervm
-# Configure the PowerVM hypervisor
-
-# Enable with:
-# VIRT_DRIVER=powervm
-
-# Dependencies:
-# ``functions`` file
-# ``nova`` configuration
-
-# install_nova_hypervisor - install any external requirements
-# configure_nova_hypervisor - make configuration changes, including those to other services
-# start_nova_hypervisor - start any external services
-# stop_nova_hypervisor - stop any external services
-# cleanup_nova_hypervisor - remove transient data and cache
-
-# Save trace setting
-MY_XTRACE=$(set +o | grep xtrace)
-set +o xtrace
-
-
-# Defaults
-# --------
-
-
-# Entry Points
-# ------------
-
-# clean_nova_hypervisor - Clean up an installation
-function cleanup_nova_hypervisor() {
- # This function intentionally left blank
- :
-}
-
-# configure_nova_hypervisor - Set config files, create data dirs, etc
-function configure_nova_hypervisor() {
- POWERVM_MGR_TYPE=${POWERVM_MGR_TYPE:-"ivm"}
- POWERVM_MGR_HOST=${POWERVM_MGR_HOST:-"powervm.host"}
- POWERVM_MGR_USER=${POWERVM_MGR_USER:-"padmin"}
- POWERVM_MGR_PASSWD=${POWERVM_MGR_PASSWD:-"password"}
- POWERVM_IMG_REMOTE_PATH=${POWERVM_IMG_REMOTE_PATH:-"/tmp"}
- POWERVM_IMG_LOCAL_PATH=${POWERVM_IMG_LOCAL_PATH:-"/tmp"}
- iniset $NOVA_CONF DEFAULT compute_driver nova.virt.powervm.PowerVMDriver
- iniset $NOVA_CONF DEFAULT powervm_mgr_type $POWERVM_MGR_TYPE
- iniset $NOVA_CONF DEFAULT powervm_mgr $POWERVM_MGR_HOST
- iniset $NOVA_CONF DEFAULT powervm_mgr_user $POWERVM_MGR_USER
- iniset $NOVA_CONF DEFAULT powervm_mgr_passwd $POWERVM_MGR_PASSWD
- iniset $NOVA_CONF DEFAULT powervm_img_remote_path $POWERVM_IMG_REMOTE_PATH
- iniset $NOVA_CONF DEFAULT powervm_img_local_path $POWERVM_IMG_LOCAL_PATH
-}
-
-# install_nova_hypervisor() - Install external components
-function install_nova_hypervisor() {
- # This function intentionally left blank
- :
-}
-
-# start_nova_hypervisor - Start any required external services
-function start_nova_hypervisor() {
- # This function intentionally left blank
- :
-}
-
-# stop_nova_hypervisor - Stop any external services
-function stop_nova_hypervisor() {
- # This function intentionally left blank
- :
-}
-
-
-# Restore xtrace
-$MY_XTRACE
-
-# Local variables:
-# mode: shell-script
-# End:
diff --git a/lib/swift b/lib/swift
index c103b5b..40722ab 100644
--- a/lib/swift
+++ b/lib/swift
@@ -96,6 +96,13 @@
SWIFT_REPLICAS=${SWIFT_REPLICAS:-1}
SWIFT_REPLICAS_SEQ=$(seq ${SWIFT_REPLICAS})
+# Set ``SWIFT_LOG_TOKEN_LENGTH`` to configure how many characters of an auth
+# token should be placed in the logs. When keystone is used with PKI tokens,
+# the token values can be huge, seemingly larger the 2K, at the least. We
+# restrict it here to a default of 12 characters, which should be enough to
+# trace through the logs when looking for its use.
+SWIFT_LOG_TOKEN_LENGTH=${SWIFT_LOG_TOKEN_LENGTH:-12}
+
# Set ``OBJECT_PORT_BASE``, ``CONTAINER_PORT_BASE``, ``ACCOUNT_PORT_BASE``
# Port bases used in port number calclution for the service "nodes"
# The specified port number will be used, the additinal ports calculated by
@@ -281,6 +288,9 @@
SWIFT_EXTRAS_MIDDLEWARE_LAST="${SWIFT_EXTRAS_MIDDLEWARE_LAST} ceilometer"
fi
+ # Restrict the length of auth tokens in the swift proxy-server logs.
+ iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:proxy-logging reveal_sensitive_prefix ${SWIFT_LOG_TOKEN_LENGTH}
+
# By default Swift will be installed with keystone and tempauth middleware
# and add the swift3 middleware if its configured for it. The token for
# tempauth would be prefixed with the reseller_prefix setting `TEMPAUTH_` the
@@ -311,6 +321,10 @@
iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken admin_user swift
iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken admin_password $SERVICE_PASSWORD
iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken signing_dir $SWIFT_AUTH_CACHE_DIR
+ # This causes the authtoken middleware to use the same python logging
+ # adapter provided by the swift proxy-server, so that request transaction
+ # IDs will included in all of its log messages.
+ iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:authtoken log_name swift
iniuncomment ${SWIFT_CONFIG_PROXY_SERVER} filter:keystoneauth use
iniuncomment ${SWIFT_CONFIG_PROXY_SERVER} filter:keystoneauth operator_roles
@@ -336,7 +350,7 @@
cp ${SWIFT_DIR}/etc/swift.conf-sample ${SWIFT_CONF_DIR}/swift.conf
iniset ${SWIFT_CONF_DIR}/swift.conf swift-hash swift_hash_path_suffix ${SWIFT_HASH}
- # This function generates an object/account/proxy configuration
+ # This function generates an object/container/account configuration
# emulating 4 nodes on different ports
function generate_swift_config() {
local swift_node_config=$1
diff --git a/lib/tempest b/lib/tempest
index b0fc9f5..803b740 100644
--- a/lib/tempest
+++ b/lib/tempest
@@ -80,6 +80,7 @@
local public_router_id
local tenant_networks_reachable
local boto_instance_type="m1.tiny"
+ local ssh_connect_method="fixed"
# TODO(afazekas):
# sudo python setup.py deploy
@@ -192,10 +193,13 @@
if [ "$Q_USE_NAMESPACE" != "False" ]; then
tenant_networks_reachable=false
+ ssh_connect_method="floating"
else
tenant_networks_reachable=true
fi
+ ssh_connect_method=${TEMPEST_SSH_CONNECT_METHOD:-$ssh_connect_method}
+
if is_service_enabled q-l3; then
public_network_id=$(neutron net-list | grep $PUBLIC_NETWORK_NAME | \
awk '{print $2}')
@@ -258,6 +262,7 @@
iniset $TEMPEST_CONF compute flavor_ref_alt $flavor_ref_alt
iniset $TEMPEST_CONF compute live_migration_available ${LIVE_MIGRATION_AVAILABLE:-False}
iniset $TEMPEST_CONF compute use_block_migration_for_live_migration ${USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION:-False}
+ iniset $TEMPEST_CONF compute ssh_connect_method $ssh_connect_method
# Compute admin
iniset $TEMPEST_CONF "compute-admin" password "$password" # DEPRECATED
@@ -306,7 +311,7 @@
iniset $TEMPEST_CONF cli cli_dir $NOVA_BIN_DIR
# service_available
- for service in nova cinder glance neutron swift heat horizon ceilometer ironic; do
+ for service in nova cinder glance neutron swift heat horizon ceilometer ironic savanna; do
if is_service_enabled $service ; then
iniset $TEMPEST_CONF service_available $service "True"
else
diff --git a/stack.sh b/stack.sh
index 47d93bd..825373e 100755
--- a/stack.sh
+++ b/stack.sh
@@ -747,6 +747,7 @@
if [[ $TRACK_DEPENDS = True ]]; then
$DEST/.venv/bin/pip freeze > $DEST/requires-post-pip
if ! diff -Nru $DEST/requires-pre-pip $DEST/requires-post-pip > $DEST/requires.diff; then
+ echo "Detect some changes for installed packages of pip, in depend tracking mode"
cat $DEST/requires.diff
fi
echo "Ran stack.sh in depend tracking mode, bailing out now"
diff --git a/tools/build_docs.sh b/tools/build_docs.sh
index 1c145e2..c566e63 100755
--- a/tools/build_docs.sh
+++ b/tools/build_docs.sh
@@ -28,22 +28,31 @@
# http://devstack.org is a GitHub gh-pages site in the https://github.com/cloudbuilders/devtack.git repo
GH_PAGES_REPO=git@github.com:cloudbuilders/devstack.git
+# Keep track of the devstack directory
+TOP_DIR=$(cd $(dirname "$0")/.. && pwd)
+
# Uses this shocco branch: https://github.com/dtroyer/shocco/tree/rst_support
SHOCCO=${SHOCCO:-shocco}
if ! which shocco; then
- if [[ ! -x shocco/shocco ]]; then
+ if [[ ! -x $TOP_DIR/shocco/shocco ]]; then
if [[ -z "$INSTALL_SHOCCO" ]]; then
echo "shocco not found in \$PATH, please set environment variable SHOCCO"
exit 1
fi
echo "Installing local copy of shocco"
+ if ! which pygmentize; then
+ sudo pip install Pygments
+ fi
+ if ! which rst2html.py; then
+ sudo pip install docutils
+ fi
git clone -b rst_support https://github.com/dtroyer/shocco shocco
cd shocco
./configure
make
cd ..
fi
- SHOCCO=shocco/shocco
+ SHOCCO=$TOP_DIR/shocco/shocco
fi
# Process command-line args
diff --git a/tools/fixup_stuff.sh b/tools/fixup_stuff.sh
index f936230..5fb47dc 100755
--- a/tools/fixup_stuff.sh
+++ b/tools/fixup_stuff.sh
@@ -51,7 +51,7 @@
# Fix prettytable 0.7.2 permissions
# Don't specify --upgrade so we use the existing package if present
-pip_install prettytable
+pip_install 'prettytable>0.7'
PACKAGE_DIR=$(get_package_path prettytable)
# Only fix version 0.7.2
dir=$(echo $PACKAGE_DIR/prettytable-0.7.2*)
diff --git a/tools/install_pip.sh b/tools/install_pip.sh
index 455323e..6b9b25e 100755
--- a/tools/install_pip.sh
+++ b/tools/install_pip.sh
@@ -26,6 +26,7 @@
# Handle arguments
+USE_GET_PIP=${USE_GET_PIP:-0}
INSTALL_PIP_VERSION=${INSTALL_PIP_VERSION:-"1.4.1"}
while [[ -n "$1" ]]; do
case $1 in
@@ -63,7 +64,7 @@
function install_get_pip() {
if [[ ! -r $FILES/get-pip.py ]]; then
(cd $FILES; \
- curl $PIP_GET_PIP_URL; \
+ curl -O $PIP_GET_PIP_URL; \
)
fi
sudo python $FILES/get-pip.py