Add PostgreSQL support to devstack
This patch adds an interface for supporting multiple database backend
types and implemnts support for PostgreSQL. It also adds a function,
use_exclusive_service, which serves as a base for enabling a service
that conflicts with other services. The use_database function uses it,
and it might also be useful for selecting messaging backends.
MySQL is still selected by default. Tested on Fedora 17 and Ubuntu
12.04 with MySQL and PostgreSQL. Implements blueprint postgresql-support
Change-Id: I4b1373e25676fd9a9809fe70cb4a6450a2479174
diff --git a/lib/cinder b/lib/cinder
index 08c840e..51acfa5 100644
--- a/lib/cinder
+++ b/lib/cinder
@@ -112,7 +112,9 @@
iniset $CINDER_CONF DEFAULT volume_group $VOLUME_GROUP
iniset $CINDER_CONF DEFAULT volume_name_template ${VOLUME_NAME_PREFIX}%s
iniset $CINDER_CONF DEFAULT iscsi_helper tgtadm
- iniset $CINDER_CONF DEFAULT sql_connection $BASE_SQL_CONN/cinder?charset=utf8
+ local dburl
+ database_connection_url dburl cinder
+ iniset $CINDER_CONF DEFAULT sql_connection $dburl
iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI
iniset $CINDER_CONF DEFAULT root_helper "sudo ${CINDER_ROOTWRAP}"
iniset $CINDER_CONF DEFAULT osapi_volume_extension cinder.api.openstack.volume.contrib.standard_extensions
@@ -141,10 +143,9 @@
# Force nova volumes off
NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/osapi_volume,//")
- if is_service_enabled mysql; then
+ if is_service_enabled $DATABASE_BACKENDS; then
# (re)create cinder database
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS cinder;'
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE cinder;'
+ recreate_database cinder utf8
# (re)create cinder database
$CINDER_BIN_DIR/cinder-manage db sync
diff --git a/lib/database b/lib/database
new file mode 100644
index 0000000..66fb36f
--- /dev/null
+++ b/lib/database
@@ -0,0 +1,103 @@
+# lib/database
+# Interface for interacting with different database backends
+
+# Dependencies:
+# DATABASE_BACKENDS variable must contain a list of available database backends
+# DATABASE_TYPE variable must be set
+
+# Each database must implement four functions:
+# recreate_database_$DATABASE_TYPE
+# install_database_$DATABASE_TYPE
+# configure_database_$DATABASE_TYPE
+# database_connection_url_$DATABASE_TYPE
+#
+# and call register_database $DATABASE_TYPE
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set +o xtrace
+
+# Register a database backend
+# $1 The name of the database backend
+function register_database {
+ [ -z "$DATABASE_BACKENDS" ] && DATABASE_BACKENDS=$1 || DATABASE_BACKENDS+=" $1"
+}
+
+for f in $TOP_DIR/lib/databases/*; do source $f; done
+
+# Set the database type based on the configuration
+function initialize_database_backends {
+ for backend in $DATABASE_BACKENDS; do
+ is_service_enabled $backend && DATABASE_TYPE=$backend
+ done
+
+ [ -z "$DATABASE_TYPE" ] && return 1
+
+ # For backward-compatibility, read in the MYSQL_HOST/USER variables and use
+ # them as the default values for the DATABASE_HOST/USER variables.
+ MYSQL_HOST=${MYSQL_HOST:-localhost}
+ MYSQL_USER=${MYSQL_USER:-root}
+
+ DATABASE_HOST=${DATABASE_HOST:-${MYSQL_HOST}}
+ DATABASE_USER=${DATABASE_USER:-${MYSQL_USER}}
+
+ if [ -n "$MYSQL_PASSWORD" ]; then
+ DATABASE_PASSWORD=$MYSQL_PASSWORD
+ else
+ read_password DATABASE_PASSWORD "ENTER A PASSWORD TO USE FOR THE DATABASE."
+ fi
+
+ # We configure Nova, Horizon, Glance and Keystone to use MySQL as their
+ # database server. While they share a single server, each has their own
+ # database and tables.
+
+ # By default this script will install and configure MySQL. If you want to
+ # use an existing server, you can pass in the user/password/host parameters.
+ # You will need to send the same ``DATABASE_PASSWORD`` to every host if you are doing
+ # a multi-node DevStack installation.
+
+ # NOTE: Don't specify ``/db`` in this string so we can use it for multiple services
+ BASE_SQL_CONN=${BASE_SQL_CONN:-${DATABASE_TYPE}://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST}
+
+ return 0
+}
+
+# Set the database backend to use
+# $1 The name of the database backend to use (mysql, postgresql, ...)
+function use_database {
+ use_exclusive_service DATABASE_BACKENDS DATABASE_TYPE $1 && return 0
+ ret=$?
+ echo "Invalid database '$1'"
+ return $ret
+}
+
+# Recreate a given database
+# $1 The name of the database
+# $2 The character set/encoding of the database
+function recreate_database {
+ local db=$1
+ local charset=$2
+ recreate_database_$DATABASE_TYPE $db $charset
+}
+
+# Install the database
+function install_database {
+ install_database_$DATABASE_TYPE
+}
+
+# Configure and start the database
+function configure_database {
+ configure_database_$DATABASE_TYPE
+}
+
+# Generate an SQLAlchemy connection URL and store it in a variable
+# $1 The variable name in which to store the connection URL
+# $2 The name of the database
+function database_connection_url {
+ local var=$1
+ local db=$2
+ database_connection_url_$DATABASE_TYPE $var $db
+}
+
+# Restore xtrace
+$XTRACE
diff --git a/lib/databases/mysql b/lib/databases/mysql
new file mode 100644
index 0000000..ed59290
--- /dev/null
+++ b/lib/databases/mysql
@@ -0,0 +1,93 @@
+# lib/mysql
+# Functions to control the configuration and operation of the MySQL database backend
+
+# Dependencies:
+# DATABASE_{HOST,USER,PASSWORD} must be defined
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set +o xtrace
+
+register_database mysql
+
+function recreate_database_mysql {
+ local db=$1
+ local charset=$2
+ mysql -u$DATABASE_USER -p$DATABASE_PASSWORD -e "DROP DATABASE IF EXISTS $db;"
+ mysql -u$DATABASE_USER -p$DATABASE_PASSWORD -e "CREATE DATABASE $db CHARACTER SET $charset;"
+}
+
+function configure_database_mysql {
+ echo_summary "Configuring and starting MySQL"
+
+ if [[ "$os_PACKAGE" = "deb" ]]; then
+ MY_CONF=/etc/mysql/my.cnf
+ MYSQL=mysql
+ else
+ MY_CONF=/etc/my.cnf
+ MYSQL=mysqld
+ fi
+
+ # Start mysql-server
+ if [[ "$os_PACKAGE" = "rpm" ]]; then
+ # RPM doesn't start the service
+ start_service $MYSQL
+ # Set the root password - only works the first time
+ sudo mysqladmin -u root password $DATABASE_PASSWORD || true
+ fi
+ # Update the DB to give user ‘$DATABASE_USER’@’%’ full control of the all databases:
+ sudo mysql -uroot -p$DATABASE_PASSWORD -h127.0.0.1 -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
+
+ # Now update ``my.cnf`` for some local needs and restart the mysql service
+
+ # Change ‘bind-address’ from localhost (127.0.0.1) to any (0.0.0.0)
+ sudo sed -i '/^bind-address/s/127.0.0.1/0.0.0.0/g' $MY_CONF
+
+ # Set default db type to InnoDB
+ if sudo grep -q "default-storage-engine" $MY_CONF; then
+ # Change it
+ sudo bash -c "source $TOP_DIR/functions; iniset $MY_CONF mysqld default-storage-engine InnoDB"
+ else
+ # Add it
+ sudo sed -i -e "/^\[mysqld\]/ a \
+default-storage-engine = InnoDB" $MY_CONF
+ fi
+
+ restart_service $MYSQL
+}
+
+function install_database_mysql {
+ if [[ "$os_PACKAGE" = "deb" ]]; then
+ # Seed configuration with mysql password so that apt-get install doesn't
+ # prompt us for a password upon install.
+ cat <<MYSQL_PRESEED | sudo debconf-set-selections
+mysql-server-5.1 mysql-server/root_password password $DATABASE_PASSWORD
+mysql-server-5.1 mysql-server/root_password_again password $DATABASE_PASSWORD
+mysql-server-5.1 mysql-server/start_on_boot boolean true
+MYSQL_PRESEED
+ fi
+
+ # while ``.my.cnf`` is not needed for OpenStack to function, it is useful
+ # as it allows you to access the mysql databases via ``mysql nova`` instead
+ # of having to specify the username/password each time.
+ if [[ ! -e $HOME/.my.cnf ]]; then
+ cat <<EOF >$HOME/.my.cnf
+[client]
+user=$DATABASE_USER
+password=$DATABASE_PASSWORD
+host=$DATABASE_HOST
+EOF
+ chmod 0600 $HOME/.my.cnf
+ fi
+ # Install mysql-server
+ install_package mysql-server
+}
+
+function database_connection_url_mysql {
+ local output=$1
+ local db=$2
+ eval "$output=$BASE_SQL_CONN/$db?charset=utf8"
+}
+
+# Restore xtrace
+$XTRACE
diff --git a/lib/databases/postgresql b/lib/databases/postgresql
new file mode 100644
index 0000000..81989f2
--- /dev/null
+++ b/lib/databases/postgresql
@@ -0,0 +1,70 @@
+# lib/postgresql
+# Functions to control the configuration and operation of the PostgreSQL database backend
+
+# Dependencies:
+# DATABASE_{HOST,USER,PASSWORD} must be defined
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set +o xtrace
+
+register_database postgresql
+
+function recreate_database_postgresql {
+ local db=$1
+ local charset=$2
+ # Avoid unsightly error when calling dropdb when the database doesn't exist
+ psql -h$DATABASE_HOST -U$DATABASE_USER -dtemplate1 -c "DROP DATABASE IF EXISTS $db"
+ createdb -h $DATABASE_HOST -U$DATABASE_USER -l C -T template0 -E $charset $db
+}
+
+function configure_database_postgresql {
+ echo_summary "Configuring and starting PostgreSQL"
+ if [[ "$os_PACKAGE" = "rpm" ]]; then
+ PG_HBA=/var/lib/pgsql/data/pg_hba.conf
+ PG_CONF=/var/lib/pgsql/data/postgresql.conf
+ else
+ PG_DIR=`find /etc/postgresql -name pg_hba.conf|xargs dirname`
+ PG_HBA=$PG_DIR/pg_hba.conf
+ PG_CONF=$PG_DIR/postgresql.conf
+ fi
+ sudo [ -e /var/lib/pgsql/data ] || sudo postgresql-setup initdb
+ # Listen on all addresses
+ sudo sed -i "/listen_addresses/s/.*/listen_addresses = '*'/" $PG_CONF
+ # Do password auth from all IPv4 clients
+ sudo sed -i "/^host/s/all\s\+127.0.0.1\/32\s\+ident/$DATABASE_USER\t0.0.0.0\/0\tpassword/" $PG_HBA
+ # Do password auth for all IPv6 clients
+ sudo sed -i "/^host/s/all\s\+::1\/128\s\+ident/$DATABASE_USER\t::0\/0\tpassword/" $PG_HBA
+ start_service postgresql
+
+ # If creating the role fails, chances are it already existed. Try to alter it.
+ sudo -u postgres -i psql -c "CREATE ROLE $DATABASE_USER WITH SUPERUSER LOGIN PASSWORD '$DATABASE_PASSWORD'" || \
+ sudo -u postgres -i psql -c "ALTER ROLE $DATABASE_USER WITH SUPERUSER LOGIN PASSWORD '$DATABASE_PASSWORD'"
+}
+
+function install_database_postgresql {
+ echo_summary "Installing postgresql"
+ PGPASS=$HOME/.pgpass
+ if [[ ! -e $PGPASS ]]; then
+ cat <<EOF > $PGPASS
+*:*:*:$DATABASE_USER:$DATABASE_PASSWORD
+EOF
+ chmod 0600 $PGPASS
+ else
+ sed -i "s/:root:\w\+/:root:$DATABASE_PASSWORD/" $PGPASS
+ fi
+ if [[ "$os_PACKAGE" = "rpm" ]]; then
+ install_package postgresql-server
+ else
+ install_package postgresql
+ fi
+}
+
+function database_connection_url_postgresql {
+ local output=$1
+ local db=$2
+ eval "$output=$BASE_SQL_CONN/$db?client_encoding=utf8"
+}
+
+# Restore xtrace
+$XTRACE
diff --git a/lib/glance b/lib/glance
index 070c80d..afddcd2 100644
--- a/lib/glance
+++ b/lib/glance
@@ -81,7 +81,9 @@
cp $GLANCE_DIR/etc/glance-registry.conf $GLANCE_REGISTRY_CONF
iniset $GLANCE_REGISTRY_CONF DEFAULT debug True
inicomment $GLANCE_REGISTRY_CONF DEFAULT log_file
- iniset $GLANCE_REGISTRY_CONF DEFAULT sql_connection $BASE_SQL_CONN/glance?charset=utf8
+ local dburl
+ database_connection_url dburl glance
+ iniset $GLANCE_REGISTRY_CONF DEFAULT sql_connection $dburl
iniset $GLANCE_REGISTRY_CONF DEFAULT use_syslog $SYSLOG
iniset $GLANCE_REGISTRY_CONF paste_deploy flavor keystone
iniset $GLANCE_REGISTRY_CONF keystone_authtoken auth_host $KEYSTONE_AUTH_HOST
@@ -95,7 +97,7 @@
cp $GLANCE_DIR/etc/glance-api.conf $GLANCE_API_CONF
iniset $GLANCE_API_CONF DEFAULT debug True
inicomment $GLANCE_API_CONF DEFAULT log_file
- iniset $GLANCE_API_CONF DEFAULT sql_connection $BASE_SQL_CONN/glance?charset=utf8
+ iniset $GLANCE_API_CONF DEFAULT sql_connection $dburl
iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG
iniset $GLANCE_API_CONF DEFAULT filesystem_store_datadir $GLANCE_IMAGE_DIR/
iniset $GLANCE_API_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
@@ -149,8 +151,7 @@
mkdir -p $GLANCE_CACHE_DIR
# (re)create glance database
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS glance;'
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE glance CHARACTER SET utf8;'
+ recreate_database glance utf8
$GLANCE_BIN_DIR/glance-manage db_sync
}
diff --git a/lib/heat b/lib/heat
index 7fb5fcc..d1f1c7c 100644
--- a/lib/heat
+++ b/lib/heat
@@ -120,7 +120,9 @@
iniset $HEAT_ENGINE_CONF DEFAULT use_syslog $SYSLOG
iniset $HEAT_ENGINE_CONF DEFAULT bind_host $HEAT_ENGINE_HOST
iniset $HEAT_ENGINE_CONF DEFAULT bind_port $HEAT_ENGINE_PORT
- iniset $HEAT_ENGINE_CONF DEFAULT sql_connection $BASE_SQL_CONN/heat?charset=utf8
+ local dburl
+ database_connection_url dburl heat
+ iniset $HEAT_ENGINE_CONF DEFAULT sql_connection $dburl
iniset $HEAT_ENGINE_CONF DEFAULT auth_encryption_key `hexdump -n 16 -v -e '/1 "%02x"' /dev/random`
if is_service_enabled rabbit; then
@@ -185,8 +187,7 @@
function init_heat() {
# (re)create heat database
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS heat;'
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE heat CHARACTER SET utf8;'
+ recreate_database heat utf8
$HEAT_DIR/bin/heat-db-setup $os_PACKAGE -r $MYSQL_PASSWORD
$HEAT_DIR/tools/nova_create_flavors.sh
diff --git a/lib/keystone b/lib/keystone
index 73d82c5..ac15cbd 100644
--- a/lib/keystone
+++ b/lib/keystone
@@ -82,9 +82,11 @@
fi
# Rewrite stock ``keystone.conf``
+ local dburl
+ database_connection_url dburl keystone
iniset $KEYSTONE_CONF DEFAULT admin_token "$SERVICE_TOKEN"
iniset $KEYSTONE_CONF signing token_format "$KEYSTONE_TOKEN_FORMAT"
- iniset $KEYSTONE_CONF sql connection "$BASE_SQL_CONN/keystone?charset=utf8"
+ iniset $KEYSTONE_CONF sql connection $dburl
iniset $KEYSTONE_CONF ec2 driver "keystone.contrib.ec2.backends.sql.Ec2"
sed -e "
/^pipeline.*ec2_extension crud_/s|ec2_extension crud_extension|ec2_extension s3_extension crud_extension|;
@@ -141,8 +143,7 @@
# init_keystone() - Initialize databases, etc.
function init_keystone() {
# (Re)create keystone database
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS keystone;'
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE keystone CHARACTER SET utf8;'
+ recreate_database keystone utf8
# Initialize keystone database
$KEYSTONE_DIR/bin/keystone-manage db_sync
diff --git a/lib/nova b/lib/nova
index 7797927..4997175 100644
--- a/lib/nova
+++ b/lib/nova
@@ -296,7 +296,9 @@
add_nova_opt "s3_port=$S3_SERVICE_PORT"
add_nova_opt "osapi_compute_extension=nova.api.openstack.compute.contrib.standard_extensions"
add_nova_opt "my_ip=$HOST_IP"
- add_nova_opt "sql_connection=$BASE_SQL_CONN/nova?charset=utf8"
+ local dburl
+ database_connection_url dburl nova
+ add_nova_opt "sql_connection=$dburl"
add_nova_opt "libvirt_type=$LIBVIRT_TYPE"
add_nova_opt "libvirt_cpu_mode=none"
add_nova_opt "instance_name_template=${INSTANCE_NAME_PREFIX}%08x"
@@ -372,14 +374,12 @@
# All nova components talk to a central database. We will need to do this step
# only once for an entire cluster.
- if is_service_enabled mysql && is_service_enabled nova; then
+ if is_service_enabled $DATABASE_BACKENDS && is_service_enabled nova; then
# (Re)create nova database
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS nova;'
-
# Explicitly use latin1: to avoid lp#829209, nova expects the database to
# use latin1 by default, and then upgrades the database to utf8 (see the
# 082_essex.py in nova)
- mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE nova CHARACTER SET latin1;'
+ recreate_database nova latin1
# (Re)create nova database
$NOVA_BIN_DIR/nova-manage db sync