B) Use keystone config files from source; move to /etc/keystone

* Put all config files in /etc/keystone
* keystone.conf rewritten
* logging.conf.sample rewritten to logging.conf
* default_catalog.templates copied from devstack/files
* iniset() now properly adds options that do not previously exist

Fixed to re-configure the catalog templated backend; sql is the
default in trunk now but DevStack needs a bit more work before
it can use it.

Change-Id: Ic7060ef897e47495cd08ca3786e49fdebadf6723
diff --git a/stack.sh b/stack.sh
index c82c296..de4e926 100755
--- a/stack.sh
+++ b/stack.sh
@@ -1514,16 +1514,42 @@
     mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS keystone;'
     mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE keystone CHARACTER SET utf8;'
 
-    # Configure keystone.conf
-    KEYSTONE_CONF=$KEYSTONE_DIR/etc/keystone.conf
-    cp $FILES/keystone.conf $KEYSTONE_CONF
-    sudo sed -e "s,%SQL_CONN%,$BASE_SQL_CONN/keystone?charset=utf8,g" -i $KEYSTONE_CONF
-    sudo sed -e "s,%DEST%,$DEST,g" -i $KEYSTONE_CONF
-    sudo sed -e "s,%SERVICE_TOKEN%,$SERVICE_TOKEN,g" -i $KEYSTONE_CONF
-    sudo sed -e "s,%KEYSTONE_DIR%,$KEYSTONE_DIR,g" -i $KEYSTONE_CONF
+    KEYSTONE_CONF_DIR=${KEYSTONE_CONF_DIR:-/etc/keystone}
+    KEYSTONE_CONF=$KEYSTONE_CONF_DIR/keystone.conf
+    KEYSTONE_CATALOG=$KEYSTONE_CONF_DIR/default_catalog.templates
 
-    KEYSTONE_CATALOG=$KEYSTONE_DIR/etc/default_catalog.templates
-    cp $FILES/default_catalog.templates $KEYSTONE_CATALOG
+    if [[ ! -d $KEYSTONE_CONF_DIR ]]; then
+        sudo mkdir -p $KEYSTONE_CONF_DIR
+        sudo chown `whoami` $KEYSTONE_CONF_DIR
+    fi
+
+    if [[ "$KEYSTONE_CONF_DIR" != "$KEYSTONE_DIR/etc" ]]; then
+        # FIXME(dtroyer): etc/keystone.conf causes trouble if the config files
+        #                 are located anywhere else (say, /etc/keystone).
+        #                 LP 966670 fixes this in keystone, we fix it
+        #                 here until the bug fix is committed.
+        if [[ -r $KEYSTONE_DIR/etc/keystone.conf ]]; then
+            # Get the sample config file out of the way
+            mv $KEYSTONE_DIR/etc/keystone.conf $KEYSTONE_DIR/etc/keystone.conf.sample
+        fi
+        cp -p $KEYSTONE_DIR/etc/keystone.conf.sample $KEYSTONE_CONF
+        cp -p $KEYSTONE_DIR/etc/policy.json $KEYSTONE_CONF_DIR
+    fi
+    cp -p $FILES/default_catalog.templates $KEYSTONE_CATALOG
+
+    # Rewrite stock keystone.conf:
+    iniset $KEYSTONE_CONF DEFAULT admin_token "$SERVICE_TOKEN"
+    iniset $KEYSTONE_CONF sql connection "$BASE_SQL_CONN/keystone?charset=utf8"
+    iniset $KEYSTONE_CONF catalog template_file "$KEYSTONE_CATALOG"
+    iniset $KEYSTONE_CONF ec2 driver "keystone.contrib.ec2.backends.sql.Ec2"
+    # Configure keystone.conf to use templates
+    iniset $KEYSTONE_CONF catalog driver "keystone.catalog.backends.templated.TemplatedCatalog"
+    iniset $KEYSTONE_CONF catalog template_file "$KEYSTONE_CATALOG"
+    sed -e "
+        /^pipeline.*ec2_extension crud_/s|ec2_extension crud_extension|ec2_extension s3_extension crud_extension|;
+    " -i $KEYSTONE_CONF
+    # Append the S3 bits
+    iniset $KEYSTONE_CONF filter:s3_extension paste.filter_factory "keystone.contrib.s3:S3Extension.factory"
 
     # Add swift endpoints to service catalog if swift is enabled
     if is_service_enabled swift; then
@@ -1541,34 +1567,32 @@
         echo "catalog.RegionOne.network.name = Quantum Service" >> $KEYSTONE_CATALOG
     fi
 
-    sudo sed -e "s,%SERVICE_HOST%,$SERVICE_HOST,g" -i $KEYSTONE_CATALOG
+    sudo sed -e "
+        s,%SERVICE_HOST%,$SERVICE_HOST,g;
+        s,%S3_SERVICE_PORT%,$S3_SERVICE_PORT,g;
+    " -i $KEYSTONE_CATALOG
 
-    sudo sed -e "s,%S3_SERVICE_PORT%,$S3_SERVICE_PORT,g" -i $KEYSTONE_CATALOG
-
+    # Set up logging
+    LOGGING_ROOT="devel"
     if [ "$SYSLOG" != "False" ]; then
-        cp $KEYSTONE_DIR/etc/logging.conf.sample $KEYSTONE_DIR/etc/logging.conf
-        sed -i -e '/^handlers=devel$/s/=devel/=production/' \
-            $KEYSTONE_DIR/etc/logging.conf
-        sed -i -e "/^log_file/s/log_file/\#log_file/" \
-            $KEYSTONE_DIR/etc/keystone.conf
-        KEYSTONE_LOG_CONFIG="--log-config $KEYSTONE_DIR/etc/logging.conf"
+        LOGGING_ROOT="$LOGGING_ROOT,production"
     fi
-fi
+    KEYSTONE_LOG_CONFIG="--log-config $KEYSTONE_CONF_DIR/logging.conf"
+    cp $KEYSTONE_DIR/etc/logging.conf.sample $KEYSTONE_CONF_DIR/logging.conf
+    iniset $KEYSTONE_CONF_DIR/logging.conf logger_root level "DEBUG"
+    iniset $KEYSTONE_CONF_DIR/logging.conf logger_root handlers "devel,production"
 
-# launch the keystone and wait for it to answer before continuing
-if is_service_enabled key; then
+    # initialize keystone database
+    $KEYSTONE_DIR/bin/keystone-manage db_sync
+
+    # launch keystone and wait for it to answer before continuing
     screen_it key "cd $KEYSTONE_DIR && $KEYSTONE_DIR/bin/keystone-all --config-file $KEYSTONE_CONF $KEYSTONE_LOG_CONFIG -d --debug"
     echo "Waiting for keystone to start..."
-    if ! timeout $SERVICE_TIMEOUT sh -c "while ! http_proxy= wget -q -O- $KEYSTONE_AUTH_PROTOCOL://$SERVICE_HOST:$KEYSTONE_API_PORT/v2.0/; do sleep 1; done"; then
+    if ! timeout $SERVICE_TIMEOUT sh -c "while http_proxy= wget -O- $KEYSTONE_AUTH_PROTOCOL://$SERVICE_HOST:$KEYSTONE_API_PORT/v2.0/ 2>&1 | grep -q 'refused'; do sleep 1; done"; then
       echo "keystone did not start"
       exit 1
     fi
 
-    # initialize keystone with default users/endpoints
-    pushd $KEYSTONE_DIR
-    $KEYSTONE_DIR/bin/keystone-manage db_sync
-    popd
-
     # keystone_data.sh creates services, admin and demo users, and roles.
     SERVICE_ENDPOINT=$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v2.0
     ADMIN_PASSWORD=$ADMIN_PASSWORD SERVICE_TENANT_NAME=$SERVICE_TENANT_NAME SERVICE_PASSWORD=$SERVICE_PASSWORD SERVICE_TOKEN=$SERVICE_TOKEN SERVICE_ENDPOINT=$SERVICE_ENDPOINT DEVSTACK_DIR=$TOP_DIR ENABLED_SERVICES=$ENABLED_SERVICES \