blob: bd029dd700d6272e9bb8225839116de5ed046acb [file] [log] [blame]
Sean Daguee263c822014-12-05 14:25:28 -05001#!/bin/bash
2#
Dean Troyerdff49a22014-01-30 15:37:40 -06003# functions-common - Common functions used by DevStack components
4#
5# The canonical copy of this file is maintained in the DevStack repo.
6# All modifications should be made there and then sync'ed to other repos
7# as required.
8#
9# This file is sorted alphabetically within the function groups.
10#
11# - Config Functions
12# - Control Functions
13# - Distro Functions
14# - Git Functions
15# - OpenStack Functions
16# - Package Functions
17# - Process Functions
Dean Troyerdff49a22014-01-30 15:37:40 -060018# - Service Functions
Masayuki Igawaf6368d32014-02-20 13:31:26 +090019# - System Functions
Dean Troyerdff49a22014-01-30 15:37:40 -060020#
21# The following variables are assumed to be defined by certain functions:
22#
23# - ``ENABLED_SERVICES``
24# - ``ERROR_ON_CLONE``
25# - ``FILES``
26# - ``OFFLINE``
Dean Troyerdff49a22014-01-30 15:37:40 -060027# - ``RECLONE``
Masayuki Igawad20f6322014-02-28 09:22:37 +090028# - ``REQUIREMENTS_DIR``
29# - ``STACK_USER``
Dean Troyerdff49a22014-01-30 15:37:40 -060030# - ``http_proxy``, ``https_proxy``, ``no_proxy``
Dean Troyer3324f192014-09-18 09:26:39 -050031#
Dean Troyerdff49a22014-01-30 15:37:40 -060032
33# Save trace setting
Ian Wienand523f4882015-10-13 11:03:03 +110034_XTRACE_FUNCTIONS_COMMON=$(set +o | grep xtrace)
Dean Troyerdff49a22014-01-30 15:37:40 -060035set +o xtrace
36
Ian Wienand4ffb4542015-06-30 11:00:32 +100037# ensure we don't re-source this in the same environment
38[[ -z "$_DEVSTACK_FUNCTIONS_COMMON" ]] || return 0
Sean Dagueafef8bf2017-03-06 14:07:23 -050039declare -r -g _DEVSTACK_FUNCTIONS_COMMON=1
Ian Wienand4ffb4542015-06-30 11:00:32 +100040
Sean Daguecc524062014-10-01 09:06:43 -040041# Global Config Variables
Sean Dagueafef8bf2017-03-06 14:07:23 -050042declare -A -g GITREPO
43declare -A -g GITBRANCH
44declare -A -g GITDIR
Sean Daguecc524062014-10-01 09:06:43 -040045
Dinesh Bhoref60f2b2017-09-05 14:40:32 +053046KILL_PATH="$(which kill)"
Sean Dague53753292014-12-04 19:38:15 -050047
Dean Troyer68162342015-05-13 15:41:03 -050048# Save these variables to .stackenv
49STACK_ENV_VARS="BASE_SQL_CONN DATA_DIR DEST ENABLED_SERVICES HOST_IP \
Jens Harbott32c00892019-04-10 10:33:39 +000050 KEYSTONE_SERVICE_URI \
Sean Dague0effa1a2017-04-18 15:16:37 -040051 LOGFILE OS_CACERT SERVICE_HOST STACK_USER TLS_IP \
Brian Haley5d04db22015-06-16 13:14:31 -040052 HOST_IPV6 SERVICE_IP_VERSION"
Dean Troyer68162342015-05-13 15:41:03 -050053
54
55# Saves significant environment variables to .stackenv for later use
56# Refers to a lot of globals, only TOP_DIR and STACK_ENV_VARS are required to
57# function, the rest are simply saved and do not cause problems if they are undefined.
58# save_stackenv [tag]
59function save_stackenv {
60 local tag=${1:-""}
61 # Save some values we generated for later use
62 time_stamp=$(date "+$TIMESTAMP_FORMAT")
63 echo "# $time_stamp $tag" >$TOP_DIR/.stackenv
64 for i in $STACK_ENV_VARS; do
65 echo $i=${!i} >>$TOP_DIR/.stackenv
66 done
67}
Dean Troyerdff49a22014-01-30 15:37:40 -060068
Monty Taylor7224eec2015-09-19 11:26:18 -040069# Update/create user clouds.yaml file.
70# clouds.yaml will have
71# - A `devstack` entry for the `demo` user for the `demo` project.
72# - A `devstack-admin` entry for the `admin` user for the `admin` project.
73# write_clouds_yaml
74function write_clouds_yaml {
Monty Tayloree9bb762015-10-19 15:16:18 -040075 # The location is a variable to allow for easier refactoring later to make it
76 # overridable. There is currently no usecase where doing so makes sense, so
77 # it's not currently configurable.
Monty Taylor7224eec2015-09-19 11:26:18 -040078
Monty Tayloree9bb762015-10-19 15:16:18 -040079 CLOUDS_YAML=/etc/openstack/clouds.yaml
80
81 sudo mkdir -p $(dirname $CLOUDS_YAML)
Lenny Verkhovsky313ddae2015-10-20 11:26:34 +030082 sudo chown -R $STACK_USER /etc/openstack
Lenny Verkhovsky313ddae2015-10-20 11:26:34 +030083
Monty Tayloree9bb762015-10-19 15:16:18 -040084 CA_CERT_ARG=''
85 if [ -f "$SSL_BUNDLE_FILE" ]; then
86 CA_CERT_ARG="--os-cacert $SSL_BUNDLE_FILE"
87 fi
Sean Daguec67d22e2016-02-02 05:51:14 -050088 # demo -> devstack
Davanum Srinivas51ecf0a2017-01-05 16:11:17 -050089 $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
Monty Tayloree9bb762015-10-19 15:16:18 -040090 --file $CLOUDS_YAML \
91 --os-cloud devstack \
92 --os-region-name $REGION_NAME \
Monty Tayloree9bb762015-10-19 15:16:18 -040093 $CA_CERT_ARG \
Monty Taylore43f60b2017-04-27 18:27:36 -050094 --os-auth-url $KEYSTONE_SERVICE_URI \
Monty Tayloree9bb762015-10-19 15:16:18 -040095 --os-username demo \
96 --os-password $ADMIN_PASSWORD \
97 --os-project-name demo
Sean Daguec67d22e2016-02-02 05:51:14 -050098
99 # alt_demo -> devstack-alt
Davanum Srinivas51ecf0a2017-01-05 16:11:17 -0500100 $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
Sean Daguec67d22e2016-02-02 05:51:14 -0500101 --file $CLOUDS_YAML \
Clark Boylane20cb432016-02-05 12:00:18 -0800102 --os-cloud devstack-alt \
Sean Daguec67d22e2016-02-02 05:51:14 -0500103 --os-region-name $REGION_NAME \
Sean Daguec67d22e2016-02-02 05:51:14 -0500104 $CA_CERT_ARG \
Monty Taylore43f60b2017-04-27 18:27:36 -0500105 --os-auth-url $KEYSTONE_SERVICE_URI \
Sean Daguec67d22e2016-02-02 05:51:14 -0500106 --os-username alt_demo \
107 --os-password $ADMIN_PASSWORD \
108 --os-project-name alt_demo
109
110 # admin -> devstack-admin
Davanum Srinivas51ecf0a2017-01-05 16:11:17 -0500111 $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
Monty Tayloree9bb762015-10-19 15:16:18 -0400112 --file $CLOUDS_YAML \
113 --os-cloud devstack-admin \
114 --os-region-name $REGION_NAME \
Monty Tayloree9bb762015-10-19 15:16:18 -0400115 $CA_CERT_ARG \
Monty Taylore43f60b2017-04-27 18:27:36 -0500116 --os-auth-url $KEYSTONE_SERVICE_URI \
Monty Tayloree9bb762015-10-19 15:16:18 -0400117 --os-username admin \
118 --os-password $ADMIN_PASSWORD \
119 --os-project-name admin
Monty Taylor74379df2016-01-21 10:20:08 -0500120
Monty Taylor56905822019-01-08 15:29:16 +0000121 # admin with a system-scoped token -> devstack-system
122 $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
123 --file $CLOUDS_YAML \
124 --os-cloud devstack-system-admin \
125 --os-region-name $REGION_NAME \
126 $CA_CERT_ARG \
127 --os-auth-url $KEYSTONE_SERVICE_URI \
128 --os-username admin \
129 --os-password $ADMIN_PASSWORD \
130 --os-system-scope all
131
Lance Bragstad021ae0b2021-03-11 15:47:50 +0000132 # system member
133 $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
134 --file $CLOUDS_YAML \
135 --os-cloud devstack-system-member \
136 --os-region-name $REGION_NAME \
137 $CA_CERT_ARG \
138 --os-auth-url $KEYSTONE_SERVICE_URI \
139 --os-username system_member \
140 --os-password $ADMIN_PASSWORD \
141 --os-system-scope all
142
143 # system reader
144 $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
145 --file $CLOUDS_YAML \
146 --os-cloud devstack-system-reader \
147 --os-region-name $REGION_NAME \
148 $CA_CERT_ARG \
149 --os-auth-url $KEYSTONE_SERVICE_URI \
150 --os-username system_reader \
151 --os-password $ADMIN_PASSWORD \
152 --os-system-scope all
153
Maciej Józefczyk7a0fa4f2020-03-05 16:55:50 +0100154 cat >> $CLOUDS_YAML <<EOF
155functional:
156 image_name: $DEFAULT_IMAGE_NAME
157EOF
158
Monty Taylor74379df2016-01-21 10:20:08 -0500159 # CLean up any old clouds.yaml files we had laying around
ZhiQiang Fan0b4a0092016-04-12 20:26:33 +0800160 rm -f $(eval echo ~"$STACK_USER")/.config/openstack/clouds.yaml
Monty Taylor7224eec2015-09-19 11:26:18 -0400161}
162
Ian Wienande82bac02015-08-25 14:29:08 +1000163# trueorfalse <True|False> <VAR>
164#
165# Normalize config-value provided in variable VAR to either "True" or
166# "False". If VAR is unset (i.e. $VAR evaluates as empty), the value
167# of the second argument will be used as the default value.
168#
169# Accepts as False: 0 no No NO false False FALSE
170# Accepts as True: 1 yes Yes YES true True TRUE
171#
172# usage:
173# VAL=$(trueorfalse False VAL)
Ian Wienandaee18c72014-02-21 15:35:08 +1100174function trueorfalse {
Ian Wienand433a9b12015-10-07 13:29:31 +1100175 local xtrace
176 xtrace=$(set +o | grep xtrace)
Sean Dague45917cc2014-02-24 16:09:14 -0500177 set +o xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -0600178
Mahito OGURA98f59aa2015-05-11 18:02:34 +0900179 local default=$1
Ian Wienande82bac02015-08-25 14:29:08 +1000180
181 if [ -z $2 ]; then
182 die $LINENO "variable to normalize required"
183 fi
Mahito OGURA98f59aa2015-05-11 18:02:34 +0900184 local testval=${!2:-}
185
186 case "$testval" in
187 "1" | [yY]es | "YES" | [tT]rue | "TRUE" ) echo "True" ;;
188 "0" | [nN]o | "NO" | [fF]alse | "FALSE" ) echo "False" ;;
189 * ) echo "$default" ;;
190 esac
191
Sean Dague45917cc2014-02-24 16:09:14 -0500192 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -0600193}
194
Attila Fazekas1bd79592015-02-24 14:06:56 +0100195function isset {
196 [[ -v "$1" ]]
197}
Dean Troyerdff49a22014-01-30 15:37:40 -0600198
Dean Troyer68162342015-05-13 15:41:03 -0500199
Dean Troyerdff49a22014-01-30 15:37:40 -0600200# Control Functions
201# =================
202
203# Prints backtrace info
204# filename:lineno:function
205# backtrace level
206function backtrace {
207 local level=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100208 local deep
209 deep=$((${#BASH_SOURCE[@]} - 1))
Dean Troyerdff49a22014-01-30 15:37:40 -0600210 echo "[Call Trace]"
211 while [ $level -le $deep ]; do
212 echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}"
213 deep=$((deep - 1))
214 done
215}
216
217# Prints line number and "message" then exits
218# die $LINENO "message"
Ian Wienandaee18c72014-02-21 15:35:08 +1100219function die {
Dean Troyerdff49a22014-01-30 15:37:40 -0600220 local exitcode=$?
221 set +o xtrace
222 local line=$1; shift
223 if [ $exitcode == 0 ]; then
224 exitcode=1
225 fi
226 backtrace 2
227 err $line "$*"
Dean Troyera25a6f62014-02-24 16:03:41 -0600228 # Give buffers a second to flush
229 sleep 1
Dean Troyerdff49a22014-01-30 15:37:40 -0600230 exit $exitcode
231}
232
233# Checks an environment variable is not set or has length 0 OR if the
234# exit code is non-zero and prints "message" and exits
235# NOTE: env-var is the variable name without a '$'
236# die_if_not_set $LINENO env-var "message"
Ian Wienandaee18c72014-02-21 15:35:08 +1100237function die_if_not_set {
Dean Troyerdff49a22014-01-30 15:37:40 -0600238 local exitcode=$?
Ian Wienand433a9b12015-10-07 13:29:31 +1100239 local xtrace
240 xtrace=$(set +o | grep xtrace)
Dean Troyerdff49a22014-01-30 15:37:40 -0600241 set +o xtrace
242 local line=$1; shift
243 local evar=$1; shift
244 if ! is_set $evar || [ $exitcode != 0 ]; then
245 die $line "$*"
246 fi
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500247 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -0600248}
249
Sean Dague1de9e332015-10-07 08:46:13 -0400250function deprecated {
251 local text=$1
252 DEPRECATED_TEXT+="\n$text"
YAMAMOTO Takashi8b1bbd62016-12-01 22:29:12 +0900253 echo "WARNING: $text" >&2
Sean Dague1de9e332015-10-07 08:46:13 -0400254}
255
Dean Troyerdff49a22014-01-30 15:37:40 -0600256# Prints line number and "message" in error format
257# err $LINENO "message"
Ian Wienandaee18c72014-02-21 15:35:08 +1100258function err {
Dean Troyerdff49a22014-01-30 15:37:40 -0600259 local exitcode=$?
Ian Wienand433a9b12015-10-07 13:29:31 +1100260 local xtrace
261 xtrace=$(set +o | grep xtrace)
Dean Troyerdff49a22014-01-30 15:37:40 -0600262 set +o xtrace
263 local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2"
Ian Wienande8a6a022018-10-08 15:20:34 +1100264 echo "$msg" 1>&2;
Dean Troyerdde41d02014-12-09 17:47:57 -0600265 if [[ -n ${LOGDIR} ]]; then
Ian Wienande8a6a022018-10-08 15:20:34 +1100266 echo "$msg" >> "${LOGDIR}/error.log"
Dean Troyerdff49a22014-01-30 15:37:40 -0600267 fi
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500268 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -0600269 return $exitcode
270}
271
272# Checks an environment variable is not set or has length 0 OR if the
273# exit code is non-zero and prints "message"
274# NOTE: env-var is the variable name without a '$'
275# err_if_not_set $LINENO env-var "message"
Ian Wienandaee18c72014-02-21 15:35:08 +1100276function err_if_not_set {
Dean Troyerdff49a22014-01-30 15:37:40 -0600277 local exitcode=$?
Ian Wienand433a9b12015-10-07 13:29:31 +1100278 local xtrace
279 xtrace=$(set +o | grep xtrace)
Dean Troyerdff49a22014-01-30 15:37:40 -0600280 set +o xtrace
281 local line=$1; shift
282 local evar=$1; shift
283 if ! is_set $evar || [ $exitcode != 0 ]; then
284 err $line "$*"
285 fi
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500286 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -0600287 return $exitcode
288}
289
290# Exit after outputting a message about the distribution not being supported.
291# exit_distro_not_supported [optional-string-telling-what-is-missing]
292function exit_distro_not_supported {
293 if [[ -z "$DISTRO" ]]; then
294 GetDistro
295 fi
296
297 if [ $# -gt 0 ]; then
298 die $LINENO "Support for $DISTRO is incomplete: no support for $@"
299 else
300 die $LINENO "Support for $DISTRO is incomplete."
301 fi
302}
303
304# Test if the named environment variable is set and not zero length
305# is_set env-var
Ian Wienandaee18c72014-02-21 15:35:08 +1100306function is_set {
Dean Troyerdff49a22014-01-30 15:37:40 -0600307 local var=\$"$1"
308 eval "[ -n \"$var\" ]" # For ex.: sh -c "[ -n \"$var\" ]" would be better, but several exercises depends on this
309}
310
311# Prints line number and "message" in warning format
312# warn $LINENO "message"
Ian Wienandaee18c72014-02-21 15:35:08 +1100313function warn {
Dean Troyerdff49a22014-01-30 15:37:40 -0600314 local exitcode=$?
Ian Wienand433a9b12015-10-07 13:29:31 +1100315 local xtrace
316 xtrace=$(set +o | grep xtrace)
Dean Troyerdff49a22014-01-30 15:37:40 -0600317 set +o xtrace
318 local msg="[WARNING] ${BASH_SOURCE[2]}:$1 $2"
Ian Wienande8a6a022018-10-08 15:20:34 +1100319 echo "$msg"
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500320 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -0600321 return $exitcode
322}
323
324
325# Distro Functions
326# ================
327
328# Determine OS Vendor, Release and Update
Ian Wienand7710e7f2014-08-27 16:15:32 +1000329
330#
331# NOTE : For portability, you almost certainly do not want to use
332# these variables directly! The "is_*" functions defined below this
333# bundle up compatible platforms under larger umbrellas that we have
334# determinted are compatible enough (e.g. is_ubuntu covers Ubuntu &
335# Debian, is_fedora covers RPM-based distros). Higher-level functions
336# such as "install_package" further abstract things in better ways.
337#
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500338# ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
Matt Riedemannff10ac32017-02-13 12:44:24 -0500339# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora)
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500340# ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
Matt Riedemannff10ac32017-02-13 12:44:24 -0500341# ``os_CODENAME`` - vendor's codename for release: ``xenial``
Ian Wienand7710e7f2014-08-27 16:15:32 +1000342
Sean Dagueafef8bf2017-03-06 14:07:23 -0500343declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME
Ian Wienand7710e7f2014-08-27 16:15:32 +1000344
345# Make a *best effort* attempt to install lsb_release packages for the
346# user if not available. Note can't use generic install_package*
347# because they depend on this!
348function _ensure_lsb_release {
Thomas Bechtolddab39012016-03-03 12:11:15 +0100349 if [[ -x $(command -v lsb_release 2>/dev/null) ]]; then
Ian Wienand7710e7f2014-08-27 16:15:32 +1000350 return
351 fi
352
Thomas Bechtolddab39012016-03-03 12:11:15 +0100353 if [[ -x $(command -v apt-get 2>/dev/null) ]]; then
Ian Wienand7710e7f2014-08-27 16:15:32 +1000354 sudo apt-get install -y lsb-release
Thomas Bechtolddab39012016-03-03 12:11:15 +0100355 elif [[ -x $(command -v zypper 2>/dev/null) ]]; then
Dirk Mueller5ef8a122017-09-25 13:08:51 +0200356 sudo zypper -n install lsb-release
Thomas Bechtolddab39012016-03-03 12:11:15 +0100357 elif [[ -x $(command -v dnf 2>/dev/null) ]]; then
Ian Wienand7710e7f2014-08-27 16:15:32 +1000358 sudo dnf install -y redhat-lsb-core
Ian Wienand7710e7f2014-08-27 16:15:32 +1000359 else
360 die $LINENO "Unable to find or auto-install lsb_release"
361 fi
362}
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500363
Dean Troyerdff49a22014-01-30 15:37:40 -0600364# GetOSVersion
Ian Wienand7710e7f2014-08-27 16:15:32 +1000365# Set the following variables:
366# - os_RELEASE
367# - os_CODENAME
368# - os_VENDOR
369# - os_PACKAGE
Ian Wienandaee18c72014-02-21 15:35:08 +1100370function GetOSVersion {
Alfredo Moralejo5ea4c3c2021-11-16 15:13:03 +0100371 # CentOS Stream 9 does not provide lsb_release
372 source /etc/os-release
373 if [[ "${ID}${VERSION}" == "centos9" ]]; then
374 os_RELEASE=${VERSION_ID}
375 os_CODENAME="n/a"
376 os_VENDOR=$(echo $NAME | tr -d '[:space:]')
377 else
378 _ensure_lsb_release
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500379
Alfredo Moralejo5ea4c3c2021-11-16 15:13:03 +0100380 os_RELEASE=$(lsb_release -r -s)
381 os_CODENAME=$(lsb_release -c -s)
382 os_VENDOR=$(lsb_release -i -s)
383 fi
Dean Troyerdff49a22014-01-30 15:37:40 -0600384
Ian Wienand7710e7f2014-08-27 16:15:32 +1000385 if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then
Dean Troyerdff49a22014-01-30 15:37:40 -0600386 os_PACKAGE="deb"
Ian Wienand7710e7f2014-08-27 16:15:32 +1000387 else
388 os_PACKAGE="rpm"
Dean Troyerdff49a22014-01-30 15:37:40 -0600389 fi
Ian Wienand7710e7f2014-08-27 16:15:32 +1000390
391 typeset -xr os_VENDOR
392 typeset -xr os_RELEASE
393 typeset -xr os_PACKAGE
394 typeset -xr os_CODENAME
Dean Troyerdff49a22014-01-30 15:37:40 -0600395}
396
397# Translate the OS version values into common nomenclature
398# Sets global ``DISTRO`` from the ``os_*`` values
Sean Dagueafef8bf2017-03-06 14:07:23 -0500399declare -g DISTRO
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500400
Ian Wienandaee18c72014-02-21 15:35:08 +1100401function GetDistro {
Dean Troyerdff49a22014-01-30 15:37:40 -0600402 GetOSVersion
ptoohill196006652016-02-15 16:07:50 -0600403 if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) || \
404 "$os_VENDOR" =~ (LinuxMint) ]]; then
405 # 'Everyone' refers to Ubuntu / Debian / Mint releases by
Ian Wienand7710e7f2014-08-27 16:15:32 +1000406 # the code name adjective
Dean Troyerdff49a22014-01-30 15:37:40 -0600407 DISTRO=$os_CODENAME
408 elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
409 # For Fedora, just use 'f' and the release
410 DISTRO="f$os_RELEASE"
Adam Spiersb6f04ca2019-01-23 18:55:16 +0000411 elif is_opensuse; then
Dean Troyerdff49a22014-01-30 15:37:40 -0600412 DISTRO="opensuse-$os_RELEASE"
Dirk Mueller4404f682018-03-02 00:37:58 +0100413 # Tumbleweed uses "n/a" as a codename, and the release is a datestring
Dirk Mueller297a50a2018-06-20 11:08:54 +0200414 # like 20180218, so not very useful. Leap however uses a release
415 # with a "dot", so for example 15.0
416 [ "$os_CODENAME" = "n/a" -a "$os_RELEASE" = "${os_RELEASE/\./}" ] && \
417 DISTRO="opensuse-tumbleweed"
Adam Spiersb6f04ca2019-01-23 18:55:16 +0000418 elif is_suse_linux_enterprise; then
Ian Wienand7710e7f2014-08-27 16:15:32 +1000419 # just use major release
420 DISTRO="sle${os_RELEASE%.*}"
421 elif [[ "$os_VENDOR" =~ (Red.*Hat) || \
anju Tiwari6c639c92014-07-15 18:11:54 +0530422 "$os_VENDOR" =~ (CentOS) || \
Mike Trimm1da4e792016-04-27 11:40:19 -0500423 "$os_VENDOR" =~ (Scientific) || \
Fernando Ribeiro4e6f8ca2016-04-12 23:34:03 -0300424 "$os_VENDOR" =~ (OracleServer) || \
Evgeny Antyshev718512c2016-03-03 14:47:58 +0000425 "$os_VENDOR" =~ (Virtuozzo) ]]; then
Dean Troyerdff49a22014-01-30 15:37:40 -0600426 # Drop the . release as we assume it's compatible
Ian Wienand7710e7f2014-08-27 16:15:32 +1000427 # XXX re-evaluate when we get RHEL10
Dean Troyerdff49a22014-01-30 15:37:40 -0600428 DISTRO="rhel${os_RELEASE::1}"
Dean Troyerdff49a22014-01-30 15:37:40 -0600429 else
Ian Wienanded92e432016-02-16 10:56:56 +1100430 # We can't make a good choice here. Setting a sensible DISTRO
431 # is part of the problem, but not the major issue -- we really
432 # only use DISTRO in the code as a fine-filter.
433 #
434 # The bigger problem is categorising the system into one of
435 # our two big categories as Ubuntu/Debian-ish or
436 # Fedora/CentOS-ish.
437 #
438 # The setting of os_PACKAGE above is only set to "deb" based
439 # on a hard-coded list of vendor names ... thus we will
440 # default to thinking unknown distros are RPM based
441 # (ie. is_ubuntu does not match). But the platform will then
442 # also not match in is_fedora, because that also has a list of
443 # names.
444 #
445 # So, if you are reading this, getting your distro supported
446 # is really about making sure it matches correctly in these
447 # functions. Then you can choose a sensible way to construct
448 # DISTRO based on your distros release approach.
449 die $LINENO "Unable to determine DISTRO, can not continue."
Dean Troyerdff49a22014-01-30 15:37:40 -0600450 fi
Ian Wienand7710e7f2014-08-27 16:15:32 +1000451 typeset -xr DISTRO
Dean Troyerdff49a22014-01-30 15:37:40 -0600452}
453
454# Utility function for checking machine architecture
455# is_arch arch-type
456function is_arch {
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500457 [[ "$(uname -m)" == "$1" ]]
Dean Troyerdff49a22014-01-30 15:37:40 -0600458}
459
Wiekus Beukesec47bc12015-03-19 08:20:38 -0700460# Determine if current distribution is an Oracle distribution
461# is_oraclelinux
462function is_oraclelinux {
463 if [[ -z "$os_VENDOR" ]]; then
464 GetOSVersion
465 fi
466
Fernando Ribeiro4e6f8ca2016-04-12 23:34:03 -0300467 [ "$os_VENDOR" = "OracleServer" ]
Wiekus Beukesec47bc12015-03-19 08:20:38 -0700468}
469
470
Dean Troyerdff49a22014-01-30 15:37:40 -0600471# Determine if current distribution is a Fedora-based distribution
472# (Fedora, RHEL, CentOS, etc).
473# is_fedora
474function is_fedora {
475 if [[ -z "$os_VENDOR" ]]; then
476 GetOSVersion
477 fi
478
anju Tiwari6c639c92014-07-15 18:11:54 +0530479 [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
tengqm78d37392016-03-15 23:08:00 -0400480 [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
Carlos Goncalves587e0a32020-08-01 21:47:55 +0200481 [ "$os_VENDOR" = "RedHatEnterprise" ] || \
Sean Mooneye7c017b2020-12-09 23:53:12 +0000482 [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "CentOSStream" ] || \
483 [ "$os_VENDOR" = "OracleServer" ] || [ "$os_VENDOR" = "Virtuozzo" ]
Dean Troyerdff49a22014-01-30 15:37:40 -0600484}
485
486
487# Determine if current distribution is a SUSE-based distribution
488# (openSUSE, SLE).
489# is_suse
490function is_suse {
Adam Spiersb6f04ca2019-01-23 18:55:16 +0000491 is_opensuse || is_suse_linux_enterprise
492}
493
494
495# Determine if current distribution is an openSUSE distribution
496# is_opensuse
497function is_opensuse {
Dean Troyerdff49a22014-01-30 15:37:40 -0600498 if [[ -z "$os_VENDOR" ]]; then
499 GetOSVersion
500 fi
501
Adam Spiersb6f04ca2019-01-23 18:55:16 +0000502 [[ "$os_VENDOR" =~ (openSUSE) ]]
503}
504
505
506# Determine if current distribution is a SUSE Linux Enterprise (SLE)
507# distribution
508# is_suse_linux_enterprise
509function is_suse_linux_enterprise {
510 if [[ -z "$os_VENDOR" ]]; then
511 GetOSVersion
512 fi
513
514 [[ "$os_VENDOR" =~ (^SUSE) ]]
Dean Troyerdff49a22014-01-30 15:37:40 -0600515}
516
517
518# Determine if current distribution is an Ubuntu-based distribution
519# It will also detect non-Ubuntu but Debian-based distros
520# is_ubuntu
521function is_ubuntu {
522 if [[ -z "$os_PACKAGE" ]]; then
523 GetOSVersion
524 fi
525 [ "$os_PACKAGE" = "deb" ]
526}
527
528
529# Git Functions
530# =============
531
Dean Troyerabc7b1d2014-02-12 12:09:22 -0600532# Returns openstack release name for a given branch name
533# ``get_release_name_from_branch branch-name``
Ian Wienandaee18c72014-02-21 15:35:08 +1100534function get_release_name_from_branch {
Dean Troyerabc7b1d2014-02-12 12:09:22 -0600535 local branch=$1
Adam Gandelman8f385722014-10-14 15:50:18 -0700536 if [[ $branch =~ "stable/" || $branch =~ "proposed/" ]]; then
Dean Troyerabc7b1d2014-02-12 12:09:22 -0600537 echo ${branch#*/}
538 else
539 echo "master"
540 fi
541}
542
Dean Troyerdff49a22014-01-30 15:37:40 -0600543# git clone only if directory doesn't exist already. Since ``DEST`` might not
544# be owned by the installation user, we create the directory and change the
545# ownership to the proper user.
Dean Troyer50cda692014-07-25 11:57:20 -0500546# Set global ``RECLONE=yes`` to simulate a clone when dest-dir exists
547# Set global ``ERROR_ON_CLONE=True`` to abort execution with an error if the git repo
Dean Troyerdff49a22014-01-30 15:37:40 -0600548# does not exist (default is False, meaning the repo will be cloned).
Sean Dague53753292014-12-04 19:38:15 -0500549# Uses globals ``ERROR_ON_CLONE``, ``OFFLINE``, ``RECLONE``
Dean Troyerdff49a22014-01-30 15:37:40 -0600550# git_clone remote dest-dir branch
551function git_clone {
Dean Troyer50cda692014-07-25 11:57:20 -0500552 local git_remote=$1
553 local git_dest=$2
554 local git_ref=$3
Ian Wienandada886d2015-10-07 14:06:26 +1100555 local orig_dir
556 orig_dir=$(pwd)
Jamie Lennox51f0de52014-10-20 16:32:34 +0200557 local git_clone_flags=""
Dean Troyer50cda692014-07-25 11:57:20 -0500558
Sean Dague53753292014-12-04 19:38:15 -0500559 RECLONE=$(trueorfalse False RECLONE)
Kevin Benton59d52f32015-01-17 11:29:12 -0800560 if [[ "${GIT_DEPTH}" -gt 0 ]]; then
Jamie Lennox51f0de52014-10-20 16:32:34 +0200561 git_clone_flags="$git_clone_flags --depth $GIT_DEPTH"
562 fi
563
Dean Troyerdff49a22014-01-30 15:37:40 -0600564 if [[ "$OFFLINE" = "True" ]]; then
565 echo "Running in offline mode, clones already exist"
566 # print out the results so we know what change was used in the logs
Dean Troyer50cda692014-07-25 11:57:20 -0500567 cd $git_dest
Dean Troyerdff49a22014-01-30 15:37:40 -0600568 git show --oneline | head -1
Sean Dague64bd0162014-03-12 13:04:22 -0400569 cd $orig_dir
Dean Troyerdff49a22014-01-30 15:37:40 -0600570 return
571 fi
572
Dean Troyer50cda692014-07-25 11:57:20 -0500573 if echo $git_ref | egrep -q "^refs"; then
Dean Troyerdff49a22014-01-30 15:37:40 -0600574 # If our branch name is a gerrit style refs/changes/...
Dean Troyer50cda692014-07-25 11:57:20 -0500575 if [[ ! -d $git_dest ]]; then
James E. Blairebe63d82015-09-24 07:43:50 -0700576 if [[ "$ERROR_ON_CLONE" = "True" ]]; then
577 echo "The $git_dest project was not found; if this is a gate job, add"
James E. Blair35a0c572017-09-10 15:37:56 -0700578 echo "the project to 'required-projects' in the job definition."
Ghanshyam Mann325792d2021-10-15 15:55:54 -0500579 die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
James E. Blairebe63d82015-09-24 07:43:50 -0700580 fi
Jamie Lennox51f0de52014-10-20 16:32:34 +0200581 git_timed clone $git_clone_flags $git_remote $git_dest
Dean Troyerdff49a22014-01-30 15:37:40 -0600582 fi
Dean Troyer50cda692014-07-25 11:57:20 -0500583 cd $git_dest
584 git_timed fetch $git_remote $git_ref && git checkout FETCH_HEAD
Dean Troyerdff49a22014-01-30 15:37:40 -0600585 else
586 # do a full clone only if the directory doesn't exist
Dean Troyer50cda692014-07-25 11:57:20 -0500587 if [[ ! -d $git_dest ]]; then
James E. Blairebe63d82015-09-24 07:43:50 -0700588 if [[ "$ERROR_ON_CLONE" = "True" ]]; then
589 echo "The $git_dest project was not found; if this is a gate job, add"
590 echo "the project to the \$PROJECTS variable in the job definition."
Ghanshyam Mann325792d2021-10-15 15:55:54 -0500591 die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
James E. Blairebe63d82015-09-24 07:43:50 -0700592 fi
Jakub Wachowski90742fc2016-11-18 14:28:47 +0100593 # '--branch' can also take tags
594 git_timed clone $git_clone_flags $git_remote $git_dest --branch $git_ref
Dean Troyerdff49a22014-01-30 15:37:40 -0600595 elif [[ "$RECLONE" = "True" ]]; then
596 # if it does exist then simulate what clone does if asked to RECLONE
Dean Troyer50cda692014-07-25 11:57:20 -0500597 cd $git_dest
Dean Troyerdff49a22014-01-30 15:37:40 -0600598 # set the url to pull from and fetch
Dean Troyer50cda692014-07-25 11:57:20 -0500599 git remote set-url origin $git_remote
Ian Wienandd53ad0b2014-02-20 13:55:13 +1100600 git_timed fetch origin
Dean Troyerdff49a22014-01-30 15:37:40 -0600601 # remove the existing ignored files (like pyc) as they cause breakage
602 # (due to the py files having older timestamps than our pyc, so python
603 # thinks the pyc files are correct using them)
Dean Troyer50cda692014-07-25 11:57:20 -0500604 find $git_dest -name '*.pyc' -delete
Dean Troyerdff49a22014-01-30 15:37:40 -0600605
Dean Troyer50cda692014-07-25 11:57:20 -0500606 # handle git_ref accordingly to type (tag, branch)
607 if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
608 git_update_tag $git_ref
609 elif [[ -n "`git show-ref refs/heads/$git_ref`" ]]; then
610 git_update_branch $git_ref
611 elif [[ -n "`git show-ref refs/remotes/origin/$git_ref`" ]]; then
612 git_update_remote_branch $git_ref
Dean Troyerdff49a22014-01-30 15:37:40 -0600613 else
Dean Troyer50cda692014-07-25 11:57:20 -0500614 die $LINENO "$git_ref is neither branch nor tag"
Dean Troyerdff49a22014-01-30 15:37:40 -0600615 fi
616
617 fi
618 fi
619
620 # print out the results so we know what change was used in the logs
Dean Troyer50cda692014-07-25 11:57:20 -0500621 cd $git_dest
Dean Troyerdff49a22014-01-30 15:37:40 -0600622 git show --oneline | head -1
Sean Dague64bd0162014-03-12 13:04:22 -0400623 cd $orig_dir
Dean Troyerdff49a22014-01-30 15:37:40 -0600624}
625
Sean Daguecc524062014-10-01 09:06:43 -0400626# A variation on git clone that lets us specify a project by it's
627# actual name, like oslo.config. This is exceptionally useful in the
628# library installation case
629function git_clone_by_name {
630 local name=$1
631 local repo=${GITREPO[$name]}
632 local dir=${GITDIR[$name]}
633 local branch=${GITBRANCH[$name]}
634 git_clone $repo $dir $branch
635}
636
637
Ian Wienandd53ad0b2014-02-20 13:55:13 +1100638# git can sometimes get itself infinitely stuck with transient network
639# errors or other issues with the remote end. This wraps git in a
640# timeout/retry loop and is intended to watch over non-local git
641# processes that might hang. GIT_TIMEOUT, if set, is passed directly
642# to timeout(1); otherwise the default value of 0 maintains the status
643# quo of waiting forever.
644# usage: git_timed <git-command>
Ian Wienandaee18c72014-02-21 15:35:08 +1100645function git_timed {
Ian Wienandd53ad0b2014-02-20 13:55:13 +1100646 local count=0
647 local timeout=0
648
649 if [[ -n "${GIT_TIMEOUT}" ]]; then
650 timeout=${GIT_TIMEOUT}
651 fi
652
Atsushi SAKAI2ca8af42015-12-08 15:36:13 +0900653 time_start "git_timed"
Ian Wienandd53ad0b2014-02-20 13:55:13 +1100654 until timeout -s SIGINT ${timeout} git "$@"; do
655 # 124 is timeout(1)'s special return code when it reached the
656 # timeout; otherwise assume fatal failure
657 if [[ $? -ne 124 ]]; then
658 die $LINENO "git call failed: [git $@]"
659 fi
660
661 count=$(($count + 1))
Sean Daguee4af9292015-04-28 08:57:57 -0400662 warn $LINENO "timeout ${count} for git call: [git $@]"
Ian Wienandd53ad0b2014-02-20 13:55:13 +1100663 if [ $count -eq 3 ]; then
664 die $LINENO "Maximum of 3 git retries reached"
665 fi
666 sleep 5
667 done
Atsushi SAKAI2ca8af42015-12-08 15:36:13 +0900668 time_stop "git_timed"
Ian Wienandd53ad0b2014-02-20 13:55:13 +1100669}
670
Dean Troyerdff49a22014-01-30 15:37:40 -0600671# git update using reference as a branch.
672# git_update_branch ref
Ian Wienandaee18c72014-02-21 15:35:08 +1100673function git_update_branch {
Dean Troyer50cda692014-07-25 11:57:20 -0500674 local git_branch=$1
Dean Troyerdff49a22014-01-30 15:37:40 -0600675
Dean Troyer50cda692014-07-25 11:57:20 -0500676 git checkout -f origin/$git_branch
Dean Troyerdff49a22014-01-30 15:37:40 -0600677 # a local branch might not exist
Dean Troyer50cda692014-07-25 11:57:20 -0500678 git branch -D $git_branch || true
679 git checkout -b $git_branch
Dean Troyerdff49a22014-01-30 15:37:40 -0600680}
681
682# git update using reference as a branch.
683# git_update_remote_branch ref
Ian Wienandaee18c72014-02-21 15:35:08 +1100684function git_update_remote_branch {
Dean Troyer50cda692014-07-25 11:57:20 -0500685 local git_branch=$1
Dean Troyerdff49a22014-01-30 15:37:40 -0600686
Dean Troyer50cda692014-07-25 11:57:20 -0500687 git checkout -b $git_branch -t origin/$git_branch
Dean Troyerdff49a22014-01-30 15:37:40 -0600688}
689
690# git update using reference as a tag. Be careful editing source at that repo
691# as working copy will be in a detached mode
692# git_update_tag ref
Ian Wienandaee18c72014-02-21 15:35:08 +1100693function git_update_tag {
Dean Troyer50cda692014-07-25 11:57:20 -0500694 local git_tag=$1
Dean Troyerdff49a22014-01-30 15:37:40 -0600695
Dean Troyer50cda692014-07-25 11:57:20 -0500696 git tag -d $git_tag
Dean Troyerdff49a22014-01-30 15:37:40 -0600697 # fetching given tag only
Dean Troyer50cda692014-07-25 11:57:20 -0500698 git_timed fetch origin tag $git_tag
699 git checkout -f $git_tag
Dean Troyerdff49a22014-01-30 15:37:40 -0600700}
701
702
703# OpenStack Functions
704# ===================
705
706# Get the default value for HOST_IP
707# get_default_host_ip fixed_range floating_range host_ip_iface host_ip
Ian Wienandaee18c72014-02-21 15:35:08 +1100708function get_default_host_ip {
Dean Troyerdff49a22014-01-30 15:37:40 -0600709 local fixed_range=$1
710 local floating_range=$2
711 local host_ip_iface=$3
712 local host_ip=$4
Brian Haley180f5eb2015-06-16 13:14:31 -0400713 local af=$5
Dean Troyerdff49a22014-01-30 15:37:40 -0600714
Dean Troyerdff49a22014-01-30 15:37:40 -0600715 # Search for an IP unless an explicit is set by ``HOST_IP`` environment variable
716 if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
717 host_ip=""
Andreas Scheuringa3430272015-03-09 16:55:32 +0100718 # Find the interface used for the default route
Brian Haley180f5eb2015-06-16 13:14:31 -0400719 host_ip_iface=${host_ip_iface:-$(ip -f $af route | awk '/default/ {print $5}' | head -1)}
Ian Wienandada886d2015-10-07 14:06:26 +1100720 local host_ips
721 host_ips=$(LC_ALL=C ip -f $af addr show ${host_ip_iface} | sed /temporary/d |awk /$af'/ {split($2,parts,"/"); print parts[1]}')
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500722 local ip
723 for ip in $host_ips; do
Dean Troyerdff49a22014-01-30 15:37:40 -0600724 # Attempt to filter out IP addresses that are part of the fixed and
725 # floating range. Note that this method only works if the ``netaddr``
726 # python library is installed. If it is not installed, an error
727 # will be printed and the first IP from the interface will be used.
728 # If that is not correct set ``HOST_IP`` in ``localrc`` to the correct
729 # address.
Brian Haley180f5eb2015-06-16 13:14:31 -0400730 if [[ "$af" == "inet6" ]]; then
731 host_ip=$ip
732 break;
733 fi
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500734 if ! (address_in_net $ip $fixed_range || address_in_net $ip $floating_range); then
735 host_ip=$ip
Dean Troyerdff49a22014-01-30 15:37:40 -0600736 break;
737 fi
738 done
739 fi
740 echo $host_ip
741}
742
Attila Fazekasf71b5002014-05-28 09:52:22 +0200743# Generates hex string from ``size`` byte of pseudo random data
744# generate_hex_string size
745function generate_hex_string {
746 local size=$1
747 hexdump -n "$size" -v -e '/1 "%02x"' /dev/urandom
748}
749
Dean Troyerdff49a22014-01-30 15:37:40 -0600750# Grab a numbered field from python prettytable output
751# Fields are numbered starting with 1
752# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
753# get_field field-number
Ian Wienandaee18c72014-02-21 15:35:08 +1100754function get_field {
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500755 local data field
Dean Troyerdff49a22014-01-30 15:37:40 -0600756 while read data; do
757 if [ "$1" -lt 0 ]; then
758 field="(\$(NF$1))"
759 else
760 field="\$$(($1 + 1))"
761 fi
762 echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
763 done
764}
765
yuntongjinf26deea2015-02-28 10:50:34 +0800766# install default policy
767# copy over a default policy.json and policy.d for projects
768function install_default_policy {
769 local project=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100770 local project_uc
771 project_uc=$(echo $1|tr a-z A-Z)
yuntongjinf26deea2015-02-28 10:50:34 +0800772 local conf_dir="${project_uc}_CONF_DIR"
773 # eval conf dir to get the variable
774 conf_dir="${!conf_dir}"
775 local project_dir="${project_uc}_DIR"
776 # eval project dir to get the variable
777 project_dir="${!project_dir}"
778 local sample_conf_dir="${project_dir}/etc/${project}"
779 local sample_policy_dir="${project_dir}/etc/${project}/policy.d"
780
781 # first copy any policy.json
782 cp -p $sample_conf_dir/policy.json $conf_dir
783 # then optionally copy over policy.d
784 if [[ -d $sample_policy_dir ]]; then
785 cp -r $sample_policy_dir $conf_dir/policy.d
786 fi
787}
788
Dean Troyerdff49a22014-01-30 15:37:40 -0600789# Add a policy to a policy.json file
790# Do nothing if the policy already exists
791# ``policy_add policy_file policy_name policy_permissions``
Ian Wienandaee18c72014-02-21 15:35:08 +1100792function policy_add {
Dean Troyerdff49a22014-01-30 15:37:40 -0600793 local policy_file=$1
794 local policy_name=$2
795 local policy_perm=$3
796
797 if grep -q ${policy_name} ${policy_file}; then
798 echo "Policy ${policy_name} already exists in ${policy_file}"
799 return
800 fi
801
802 # Add a terminating comma to policy lines without one
803 # Remove the closing '}' and all lines following to the end-of-file
Ian Wienandada886d2015-10-07 14:06:26 +1100804 local tmpfile
805 tmpfile=$(mktemp)
Dean Troyerdff49a22014-01-30 15:37:40 -0600806 uniq ${policy_file} | sed -e '
807 s/]$/],/
808 /^[}]/,$d
809 ' > ${tmpfile}
810
811 # Append policy and closing brace
812 echo " \"${policy_name}\": ${policy_perm}" >>${tmpfile}
813 echo "}" >>${tmpfile}
814
815 mv ${tmpfile} ${policy_file}
816}
817
Alistair Coles24779f62014-10-15 18:57:59 +0100818# Gets or creates a domain
819# Usage: get_or_create_domain <name> <description>
820function get_or_create_domain {
Ian Wienand11d276c2015-07-02 09:34:34 +1000821 local domain_id
Alistair Coles24779f62014-10-15 18:57:59 +0100822 # Gets domain id
Ian Wienand11d276c2015-07-02 09:34:34 +1000823 domain_id=$(
Alistair Coles24779f62014-10-15 18:57:59 +0100824 # Gets domain id
Steve Martinelli050a0d52015-09-06 22:03:54 +0000825 openstack domain show $1 \
Alistair Coles24779f62014-10-15 18:57:59 +0100826 -f value -c id 2>/dev/null ||
827 # Creates new domain
Steve Martinelli050a0d52015-09-06 22:03:54 +0000828 openstack domain create $1 \
Alistair Coles24779f62014-10-15 18:57:59 +0100829 --description "$2" \
830 -f value -c id
831 )
832 echo $domain_id
833}
834
Steve Martinellib74e01c2014-12-18 01:35:35 -0500835# Gets or creates group
Jamie Lennox9d7e7762015-05-29 01:08:53 +0000836# Usage: get_or_create_group <groupname> <domain> [<description>]
Steve Martinellib74e01c2014-12-18 01:35:35 -0500837function get_or_create_group {
Steve Martinellib74e01c2014-12-18 01:35:35 -0500838 local desc="${3:-}"
Ian Wienand11d276c2015-07-02 09:34:34 +1000839 local group_id
Steve Martinellib74e01c2014-12-18 01:35:35 -0500840 # Gets group id
Ian Wienand11d276c2015-07-02 09:34:34 +1000841 group_id=$(
Steve Martinellib74e01c2014-12-18 01:35:35 -0500842 # Creates new group with --or-show
Steve Martinelli050a0d52015-09-06 22:03:54 +0000843 openstack group create $1 \
Jamie Lennox9d7e7762015-05-29 01:08:53 +0000844 --domain $2 --description "$desc" --or-show \
Steve Martinellib74e01c2014-12-18 01:35:35 -0500845 -f value -c id
846 )
847 echo $group_id
848}
849
Bartosz Górski0abde392014-02-28 14:15:19 +0100850# Gets or creates user
Jamie Lennox9d7e7762015-05-29 01:08:53 +0000851# Usage: get_or_create_user <username> <password> <domain> [<email>]
Bartosz Górski0abde392014-02-28 14:15:19 +0100852function get_or_create_user {
Ian Wienand11d276c2015-07-02 09:34:34 +1000853 local user_id
Jamie Lennox9d7e7762015-05-29 01:08:53 +0000854 if [[ ! -z "$4" ]]; then
855 local email="--email=$4"
Gael Chamoulaud6dd8a8b2014-07-22 01:12:12 +0200856 else
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500857 local email=""
Gael Chamoulaud6dd8a8b2014-07-22 01:12:12 +0200858 fi
Bartosz Górski0abde392014-02-28 14:15:19 +0100859 # Gets user id
Ian Wienand11d276c2015-07-02 09:34:34 +1000860 user_id=$(
Steve Martinelli245daa22014-11-14 02:17:22 -0500861 # Creates new user with --or-show
Jamie Lennox9d7e7762015-05-29 01:08:53 +0000862 openstack user create \
Bartosz Górski0abde392014-02-28 14:15:19 +0100863 $1 \
864 --password "$2" \
Jamie Lennox9d7e7762015-05-29 01:08:53 +0000865 --domain=$3 \
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500866 $email \
Steve Martinelli245daa22014-11-14 02:17:22 -0500867 --or-show \
Bartosz Górski0abde392014-02-28 14:15:19 +0100868 -f value -c id
869 )
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500870 echo $user_id
Bartosz Górski0abde392014-02-28 14:15:19 +0100871}
872
873# Gets or creates project
Jamie Lennoxb632c9e2015-05-28 23:36:15 +0000874# Usage: get_or_create_project <name> <domain>
Bartosz Górski0abde392014-02-28 14:15:19 +0100875function get_or_create_project {
Ian Wienand11d276c2015-07-02 09:34:34 +1000876 local project_id
877 project_id=$(
Steve Martinelli245daa22014-11-14 02:17:22 -0500878 # Creates new project with --or-show
Steve Martinelli050a0d52015-09-06 22:03:54 +0000879 openstack project create $1 \
Jamie Lennoxb632c9e2015-05-28 23:36:15 +0000880 --domain=$2 \
881 --or-show -f value -c id
Bartosz Górski0abde392014-02-28 14:15:19 +0100882 )
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500883 echo $project_id
Bartosz Górski0abde392014-02-28 14:15:19 +0100884}
885
886# Gets or creates role
887# Usage: get_or_create_role <name>
888function get_or_create_role {
Ian Wienand11d276c2015-07-02 09:34:34 +1000889 local role_id
890 role_id=$(
Steve Martinelli245daa22014-11-14 02:17:22 -0500891 # Creates role with --or-show
Jamie Lennox72ce6ac2015-07-02 09:19:01 +1000892 openstack role create $1 \
Jamie Lennox72ce6ac2015-07-02 09:19:01 +1000893 --or-show -f value -c id
Bartosz Górski0abde392014-02-28 14:15:19 +0100894 )
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500895 echo $role_id
Bartosz Górski0abde392014-02-28 14:15:19 +0100896}
897
Jamie Lennoxcbcbd8f2016-01-21 16:08:14 -0600898# Returns the domain parts of a function call if present
899# Usage: _get_domain_args [<user_domain> <project_domain>]
900function _get_domain_args {
901 local domain
902 domain=""
903
904 if [[ -n "$1" ]]; then
905 domain="$domain --user-domain $1"
906 fi
907 if [[ -n "$2" ]]; then
908 domain="$domain --project-domain $2"
909 fi
910
911 echo $domain
912}
913
Jamie Lennox9b215db2015-02-10 18:19:57 +1100914# Gets or adds user role to project
Jamie Lennoxcbcbd8f2016-01-21 16:08:14 -0600915# Usage: get_or_add_user_project_role <role> <user> <project> [<user_domain> <project_domain>]
Jamie Lennox9b215db2015-02-10 18:19:57 +1100916function get_or_add_user_project_role {
Ian Wienand11d276c2015-07-02 09:34:34 +1000917 local user_role_id
Jamie Lennoxcbcbd8f2016-01-21 16:08:14 -0600918
919 domain_args=$(_get_domain_args $4 $5)
920
Bartosz Górski0abde392014-02-28 14:15:19 +0100921 # Gets user role id
Mike Perezc271b3e2016-10-03 16:00:33 -0700922 user_role_id=$(openstack role assignment list \
Clark Boylan3e9b5622017-06-14 15:29:47 -0700923 --role $1 \
Steve Martinelli5541a612015-01-19 15:58:49 -0500924 --user $2 \
Jamie Lennox72ce6ac2015-07-02 09:19:01 +1000925 --project $3 \
Jamie Lennoxcbcbd8f2016-01-21 16:08:14 -0600926 $domain_args \
Clark Boylan3e9b5622017-06-14 15:29:47 -0700927 | grep '^|\s[a-f0-9]\+' | get_field 1)
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500928 if [[ -z "$user_role_id" ]]; then
Roxana Gherle50821be2015-09-22 10:52:46 -0700929 # Adds role to user and get it
930 openstack role add $1 \
Bartosz Górski0abde392014-02-28 14:15:19 +0100931 --user $2 \
Jamie Lennoxcbcbd8f2016-01-21 16:08:14 -0600932 --project $3 \
933 $domain_args
Mike Perezc271b3e2016-10-03 16:00:33 -0700934 user_role_id=$(openstack role assignment list \
Clark Boylan3e9b5622017-06-14 15:29:47 -0700935 --role $1 \
Roxana Gherle50821be2015-09-22 10:52:46 -0700936 --user $2 \
Roxana Gherle50821be2015-09-22 10:52:46 -0700937 --project $3 \
Jamie Lennoxcbcbd8f2016-01-21 16:08:14 -0600938 $domain_args \
Clark Boylan3e9b5622017-06-14 15:29:47 -0700939 | grep '^|\s[a-f0-9]\+' | get_field 1)
Bartosz Górski0abde392014-02-28 14:15:19 +0100940 fi
Dean Troyerd5dfa4c2014-07-25 11:13:11 -0500941 echo $user_role_id
Bartosz Górski0abde392014-02-28 14:15:19 +0100942}
943
Steve Martinelli4ce859a2015-12-20 01:27:30 -0500944# Gets or adds user role to domain
945# Usage: get_or_add_user_domain_role <role> <user> <domain>
946function get_or_add_user_domain_role {
947 local user_role_id
948 # Gets user role id
Mike Perezc271b3e2016-10-03 16:00:33 -0700949 user_role_id=$(openstack role assignment list \
Clark Boylan3e9b5622017-06-14 15:29:47 -0700950 --role $1 \
Steve Martinelli4ce859a2015-12-20 01:27:30 -0500951 --user $2 \
Steve Martinelli4ce859a2015-12-20 01:27:30 -0500952 --domain $3 \
Clark Boylan3e9b5622017-06-14 15:29:47 -0700953 | grep '^|\s[a-f0-9]\+' | get_field 1)
Steve Martinelli4ce859a2015-12-20 01:27:30 -0500954 if [[ -z "$user_role_id" ]]; then
955 # Adds role to user and get it
956 openstack role add $1 \
957 --user $2 \
958 --domain $3
Mike Perezc271b3e2016-10-03 16:00:33 -0700959 user_role_id=$(openstack role assignment list \
Clark Boylan3e9b5622017-06-14 15:29:47 -0700960 --role $1 \
Steve Martinelli4ce859a2015-12-20 01:27:30 -0500961 --user $2 \
Steve Martinelli4ce859a2015-12-20 01:27:30 -0500962 --domain $3 \
Clark Boylan3e9b5622017-06-14 15:29:47 -0700963 | grep '^|\s[a-f0-9]\+' | get_field 1)
Steve Martinelli4ce859a2015-12-20 01:27:30 -0500964 fi
965 echo $user_role_id
966}
967
Lance Bragstad021ae0b2021-03-11 15:47:50 +0000968# Gets or adds user role to system
969# Usage: get_or_add_user_system_role <role> <user> <system> [<user_domain>]
970function get_or_add_user_system_role {
971 local user_role_id
972 local domain_args
973
974 domain_args=$(_get_domain_args $4)
975
976 # Gets user role id
977 user_role_id=$(openstack role assignment list \
978 --role $1 \
979 --user $2 \
980 --system $3 \
981 $domain_args \
982 -f value -c Role)
983 if [[ -z "$user_role_id" ]]; then
984 # Adds role to user and get it
985 openstack role add $1 \
986 --user $2 \
987 --system $3 \
988 $domain_args
989 user_role_id=$(openstack role assignment list \
990 --role $1 \
991 --user $2 \
992 --system $3 \
993 $domain_args \
994 -f value -c Role)
995 fi
996 echo $user_role_id
997}
998
Steve Martinelli4599fd12015-03-12 21:30:58 -0400999# Gets or adds group role to project
1000# Usage: get_or_add_group_project_role <role> <group> <project>
1001function get_or_add_group_project_role {
Ian Wienand11d276c2015-07-02 09:34:34 +10001002 local group_role_id
Steve Martinelli4599fd12015-03-12 21:30:58 -04001003 # Gets group role id
Mike Perezc271b3e2016-10-03 16:00:33 -07001004 group_role_id=$(openstack role assignment list \
Clark Boylan3e9b5622017-06-14 15:29:47 -07001005 --role $1 \
Steve Martinelli4599fd12015-03-12 21:30:58 -04001006 --group $2 \
1007 --project $3 \
Mike Perezc271b3e2016-10-03 16:00:33 -07001008 -f value)
Steve Martinelli4599fd12015-03-12 21:30:58 -04001009 if [[ -z "$group_role_id" ]]; then
Jamie Lennox72ce6ac2015-07-02 09:19:01 +10001010 # Adds role to group and get it
1011 openstack role add $1 \
Jamie Lennox72ce6ac2015-07-02 09:19:01 +10001012 --group $2 \
1013 --project $3
Mike Perezc271b3e2016-10-03 16:00:33 -07001014 group_role_id=$(openstack role assignment list \
Clark Boylan3e9b5622017-06-14 15:29:47 -07001015 --role $1 \
Steve Martinelli4599fd12015-03-12 21:30:58 -04001016 --group $2 \
1017 --project $3 \
Mike Perezc271b3e2016-10-03 16:00:33 -07001018 -f value)
Steve Martinelli4599fd12015-03-12 21:30:58 -04001019 fi
1020 echo $group_role_id
1021}
1022
Bartosz Górski0abde392014-02-28 14:15:19 +01001023# Gets or creates service
1024# Usage: get_or_create_service <name> <type> <description>
1025function get_or_create_service {
Ian Wienand11d276c2015-07-02 09:34:34 +10001026 local service_id
Bartosz Górski0abde392014-02-28 14:15:19 +01001027 # Gets service id
Ian Wienand11d276c2015-07-02 09:34:34 +10001028 service_id=$(
Bartosz Górski0abde392014-02-28 14:15:19 +01001029 # Gets service id
Jamie Lennoxaedb8b92015-07-02 17:39:07 +10001030 openstack service show $2 -f value -c id 2>/dev/null ||
Bartosz Górski0abde392014-02-28 14:15:19 +01001031 # Creates new service if not exists
1032 openstack service create \
Steve Martinelli789af5c2015-01-19 16:11:44 -05001033 $2 \
1034 --name $1 \
Bartosz Górski0abde392014-02-28 14:15:19 +01001035 --description="$3" \
1036 -f value -c id
1037 )
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05001038 echo $service_id
Bartosz Górski0abde392014-02-28 14:15:19 +01001039}
1040
Jamie Lennoxb17ad752015-05-29 06:04:47 +00001041# Create an endpoint with a specific interface
Matt Riedemannae4578b2016-04-23 01:45:40 +00001042# Usage: _get_or_create_endpoint_with_interface <service> <interface> <url> <region>
1043function _get_or_create_endpoint_with_interface {
Ian Wienand11d276c2015-07-02 09:34:34 +10001044 local endpoint_id
Matt Riedemannae4578b2016-04-23 01:45:40 +00001045 endpoint_id=$(openstack endpoint list \
1046 --service $1 \
1047 --interface $2 \
1048 --region $4 \
1049 -c ID -f value)
1050 if [[ -z "$endpoint_id" ]]; then
1051 # Creates new endpoint
1052 endpoint_id=$(openstack endpoint create \
1053 $1 $2 $3 --region $4 -f value -c id)
1054 fi
1055
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05001056 echo $endpoint_id
Bartosz Górski0abde392014-02-28 14:15:19 +01001057}
Dean Troyerdff49a22014-01-30 15:37:40 -06001058
Sean Dague7d1ec432016-04-22 09:19:10 -04001059# Gets or creates endpoint
Sean Dague11eb2012017-02-13 16:16:59 -05001060# Usage: get_or_create_endpoint <service> <region> <publicurl> [adminurl] [internalurl]
Matt Riedemannae4578b2016-04-23 01:45:40 +00001061function get_or_create_endpoint {
Jamie Lennoxb17ad752015-05-29 06:04:47 +00001062 # NOTE(jamielennnox): when converting to v3 endpoint creation we go from
1063 # creating one endpoint with multiple urls to multiple endpoints each with
1064 # a different interface. To maintain the existing function interface we
Matt Riedemannae4578b2016-04-23 01:45:40 +00001065 # create 3 endpoints and return the id of the public one. In reality
Jamie Lennoxb17ad752015-05-29 06:04:47 +00001066 # returning the public id will not make a lot of difference as there are no
1067 # scenarios currently that use the returned id. Ideally this behaviour
1068 # should be pushed out to the service setups and let them create the
1069 # endpoints they need.
Ian Wienandada886d2015-10-07 14:06:26 +11001070 local public_id
Matt Riedemannae4578b2016-04-23 01:45:40 +00001071 public_id=$(_get_or_create_endpoint_with_interface $1 public $3 $2)
Sean Dague11eb2012017-02-13 16:16:59 -05001072 # only create admin/internal urls if provided content for them
1073 if [[ -n "$4" ]]; then
1074 _get_or_create_endpoint_with_interface $1 admin $4 $2
1075 fi
1076 if [[ -n "$5" ]]; then
1077 _get_or_create_endpoint_with_interface $1 internal $5 $2
1078 fi
Jamie Lennoxb17ad752015-05-29 06:04:47 +00001079 # return the public id to indicate success, and this is the endpoint most likely wanted
1080 echo $public_id
1081}
1082
1083# Get a URL from the identity service
1084# Usage: get_endpoint_url <service> <interface>
1085function get_endpoint_url {
1086 echo $(openstack endpoint list \
1087 --service $1 --interface $2 \
Jamie Lennoxb17ad752015-05-29 06:04:47 +00001088 -c URL -f value)
1089}
1090
Jim Rollenhagen47367072015-12-10 14:24:00 +00001091# check if we are using ironic with hardware
1092# TODO(jroll) this is a kludge left behind when ripping ironic code
1093# out of tree, as it is used by nova and neutron.
1094# figure out a way to refactor nova/neutron code to eliminate this
1095function is_ironic_hardware {
Lucas Alvares Gomes8a4dea22016-02-18 15:52:22 +00001096 is_service_enabled ironic && [[ "$IRONIC_IS_HARDWARE" == "True" ]] && return 0
Jim Rollenhagen47367072015-12-10 14:24:00 +00001097 return 1
1098}
1099
Julia Kreger6af3cb92021-03-11 11:28:47 -08001100function is_ironic_enforce_scope {
1101 is_service_enabled ironic && [[ "$IRONIC_ENFORCE_SCOPE" == "True" ]] && return 0
1102 return 1
1103}
1104
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05001105
Dean Troyerdff49a22014-01-30 15:37:40 -06001106# Package Functions
1107# =================
1108
1109# _get_package_dir
Ian Wienandaee18c72014-02-21 15:35:08 +11001110function _get_package_dir {
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001111 local base_dir=$1
Dean Troyerdff49a22014-01-30 15:37:40 -06001112 local pkg_dir
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001113
1114 if [[ -z "$base_dir" ]]; then
1115 base_dir=$FILES
1116 fi
Dean Troyerdff49a22014-01-30 15:37:40 -06001117 if is_ubuntu; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001118 pkg_dir=$base_dir/debs
Dean Troyerdff49a22014-01-30 15:37:40 -06001119 elif is_fedora; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001120 pkg_dir=$base_dir/rpms
Dean Troyerdff49a22014-01-30 15:37:40 -06001121 elif is_suse; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001122 pkg_dir=$base_dir/rpms-suse
Dean Troyerdff49a22014-01-30 15:37:40 -06001123 else
1124 exit_distro_not_supported "list of packages"
1125 fi
1126 echo "$pkg_dir"
1127}
1128
Sean Dague88ee8ce2015-12-02 07:47:31 -05001129# Wrapper for ``apt-get update`` to try multiple times on the update
1130# to address bad package mirrors (which happen all the time).
1131function apt_get_update {
1132 # only do this once per run
1133 if [[ "$REPOS_UPDATED" == "True" && "$RETRY_UPDATE" != "True" ]]; then
1134 return
1135 fi
1136
1137 # bail if we are offline
1138 [[ "$OFFLINE" = "True" ]] && return
1139
1140 local sudo="sudo"
1141 [[ "$(id -u)" = "0" ]] && sudo="env"
1142
1143 # time all the apt operations
1144 time_start "apt-get-update"
1145
1146 local proxies="http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} no_proxy=${no_proxy:-} "
1147 local update_cmd="$sudo $proxies apt-get update"
1148 if ! timeout 300 sh -c "while ! $update_cmd; do sleep 30; done"; then
1149 die $LINENO "Failed to update apt repos, we're dead now"
1150 fi
1151
1152 REPOS_UPDATED=True
1153 # stop the clock
1154 time_stop "apt-get-update"
1155}
1156
Dean Troyerdff49a22014-01-30 15:37:40 -06001157# Wrapper for ``apt-get`` to set cache and proxy environment variables
1158# Uses globals ``OFFLINE``, ``*_proxy``
1159# apt_get operation package [package ...]
Ian Wienandaee18c72014-02-21 15:35:08 +11001160function apt_get {
Federico Ressie208d062015-11-21 11:15:39 +00001161 local xtrace result
Ian Wienand433a9b12015-10-07 13:29:31 +11001162 xtrace=$(set +o | grep xtrace)
Sean Dague45917cc2014-02-24 16:09:14 -05001163 set +o xtrace
1164
Dean Troyerdff49a22014-01-30 15:37:40 -06001165 [[ "$OFFLINE" = "True" || -z "$@" ]] && return
1166 local sudo="sudo"
1167 [[ "$(id -u)" = "0" ]] && sudo="env"
Sean Dague45917cc2014-02-24 16:09:14 -05001168
Sean Dague95c33d52015-10-07 11:05:59 -04001169 # time all the apt operations
1170 time_start "apt-get"
1171
Sean Dague45917cc2014-02-24 16:09:14 -05001172 $xtrace
Sean Dague53753292014-12-04 19:38:15 -05001173
Dean Troyerdff49a22014-01-30 15:37:40 -06001174 $sudo DEBIAN_FRONTEND=noninteractive \
Sean Dague53753292014-12-04 19:38:15 -05001175 http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} \
1176 no_proxy=${no_proxy:-} \
John L. Villalovosf568c3a2016-01-07 19:10:50 -08001177 apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@" < /dev/null
Federico Ressie208d062015-11-21 11:15:39 +00001178 result=$?
Sean Dague95c33d52015-10-07 11:05:59 -04001179
1180 # stop the clock
1181 time_stop "apt-get"
Federico Ressie208d062015-11-21 11:15:39 +00001182 return $result
Dean Troyerdff49a22014-01-30 15:37:40 -06001183}
1184
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001185function _parse_package_files {
1186 local files_to_parse=$@
Dean Troyerdff49a22014-01-30 15:37:40 -06001187
Dean Troyerdff49a22014-01-30 15:37:40 -06001188 if [[ -z "$DISTRO" ]]; then
1189 GetDistro
1190 fi
Dean Troyerdff49a22014-01-30 15:37:40 -06001191
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001192 for fname in ${files_to_parse}; do
Dean Troyerdff49a22014-01-30 15:37:40 -06001193 local OIFS line package distros distro
1194 [[ -e $fname ]] || continue
1195
1196 OIFS=$IFS
1197 IFS=$'\n'
1198 for line in $(<${fname}); do
1199 if [[ $line =~ "NOPRIME" ]]; then
1200 continue
1201 fi
1202
Ian Wienand1ff75ff2016-02-19 14:28:37 +11001203 # Assume we want this package; free-form
1204 # comments allowed after a #
1205 package=${line%%#*}
Dean Troyerdff49a22014-01-30 15:37:40 -06001206 inst_pkg=1
1207
1208 # Look for # dist:xxx in comment
1209 if [[ $line =~ (.*)#.*dist:([^ ]*) ]]; then
1210 # We are using BASH regexp matching feature.
1211 package=${BASH_REMATCH[1]}
1212 distros=${BASH_REMATCH[2]}
Atsushi SAKAI33c9a672015-11-12 19:50:00 +09001213 # In bash ${VAR,,} will lowercase VAR
Dean Troyerdff49a22014-01-30 15:37:40 -06001214 # Look for a match in the distro list
1215 if [[ ! ${distros,,} =~ ${DISTRO,,} ]]; then
1216 # If no match then skip this package
1217 inst_pkg=0
1218 fi
1219 fi
1220
David Rabel682e0ab2017-03-17 19:19:00 +01001221 # Look for # not:xxx in comment
1222 if [[ $line =~ (.*)#.*not:([^ ]*) ]]; then
1223 # We are using BASH regexp matching feature.
1224 package=${BASH_REMATCH[1]}
1225 distros=${BASH_REMATCH[2]}
1226 # In bash ${VAR,,} will lowercase VAR
1227 # Look for a match in the distro list
1228 if [[ ${distros,,} =~ ${DISTRO,,} ]]; then
1229 # If match then skip this package
1230 inst_pkg=0
1231 fi
1232 fi
1233
Dean Troyerdff49a22014-01-30 15:37:40 -06001234 if [[ $inst_pkg = 1 ]]; then
1235 echo $package
1236 fi
1237 done
1238 IFS=$OIFS
1239 done
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001240}
1241
1242# get_packages() collects a list of package names of any type from the
1243# prerequisite files in ``files/{debs|rpms}``. The list is intended
1244# to be passed to a package installer such as apt or yum.
1245#
1246# Only packages required for the services in 1st argument will be
1247# included. Two bits of metadata are recognized in the prerequisite files:
1248#
1249# - ``# NOPRIME`` defers installation to be performed later in `stack.sh`
1250# - ``# dist:DISTRO`` or ``dist:DISTRO1,DISTRO2`` limits the selection
1251# of the package to the distros listed. The distro names are case insensitive.
David Rabel682e0ab2017-03-17 19:19:00 +01001252# - ``# not:DISTRO`` or ``not:DISTRO1,DISTRO2`` limits the selection
1253# of the package to the distros not listed. The distro names are case insensitive.
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001254function get_packages {
Ian Wienand433a9b12015-10-07 13:29:31 +11001255 local xtrace
1256 xtrace=$(set +o | grep xtrace)
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001257 set +o xtrace
1258 local services=$@
Ian Wienandada886d2015-10-07 14:06:26 +11001259 local package_dir
1260 package_dir=$(_get_package_dir)
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001261 local file_to_parse=""
1262 local service=""
1263
Ian Wienand7d515b52015-11-09 15:04:32 +11001264 if [ $# -ne 1 ]; then
1265 die $LINENO "get_packages takes a single, comma-separated argument"
1266 fi
1267
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001268 if [[ -z "$package_dir" ]]; then
1269 echo "No package directory supplied"
1270 return 1
1271 fi
1272 for service in ${services//,/ }; do
1273 # Allow individual services to specify dependencies
1274 if [[ -e ${package_dir}/${service} ]]; then
1275 file_to_parse="${file_to_parse} ${package_dir}/${service}"
1276 fi
1277 # NOTE(sdague) n-api needs glance for now because that's where
1278 # glance client is
1279 if [[ $service == n-api ]]; then
Ryan Hsu6f3f3102015-03-19 16:26:45 -07001280 if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001281 file_to_parse="${file_to_parse} ${package_dir}/nova"
1282 fi
Ryan Hsu6f3f3102015-03-19 16:26:45 -07001283 if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001284 file_to_parse="${file_to_parse} ${package_dir}/glance"
1285 fi
Luigi Toscano52e52d82020-08-13 09:55:08 +02001286 if [[ ! $file_to_parse =~ $package_dir/os-brick ]]; then
1287 file_to_parse="${file_to_parse} ${package_dir}/os-brick"
1288 fi
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001289 elif [[ $service == c-* ]]; then
Ryan Hsu6f3f3102015-03-19 16:26:45 -07001290 if [[ ! $file_to_parse =~ $package_dir/cinder ]]; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001291 file_to_parse="${file_to_parse} ${package_dir}/cinder"
1292 fi
Luigi Toscano52e52d82020-08-13 09:55:08 +02001293 if [[ ! $file_to_parse =~ $package_dir/os-brick ]]; then
1294 file_to_parse="${file_to_parse} ${package_dir}/os-brick"
1295 fi
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001296 elif [[ $service == s-* ]]; then
Ryan Hsu6f3f3102015-03-19 16:26:45 -07001297 if [[ ! $file_to_parse =~ $package_dir/swift ]]; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001298 file_to_parse="${file_to_parse} ${package_dir}/swift"
1299 fi
1300 elif [[ $service == n-* ]]; then
Ryan Hsu6f3f3102015-03-19 16:26:45 -07001301 if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001302 file_to_parse="${file_to_parse} ${package_dir}/nova"
1303 fi
Luigi Toscano52e52d82020-08-13 09:55:08 +02001304 if [[ ! $file_to_parse =~ $package_dir/os-brick ]]; then
1305 file_to_parse="${file_to_parse} ${package_dir}/os-brick"
1306 fi
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001307 elif [[ $service == g-* ]]; then
Ryan Hsu6f3f3102015-03-19 16:26:45 -07001308 if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001309 file_to_parse="${file_to_parse} ${package_dir}/glance"
1310 fi
1311 elif [[ $service == key* ]]; then
Ryan Hsu6f3f3102015-03-19 16:26:45 -07001312 if [[ ! $file_to_parse =~ $package_dir/keystone ]]; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001313 file_to_parse="${file_to_parse} ${package_dir}/keystone"
1314 fi
Ihar Hrachyshka387aadd2017-09-14 09:25:59 -06001315 elif [[ $service == q-* || $service == neutron-* ]]; then
1316 if [[ ! $file_to_parse =~ $package_dir/neutron-common ]]; then
1317 file_to_parse="${file_to_parse} ${package_dir}/neutron-common"
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001318 fi
1319 elif [[ $service == ir-* ]]; then
Ryan Hsu6f3f3102015-03-19 16:26:45 -07001320 if [[ ! $file_to_parse =~ $package_dir/ironic ]]; then
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001321 file_to_parse="${file_to_parse} ${package_dir}/ironic"
1322 fi
1323 fi
1324 done
1325 echo "$(_parse_package_files $file_to_parse)"
1326 $xtrace
1327}
1328
1329# get_plugin_packages() collects a list of package names of any type from a
1330# plugin's prerequisite files in ``$PLUGIN/devstack/files/{debs|rpms}``. The
1331# list is intended to be passed to a package installer such as apt or yum.
1332#
1333# Only packages required for enabled and collected plugins will included.
1334#
Dean Troyerdc97cb72015-03-28 08:20:50 -05001335# The same metadata used in the main DevStack prerequisite files may be used
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001336# in these prerequisite files, see get_packages() for more info.
1337function get_plugin_packages {
Ian Wienand433a9b12015-10-07 13:29:31 +11001338 local xtrace
1339 xtrace=$(set +o | grep xtrace)
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001340 set +o xtrace
1341 local files_to_parse=""
1342 local package_dir=""
1343 for plugin in ${DEVSTACK_PLUGINS//,/ }; do
Ian Wienand7ae97292016-02-16 14:50:53 +11001344 package_dir="$(_get_package_dir ${GITDIR[$plugin]}/devstack/files)"
Dmitry Tantsur5979f472016-01-03 18:08:14 +01001345 files_to_parse+=" $package_dir/$plugin"
Adam Gandelman7ca90cd2015-03-04 17:25:07 -08001346 done
1347 echo "$(_parse_package_files $files_to_parse)"
Sean Dague45917cc2014-02-24 16:09:14 -05001348 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -06001349}
1350
Ian Wienandfa9aadf2019-01-15 18:31:05 +11001351# Search plugins for a bindep.txt file
1352#
1353# Uses globals ``BINDEP_CMD``, ``GITDIR``, ``DEVSTACK_PLUGINS``
1354#
1355# Note this is only valid after BINDEP_CMD is setup in stack.sh, and
1356# is thus not really intended to be called externally.
1357function _get_plugin_bindep_packages {
1358 local xtrace
1359 xtrace=$(set +o | grep xtrace)
1360 set +o xtrace
1361
1362 local bindep_file
1363 local packages
1364
1365 for plugin in ${DEVSTACK_PLUGINS//,/ }; do
1366 bindep_file=${GITDIR[$plugin]}/devstack/files/bindep.txt
1367 if [[ -f ${bindep_file} ]]; then
1368 packages+=$($BINDEP_CMD -b --file ${bindep_file} || true)
1369 fi
1370 done
1371 echo "${packages}"
1372 $xtrace
1373}
1374
Dean Troyerdff49a22014-01-30 15:37:40 -06001375# Distro-agnostic package installer
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05001376# Uses globals ``NO_UPDATE_REPOS``, ``REPOS_UPDATED``, ``RETRY_UPDATE``
Dean Troyerdff49a22014-01-30 15:37:40 -06001377# install_package package [package ...]
Monty Taylor5cc6d2c2014-06-06 08:45:16 -04001378function update_package_repo {
Sean Dague53753292014-12-04 19:38:15 -05001379 NO_UPDATE_REPOS=${NO_UPDATE_REPOS:-False}
1380 REPOS_UPDATED=${REPOS_UPDATED:-False}
1381 RETRY_UPDATE=${RETRY_UPDATE:-False}
1382
Paul Linchpiner9e179742014-07-13 22:23:00 -07001383 if [[ "$NO_UPDATE_REPOS" = "True" ]]; then
Monty Taylor5cc6d2c2014-06-06 08:45:16 -04001384 return 0
1385 fi
Dean Troyerdff49a22014-01-30 15:37:40 -06001386
Monty Taylor5cc6d2c2014-06-06 08:45:16 -04001387 if is_ubuntu; then
Sean Dague88ee8ce2015-12-02 07:47:31 -05001388 apt_get_update
Monty Taylor5cc6d2c2014-06-06 08:45:16 -04001389 fi
1390}
1391
1392function real_install_package {
1393 if is_ubuntu; then
Dean Troyerdff49a22014-01-30 15:37:40 -06001394 apt_get install "$@"
1395 elif is_fedora; then
1396 yum_install "$@"
1397 elif is_suse; then
1398 zypper_install "$@"
1399 else
1400 exit_distro_not_supported "installing packages"
1401 fi
1402}
1403
Monty Taylor5cc6d2c2014-06-06 08:45:16 -04001404# Distro-agnostic package installer
1405# install_package package [package ...]
1406function install_package {
1407 update_package_repo
Federico Ressiecc1f412015-12-28 15:14:13 +01001408 if ! real_install_package "$@"; then
1409 RETRY_UPDATE=True update_package_repo && real_install_package "$@"
1410 fi
Monty Taylor5cc6d2c2014-06-06 08:45:16 -04001411}
1412
Dean Troyerdff49a22014-01-30 15:37:40 -06001413# Distro-agnostic function to tell if a package is installed
1414# is_package_installed package [package ...]
Ian Wienandaee18c72014-02-21 15:35:08 +11001415function is_package_installed {
Dean Troyerdff49a22014-01-30 15:37:40 -06001416 if [[ -z "$@" ]]; then
1417 return 1
1418 fi
1419
1420 if [[ -z "$os_PACKAGE" ]]; then
1421 GetOSVersion
1422 fi
1423
1424 if [[ "$os_PACKAGE" = "deb" ]]; then
1425 dpkg -s "$@" > /dev/null 2> /dev/null
1426 elif [[ "$os_PACKAGE" = "rpm" ]]; then
1427 rpm --quiet -q "$@"
1428 else
1429 exit_distro_not_supported "finding if a package is installed"
1430 fi
1431}
1432
1433# Distro-agnostic package uninstaller
1434# uninstall_package package [package ...]
Ian Wienandaee18c72014-02-21 15:35:08 +11001435function uninstall_package {
Dean Troyerdff49a22014-01-30 15:37:40 -06001436 if is_ubuntu; then
1437 apt_get purge "$@"
1438 elif is_fedora; then
Ian Wienand67fd81a2020-04-30 09:24:04 +10001439 sudo dnf remove -y "$@" ||:
Dean Troyerdff49a22014-01-30 15:37:40 -06001440 elif is_suse; then
Gary W. Smith56b39122016-11-16 22:03:43 -08001441 sudo zypper remove -y "$@" ||:
Dean Troyerdff49a22014-01-30 15:37:40 -06001442 else
1443 exit_distro_not_supported "uninstalling packages"
1444 fi
1445}
1446
Ian Wienand67fd81a2020-04-30 09:24:04 +10001447# Wrapper for ``dnf`` to set proxy environment variables
1448# Uses globals ``OFFLINE``, ``*_proxy``
1449# The name is kept for backwards compatability with external
1450# callers, despite none of our supported platforms using yum
1451# any more.
Dean Troyerdff49a22014-01-30 15:37:40 -06001452# yum_install package [package ...]
Ian Wienandaee18c72014-02-21 15:35:08 +11001453function yum_install {
Federico Ressi5dcbf7a2016-01-13 12:17:32 +00001454 local result parse_yum_result
1455
Dean Troyerdff49a22014-01-30 15:37:40 -06001456 [[ "$OFFLINE" = "True" ]] && return
Federico Ressi5dcbf7a2016-01-13 12:17:32 +00001457
1458 time_start "yum_install"
Ian Wienand67fd81a2020-04-30 09:24:04 +10001459 sudo_with_proxies dnf install -y "$@"
Federico Ressi5dcbf7a2016-01-13 12:17:32 +00001460 time_stop "yum_install"
Dean Troyerdff49a22014-01-30 15:37:40 -06001461}
1462
1463# zypper wrapper to set arguments correctly
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05001464# Uses globals ``OFFLINE``, ``*_proxy``
Dean Troyerdff49a22014-01-30 15:37:40 -06001465# zypper_install package [package ...]
Ian Wienandaee18c72014-02-21 15:35:08 +11001466function zypper_install {
Dean Troyerdff49a22014-01-30 15:37:40 -06001467 [[ "$OFFLINE" = "True" ]] && return
1468 local sudo="sudo"
1469 [[ "$(id -u)" = "0" ]] && sudo="env"
Ian Wienandfdf00f22015-03-13 11:50:02 +11001470 $sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
1471 no_proxy="${no_proxy:-}" \
Dirk Mueller297a50a2018-06-20 11:08:54 +02001472 zypper --non-interactive install --auto-agree-with-licenses --no-recommends "$@"
Dean Troyerdff49a22014-01-30 15:37:40 -06001473}
1474
Ian Wienand58243f62018-12-13 14:05:53 +11001475# Run bindep and install packages it outputs
1476#
1477# Usage:
1478# install_bindep <path-to-bindep.txt> [profile,profile]
1479#
1480# Note unlike the bindep command itself, profile(s) specified should
1481# be a single, comma-separated string, no spaces.
1482function install_bindep {
1483 local file=$1
1484 local profiles=${2:-""}
1485 local pkgs
1486
1487 if [[ ! -f $file ]]; then
Sean McGinnisdd3731c2020-07-28 08:51:41 -05001488 warn $LINENO "Can not find bindep file: $file"
1489 return
Ian Wienand58243f62018-12-13 14:05:53 +11001490 fi
1491
1492 # converting here makes it much easier to work with passing
1493 # arguments
1494 profiles=${profiles/,/ /}
1495
1496 # Note bindep returns 1 when packages need to be installed, so we
1497 # have to ignore it's return for "-e"
1498 pkgs=$($DEST/bindep-venv/bin/bindep -b --file $file $profiles || true)
1499
1500 if [[ -n "${pkgs}" ]]; then
1501 install_package ${pkgs}
1502 fi
1503}
1504
Sean Dague5edae542017-03-21 20:50:24 -04001505function write_user_unit_file {
1506 local service=$1
1507 local command="$2"
1508 local group=$3
1509 local user=$4
1510 local extra=""
1511 if [[ -n "$group" ]]; then
1512 extra="Group=$group"
1513 fi
1514 local unitfile="$SYSTEMD_DIR/$service"
1515 mkdir -p $SYSTEMD_DIR
1516
1517 iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
1518 iniset -sudo $unitfile "Service" "User" "$user"
1519 iniset -sudo $unitfile "Service" "ExecStart" "$command"
Matthew Treinish477a9622017-08-04 11:09:26 -04001520 iniset -sudo $unitfile "Service" "KillMode" "process"
Michał Dulko3b815a32017-11-14 16:04:51 +01001521 iniset -sudo $unitfile "Service" "TimeoutStopSec" "300"
Dinesh Bhoref60f2b2017-09-05 14:40:32 +05301522 iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID"
Sean Dague5edae542017-03-21 20:50:24 -04001523 if [[ -n "$group" ]]; then
1524 iniset -sudo $unitfile "Service" "Group" "$group"
1525 fi
1526 iniset -sudo $unitfile "Install" "WantedBy" "multi-user.target"
1527
1528 # changes to existing units sometimes need a refresh
1529 $SYSTEMCTL daemon-reload
1530}
1531
Matthew Treinish9c5ffd82017-03-29 16:47:57 -04001532function write_uwsgi_user_unit_file {
1533 local service=$1
1534 local command="$2"
1535 local group=$3
1536 local user=$4
1537 local unitfile="$SYSTEMD_DIR/$service"
1538 mkdir -p $SYSTEMD_DIR
1539
1540 iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
Sean Daguef41bf2a2017-05-05 07:50:52 -04001541 iniset -sudo $unitfile "Service" "SyslogIdentifier" "$service"
Matthew Treinish9c5ffd82017-03-29 16:47:57 -04001542 iniset -sudo $unitfile "Service" "User" "$user"
1543 iniset -sudo $unitfile "Service" "ExecStart" "$command"
Dinesh Bhoref60f2b2017-09-05 14:40:32 +05301544 iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID"
Matthew Treinish9c5ffd82017-03-29 16:47:57 -04001545 iniset -sudo $unitfile "Service" "Type" "notify"
Matthew Treinish477a9622017-08-04 11:09:26 -04001546 iniset -sudo $unitfile "Service" "KillMode" "process"
gong yong sheng07b3bc22017-06-05 14:02:28 +08001547 iniset -sudo $unitfile "Service" "Restart" "always"
Matthew Treinish9c5ffd82017-03-29 16:47:57 -04001548 iniset -sudo $unitfile "Service" "NotifyAccess" "all"
1549 iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100"
1550
1551 if [[ -n "$group" ]]; then
1552 iniset -sudo $unitfile "Service" "Group" "$group"
1553 fi
1554 iniset -sudo $unitfile "Install" "WantedBy" "multi-user.target"
1555
1556 # changes to existing units sometimes need a refresh
1557 $SYSTEMCTL daemon-reload
1558}
1559
Sean Dague148d58c2017-05-04 13:08:25 -04001560function _common_systemd_pitfalls {
1561 local cmd=$1
1562 # do some sanity checks on $cmd to see things we don't expect to work
1563
1564 if [[ "$cmd" =~ "sudo" ]]; then
Paweł Suder4e16c3d2018-09-26 15:52:13 +02001565 read -r -d '' msg << EOF || true # read returns 1 for EOF, but it is ok here
Sean Dague148d58c2017-05-04 13:08:25 -04001566You are trying to use run_process with sudo, this is not going to work under systemd.
1567
Ian Wienande8a6a022018-10-08 15:20:34 +11001568If you need to run a service as a user other than \$STACK_USER call it with:
Sean Dague148d58c2017-05-04 13:08:25 -04001569
1570 run_process \$name \$cmd \$group \$user
1571EOF
Ian Wienande8a6a022018-10-08 15:20:34 +11001572 die $LINENO "$msg"
Sean Dague148d58c2017-05-04 13:08:25 -04001573 fi
1574
1575 if [[ ! "$cmd" =~ ^/ ]]; then
Paweł Suder4e16c3d2018-09-26 15:52:13 +02001576 read -r -d '' msg << EOF || true # read returns 1 for EOF, but it is ok here
Sean Dague148d58c2017-05-04 13:08:25 -04001577The cmd="$cmd" does not start with an absolute path. It will fail to
1578start under systemd.
1579
1580Please update your run_process stanza to have an absolute path.
1581EOF
Ian Wienande8a6a022018-10-08 15:20:34 +11001582 die $LINENO "$msg"
Sean Dague148d58c2017-05-04 13:08:25 -04001583 fi
1584
1585}
1586
YAMAMOTO Takashic087c712017-06-15 12:10:45 +00001587# Helper function to build a basic unit file and run it under systemd.
1588function _run_under_systemd {
Sean Dague5edae542017-03-21 20:50:24 -04001589 local service=$1
1590 local command="$2"
1591 local cmd=$command
Sean Dague148d58c2017-05-04 13:08:25 -04001592 # sanity check the command
1593 _common_systemd_pitfalls "$cmd"
1594
Sean Dague5edae542017-03-21 20:50:24 -04001595 local systemd_service="devstack@$service.service"
1596 local group=$3
1597 local user=${4:-$STACK_USER}
Matthew Treinish9c5ffd82017-03-29 16:47:57 -04001598 if [[ "$command" =~ "uwsgi" ]] ; then
1599 write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user"
1600 else
1601 write_user_unit_file $systemd_service "$cmd" "$group" "$user"
1602 fi
YAMAMOTO Takashic087c712017-06-15 12:10:45 +00001603
1604 $SYSTEMCTL enable $systemd_service
1605 $SYSTEMCTL start $systemd_service
Sean Dague5edae542017-03-21 20:50:24 -04001606}
1607
Dean Troyerdff49a22014-01-30 15:37:40 -06001608# Find out if a process exists by partial name.
1609# is_running name
Ian Wienandaee18c72014-02-21 15:35:08 +11001610function is_running {
Dean Troyerdff49a22014-01-30 15:37:40 -06001611 local name=$1
1612 ps auxw | grep -v grep | grep ${name} > /dev/null
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05001613 local exitcode=$?
Dean Troyerdff49a22014-01-30 15:37:40 -06001614 # some times I really hate bash reverse binary logic
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05001615 return $exitcode
Dean Troyerdff49a22014-01-30 15:37:40 -06001616}
1617
Dean Troyer3159a822014-08-27 14:13:58 -05001618# Run a single service under screen or directly
1619# If the command includes shell metachatacters (;<>*) it must be run using a shell
Chris Dent2f27a0e2014-09-09 13:46:02 +01001620# If an optional group is provided sg will be used to run the
1621# command as that group.
Sean Dague148d58c2017-05-04 13:08:25 -04001622# run_process service "command-line" [group] [user]
Ian Wienandaee18c72014-02-21 15:35:08 +11001623function run_process {
Dean Troyerdff49a22014-01-30 15:37:40 -06001624 local service=$1
1625 local command="$2"
Chris Dent2f27a0e2014-09-09 13:46:02 +01001626 local group=$3
Sean Dague5edae542017-03-21 20:50:24 -04001627 local user=$4
Brant Knudsonedc11c22015-12-14 15:32:05 -06001628
Sean Dague5edae542017-03-21 20:50:24 -04001629 local name=$service
Dean Troyerdff49a22014-01-30 15:37:40 -06001630
Atsushi SAKAI2ca8af42015-12-08 15:36:13 +09001631 time_start "run_process"
Dean Troyer3159a822014-08-27 14:13:58 -05001632 if is_service_enabled $service; then
Sean Daguecdba1b32017-08-30 11:11:06 -04001633 _run_under_systemd "$name" "$command" "$group" "$user"
Dean Troyer3159a822014-08-27 14:13:58 -05001634 fi
Atsushi SAKAI2ca8af42015-12-08 15:36:13 +09001635 time_stop "run_process"
Dean Troyerdff49a22014-01-30 15:37:40 -06001636}
1637
Dean Troyer3159a822014-08-27 14:13:58 -05001638# Stop a service process
1639# If a PID is available use it, kill the whole process group via TERM
1640# If screen is being used kill the screen window; this will catch processes
1641# that did not leave a PID behind
Radoslav Gerganov2b97a812017-10-10 16:51:12 +03001642# Uses globals ``SERVICE_DIR``
Dean Troyer3159a822014-08-27 14:13:58 -05001643# stop_process service
1644function stop_process {
1645 local service=$1
1646
1647 SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
Dean Troyer3159a822014-08-27 14:13:58 -05001648
1649 if is_service_enabled $service; then
Sean Dague803acff2017-05-01 10:52:38 -04001650 # Only do this for units which appear enabled, this also
1651 # catches units that don't really exist for cases like
1652 # keystone without a failure.
1653 if $SYSTEMCTL is-enabled devstack@$service.service; then
1654 $SYSTEMCTL stop devstack@$service.service
1655 $SYSTEMCTL disable devstack@$service.service
Sean Dague5edae542017-03-21 20:50:24 -04001656 fi
Dean Troyerdff49a22014-01-30 15:37:40 -06001657 fi
1658}
1659
Sean Daguecdba1b32017-08-30 11:11:06 -04001660# use systemctl to check service status
Ian Wienandaee18c72014-02-21 15:35:08 +11001661function service_check {
Dean Troyerdff49a22014-01-30 15:37:40 -06001662 local service
Sean Daguecdba1b32017-08-30 11:11:06 -04001663 for service in ${ENABLED_SERVICES//,/ }; do
1664 # because some things got renamed like key => keystone
1665 if $SYSTEMCTL is-enabled devstack@$service.service; then
1666 # no-pager is needed because otherwise status dumps to a
1667 # pager when in interactive mode, which will stop a manual
1668 # devstack run.
1669 $SYSTEMCTL status devstack@$service.service --no-pager
Dean Troyer3159a822014-08-27 14:13:58 -05001670 fi
Sean Daguecdba1b32017-08-30 11:11:06 -04001671 done
Dean Troyer3159a822014-08-27 14:13:58 -05001672}
1673
Dean Troyer3159a822014-08-27 14:13:58 -05001674
Sean Dague2c65e712014-12-18 09:44:56 -05001675# Plugin Functions
1676# =================
1677
1678DEVSTACK_PLUGINS=${DEVSTACK_PLUGINS:-""}
1679
1680# enable_plugin <name> <url> [branch]
1681#
1682# ``name`` is an arbitrary name - (aka: glusterfs, nova-docker, zaqar)
1683# ``url`` is a git url
1684# ``branch`` is a gitref. If it's not set, defaults to master
1685function enable_plugin {
1686 local name=$1
1687 local url=$2
1688 local branch=${3:-master}
Omer Anson51584862017-08-24 17:47:37 +03001689 if is_plugin_enabled $name; then
John L. Villalovos21d84c22016-11-11 15:26:11 -08001690 die $LINENO "Plugin attempted to be enabled twice: ${name} ${url} ${branch}"
1691 fi
Sean Dague2c65e712014-12-18 09:44:56 -05001692 DEVSTACK_PLUGINS+=",$name"
1693 GITREPO[$name]=$url
1694 GITDIR[$name]=$DEST/$name
1695 GITBRANCH[$name]=$branch
1696}
1697
Omer Anson51584862017-08-24 17:47:37 +03001698# is_plugin_enabled <name>
1699#
1700# Check if the plugin was enabled, e.g. using enable_plugin
1701#
1702# ``name`` The name with which the plugin was enabled
1703function is_plugin_enabled {
1704 local name=$1
1705 if [[ ",${DEVSTACK_PLUGINS}," =~ ",${name}," ]]; then
1706 return 0
1707 fi
1708 return 1
1709}
1710
Sean Dague2c65e712014-12-18 09:44:56 -05001711# fetch_plugins
1712#
1713# clones all plugins
1714function fetch_plugins {
1715 local plugins="${DEVSTACK_PLUGINS}"
1716 local plugin
1717
1718 # short circuit if nothing to do
1719 if [[ -z $plugins ]]; then
1720 return
1721 fi
1722
Dean Troyerdc97cb72015-03-28 08:20:50 -05001723 echo "Fetching DevStack plugins"
Sean Dague2c65e712014-12-18 09:44:56 -05001724 for plugin in ${plugins//,/ }; do
1725 git_clone_by_name $plugin
1726 done
1727}
1728
1729# load_plugin_settings
1730#
1731# Load settings from plugins in the order that they were registered
1732function load_plugin_settings {
1733 local plugins="${DEVSTACK_PLUGINS}"
1734 local plugin
1735
1736 # short circuit if nothing to do
1737 if [[ -z $plugins ]]; then
1738 return
1739 fi
1740
1741 echo "Loading plugin settings"
1742 for plugin in ${plugins//,/ }; do
1743 local dir=${GITDIR[$plugin]}
1744 # source any known settings
1745 if [[ -f $dir/devstack/settings ]]; then
1746 source $dir/devstack/settings
1747 fi
1748 done
1749}
1750
Sean Dague6e275e12015-03-26 05:54:28 -04001751# plugin_override_defaults
1752#
1753# Run an extremely early setting phase for plugins that allows default
1754# overriding of services.
1755function plugin_override_defaults {
1756 local plugins="${DEVSTACK_PLUGINS}"
1757 local plugin
1758
1759 # short circuit if nothing to do
1760 if [[ -z $plugins ]]; then
1761 return
1762 fi
1763
1764 echo "Overriding Configuration Defaults"
1765 for plugin in ${plugins//,/ }; do
1766 local dir=${GITDIR[$plugin]}
1767 # source any overrides
1768 if [[ -f $dir/devstack/override-defaults ]]; then
1769 # be really verbose that an override is happening, as it
1770 # may not be obvious if things fail later.
Atsushi SAKAI33c9a672015-11-12 19:50:00 +09001771 echo "$plugin has overridden the following defaults"
Sean Dague6e275e12015-03-26 05:54:28 -04001772 cat $dir/devstack/override-defaults
1773 source $dir/devstack/override-defaults
1774 fi
1775 done
1776}
1777
Sean Dague2c65e712014-12-18 09:44:56 -05001778# run_plugins
1779#
1780# Run the devstack/plugin.sh in all the plugin directories. These are
1781# run in registration order.
1782function run_plugins {
1783 local mode=$1
1784 local phase=$2
Bharat Kumar Kobagana441ff072015-01-08 12:26:26 +05301785
1786 local plugins="${DEVSTACK_PLUGINS}"
1787 local plugin
Sean Dague2c65e712014-12-18 09:44:56 -05001788 for plugin in ${plugins//,/ }; do
1789 local dir=${GITDIR[$plugin]}
1790 if [[ -f $dir/devstack/plugin.sh ]]; then
1791 source $dir/devstack/plugin.sh $mode $phase
1792 fi
1793 done
1794}
1795
1796function run_phase {
1797 local mode=$1
1798 local phase=$2
1799 if [[ -d $TOP_DIR/extras.d ]]; then
Dmitry Tantsur64be3212015-10-12 13:10:24 +02001800 local extra_plugin_file_name
1801 for extra_plugin_file_name in $TOP_DIR/extras.d/*.sh; do
Sean Dague41d01102015-12-03 08:12:23 -05001802 # NOTE(sdague): only process extras.d for the 3 explicitly
1803 # white listed elements in tree. We want these to move out
1804 # over time as well, but they are in tree, so we need to
1805 # manage that.
Matt Riedemannc9f63272016-08-30 17:21:30 -04001806 local exceptions="80-tempest.sh"
Ian Wienand5cdee8d2015-10-19 14:17:18 +11001807 local extra
1808 extra=$(basename $extra_plugin_file_name)
Sean Dague1de9e332015-10-07 08:46:13 -04001809 if [[ ! ( $exceptions =~ "$extra" ) ]]; then
Sean Dague41d01102015-12-03 08:12:23 -05001810 warn "use of extras.d is no longer supported"
1811 warn "processing of project $extra is skipped"
1812 else
1813 [[ -r $extra_plugin_file_name ]] && source $extra_plugin_file_name $mode $phase
Sean Dague1de9e332015-10-07 08:46:13 -04001814 fi
Sean Dague2c65e712014-12-18 09:44:56 -05001815 done
1816 fi
1817 # the source phase corresponds to settings loading in plugins
1818 if [[ "$mode" == "source" ]]; then
1819 load_plugin_settings
Chris Dentc6d47012015-10-09 14:57:05 +00001820 verify_disabled_services
Sean Dague6e275e12015-03-26 05:54:28 -04001821 elif [[ "$mode" == "override_defaults" ]]; then
1822 plugin_override_defaults
Sean Dague2c65e712014-12-18 09:44:56 -05001823 else
1824 run_plugins $mode $phase
1825 fi
1826}
1827
James E. Blairc5853ac2017-11-21 09:44:42 -08001828# define_plugin <name>
1829#
1830# This function is a no-op. It allows a plugin to define its name So
1831# that other plugins may reference it by name. It should generally be
1832# the last component of the canonical git repo name. E.g.,
1833# openstack/devstack-foo should use "devstack-foo" as the name here.
1834#
1835# This function is currently a noop, but the value may still be used
1836# by external tools (as in plugin_requires) and may be used by
1837# devstack in the future.
1838#
1839# ``name`` is an arbitrary name - (aka: glusterfs, nova-docker, zaqar)
1840function define_plugin {
1841 :
1842}
1843
1844# plugin_requires <name> <other>
1845#
1846# This function is a no-op. It is currently used by external tools
1847# (such as the devstack module for Ansible) to automatically generate
1848# local.conf files. It is not currently used by devstack itself to
1849# resolve dependencies.
1850#
1851# ``name`` is an arbitrary name - (aka: glusterfs, nova-docker, zaqar)
1852# ``other`` is the name of another plugin
1853function plugin_requires {
1854 :
1855}
1856
Dean Troyerdff49a22014-01-30 15:37:40 -06001857
1858# Service Functions
1859# =================
1860
1861# remove extra commas from the input string (i.e. ``ENABLED_SERVICES``)
1862# _cleanup_service_list service-list
Ian Wienandaee18c72014-02-21 15:35:08 +11001863function _cleanup_service_list {
Ian Wienand8043bfa2015-10-14 14:53:18 +11001864 local xtrace
1865 xtrace=$(set +o | grep xtrace)
1866 set +o xtrace
1867
Dean Troyerdff49a22014-01-30 15:37:40 -06001868 echo "$1" | sed -e '
1869 s/,,/,/g;
1870 s/^,//;
1871 s/,$//
1872 '
Ian Wienand8043bfa2015-10-14 14:53:18 +11001873
1874 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -06001875}
1876
1877# disable_all_services() removes all current services
1878# from ``ENABLED_SERVICES`` to reset the configuration
1879# before a minimal installation
1880# Uses global ``ENABLED_SERVICES``
1881# disable_all_services
Ian Wienandaee18c72014-02-21 15:35:08 +11001882function disable_all_services {
Dean Troyerdff49a22014-01-30 15:37:40 -06001883 ENABLED_SERVICES=""
1884}
1885
1886# Remove all services starting with '-'. For example, to install all default
1887# services except rabbit (rabbit) set in ``localrc``:
1888# ENABLED_SERVICES+=",-rabbit"
1889# Uses global ``ENABLED_SERVICES``
1890# disable_negated_services
Ian Wienandaee18c72014-02-21 15:35:08 +11001891function disable_negated_services {
Ian Wienand8043bfa2015-10-14 14:53:18 +11001892 local xtrace
1893 xtrace=$(set +o | grep xtrace)
1894 set +o xtrace
1895
Ian Wienand2796a822015-04-15 08:59:04 +10001896 local to_remove=""
1897 local remaining=""
Dean Troyerdff49a22014-01-30 15:37:40 -06001898 local service
Ian Wienand2796a822015-04-15 08:59:04 +10001899
1900 # build up list of services that should be removed; i.e. they
1901 # begin with "-"
1902 for service in ${ENABLED_SERVICES//,/ }; do
Dean Troyerdff49a22014-01-30 15:37:40 -06001903 if [[ ${service} == -* ]]; then
Ian Wienand2796a822015-04-15 08:59:04 +10001904 to_remove+=",${service#-}"
1905 else
1906 remaining+=",${service}"
Dean Troyerdff49a22014-01-30 15:37:40 -06001907 fi
1908 done
Ian Wienand2796a822015-04-15 08:59:04 +10001909
1910 # go through the service list. if this service appears in the "to
1911 # be removed" list, drop it
fumihiko kakuma8606c982015-04-13 09:55:06 +09001912 ENABLED_SERVICES=$(remove_disabled_services "$remaining" "$to_remove")
Ian Wienand8043bfa2015-10-14 14:53:18 +11001913
1914 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -06001915}
1916
Chris Dentc6d47012015-10-09 14:57:05 +00001917# disable_service() prepares the services passed as argument to be
1918# removed from the ``ENABLED_SERVICES`` list, if they are present.
Dean Troyerdff49a22014-01-30 15:37:40 -06001919#
1920# For example:
1921# disable_service rabbit
1922#
Chris Dentc6d47012015-10-09 14:57:05 +00001923# Uses global ``DISABLED_SERVICES``
Dean Troyerdff49a22014-01-30 15:37:40 -06001924# disable_service service [service ...]
Ian Wienandaee18c72014-02-21 15:35:08 +11001925function disable_service {
Ian Wienand8043bfa2015-10-14 14:53:18 +11001926 local xtrace
1927 xtrace=$(set +o | grep xtrace)
1928 set +o xtrace
1929
Chris Dentc6d47012015-10-09 14:57:05 +00001930 local disabled_svcs="${DISABLED_SERVICES}"
1931 local enabled_svcs=",${ENABLED_SERVICES},"
Dean Troyerdff49a22014-01-30 15:37:40 -06001932 local service
1933 for service in $@; do
Chris Dentc6d47012015-10-09 14:57:05 +00001934 disabled_svcs+=",$service"
Dean Troyerdff49a22014-01-30 15:37:40 -06001935 if is_service_enabled $service; then
Chris Dentc6d47012015-10-09 14:57:05 +00001936 enabled_svcs=${enabled_svcs//,$service,/,}
Dean Troyerdff49a22014-01-30 15:37:40 -06001937 fi
1938 done
Chris Dentc6d47012015-10-09 14:57:05 +00001939 DISABLED_SERVICES=$(_cleanup_service_list "$disabled_svcs")
1940 ENABLED_SERVICES=$(_cleanup_service_list "$enabled_svcs")
Ian Wienand8043bfa2015-10-14 14:53:18 +11001941
1942 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -06001943}
1944
1945# enable_service() adds the services passed as argument to the
1946# ``ENABLED_SERVICES`` list, if they are not already present.
1947#
1948# For example:
Sean Dague37eca482015-06-16 07:19:22 -04001949# enable_service q-svc
Dean Troyerdff49a22014-01-30 15:37:40 -06001950#
1951# This function does not know about the special cases
1952# for nova, glance, and neutron built into is_service_enabled().
1953# Uses global ``ENABLED_SERVICES``
1954# enable_service service [service ...]
Ian Wienandaee18c72014-02-21 15:35:08 +11001955function enable_service {
Ian Wienand8043bfa2015-10-14 14:53:18 +11001956 local xtrace
1957 xtrace=$(set +o | grep xtrace)
1958 set +o xtrace
1959
Dean Troyerdff49a22014-01-30 15:37:40 -06001960 local tmpsvcs="${ENABLED_SERVICES}"
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05001961 local service
Dean Troyerdff49a22014-01-30 15:37:40 -06001962 for service in $@; do
Chris Dentc6d47012015-10-09 14:57:05 +00001963 if [[ ,${DISABLED_SERVICES}, =~ ,${service}, ]]; then
1964 warn $LINENO "Attempt to enable_service ${service} when it has been disabled"
1965 continue
1966 fi
Dean Troyerdff49a22014-01-30 15:37:40 -06001967 if ! is_service_enabled $service; then
1968 tmpsvcs+=",$service"
1969 fi
1970 done
1971 ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
1972 disable_negated_services
Ian Wienand8043bfa2015-10-14 14:53:18 +11001973
1974 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -06001975}
1976
1977# is_service_enabled() checks if the service(s) specified as arguments are
1978# enabled by the user in ``ENABLED_SERVICES``.
1979#
1980# Multiple services specified as arguments are ``OR``'ed together; the test
1981# is a short-circuit boolean, i.e it returns on the first match.
1982#
1983# There are special cases for some 'catch-all' services::
1984# **nova** returns true if any service enabled start with **n-**
1985# **cinder** returns true if any service enabled start with **c-**
Dean Troyerdff49a22014-01-30 15:37:40 -06001986# **glance** returns true if any service enabled start with **g-**
1987# **neutron** returns true if any service enabled start with **q-**
1988# **swift** returns true if any service enabled start with **s-**
1989# **trove** returns true if any service enabled start with **tr-**
1990# For backward compatibility if we have **swift** in ENABLED_SERVICES all the
1991# **s-** services will be enabled. This will be deprecated in the future.
1992#
Dean Troyerdff49a22014-01-30 15:37:40 -06001993# Uses global ``ENABLED_SERVICES``
1994# is_service_enabled service [service ...]
Ian Wienandaee18c72014-02-21 15:35:08 +11001995function is_service_enabled {
Ian Wienand433a9b12015-10-07 13:29:31 +11001996 local xtrace
1997 xtrace=$(set +o | grep xtrace)
Sean Dague45917cc2014-02-24 16:09:14 -05001998 set +o xtrace
Ian Wienand8043bfa2015-10-14 14:53:18 +11001999
Sean Dague45917cc2014-02-24 16:09:14 -05002000 local enabled=1
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05002001 local services=$@
2002 local service
Dean Troyerdff49a22014-01-30 15:37:40 -06002003 for service in ${services}; do
Sean Dague45917cc2014-02-24 16:09:14 -05002004 [[ ,${ENABLED_SERVICES}, =~ ,${service}, ]] && enabled=0
Dean Troyerdff49a22014-01-30 15:37:40 -06002005
2006 # Look for top-level 'enabled' function for this service
2007 if type is_${service}_enabled >/dev/null 2>&1; then
2008 # A function exists for this service, use it
Christian Schwede3db0aad2015-09-10 11:15:39 +00002009 is_${service}_enabled && enabled=0
Dean Troyerdff49a22014-01-30 15:37:40 -06002010 fi
2011
2012 # TODO(dtroyer): Remove these legacy special-cases after the is_XXX_enabled()
2013 # are implemented
2014
John Kasperski7b3ae532016-04-24 22:56:04 -05002015 [[ ${service} == n-cpu-* && ,${ENABLED_SERVICES} =~ ,"n-cpu" ]] && enabled=0
2016 [[ ${service} == "nova" && ,${ENABLED_SERVICES} =~ ,"n-" ]] && enabled=0
2017 [[ ${service} == "glance" && ,${ENABLED_SERVICES} =~ ,"g-" ]] && enabled=0
2018 [[ ${service} == "neutron" && ,${ENABLED_SERVICES} =~ ,"q-" ]] && enabled=0
2019 [[ ${service} == "trove" && ,${ENABLED_SERVICES} =~ ,"tr-" ]] && enabled=0
2020 [[ ${service} == "swift" && ,${ENABLED_SERVICES} =~ ,"s-" ]] && enabled=0
2021 [[ ${service} == s-* && ,${ENABLED_SERVICES} =~ ,"swift" ]] && enabled=0
Dean Troyerdff49a22014-01-30 15:37:40 -06002022 done
Ian Wienand8043bfa2015-10-14 14:53:18 +11002023
Sean Dague45917cc2014-02-24 16:09:14 -05002024 $xtrace
2025 return $enabled
Dean Troyerdff49a22014-01-30 15:37:40 -06002026}
2027
fumihiko kakuma8606c982015-04-13 09:55:06 +09002028# remove specified list from the input string
2029# remove_disabled_services service-list remove-list
2030function remove_disabled_services {
Ian Wienand8043bfa2015-10-14 14:53:18 +11002031 local xtrace
2032 xtrace=$(set +o | grep xtrace)
2033 set +o xtrace
2034
fumihiko kakuma8606c982015-04-13 09:55:06 +09002035 local service_list=$1
2036 local remove_list=$2
2037 local service
2038 local enabled=""
2039
2040 for service in ${service_list//,/ }; do
2041 local remove
2042 local add=1
2043 for remove in ${remove_list//,/ }; do
2044 if [[ ${remove} == ${service} ]]; then
2045 add=0
2046 break
2047 fi
2048 done
2049 if [[ $add == 1 ]]; then
2050 enabled="${enabled},$service"
2051 fi
2052 done
Ian Wienand8043bfa2015-10-14 14:53:18 +11002053
2054 $xtrace
2055
fumihiko kakuma8606c982015-04-13 09:55:06 +09002056 _cleanup_service_list "$enabled"
2057}
2058
Dean Troyerdff49a22014-01-30 15:37:40 -06002059# Toggle enable/disable_service for services that must run exclusive of each other
2060# $1 The name of a variable containing a space-separated list of services
2061# $2 The name of a variable in which to store the enabled service's name
2062# $3 The name of the service to enable
2063function use_exclusive_service {
2064 local options=${!1}
2065 local selection=$3
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05002066 local out=$2
Dean Troyerdff49a22014-01-30 15:37:40 -06002067 [ -z $selection ] || [[ ! "$options" =~ "$selection" ]] && return 1
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05002068 local opt
Dean Troyerdff49a22014-01-30 15:37:40 -06002069 for opt in $options;do
2070 [[ "$opt" = "$selection" ]] && enable_service $opt || disable_service $opt
2071 done
2072 eval "$out=$selection"
2073 return 0
2074}
2075
Chris Dentc6d47012015-10-09 14:57:05 +00002076# Make sure that nothing has manipulated ENABLED_SERVICES in a way
2077# that conflicts with prior calls to disable_service.
2078# Uses global ``ENABLED_SERVICES``
2079function verify_disabled_services {
2080 local service
2081 for service in ${ENABLED_SERVICES//,/ }; do
2082 if [[ ,${DISABLED_SERVICES}, =~ ,${service}, ]]; then
2083 die $LINENO "ENABLED_SERVICES directly modified to overcome 'disable_service ${service}'"
2084 fi
2085 done
2086}
2087
Dean Troyerdff49a22014-01-30 15:37:40 -06002088
Masayuki Igawaf6368d32014-02-20 13:31:26 +09002089# System Functions
2090# ================
Dean Troyerdff49a22014-01-30 15:37:40 -06002091
2092# Only run the command if the target file (the last arg) is not on an
2093# NFS filesystem.
Ian Wienandaee18c72014-02-21 15:35:08 +11002094function _safe_permission_operation {
Ian Wienand433a9b12015-10-07 13:29:31 +11002095 local xtrace
2096 xtrace=$(set +o | grep xtrace)
Sean Dague45917cc2014-02-24 16:09:14 -05002097 set +o xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -06002098 local args=( $@ )
2099 local last
2100 local sudo_cmd
2101 local dir_to_check
2102
2103 let last="${#args[*]} - 1"
2104
Dean Troyerd5dfa4c2014-07-25 11:13:11 -05002105 local dir_to_check=${args[$last]}
Dean Troyerdff49a22014-01-30 15:37:40 -06002106 if [ ! -d "$dir_to_check" ]; then
2107 dir_to_check=`dirname "$dir_to_check"`
2108 fi
2109
2110 if is_nfs_directory "$dir_to_check" ; then
Sean Dague45917cc2014-02-24 16:09:14 -05002111 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -06002112 return 0
2113 fi
2114
Ian Wienandbcb2c302020-01-13 16:31:20 +11002115 sudo_cmd="sudo"
Dean Troyerdff49a22014-01-30 15:37:40 -06002116
Sean Dague45917cc2014-02-24 16:09:14 -05002117 $xtrace
Dean Troyerdff49a22014-01-30 15:37:40 -06002118 $sudo_cmd $@
2119}
2120
2121# Exit 0 if address is in network or 1 if address is not in network
2122# ip-range is in CIDR notation: 1.2.3.4/20
2123# address_in_net ip-address ip-range
Ian Wienandaee18c72014-02-21 15:35:08 +11002124function address_in_net {
Dean Troyerdff49a22014-01-30 15:37:40 -06002125 local ip=$1
2126 local range=$2
2127 local masklen=${range#*/}
Ian Wienandada886d2015-10-07 14:06:26 +11002128 local network
2129 network=$(maskip ${range%/*} $(cidr2netmask $masklen))
2130 local subnet
2131 subnet=$(maskip $ip $(cidr2netmask $masklen))
Dean Troyerdff49a22014-01-30 15:37:40 -06002132 [[ $network == $subnet ]]
2133}
2134
2135# Add a user to a group.
2136# add_user_to_group user group
Ian Wienandaee18c72014-02-21 15:35:08 +11002137function add_user_to_group {
Dean Troyerdff49a22014-01-30 15:37:40 -06002138 local user=$1
2139 local group=$2
2140
Thomas Bechtolda8580852015-05-31 00:04:33 +02002141 sudo usermod -a -G "$group" "$user"
Dean Troyerdff49a22014-01-30 15:37:40 -06002142}
2143
2144# Convert CIDR notation to a IPv4 netmask
2145# cidr2netmask cidr-bits
Ian Wienandaee18c72014-02-21 15:35:08 +11002146function cidr2netmask {
Dean Troyerdff49a22014-01-30 15:37:40 -06002147 local maskpat="255 255 255 255"
2148 local maskdgt="254 252 248 240 224 192 128"
2149 set -- ${maskpat:0:$(( ($1 / 8) * 4 ))}${maskdgt:$(( (7 - ($1 % 8)) * 4 )):3}
2150 echo ${1-0}.${2-0}.${3-0}.${4-0}
2151}
2152
Ian Cordasco69e3c0a2016-09-26 12:53:14 -05002153# Check if this is a valid ipv4 address string
2154function is_ipv4_address {
2155 local address=$1
Jens Harbott7617ac22017-09-19 17:43:48 +00002156 local regex='([0-9]{1,3}\.){3}[0-9]{1,3}'
Ian Cordasco69e3c0a2016-09-26 12:53:14 -05002157 # TODO(clarkb) make this more robust
2158 if [[ "$address" =~ $regex ]] ; then
2159 return 0
2160 else
2161 return 1
2162 fi
2163}
2164
Jens Harbottdc7b4292017-09-19 10:52:32 +00002165# Remove "[]" around urlquoted IPv6 addresses
2166function ipv6_unquote {
2167 echo $1 | tr -d []
2168}
2169
Dean Troyerdff49a22014-01-30 15:37:40 -06002170# Gracefully cp only if source file/dir exists
2171# cp_it source destination
2172function cp_it {
2173 if [ -e $1 ] || [ -d $1 ]; then
2174 cp -pRL $1 $2
2175 fi
2176}
2177
2178# HTTP and HTTPS proxy servers are supported via the usual environment variables [1]
2179# ``http_proxy``, ``https_proxy`` and ``no_proxy``. They can be set in
2180# ``localrc`` or on the command line if necessary::
2181#
2182# [1] http://www.w3.org/Daemon/User/Proxies/ProxyClients.html
2183#
2184# http_proxy=http://proxy.example.com:3128/ no_proxy=repo.example.net ./stack.sh
2185
Ian Wienandaee18c72014-02-21 15:35:08 +11002186function export_proxy_variables {
Sean Dague53753292014-12-04 19:38:15 -05002187 if isset http_proxy ; then
Dean Troyerdff49a22014-01-30 15:37:40 -06002188 export http_proxy=$http_proxy
2189 fi
Sean Dague53753292014-12-04 19:38:15 -05002190 if isset https_proxy ; then
Dean Troyerdff49a22014-01-30 15:37:40 -06002191 export https_proxy=$https_proxy
2192 fi
Sean Dague53753292014-12-04 19:38:15 -05002193 if isset no_proxy ; then
Dean Troyerdff49a22014-01-30 15:37:40 -06002194 export no_proxy=$no_proxy
2195 fi
2196}
2197
2198# Returns true if the directory is on a filesystem mounted via NFS.
Ian Wienandaee18c72014-02-21 15:35:08 +11002199function is_nfs_directory {
Ian Wienandada886d2015-10-07 14:06:26 +11002200 local mount_type
2201 mount_type=`stat -f -L -c %T $1`
Dean Troyerdff49a22014-01-30 15:37:40 -06002202 test "$mount_type" == "nfs"
2203}
2204
2205# Return the network portion of the given IP address using netmask
2206# netmask is in the traditional dotted-quad format
2207# maskip ip-address netmask
Ian Wienandaee18c72014-02-21 15:35:08 +11002208function maskip {
Dean Troyerdff49a22014-01-30 15:37:40 -06002209 local ip=$1
2210 local mask=$2
2211 local l="${ip%.*}"; local r="${ip#*.}"; local n="${mask%.*}"; local m="${mask#*.}"
Ian Wienandada886d2015-10-07 14:06:26 +11002212 local subnet
2213 subnet=$((${ip%%.*}&${mask%%.*})).$((${r%%.*}&${m%%.*})).$((${l##*.}&${n##*.})).$((${ip##*.}&${mask##*.}))
Dean Troyerdff49a22014-01-30 15:37:40 -06002214 echo $subnet
2215}
2216
Michael Turek7938d832016-04-12 14:55:21 -04002217function is_provider_network {
2218 if [ "$Q_USE_PROVIDER_NETWORKING" == "True" ]; then
2219 return 0
2220 fi
2221 return 1
2222}
2223
2224
Ian Wienand6d213df2017-08-22 16:05:16 +10002225# Return just the <major>.<minor> for the given python interpreter
2226function _get_python_version {
2227 local interp=$1
2228 local version
Sean Dague4324f4e2017-09-14 12:59:25 -06002229 # disable erroring out here, otherwise if python 3 doesn't exist we fail hard.
Javier Pena6bd49242017-09-15 15:55:00 +02002230 if [[ -x $(which $interp 2> /dev/null) ]]; then
Sean Dague4324f4e2017-09-14 12:59:25 -06002231 version=$($interp -c 'import sys; print("%s.%s" % sys.version_info[0:2])')
2232 fi
Ian Wienand6d213df2017-08-22 16:05:16 +10002233 echo ${version}
2234}
2235
Chris Dent3a2c86a2015-05-12 13:41:25 +00002236# Return the current python as "python<major>.<minor>"
2237function python_version {
Ian Wienandada886d2015-10-07 14:06:26 +11002238 local python_version
Ian Wienand6d213df2017-08-22 16:05:16 +10002239 python_version=$(_get_python_version python2)
Chris Dent3a2c86a2015-05-12 13:41:25 +00002240 echo "python${python_version}"
2241}
2242
Ian Wienand6d213df2017-08-22 16:05:16 +10002243function python3_version {
2244 local python3_version
2245 python3_version=$(_get_python_version python3)
Doug Hellmann04178582018-06-12 15:37:00 -04002246 echo "python${python3_version}"
Ian Wienand6d213df2017-08-22 16:05:16 +10002247}
2248
2249
Dean Troyerdff49a22014-01-30 15:37:40 -06002250# Service wrapper to restart services
2251# restart_service service-name
Ian Wienandaee18c72014-02-21 15:35:08 +11002252function restart_service {
Kashyap Chamarthy247e4462016-05-20 13:34:41 +02002253 if [ -x /bin/systemctl ]; then
2254 sudo /bin/systemctl restart $1
Dean Troyerdff49a22014-01-30 15:37:40 -06002255 else
Kashyap Chamarthy247e4462016-05-20 13:34:41 +02002256 sudo service $1 restart
Dean Troyerdff49a22014-01-30 15:37:40 -06002257 fi
Kashyap Chamarthy247e4462016-05-20 13:34:41 +02002258
Dean Troyerdff49a22014-01-30 15:37:40 -06002259}
2260
2261# Only change permissions of a file or directory if it is not on an
2262# NFS filesystem.
Ian Wienandaee18c72014-02-21 15:35:08 +11002263function safe_chmod {
Dean Troyerdff49a22014-01-30 15:37:40 -06002264 _safe_permission_operation chmod $@
2265}
2266
2267# Only change ownership of a file or directory if it is not on an NFS
2268# filesystem.
Ian Wienandaee18c72014-02-21 15:35:08 +11002269function safe_chown {
Dean Troyerdff49a22014-01-30 15:37:40 -06002270 _safe_permission_operation chown $@
2271}
2272
2273# Service wrapper to start services
2274# start_service service-name
Ian Wienandaee18c72014-02-21 15:35:08 +11002275function start_service {
Kashyap Chamarthy247e4462016-05-20 13:34:41 +02002276 if [ -x /bin/systemctl ]; then
2277 sudo /bin/systemctl start $1
Dean Troyerdff49a22014-01-30 15:37:40 -06002278 else
Kashyap Chamarthy247e4462016-05-20 13:34:41 +02002279 sudo service $1 start
Dean Troyerdff49a22014-01-30 15:37:40 -06002280 fi
2281}
2282
2283# Service wrapper to stop services
2284# stop_service service-name
Ian Wienandaee18c72014-02-21 15:35:08 +11002285function stop_service {
Kashyap Chamarthy247e4462016-05-20 13:34:41 +02002286 if [ -x /bin/systemctl ]; then
2287 sudo /bin/systemctl stop $1
Dean Troyerdff49a22014-01-30 15:37:40 -06002288 else
Kashyap Chamarthy247e4462016-05-20 13:34:41 +02002289 sudo service $1 stop
Dean Troyerdff49a22014-01-30 15:37:40 -06002290 fi
2291}
2292
Attila Fazekasbcaadd62016-11-23 12:43:02 +01002293# Service wrapper to reload services
2294# If the service was not in running state it will start it
Gregory Haynes4b49e402016-08-31 18:19:51 -07002295# reload_service service-name
2296function reload_service {
2297 if [ -x /bin/systemctl ]; then
Attila Fazekasbcaadd62016-11-23 12:43:02 +01002298 sudo /bin/systemctl reload-or-restart $1
Gregory Haynes4b49e402016-08-31 18:19:51 -07002299 else
2300 sudo service $1 reload
2301 fi
2302}
2303
Sean Dague442e4e92015-06-24 13:24:02 -04002304# Test with a finite retry loop.
2305#
2306function test_with_retry {
2307 local testcmd=$1
2308 local failmsg=$2
2309 local until=${3:-10}
2310 local sleep=${4:-0.5}
2311
Atsushi SAKAI2ca8af42015-12-08 15:36:13 +09002312 time_start "test_with_retry"
Sean Dague442e4e92015-06-24 13:24:02 -04002313 if ! timeout $until sh -c "while ! $testcmd; do sleep $sleep; done"; then
2314 die $LINENO "$failmsg"
2315 fi
Atsushi SAKAI2ca8af42015-12-08 15:36:13 +09002316 time_stop "test_with_retry"
Sean Dague442e4e92015-06-24 13:24:02 -04002317}
2318
Federico Ressi5dcbf7a2016-01-13 12:17:32 +00002319# Like sudo but forwarding http_proxy https_proxy no_proxy environment vars.
2320# If it is run as superuser then sudo is replaced by env.
2321#
2322function sudo_with_proxies {
2323 local sudo
2324
2325 [[ "$(id -u)" = "0" ]] && sudo="env" || sudo="sudo"
2326
2327 $sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}"\
2328 no_proxy="${no_proxy:-}" "$@"
2329}
2330
Sean Dague95c33d52015-10-07 11:05:59 -04002331# Timing infrastructure - figure out where large blocks of time are
2332# used in DevStack
2333#
2334# The timing infrastructure for DevStack is about collecting buckets
2335# of time that are spend in some subtask. For instance, that might be
2336# 'apt', 'pip', 'osc', even database migrations. We do this by a pair
2337# of functions: time_start / time_stop.
2338#
2339# These take a single parameter: $name - which specifies the name of
2340# the bucket to be accounted against. time_totals function spits out
2341# the results.
2342#
2343# Resolution is only in whole seconds, so should be used for long
2344# running activities.
2345
Sean Dagueafef8bf2017-03-06 14:07:23 -05002346declare -A -g _TIME_TOTAL
2347declare -A -g _TIME_START
2348declare -r -g _TIME_BEGIN=$(date +%s)
Sean Dague95c33d52015-10-07 11:05:59 -04002349
2350# time_start $name
2351#
2352# starts the clock for a timer by name. Errors if that clock is
2353# already started.
2354function time_start {
2355 local name=$1
Ian Wienand908a3a92016-03-29 14:47:09 +11002356 local start_time=${_TIME_START[$name]}
Sean Dague95c33d52015-10-07 11:05:59 -04002357 if [[ -n "$start_time" ]]; then
2358 die $LINENO "Trying to start the clock on $name, but it's already been started"
2359 fi
Sean Dague85cf2932017-03-27 15:35:13 -04002360 _TIME_START[$name]=$(date +%s%3N)
Sean Dague95c33d52015-10-07 11:05:59 -04002361}
2362
2363# time_stop $name
2364#
2365# stops the clock for a timer by name, and accumulate that time in the
2366# global counter for that name. Errors if that clock had not
2367# previously been started.
2368function time_stop {
Ian Wienand5cdee8d2015-10-19 14:17:18 +11002369 local name
2370 local end_time
Rob Crittenden38245da2016-05-26 17:55:14 -04002371 local elapsed_time
Ian Wienand5cdee8d2015-10-19 14:17:18 +11002372 local total
2373 local start_time
2374
2375 name=$1
Ian Wienand908a3a92016-03-29 14:47:09 +11002376 start_time=${_TIME_START[$name]}
Ian Wienand5cdee8d2015-10-19 14:17:18 +11002377
Sean Dague95c33d52015-10-07 11:05:59 -04002378 if [[ -z "$start_time" ]]; then
2379 die $LINENO "Trying to stop the clock on $name, but it was never started"
2380 fi
Sean Dague85cf2932017-03-27 15:35:13 -04002381 end_time=$(date +%s%3N)
Ian Wienand5cdee8d2015-10-19 14:17:18 +11002382 elapsed_time=$(($end_time - $start_time))
Ian Wienand908a3a92016-03-29 14:47:09 +11002383 total=${_TIME_TOTAL[$name]:-0}
Sean Dague95c33d52015-10-07 11:05:59 -04002384 # reset the clock so we can start it in the future
Ian Wienand908a3a92016-03-29 14:47:09 +11002385 _TIME_START[$name]=""
2386 _TIME_TOTAL[$name]=$(($total + $elapsed_time))
Sean Dague95c33d52015-10-07 11:05:59 -04002387}
2388
Sean Dague85cf2932017-03-27 15:35:13 -04002389function oscwrap {
Ian Wienand474f5352019-08-08 09:15:11 +10002390 local xtrace
2391 xtrace=$(set +o | grep xtrace)
2392 set +o xtrace
2393
Sean Dague85cf2932017-03-27 15:35:13 -04002394 local out
2395 local rc
2396 local start
2397 local end
2398 # Cannot use timer_start and timer_stop as we run in subshells
2399 # and those rely on modifying vars in the same process (which cannot
2400 # happen from a subshell.
2401 start=$(date +%s%3N)
2402 out=$(command openstack "$@")
2403 rc=$?
2404 end=$(date +%s%3N)
2405 echo $((end - start)) >> $OSCWRAP_TIMER_FILE
2406
2407 echo "$out"
Ian Wienand474f5352019-08-08 09:15:11 +10002408 $xtrace
Sean Dague85cf2932017-03-27 15:35:13 -04002409 return $rc
2410}
2411
2412function install_oscwrap {
2413 # File to accumulate our timing data
2414 OSCWRAP_TIMER_FILE=$(mktemp)
2415 # Bash by default doesn't expand aliases, allow it for the aliases
2416 # we want to whitelist.
2417 shopt -s expand_aliases
2418 # Remove all aliases that might be expanded to preserve old unexpanded
2419 # behavior
2420 unalias -a
2421 # Add only the alias we want for openstack
2422 alias openstack=oscwrap
2423}
2424
2425function cleanup_oscwrap {
2426 local total=0
Stephen Finucaneffd00472018-01-18 15:12:29 +00002427 total=$(cat $OSCWRAP_TIMER_FILE | $PYTHON -c "import sys; print(sum(int(l) for l in sys.stdin))")
Sean Dague85cf2932017-03-27 15:35:13 -04002428 _TIME_TOTAL["osc"]=$total
2429 rm $OSCWRAP_TIMER_FILE
2430}
2431
Sean Dague95c33d52015-10-07 11:05:59 -04002432# time_totals
Ian Wienand908a3a92016-03-29 14:47:09 +11002433# Print out total time summary
Sean Dague95c33d52015-10-07 11:05:59 -04002434function time_totals {
Ian Wienand908a3a92016-03-29 14:47:09 +11002435 local elapsed_time
2436 local end_time
John L. Villalovos5ad4e582017-09-07 15:33:57 -07002437 local len=20
Ian Wienand908a3a92016-03-29 14:47:09 +11002438 local xtrace
John L. Villalovos5ad4e582017-09-07 15:33:57 -07002439 local unaccounted_time
Ian Wienand908a3a92016-03-29 14:47:09 +11002440
2441 end_time=$(date +%s)
2442 elapsed_time=$(($end_time - $_TIME_BEGIN))
John L. Villalovos5ad4e582017-09-07 15:33:57 -07002443 unaccounted_time=$elapsed_time
Ian Wienand908a3a92016-03-29 14:47:09 +11002444
2445 # pad 1st column this far
2446 for t in ${!_TIME_TOTAL[*]}; do
2447 if [[ ${#t} -gt $len ]]; then
2448 len=${#t}
2449 fi
Sean Dague95c33d52015-10-07 11:05:59 -04002450 done
Ian Wienand908a3a92016-03-29 14:47:09 +11002451
Sean Dague85cf2932017-03-27 15:35:13 -04002452 cleanup_oscwrap
2453
Ian Wienand908a3a92016-03-29 14:47:09 +11002454 xtrace=$(set +o | grep xtrace)
2455 set +o xtrace
2456
2457 echo
2458 echo "========================="
2459 echo "DevStack Component Timing"
John L. Villalovos5ad4e582017-09-07 15:33:57 -07002460 echo " (times are in seconds) "
Ian Wienand908a3a92016-03-29 14:47:09 +11002461 echo "========================="
Ian Wienand908a3a92016-03-29 14:47:09 +11002462 for t in ${!_TIME_TOTAL[*]}; do
2463 local v=${_TIME_TOTAL[$t]}
Sean Dague85cf2932017-03-27 15:35:13 -04002464 # because we're recording in milliseconds
2465 v=$(($v / 1000))
Ian Wienand908a3a92016-03-29 14:47:09 +11002466 printf "%-${len}s %3d\n" "$t" "$v"
John L. Villalovos5ad4e582017-09-07 15:33:57 -07002467 unaccounted_time=$(($unaccounted_time - $v))
Ian Wienand908a3a92016-03-29 14:47:09 +11002468 done
John L. Villalovos5ad4e582017-09-07 15:33:57 -07002469 echo "-------------------------"
2470 printf "%-${len}s %3d\n" "Unaccounted time" "$unaccounted_time"
Ian Wienand908a3a92016-03-29 14:47:09 +11002471 echo "========================="
John L. Villalovos5ad4e582017-09-07 15:33:57 -07002472 printf "%-${len}s %3d\n" "Total runtime" "$elapsed_time"
Ian Wienand908a3a92016-03-29 14:47:09 +11002473
2474 $xtrace
Sean Dague95c33d52015-10-07 11:05:59 -04002475}
Dean Troyerdff49a22014-01-30 15:37:40 -06002476
Sean Mooneyae21b352020-09-01 14:11:45 +00002477function clean_pyc_files {
2478 # Clean up all *.pyc files
2479 if [[ -n "$DEST" ]] && [[ -d "$DEST" ]]; then
2480 sudo find $DEST -name "*.pyc" -delete
2481 fi
2482}
2483
Dean Troyerdff49a22014-01-30 15:37:40 -06002484# Restore xtrace
Ian Wienand523f4882015-10-13 11:03:03 +11002485$_XTRACE_FUNCTIONS_COMMON
Dean Troyerdff49a22014-01-30 15:37:40 -06002486
2487# Local variables:
2488# mode: shell-script
2489# End: