inc/python - devstack - Gitiles

 #!/bin/bash
 #
 # **inc/python** - Python-related functions
 #
 # Support for pip/setuptools interfaces and virtual environments
 #
 # External functions used:
 # - GetOSVersion
 # - is_fedora
 # - safe_chown

 # Save trace setting
 INC_PY_TRACE=$(set +o | grep xtrace)
 set +o xtrace


 # Global Config Variables

 # PROJECT_VENV contains the name of the virtual environment for each
 # project.  A null value installs to the system Python directories.
 declare -A -g PROJECT_VENV

 # Utility Functions
 # =================

 # Joins bash array of extras with commas as expected by other functions
 function join_extras {
     local IFS=","
     echo "$*"
 }

 # Python Functions
 # ================

 # Setup the global devstack virtualenvs and the associated environment
 # updates.
 function setup_devstack_virtualenv {
     # We run devstack out of a global virtualenv.
     if [[ ! -d $DEVSTACK_VENV ]] ; then
         # Using system site packages to enable nova to use libguestfs.
         # This package is currently installed via the distro and not
         # available on pypi.
         python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
         pip_install -U pip
     fi
     if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
         export PATH="$DEVSTACK_VENV/bin:$PATH"
         export PYTHON="$DEVSTACK_VENV/bin/python3"
     fi
 }

 # Get the path to the pip command.
 # get_pip_command
 function get_pip_command {
     local version="$1"
     if [ -z "$version" ]; then
         die $LINENO "pip python version is not set."
     fi

     # NOTE(dhellmann): I don't know if we actually get a pip3.4-python
     # under any circumstances.
     which pip${version} || which pip${version}-python

     if [ $? -ne 0 ]; then
         die $LINENO "Unable to find pip${version}; cannot continue"
     fi
 }

 # Get the path to the directory where python executables are installed.
 # get_python_exec_prefix
 function get_python_exec_prefix {
     local xtrace
     xtrace=$(set +o | grep xtrace)
     set +o xtrace
     if [[ -z "$os_PACKAGE" ]]; then
         GetOSVersion
     fi
     $xtrace

     if [[ "$GLOBAL_VENV" == "True" ]] ; then
         echo "$DEVSTACK_VENV/bin"
     else
         echo "/usr/local/bin"
     fi
 }

 # Wrapper for ``pip install`` that only installs versions of libraries
 # from the global-requirements specification.
 #
 # Uses globals ``REQUIREMENTS_DIR``
 #
 # pip_install_gr packagename
 function pip_install_gr {
     local name=$1
     local clean_name
     clean_name=$(get_from_global_requirements $name)
     pip_install $clean_name
 }

 # Wrapper for ``pip install`` that only installs versions of libraries
 # from the global-requirements specification with extras.
 #
 # Uses globals ``REQUIREMENTS_DIR``
 #
 # pip_install_gr_extras packagename extra1,extra2,...
 function pip_install_gr_extras {
     local name=$1
     local extras=$2
     local version_constraints
     version_constraints=$(get_version_constraints_from_global_requirements $name)
     pip_install $name[$extras]$version_constraints
 }

 # enable_python3_package() -- no-op for backwards compatibility
 #
 # enable_python3_package dir [dir ...]
 function enable_python3_package {
     local xtrace
     xtrace=$(set +o | grep xtrace)
     set +o xtrace

     echo "It is no longer necessary to call enable_python3_package()."

     $xtrace
 }

 # disable_python3_package() -- no-op for backwards compatibility
 #
 # disable_python3_package dir [dir ...]
 function disable_python3_package {
     local xtrace
     xtrace=$(set +o | grep xtrace)
     set +o xtrace

     echo "It is no longer possible to call disable_python3_package()."

     $xtrace
 }

 # Wrapper for ``pip install`` to set cache and proxy environment variables
 # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
 # ``PIP_UPGRADE``, ``*_proxy``,
 # Usage:
 #  pip_install pip_arguments
 function pip_install {
     local xtrace result
     xtrace=$(set +o | grep xtrace)
     set +o xtrace
     local upgrade=""
     local offline=${OFFLINE:-False}
     if [[ "$offline" == "True" || -z "$@" ]]; then
         $xtrace
         return
     fi

     time_start "pip_install"

     PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
     if [[ "$PIP_UPGRADE" = "True" ]] ; then
         upgrade="--upgrade"
     fi

     if [[ -z "$os_PACKAGE" ]]; then
         GetOSVersion
     fi

     # Try to extract the path of the package we are installing into
     # package_dir.  We need this to check for test-requirements.txt,
     # at least.
     #
     # ${!#} expands to the last positional argument to this function.
     # With "extras" syntax included, our arguments might be something
     # like:
     #  -e /path/to/fooproject[extra]
     # Thus this magic line grabs just the path without extras
     #
     # Note that this makes no sense if this is a pypi (rather than
     # local path) install; ergo you must check this path exists before
     # use.  Also, if we had multiple or mixed installs, we would also
     # likely break.  But for historical reasons, it's basically only
     # the other wrapper functions in here calling this to install
     # local packages, and they do so with single call per install.  So
     # this works (for now...)
     local package_dir=${!#%\[*\]}

     if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
         local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
         local sudo_pip="env"
     elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
         # We have to check that the DEVSTACK_VENV exists because early
         # devstack boostrapping needs to operate in a system context
         # too bootstrap pip. Once pip is bootstrapped we create the
         # global venv and can start to use it.
         local cmd_pip=$DEVSTACK_VENV/bin/pip
         local sudo_pip="env"
         echo "Using python $PYTHON3_VERSION to install $package_dir"
     else
         local cmd_pip="python$PYTHON3_VERSION -m pip"
         # See
         #  https://github.com/pypa/setuptools/issues/2232
         #  http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html
         # this makes setuptools >=50 use the platform distutils.
         # We only want to do this on global pip installs, not if
         # installing in a virtualenv
         local sudo_pip="sudo -H LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib "
         echo "Using python $PYTHON3_VERSION to install $package_dir"
     fi

     cmd_pip="$cmd_pip install"
     # Always apply constraints
     cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"

     $xtrace

     $sudo_pip \
         http_proxy="${http_proxy:-}" \
         https_proxy="${https_proxy:-}" \
         no_proxy="${no_proxy:-}" \
         PIP_FIND_LINKS=$PIP_FIND_LINKS \
         $cmd_pip $upgrade \
         $@
     result=$?

     time_stop "pip_install"
     return $result
 }

 function pip_uninstall {
     # Skip uninstall if offline
     [[ "${OFFLINE}" = "True" ]] && return

     local name=$1
     if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
         local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
         local sudo_pip="env"
     else
         local cmd_pip="python$PYTHON3_VERSION -m pip"
         local sudo_pip="sudo -H LC_ALL=en_US.UTF-8"
     fi
     # don't error if we can't uninstall, it might not be there
     $sudo_pip $cmd_pip uninstall -y $name || /bin/true
 }

 # get version of a package from global requirements file
 # get_from_global_requirements <package>
 function get_from_global_requirements {
     local package=$1
     local required_pkg
     required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
     if [[ $required_pkg == ""  ]]; then
         die $LINENO "Can't find package $package in requirements"
     fi
     echo $required_pkg
 }

 # get only version constraints of a package from global requirements file
 # get_version_constraints_from_global_requirements <package>
 function get_version_constraints_from_global_requirements {
     local package=$1
     local required_pkg_version_constraint
     # drop the package name from output (\K)
     required_pkg_version_constraint=$(grep -i -h -o -P "^${package}\K.*" $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
     if [[ $required_pkg_version_constraint == ""  ]]; then
         die $LINENO "Can't find package $package in requirements"
     fi
     echo $required_pkg_version_constraint
 }

 # should we use this library from their git repo, or should we let it
 # get pulled in via pip dependencies.
 function use_library_from_git {
     local name=$1
     local enabled=1
     [[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
     return $enabled
 }

 # determine if a package was installed from git
 function lib_installed_from_git {
     local name=$1
     local safe_name
     safe_name=$(python -c "from pkg_resources import safe_name; \
         print(safe_name('${name}'))")
     # Note "pip freeze" doesn't always work here, because it tries to
     # be smart about finding the remote of the git repo the package
     # was installed from.  This doesn't work with zuul which clones
     # repos with no remote.
     #
     # The best option seems to be to use "pip list" which will tell
     # you the path an editable install was installed from; for example
     # in response to something like
     #  pip install -e 'git+https://opendev.org/openstack/bashate#egg=bashate'
     # pip list --format columns shows
     #  bashate 0.5.2.dev19 /tmp/env/src/bashate
     # Thus we check the third column to see if we're installed from
     # some local place.
     [[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]]
 }

 # setup a library by name. If we are trying to use the library from
 # git, we'll do a git based install, otherwise we'll punt and the
 # library should be installed by a requirements pull from another
 # project.
 function setup_lib {
     local name=$1
     local dir=${GITDIR[$name]}
     setup_install $dir
 }

 # setup a library by name in editable mode. If we are trying to use
 # the library from git, we'll do a git based install, otherwise we'll
 # punt and the library should be installed by a requirements pull from
 # another project.
 #
 # use this for non namespaced libraries
 #
 # setup_dev_lib [-bindep] <name> [<extras>]
 function setup_dev_lib {
     local bindep
     if [[ $1 == -bindep* ]]; then
         bindep="${1}"
         shift
     fi
     local name=$1
     local dir=${GITDIR[$name]}
     local extras=$2
     setup_develop $bindep $dir $extras
 }

 # this should be used if you want to install globally, all libraries should
 # use this, especially *oslo* ones
 #
 # setup_install project_dir [extras]
 # project_dir: directory of project repo (e.g., /opt/stack/keystone)
 # extras: comma-separated list of optional dependencies to install
 #         (e.g., ldap,memcache).
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 # bindep: Set "-bindep" as first argument to install bindep.txt packages
 # The command is like "pip install <project_dir>[<extras>]"
 function setup_install {
     local bindep
     if [[ $1 == -bindep* ]]; then
         bindep="${1}"
         shift
     fi
     local project_dir=$1
     local extras=$2
     _setup_package_with_constraints_edit $bindep $project_dir "" $extras
 }

 # this should be used for projects which run services, like all services
 #
 # setup_develop project_dir [extras]
 # project_dir: directory of project repo (e.g., /opt/stack/keystone)
 # extras: comma-separated list of optional dependencies to install
 #         (e.g., ldap,memcache).
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 # The command is like "pip install -e <project_dir>[<extras>]"
 function setup_develop {
     local bindep
     if [[ $1 == -bindep* ]]; then
         bindep="${1}"
         shift
     fi
     local project_dir=$1
     local extras=$2
     _setup_package_with_constraints_edit $bindep $project_dir -e $extras
 }

 # ``pip install -e`` the package, which processes the dependencies
 # using pip before running `setup.py develop`
 #
 # Updates the constraints from REQUIREMENTS_DIR to reflect the
 # future installed state of this package. This ensures when we
 # install this package we get the from source version.
 #
 # Uses globals ``REQUIREMENTS_DIR``
 # _setup_package_with_constraints_edit project_dir flags [extras]
 # project_dir: directory of project repo (e.g., /opt/stack/keystone)
 # flags: pip CLI options/flags
 # extras: comma-separated list of optional dependencies to install
 #         (e.g., ldap,memcache).
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 # The command is like "pip install <flags> <project_dir>[<extras>]"
 function _setup_package_with_constraints_edit {
     local bindep
     if [[ $1 == -bindep* ]]; then
         bindep="${1}"
         shift
     fi
     local project_dir=$1
     local flags=$2
     local extras=$3

     # Normalize the directory name to avoid
     # "installation from path or url cannot be constrained to a version"
     # error.
     # REVISIT(yamamoto): Remove this when fixed in pip.
     # https://github.com/pypa/pip/pull/3582
     project_dir=$(cd $project_dir && pwd)

     if [ -n "$REQUIREMENTS_DIR" ]; then
         # Remove this package from constraints before we install it.
         # That way, later installs won't "downgrade" the install from
         # source we are about to do.
         local name
         name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
         if [ -z $name ]; then
             name=$(awk '/^name =/ {gsub(/"/, "", $3); print $3}' $project_dir/pyproject.toml)
         fi
         $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
             $REQUIREMENTS_DIR/upper-constraints.txt -- $name
     fi

     setup_package $bindep $project_dir "$flags" $extras

     # If this project is in LIBS_FROM_GIT, verify it was actually installed
     # correctly.  This helps catch errors caused by constraints mismatches.
     if use_library_from_git "$project_dir"; then
         if ! lib_installed_from_git "$project_dir"; then
             die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
         fi
     fi
 }

 # ``pip install -e`` the package, which processes the dependencies
 # using pip before running `setup.py develop`.  The command is like
 # "pip install <flags> <project_dir>[<extras>]"
 #
 # Uses globals ``STACK_USER``
 #
 # Usage:
 #  setup_package [-bindep[=profile,profile]] <project_dir> <flags> [extras]
 #
 # -bindep     : Use bindep to install dependencies; select extra profiles
 #               as comma separated arguments after "="
 # project_dir : directory of project repo (e.g., /opt/stack/keystone)
 # flags       : pip CLI options/flags
 # extras      : comma-separated list of optional dependencies to install
 #               (e.g., ldap,memcache).
 #               See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 function setup_package {
     local bindep=0
     local bindep_flag=""
     local bindep_profiles=""
     if [[ $1 == -bindep* ]]; then
         bindep=1
         IFS="=" read bindep_flag bindep_profiles <<< ${1}
         shift
     fi
     local project_dir=$1
     local flags=$2
     local extras=$3

     # if the flags variable exists, and it doesn't look like a flag,
     # assume it's actually the extras list.
     if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
         extras=$flags
         flags=""
     fi

     if [[ ! -z "$extras" ]]; then
         extras="[$extras]"
     fi

     # install any bindep packages
     if [[ $bindep == 1 ]]; then
         install_bindep $project_dir/bindep.txt $bindep_profiles
     fi

     pip_install $flags "$project_dir$extras"
     # ensure that further actions can do things like setup.py sdist
     if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
         safe_chown -R $STACK_USER $1/*.egg-info
     fi
 }

 # Report whether python 3 should be used
 # TODO(frickler): drop this once all legacy uses are removed
 function python3_enabled {
     return 0
 }

 # Provide requested python version and sets PYTHON variable
 function install_python {
     install_python3
     export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null)
 }

 # Install python3 packages
 function install_python3 {
     if is_ubuntu; then
         apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
     elif is_fedora; then
         if [ "$os_VENDOR" = "Fedora" ]; then
             install_package python${PYTHON3_VERSION//.}
         else
             install_package python${PYTHON3_VERSION//.} python${PYTHON3_VERSION//.}-devel
         fi
     fi
 }

 function install_devstack_tools {
     # intentionally old to ensure devstack-gate has control
     local dstools_version=${DSTOOLS_VERSION:-0.1.2}
     install_python3
     sudo pip3 install -U devstack-tools==${dstools_version}
 }

 # Restore xtrace
 $INC_PY_TRACE

 # Local variables:
 # mode: shell-script
 # End:
	#!/bin/bash
	#
	# inc/python - Python-related functions
	#
	# Support for pip/setuptools interfaces and virtual environments
	#
	# External functions used:
	# - GetOSVersion
	# - is_fedora
	# - safe_chown

	# Save trace setting
	INC_PY_TRACE=$(set +o \| grep xtrace)
	set +o xtrace


	# Global Config Variables

	# PROJECT_VENV contains the name of the virtual environment for each
	# project. A null value installs to the system Python directories.
	declare -A -g PROJECT_VENV

	# Utility Functions
	# =================

	# Joins bash array of extras with commas as expected by other functions
	function join_extras {
	local IFS=","
	echo "$*"
	}

	# Python Functions
	# ================

	# Setup the global devstack virtualenvs and the associated environment
	# updates.
	function setup_devstack_virtualenv {
	# We run devstack out of a global virtualenv.
	if [[ ! -d $DEVSTACK_VENV ]] ; then
	# Using system site packages to enable nova to use libguestfs.
	# This package is currently installed via the distro and not
	# available on pypi.
	python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
	pip_install -U pip
	fi
	if [[ ":$PATH:" != ":$DEVSTACK_VENV/bin:" ]] ; then
	export PATH="$DEVSTACK_VENV/bin:$PATH"
	export PYTHON="$DEVSTACK_VENV/bin/python3"
	fi
	}

	# Get the path to the pip command.
	# get_pip_command
	function get_pip_command {
	local version="$1"
	if [ -z "$version" ]; then
	die $LINENO "pip python version is not set."
	fi

	# NOTE(dhellmann): I don't know if we actually get a pip3.4-python
	# under any circumstances.
	which pip${version} \|\| which pip${version}-python

	if [ $? -ne 0 ]; then
	die $LINENO "Unable to find pip${version}; cannot continue"
	fi
	}

	# Get the path to the directory where python executables are installed.
	# get_python_exec_prefix
	function get_python_exec_prefix {
	local xtrace
	xtrace=$(set +o \| grep xtrace)
	set +o xtrace
	if [[ -z "$os_PACKAGE" ]]; then
	GetOSVersion
	fi
	$xtrace

	if [[ "$GLOBAL_VENV" == "True" ]] ; then
	echo "$DEVSTACK_VENV/bin"
	else
	echo "/usr/local/bin"
	fi
	}

	# Wrapper for ``pip install`` that only installs versions of libraries
	# from the global-requirements specification.
	#
	# Uses globals ``REQUIREMENTS_DIR``
	#
	# pip_install_gr packagename
	function pip_install_gr {
	local name=$1
	local clean_name
	clean_name=$(get_from_global_requirements $name)
	pip_install $clean_name
	}

	# Wrapper for ``pip install`` that only installs versions of libraries
	# from the global-requirements specification with extras.
	#
	# Uses globals ``REQUIREMENTS_DIR``
	#
	# pip_install_gr_extras packagename extra1,extra2,...
	function pip_install_gr_extras {
	local name=$1
	local extras=$2
	local version_constraints
	version_constraints=$(get_version_constraints_from_global_requirements $name)
	pip_install $name[$extras]$version_constraints
	}

	# enable_python3_package() -- no-op for backwards compatibility
	#
	# enable_python3_package dir [dir ...]
	function enable_python3_package {
	local xtrace
	xtrace=$(set +o \| grep xtrace)
	set +o xtrace

	echo "It is no longer necessary to call enable_python3_package()."

	$xtrace
	}

	# disable_python3_package() -- no-op for backwards compatibility
	#
	# disable_python3_package dir [dir ...]
	function disable_python3_package {
	local xtrace
	xtrace=$(set +o \| grep xtrace)
	set +o xtrace

	echo "It is no longer possible to call disable_python3_package()."

	$xtrace
	}

	# Wrapper for ``pip install`` to set cache and proxy environment variables
	# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
	# ``PIP_UPGRADE``, ``*_proxy``,
	# Usage:
	# pip_install pip_arguments
	function pip_install {
	local xtrace result
	xtrace=$(set +o \| grep xtrace)
	set +o xtrace
	local upgrade=""
	local offline=${OFFLINE:-False}
	if [[ "$offline" == "True" \|\| -z "$@" ]]; then
	$xtrace
	return
	fi

	time_start "pip_install"

	PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
	if [[ "$PIP_UPGRADE" = "True" ]] ; then
	upgrade="--upgrade"
	fi

	if [[ -z "$os_PACKAGE" ]]; then
	GetOSVersion
	fi

	# Try to extract the path of the package we are installing into
	# package_dir. We need this to check for test-requirements.txt,
	# at least.
	#
	# ${!#} expands to the last positional argument to this function.
	# With "extras" syntax included, our arguments might be something
	# like:
	# -e /path/to/fooproject[extra]
	# Thus this magic line grabs just the path without extras
	#
	# Note that this makes no sense if this is a pypi (rather than
	# local path) install; ergo you must check this path exists before
	# use. Also, if we had multiple or mixed installs, we would also
	# likely break. But for historical reasons, it's basically only
	# the other wrapper functions in here calling this to install
	# local packages, and they do so with single call per install. So
	# this works (for now...)
	local package_dir=${!#%\[*\]}

	if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
	local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
	local sudo_pip="env"
	elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
	# We have to check that the DEVSTACK_VENV exists because early
	# devstack boostrapping needs to operate in a system context
	# too bootstrap pip. Once pip is bootstrapped we create the
	# global venv and can start to use it.
	local cmd_pip=$DEVSTACK_VENV/bin/pip
	local sudo_pip="env"
	echo "Using python $PYTHON3_VERSION to install $package_dir"
	else
	local cmd_pip="python$PYTHON3_VERSION -m pip"
	# See
	# https://github.com/pypa/setuptools/issues/2232
	# http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html
	# this makes setuptools >=50 use the platform distutils.
	# We only want to do this on global pip installs, not if
	# installing in a virtualenv
	local sudo_pip="sudo -H LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib "
	echo "Using python $PYTHON3_VERSION to install $package_dir"
	fi

	cmd_pip="$cmd_pip install"
	# Always apply constraints
	cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"

	$xtrace

	$sudo_pip \
	http_proxy="${http_proxy:-}" \
	https_proxy="${https_proxy:-}" \
	no_proxy="${no_proxy:-}" \
	PIP_FIND_LINKS=$PIP_FIND_LINKS \
	$cmd_pip $upgrade \
	$@
	result=$?

	time_stop "pip_install"
	return $result
	}

	function pip_uninstall {
	# Skip uninstall if offline
	[[ "${OFFLINE}" = "True" ]] && return

	local name=$1
	if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
	local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
	local sudo_pip="env"
	else
	local cmd_pip="python$PYTHON3_VERSION -m pip"
	local sudo_pip="sudo -H LC_ALL=en_US.UTF-8"
	fi
	# don't error if we can't uninstall, it might not be there
	$sudo_pip $cmd_pip uninstall -y $name \|\| /bin/true
	}

	# get version of a package from global requirements file
	# get_from_global_requirements <package>
	function get_from_global_requirements {
	local package=$1
	local required_pkg
	required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt \| cut -d\# -f1)
	if [[ $required_pkg == "" ]]; then
	die $LINENO "Can't find package $package in requirements"
	fi
	echo $required_pkg
	}

	# get only version constraints of a package from global requirements file
	# get_version_constraints_from_global_requirements <package>
	function get_version_constraints_from_global_requirements {
	local package=$1
	local required_pkg_version_constraint
	# drop the package name from output (\K)
	required_pkg_version_constraint=$(grep -i -h -o -P "^${package}\K.*" $REQUIREMENTS_DIR/global-requirements.txt \| cut -d\# -f1)
	if [[ $required_pkg_version_constraint == "" ]]; then
	die $LINENO "Can't find package $package in requirements"
	fi
	echo $required_pkg_version_constraint
	}

	# should we use this library from their git repo, or should we let it
	# get pulled in via pip dependencies.
	function use_library_from_git {
	local name=$1
	local enabled=1
	[[ ${LIBS_FROM_GIT} = 'ALL' ]] \|\| [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
	return $enabled
	}

	# determine if a package was installed from git
	function lib_installed_from_git {
	local name=$1
	local safe_name
	safe_name=$(python -c "from pkg_resources import safe_name; \
	print(safe_name('${name}'))")
	# Note "pip freeze" doesn't always work here, because it tries to
	# be smart about finding the remote of the git repo the package
	# was installed from. This doesn't work with zuul which clones
	# repos with no remote.
	#
	# The best option seems to be to use "pip list" which will tell
	# you the path an editable install was installed from; for example
	# in response to something like
	# pip install -e 'git+https://opendev.org/openstack/bashate#egg=bashate'
	# pip list --format columns shows
	# bashate 0.5.2.dev19 /tmp/env/src/bashate
	# Thus we check the third column to see if we're installed from
	# some local place.
	[[ -n $(pip list --format=columns 2>/dev/null \| awk "/^$safe_name/ {print \$3}") ]]
	}

	# setup a library by name. If we are trying to use the library from
	# git, we'll do a git based install, otherwise we'll punt and the
	# library should be installed by a requirements pull from another
	# project.
	function setup_lib {
	local name=$1
	local dir=${GITDIR[$name]}
	setup_install $dir
	}

	# setup a library by name in editable mode. If we are trying to use
	# the library from git, we'll do a git based install, otherwise we'll
	# punt and the library should be installed by a requirements pull from
	# another project.
	#
	# use this for non namespaced libraries
	#
	# setup_dev_lib [-bindep] <name> [<extras>]
	function setup_dev_lib {
	local bindep
	if [[ $1 == -bindep* ]]; then
	bindep="${1}"
	shift
	fi
	local name=$1
	local dir=${GITDIR[$name]}
	local extras=$2
	setup_develop $bindep $dir $extras
	}

	# this should be used if you want to install globally, all libraries should
	# use this, especially oslo ones
	#
	# setup_install project_dir [extras]
	# project_dir: directory of project repo (e.g., /opt/stack/keystone)
	# extras: comma-separated list of optional dependencies to install
	# (e.g., ldap,memcache).
	# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
	# bindep: Set "-bindep" as first argument to install bindep.txt packages
	# The command is like "pip install <project_dir>[<extras>]"
	function setup_install {
	local bindep
	if [[ $1 == -bindep* ]]; then
	bindep="${1}"
	shift
	fi
	local project_dir=$1
	local extras=$2
	_setup_package_with_constraints_edit $bindep $project_dir "" $extras
	}

	# this should be used for projects which run services, like all services
	#
	# setup_develop project_dir [extras]
	# project_dir: directory of project repo (e.g., /opt/stack/keystone)
	# extras: comma-separated list of optional dependencies to install
	# (e.g., ldap,memcache).
	# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
	# The command is like "pip install -e <project_dir>[<extras>]"
	function setup_develop {
	local bindep
	if [[ $1 == -bindep* ]]; then
	bindep="${1}"
	shift
	fi
	local project_dir=$1
	local extras=$2
	_setup_package_with_constraints_edit $bindep $project_dir -e $extras
	}

	# ``pip install -e`` the package, which processes the dependencies
	# using pip before running `setup.py develop`
	#
	# Updates the constraints from REQUIREMENTS_DIR to reflect the
	# future installed state of this package. This ensures when we
	# install this package we get the from source version.
	#
	# Uses globals ``REQUIREMENTS_DIR``
	# _setup_package_with_constraints_edit project_dir flags [extras]
	# project_dir: directory of project repo (e.g., /opt/stack/keystone)
	# flags: pip CLI options/flags
	# extras: comma-separated list of optional dependencies to install
	# (e.g., ldap,memcache).
	# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
	# The command is like "pip install <flags> <project_dir>[<extras>]"
	function _setup_package_with_constraints_edit {
	local bindep
	if [[ $1 == -bindep* ]]; then
	bindep="${1}"
	shift
	fi
	local project_dir=$1
	local flags=$2
	local extras=$3

	# Normalize the directory name to avoid
	# "installation from path or url cannot be constrained to a version"
	# error.
	# REVISIT(yamamoto): Remove this when fixed in pip.
	# https://github.com/pypa/pip/pull/3582
	project_dir=$(cd $project_dir && pwd)

	if [ -n "$REQUIREMENTS_DIR" ]; then
	# Remove this package from constraints before we install it.
	# That way, later installs won't "downgrade" the install from
	# source we are about to do.
	local name
	name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
	if [ -z $name ]; then
	name=$(awk '/^name =/ {gsub(/"/, "", $3); print $3}' $project_dir/pyproject.toml)
	fi
	$REQUIREMENTS_DIR/.venv/bin/edit-constraints \
	$REQUIREMENTS_DIR/upper-constraints.txt -- $name
	fi

	setup_package $bindep $project_dir "$flags" $extras

	# If this project is in LIBS_FROM_GIT, verify it was actually installed
	# correctly. This helps catch errors caused by constraints mismatches.
	if use_library_from_git "$project_dir"; then
	if ! lib_installed_from_git "$project_dir"; then
	die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
	fi
	fi
	}

	# ``pip install -e`` the package, which processes the dependencies
	# using pip before running `setup.py develop`. The command is like
	# "pip install <flags> <project_dir>[<extras>]"
	#
	# Uses globals ``STACK_USER``
	#
	# Usage:
	# setup_package [-bindep[=profile,profile]] <project_dir> <flags> [extras]
	#
	# -bindep : Use bindep to install dependencies; select extra profiles
	# as comma separated arguments after "="
	# project_dir : directory of project repo (e.g., /opt/stack/keystone)
	# flags : pip CLI options/flags
	# extras : comma-separated list of optional dependencies to install
	# (e.g., ldap,memcache).
	# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
	function setup_package {
	local bindep=0
	local bindep_flag=""
	local bindep_profiles=""
	if [[ $1 == -bindep* ]]; then
	bindep=1
	IFS="=" read bindep_flag bindep_profiles <<< ${1}
	shift
	fi
	local project_dir=$1
	local flags=$2
	local extras=$3

	# if the flags variable exists, and it doesn't look like a flag,
	# assume it's actually the extras list.
	if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
	extras=$flags
	flags=""
	fi

	if [[ ! -z "$extras" ]]; then
	extras="[$extras]"
	fi

	# install any bindep packages
	if [[ $bindep == 1 ]]; then
	install_bindep $project_dir/bindep.txt $bindep_profiles
	fi

	pip_install $flags "$project_dir$extras"
	# ensure that further actions can do things like setup.py sdist
	if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
	safe_chown -R $STACK_USER $1/*.egg-info
	fi
	}

	# Report whether python 3 should be used
	# TODO(frickler): drop this once all legacy uses are removed
	function python3_enabled {
	return 0
	}

	# Provide requested python version and sets PYTHON variable
	function install_python {
	install_python3
	export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null)
	}

	# Install python3 packages
	function install_python3 {
	if is_ubuntu; then
	apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
	elif is_fedora; then
	if [ "$os_VENDOR" = "Fedora" ]; then
	install_package python${PYTHON3_VERSION//.}
	else
	install_package python${PYTHON3_VERSION//.} python${PYTHON3_VERSION//.}-devel
	fi
	fi
	}

	function install_devstack_tools {
	# intentionally old to ensure devstack-gate has control
	local dstools_version=${DSTOOLS_VERSION:-0.1.2}
	install_python3
	sudo pip3 install -U devstack-tools==${dstools_version}
	}

	# Restore xtrace
	$INC_PY_TRACE

	# Local variables:
	# mode: shell-script
	# End: