Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | # |
| 3 | # **inc/python** - Python-related functions |
| 4 | # |
| 5 | # Support for pip/setuptools interfaces and virtual environments |
| 6 | # |
| 7 | # External functions used: |
| 8 | # - GetOSVersion |
| 9 | # - is_fedora |
| 10 | # - is_suse |
| 11 | # - safe_chown |
| 12 | |
| 13 | # Save trace setting |
| 14 | INC_PY_TRACE=$(set +o | grep xtrace) |
| 15 | set +o xtrace |
| 16 | |
| 17 | |
Dean Troyer | 8c2ce6e | 2015-02-18 14:47:54 -0600 | [diff] [blame] | 18 | # Global Config Variables |
| 19 | |
Atsushi SAKAI | 5509ed5 | 2015-11-30 20:20:21 +0900 | [diff] [blame] | 20 | # PROJECT_VENV contains the name of the virtual environment for each |
Dean Troyer | 8c2ce6e | 2015-02-18 14:47:54 -0600 | [diff] [blame] | 21 | # project. A null value installs to the system Python directories. |
Sean Dague | afef8bf | 2017-03-06 14:07:23 -0500 | [diff] [blame] | 22 | declare -A -g PROJECT_VENV |
Dean Troyer | 8c2ce6e | 2015-02-18 14:47:54 -0600 | [diff] [blame] | 23 | |
| 24 | |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 25 | # Python Functions |
| 26 | # ================ |
| 27 | |
| 28 | # Get the path to the pip command. |
| 29 | # get_pip_command |
| 30 | function get_pip_command { |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 31 | local version="$1" |
| 32 | # NOTE(dhellmann): I don't know if we actually get a pip3.4-python |
| 33 | # under any circumstances. |
| 34 | which pip${version} || which pip${version}-python |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 35 | |
| 36 | if [ $? -ne 0 ]; then |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 37 | die $LINENO "Unable to find pip${version}; cannot continue" |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 38 | fi |
| 39 | } |
| 40 | |
Atsushi SAKAI | 5509ed5 | 2015-11-30 20:20:21 +0900 | [diff] [blame] | 41 | # Get the path to the directory where python executables are installed. |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 42 | # get_python_exec_prefix |
| 43 | function get_python_exec_prefix { |
Ian Wienand | 433a9b1 | 2015-10-07 13:29:31 +1100 | [diff] [blame] | 44 | local xtrace |
| 45 | xtrace=$(set +o | grep xtrace) |
Dean Troyer | 2b56476 | 2015-02-11 17:01:02 -0600 | [diff] [blame] | 46 | set +o xtrace |
| 47 | if [[ -z "$os_PACKAGE" ]]; then |
| 48 | GetOSVersion |
| 49 | fi |
| 50 | $xtrace |
| 51 | |
Lenny Verkhovsky | a30dd1c | 2019-03-14 13:19:36 +0200 | [diff] [blame] | 52 | local PYTHON_PATH=/usr/local/bin |
| 53 | ( is_fedora && ! python3_enabled ) || is_suse && PYTHON_PATH=/usr/bin |
| 54 | echo $PYTHON_PATH |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 55 | } |
| 56 | |
Sean Dague | 60996b1 | 2015-04-08 09:06:49 -0400 | [diff] [blame] | 57 | # Wrapper for ``pip install`` that only installs versions of libraries |
| 58 | # from the global-requirements specification. |
| 59 | # |
| 60 | # Uses globals ``REQUIREMENTS_DIR`` |
| 61 | # |
| 62 | # pip_install_gr packagename |
| 63 | function pip_install_gr { |
| 64 | local name=$1 |
Ian Wienand | ada886d | 2015-10-07 14:06:26 +1100 | [diff] [blame] | 65 | local clean_name |
| 66 | clean_name=$(get_from_global_requirements $name) |
Sean Dague | 60996b1 | 2015-04-08 09:06:49 -0400 | [diff] [blame] | 67 | pip_install $clean_name |
| 68 | } |
| 69 | |
Mehdi Abaakouk | 52b1074 | 2016-12-01 16:11:17 +0100 | [diff] [blame] | 70 | # Wrapper for ``pip install`` that only installs versions of libraries |
| 71 | # from the global-requirements specification with extras. |
| 72 | # |
| 73 | # Uses globals ``REQUIREMENTS_DIR`` |
| 74 | # |
| 75 | # pip_install_gr_extras packagename extra1,extra2,... |
| 76 | function pip_install_gr_extras { |
| 77 | local name=$1 |
| 78 | local extras=$2 |
| 79 | local clean_name |
| 80 | clean_name=$(get_from_global_requirements $name) |
| 81 | pip_install $clean_name[$extras] |
| 82 | } |
| 83 | |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 84 | # Determine the python versions supported by a package |
| 85 | function get_python_versions_for_package { |
| 86 | local name=$1 |
| 87 | cd $name && python setup.py --classifiers \ |
| 88 | | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' ' |
| 89 | } |
| 90 | |
Davanum Srinivas | afa8a00 | 2016-12-19 09:51:01 -0500 | [diff] [blame] | 91 | # Check for python3 classifier in local directory |
| 92 | function check_python3_support_for_package_local { |
| 93 | local name=$1 |
| 94 | cd $name |
| 95 | set +e |
| 96 | classifier=$(python setup.py --classifiers \ |
Matt Riedemann | ddb6179 | 2019-03-19 15:04:12 -0400 | [diff] [blame] | 97 | | grep 'Programming Language :: Python :: 3') |
Davanum Srinivas | afa8a00 | 2016-12-19 09:51:01 -0500 | [diff] [blame] | 98 | set -e |
| 99 | echo $classifier |
| 100 | } |
| 101 | |
| 102 | # Check for python3 classifier on pypi |
| 103 | function check_python3_support_for_package_remote { |
| 104 | local name=$1 |
| 105 | set +e |
| 106 | classifier=$(curl -s -L "https://pypi.python.org/pypi/$name/json" \ |
| 107 | | grep '"Programming Language :: Python :: 3"') |
| 108 | set -e |
| 109 | echo $classifier |
| 110 | } |
| 111 | |
Doug Hellmann | 36377f6 | 2018-12-04 11:33:03 -0500 | [diff] [blame] | 112 | # python3_enabled_for() assumes the service(s) specified as arguments are |
| 113 | # enabled for python 3 unless explicitly disabled. See python3_disabled_for(). |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 114 | # |
| 115 | # Multiple services specified as arguments are ``OR``'ed together; the test |
| 116 | # is a short-circuit boolean, i.e it returns on the first match. |
| 117 | # |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 118 | # python3_enabled_for dir [dir ...] |
| 119 | function python3_enabled_for { |
| 120 | local xtrace |
| 121 | xtrace=$(set +o | grep xtrace) |
| 122 | set +o xtrace |
| 123 | |
| 124 | local enabled=1 |
| 125 | local dirs=$@ |
| 126 | local dir |
| 127 | for dir in ${dirs}; do |
Doug Hellmann | 36377f6 | 2018-12-04 11:33:03 -0500 | [diff] [blame] | 128 | if ! python3_disabled_for "${dir}"; then |
| 129 | enabled=0 |
| 130 | fi |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 131 | done |
| 132 | |
| 133 | $xtrace |
| 134 | return $enabled |
| 135 | } |
| 136 | |
| 137 | # python3_disabled_for() checks if the service(s) specified as arguments are |
| 138 | # disabled by the user in ``DISABLED_PYTHON3_PACKAGES``. |
| 139 | # |
| 140 | # Multiple services specified as arguments are ``OR``'ed together; the test |
| 141 | # is a short-circuit boolean, i.e it returns on the first match. |
| 142 | # |
| 143 | # Uses global ``DISABLED_PYTHON3_PACKAGES`` |
| 144 | # python3_disabled_for dir [dir ...] |
| 145 | function python3_disabled_for { |
| 146 | local xtrace |
| 147 | xtrace=$(set +o | grep xtrace) |
| 148 | set +o xtrace |
| 149 | |
| 150 | local enabled=1 |
| 151 | local dirs=$@ |
| 152 | local dir |
| 153 | for dir in ${dirs}; do |
| 154 | [[ ,${DISABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]] && enabled=0 |
| 155 | done |
| 156 | |
| 157 | $xtrace |
| 158 | return $enabled |
| 159 | } |
| 160 | |
Doug Hellmann | 36377f6 | 2018-12-04 11:33:03 -0500 | [diff] [blame] | 161 | # enable_python3_package() -- no-op for backwards compatibility |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 162 | # |
| 163 | # For example: |
| 164 | # enable_python3_package nova |
| 165 | # |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 166 | # enable_python3_package dir [dir ...] |
| 167 | function enable_python3_package { |
| 168 | local xtrace |
| 169 | xtrace=$(set +o | grep xtrace) |
| 170 | set +o xtrace |
| 171 | |
Doug Hellmann | 36377f6 | 2018-12-04 11:33:03 -0500 | [diff] [blame] | 172 | echo "It is no longer necessary to call enable_python3_package()." |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 173 | |
| 174 | $xtrace |
| 175 | } |
| 176 | |
Doug Hellmann | 36377f6 | 2018-12-04 11:33:03 -0500 | [diff] [blame] | 177 | # disable_python3_package() adds the services passed as argument to |
| 178 | # the ``DISABLED_PYTHON3_PACKAGES`` list. |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 179 | # |
| 180 | # For example: |
| 181 | # disable_python3_package swift |
| 182 | # |
Doug Hellmann | 36377f6 | 2018-12-04 11:33:03 -0500 | [diff] [blame] | 183 | # Uses global ``DISABLED_PYTHON3_PACKAGES`` |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 184 | # disable_python3_package dir [dir ...] |
| 185 | function disable_python3_package { |
| 186 | local xtrace |
| 187 | xtrace=$(set +o | grep xtrace) |
| 188 | set +o xtrace |
| 189 | |
| 190 | local disabled_svcs="${DISABLED_PYTHON3_PACKAGES}" |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 191 | local dir |
| 192 | for dir in $@; do |
| 193 | disabled_svcs+=",$dir" |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 194 | done |
| 195 | DISABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$disabled_svcs") |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 196 | |
| 197 | $xtrace |
| 198 | } |
| 199 | |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 200 | # Wrapper for ``pip install`` to set cache and proxy environment variables |
Dean Troyer | 41d6f85 | 2015-03-25 22:42:46 -0500 | [diff] [blame] | 201 | # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``, |
Robert Collins | 635a5ba | 2015-06-10 08:48:06 +1200 | [diff] [blame] | 202 | # ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``, |
Zane Bitter | 9e7ead9 | 2017-10-05 16:51:09 -0400 | [diff] [blame] | 203 | # Usage: |
| 204 | # pip_install pip_arguments |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 205 | function pip_install { |
Federico Ressi | e208d06 | 2015-11-21 11:15:39 +0000 | [diff] [blame] | 206 | local xtrace result |
Ian Wienand | 433a9b1 | 2015-10-07 13:29:31 +1100 | [diff] [blame] | 207 | xtrace=$(set +o | grep xtrace) |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 208 | set +o xtrace |
Chris Dent | ebdd9ac | 2015-03-04 12:35:14 +0000 | [diff] [blame] | 209 | local upgrade="" |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 210 | local offline=${OFFLINE:-False} |
| 211 | if [[ "$offline" == "True" || -z "$@" ]]; then |
| 212 | $xtrace |
| 213 | return |
| 214 | fi |
| 215 | |
Sean Dague | cb658fa | 2015-10-08 17:12:03 -0400 | [diff] [blame] | 216 | time_start "pip_install" |
| 217 | |
Chris Dent | ebdd9ac | 2015-03-04 12:35:14 +0000 | [diff] [blame] | 218 | PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE) |
| 219 | if [[ "$PIP_UPGRADE" = "True" ]] ; then |
| 220 | upgrade="--upgrade" |
| 221 | fi |
| 222 | |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 223 | if [[ -z "$os_PACKAGE" ]]; then |
| 224 | GetOSVersion |
| 225 | fi |
Zane Bitter | 9e7ead9 | 2017-10-05 16:51:09 -0400 | [diff] [blame] | 226 | |
| 227 | # Try to extract the path of the package we are installing into |
| 228 | # package_dir. We need this to check for test-requirements.txt, |
| 229 | # at least. |
| 230 | # |
| 231 | # ${!#} expands to the last positional argument to this function. |
| 232 | # With "extras" syntax included, our arguments might be something |
| 233 | # like: |
| 234 | # -e /path/to/fooproject[extra] |
| 235 | # Thus this magic line grabs just the path without extras |
| 236 | # |
| 237 | # Note that this makes no sense if this is a pypi (rather than |
| 238 | # local path) install; ergo you must check this path exists before |
| 239 | # use. Also, if we had multiple or mixed installs, we would also |
| 240 | # likely break. But for historical reasons, it's basically only |
| 241 | # the other wrapper functions in here calling this to install |
| 242 | # local packages, and they do so with single call per install. So |
| 243 | # this works (for now...) |
| 244 | local package_dir=${!#%\[*\]} |
| 245 | |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 246 | if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then |
| 247 | # TRACK_DEPENDS=True installation creates a circular dependency when |
Atsushi SAKAI | 5509ed5 | 2015-11-30 20:20:21 +0900 | [diff] [blame] | 248 | # we attempt to install virtualenv into a virtualenv, so we must global |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 249 | # that installation. |
| 250 | source $DEST/.venv/bin/activate |
| 251 | local cmd_pip=$DEST/.venv/bin/pip |
| 252 | local sudo_pip="env" |
| 253 | else |
Dean Troyer | 2b56476 | 2015-02-11 17:01:02 -0600 | [diff] [blame] | 254 | if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then |
| 255 | local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip |
| 256 | local sudo_pip="env" |
| 257 | else |
Ian Wienand | ada886d | 2015-10-07 14:06:26 +1100 | [diff] [blame] | 258 | local cmd_pip |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 259 | cmd_pip=$(get_pip_command $PYTHON2_VERSION) |
Dean Troyer | 2b56476 | 2015-02-11 17:01:02 -0600 | [diff] [blame] | 260 | local sudo_pip="sudo -H" |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 261 | if python3_enabled; then |
| 262 | # Look at the package classifiers to find the python |
| 263 | # versions supported, and if we find the version of |
| 264 | # python3 we've been told to use, use that instead of the |
| 265 | # default pip |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 266 | local python_versions |
Davanum Srinivas | afa8a00 | 2016-12-19 09:51:01 -0500 | [diff] [blame] | 267 | |
| 268 | # Special case some services that have experimental |
| 269 | # support for python3 in progress, but don't claim support |
| 270 | # in their classifier |
| 271 | echo "Check python version for : $package_dir" |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 272 | if python3_disabled_for ${package_dir##*/}; then |
| 273 | echo "Explicitly using $PYTHON2_VERSION version to install $package_dir based on DISABLED_PYTHON3_PACKAGES" |
| 274 | elif python3_enabled_for ${package_dir##*/}; then |
Doug Hellmann | 36377f6 | 2018-12-04 11:33:03 -0500 | [diff] [blame] | 275 | echo "Using $PYTHON3_VERSION version to install $package_dir based on default behavior" |
Davanum Srinivas | afa8a00 | 2016-12-19 09:51:01 -0500 | [diff] [blame] | 276 | sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" |
| 277 | cmd_pip=$(get_pip_command $PYTHON3_VERSION) |
| 278 | elif [[ -d "$package_dir" ]]; then |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 279 | python_versions=$(get_python_versions_for_package $package_dir) |
| 280 | if [[ $python_versions =~ $PYTHON3_VERSION ]]; then |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 281 | echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on classifiers" |
Davanum Srinivas | afa8a00 | 2016-12-19 09:51:01 -0500 | [diff] [blame] | 282 | sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" |
| 283 | cmd_pip=$(get_pip_command $PYTHON3_VERSION) |
| 284 | else |
| 285 | # The package may not have yet advertised python3.5 |
| 286 | # support so check for just python3 classifier and log |
| 287 | # a warning. |
| 288 | python3_classifier=$(check_python3_support_for_package_local $package_dir) |
| 289 | if [[ ! -z "$python3_classifier" ]]; then |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 290 | echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on local package settings" |
Davanum Srinivas | afa8a00 | 2016-12-19 09:51:01 -0500 | [diff] [blame] | 291 | sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" |
| 292 | cmd_pip=$(get_pip_command $PYTHON3_VERSION) |
Matt Riedemann | ddb6179 | 2019-03-19 15:04:12 -0400 | [diff] [blame] | 293 | else |
| 294 | echo "WARNING: Did not find python 3 classifier for local package $package_dir" |
Davanum Srinivas | afa8a00 | 2016-12-19 09:51:01 -0500 | [diff] [blame] | 295 | fi |
| 296 | fi |
| 297 | else |
| 298 | # Check pypi as we don't have the package on disk |
| 299 | package=$(echo $package_dir | grep -o '^[.a-zA-Z0-9_-]*') |
| 300 | python3_classifier=$(check_python3_support_for_package_remote $package) |
| 301 | if [[ ! -z "$python3_classifier" ]]; then |
Doug Hellmann | 94129c7 | 2017-01-09 21:24:24 +0000 | [diff] [blame] | 302 | echo "Automatically using $PYTHON3_VERSION version to install $package based on remote package settings" |
Davanum Srinivas | afa8a00 | 2016-12-19 09:51:01 -0500 | [diff] [blame] | 303 | sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8" |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 304 | cmd_pip=$(get_pip_command $PYTHON3_VERSION) |
Matt Riedemann | ddb6179 | 2019-03-19 15:04:12 -0400 | [diff] [blame] | 305 | else |
| 306 | echo "WARNING: Did not find python 3 classifier for remote package $package_dir" |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 307 | fi |
| 308 | fi |
| 309 | fi |
Dean Troyer | 2b56476 | 2015-02-11 17:01:02 -0600 | [diff] [blame] | 310 | fi |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 311 | fi |
| 312 | |
Robert Collins | 635a5ba | 2015-06-10 08:48:06 +1200 | [diff] [blame] | 313 | cmd_pip="$cmd_pip install" |
Clark Boylan | 05aa384 | 2015-08-03 11:14:13 -0700 | [diff] [blame] | 314 | # Always apply constraints |
| 315 | cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt" |
Robert Collins | 635a5ba | 2015-06-10 08:48:06 +1200 | [diff] [blame] | 316 | |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 317 | # FIXME(dhellmann): Need to force multiple versions of pip for |
| 318 | # packages like setuptools? |
Ian Wienand | ada886d | 2015-10-07 14:06:26 +1100 | [diff] [blame] | 319 | local pip_version |
| 320 | pip_version=$(python -c "import pip; \ |
Clark Boylan | 0657795 | 2017-10-20 12:14:29 -0700 | [diff] [blame] | 321 | print(pip.__version__.split('.')[0])") |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 322 | if (( pip_version<6 )); then |
| 323 | die $LINENO "Currently installed pip version ${pip_version} does not" \ |
| 324 | "meet minimum requirements (>=6)." |
| 325 | fi |
| 326 | |
| 327 | $xtrace |
Clark Boylan | f266a2d | 2017-06-12 14:57:59 -0700 | [diff] [blame] | 328 | |
| 329 | # Also install test requirements |
| 330 | local install_test_reqs="" |
Zane Bitter | 9e7ead9 | 2017-10-05 16:51:09 -0400 | [diff] [blame] | 331 | local test_req="${package_dir}/test-requirements.txt" |
Clark Boylan | f266a2d | 2017-06-12 14:57:59 -0700 | [diff] [blame] | 332 | if [[ -e "$test_req" ]]; then |
| 333 | install_test_reqs="-r $test_req" |
| 334 | fi |
| 335 | |
Spyros Trigazis | 88ccd47 | 2016-07-24 22:13:57 +0200 | [diff] [blame] | 336 | # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep |
| 337 | # the same behaviour of setuptools before version 25.0.0. |
| 338 | # related issue: https://github.com/pypa/pip/issues/3874 |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 339 | $sudo_pip \ |
Eli Qiao | 6a83c42 | 2015-03-17 16:54:16 +0800 | [diff] [blame] | 340 | http_proxy="${http_proxy:-}" \ |
| 341 | https_proxy="${https_proxy:-}" \ |
| 342 | no_proxy="${no_proxy:-}" \ |
Joe Gordon | cd8824a | 2015-03-04 16:40:19 -0800 | [diff] [blame] | 343 | PIP_FIND_LINKS=$PIP_FIND_LINKS \ |
Spyros Trigazis | 88ccd47 | 2016-07-24 22:13:57 +0200 | [diff] [blame] | 344 | SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \ |
Clark Boylan | f266a2d | 2017-06-12 14:57:59 -0700 | [diff] [blame] | 345 | $cmd_pip $upgrade $install_test_reqs \ |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 346 | $@ |
Federico Ressi | e208d06 | 2015-11-21 11:15:39 +0000 | [diff] [blame] | 347 | result=$? |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 348 | |
Sean Dague | cb658fa | 2015-10-08 17:12:03 -0400 | [diff] [blame] | 349 | time_stop "pip_install" |
Federico Ressi | e208d06 | 2015-11-21 11:15:39 +0000 | [diff] [blame] | 350 | return $result |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 351 | } |
| 352 | |
Sean Dague | f28e7ef | 2017-05-07 22:02:10 -0400 | [diff] [blame] | 353 | function pip_uninstall { |
Sampath Priyankara | 87d2396 | 2017-08-03 16:12:40 +0900 | [diff] [blame] | 354 | # Skip uninstall if offline |
| 355 | [[ "${OFFLINE}" = "True" ]] && return |
| 356 | |
Sean Dague | f28e7ef | 2017-05-07 22:02:10 -0400 | [diff] [blame] | 357 | local name=$1 |
| 358 | if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then |
| 359 | local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip |
| 360 | local sudo_pip="env" |
| 361 | else |
| 362 | local cmd_pip |
| 363 | cmd_pip=$(get_pip_command $PYTHON2_VERSION) |
| 364 | local sudo_pip="sudo -H" |
| 365 | fi |
| 366 | # don't error if we can't uninstall, it might not be there |
Brian Haley | 954fd1b | 2017-05-16 12:24:45 -0400 | [diff] [blame] | 367 | $sudo_pip $cmd_pip uninstall -y $name || /bin/true |
Sean Dague | f28e7ef | 2017-05-07 22:02:10 -0400 | [diff] [blame] | 368 | } |
| 369 | |
Joe Gordon | d5ac785 | 2015-02-06 19:29:23 -0800 | [diff] [blame] | 370 | # get version of a package from global requirements file |
| 371 | # get_from_global_requirements <package> |
| 372 | function get_from_global_requirements { |
| 373 | local package=$1 |
Ian Wienand | ada886d | 2015-10-07 14:06:26 +1100 | [diff] [blame] | 374 | local required_pkg |
| 375 | required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1) |
Joe Gordon | d5ac785 | 2015-02-06 19:29:23 -0800 | [diff] [blame] | 376 | if [[ $required_pkg == "" ]]; then |
| 377 | die $LINENO "Can't find package $package in requirements" |
| 378 | fi |
| 379 | echo $required_pkg |
| 380 | } |
| 381 | |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 382 | # should we use this library from their git repo, or should we let it |
| 383 | # get pulled in via pip dependencies. |
| 384 | function use_library_from_git { |
| 385 | local name=$1 |
| 386 | local enabled=1 |
Marc Koderer | 46f8cb7 | 2016-05-13 09:08:16 +0200 | [diff] [blame] | 387 | [[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0 |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 388 | return $enabled |
| 389 | } |
| 390 | |
Sean Dague | c71973e | 2015-09-08 07:12:48 -0400 | [diff] [blame] | 391 | # determine if a package was installed from git |
| 392 | function lib_installed_from_git { |
| 393 | local name=$1 |
DamonLi | 007f588 | 2017-11-23 10:05:46 +0800 | [diff] [blame] | 394 | local safe_name |
| 395 | safe_name=$(python -c "from pkg_resources import safe_name; \ |
| 396 | print(safe_name('${name}'))") |
Ian Wienand | ae9c6ab | 2017-09-29 10:16:47 +1000 | [diff] [blame] | 397 | # Note "pip freeze" doesn't always work here, because it tries to |
| 398 | # be smart about finding the remote of the git repo the package |
| 399 | # was installed from. This doesn't work with zuul which clones |
| 400 | # repos with no remote. |
| 401 | # |
| 402 | # The best option seems to be to use "pip list" which will tell |
| 403 | # you the path an editable install was installed from; for example |
| 404 | # in response to something like |
| 405 | # pip install -e 'git+http://git.openstack.org/openstack-dev/bashate#egg=bashate' |
Monty Taylor | f0cd9a8 | 2017-10-06 13:11:48 -0500 | [diff] [blame] | 406 | # pip list --format columns shows |
| 407 | # bashate 0.5.2.dev19 /tmp/env/src/bashate |
| 408 | # Thus we check the third column to see if we're installed from |
| 409 | # some local place. |
DamonLi | 007f588 | 2017-11-23 10:05:46 +0800 | [diff] [blame] | 410 | [[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]] |
Sean Dague | c71973e | 2015-09-08 07:12:48 -0400 | [diff] [blame] | 411 | } |
| 412 | |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 413 | # setup a library by name. If we are trying to use the library from |
| 414 | # git, we'll do a git based install, otherwise we'll punt and the |
| 415 | # library should be installed by a requirements pull from another |
| 416 | # project. |
| 417 | function setup_lib { |
| 418 | local name=$1 |
| 419 | local dir=${GITDIR[$name]} |
| 420 | setup_install $dir |
| 421 | } |
| 422 | |
Atsushi SAKAI | 5509ed5 | 2015-11-30 20:20:21 +0900 | [diff] [blame] | 423 | # setup a library by name in editable mode. If we are trying to use |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 424 | # the library from git, we'll do a git based install, otherwise we'll |
| 425 | # punt and the library should be installed by a requirements pull from |
| 426 | # another project. |
| 427 | # |
| 428 | # use this for non namespaced libraries |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 429 | # |
| 430 | # setup_dev_lib [-bindep] <name> |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 431 | function setup_dev_lib { |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 432 | local bindep |
| 433 | if [[ $1 == -bindep* ]]; then |
| 434 | bindep="${1}" |
| 435 | shift |
| 436 | fi |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 437 | local name=$1 |
| 438 | local dir=${GITDIR[$name]} |
Doug Hellmann | a2eb894 | 2017-01-09 22:11:49 +0000 | [diff] [blame] | 439 | if python3_enabled; then |
| 440 | # Turn off Python 3 mode and install the package again, |
| 441 | # forcing a Python 2 installation. This ensures that all libs |
| 442 | # being used for development are installed under both versions |
| 443 | # of Python. |
| 444 | echo "Installing $name again without Python 3 enabled" |
| 445 | USE_PYTHON3=False |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 446 | setup_develop $bindep $dir |
Doug Hellmann | a2eb894 | 2017-01-09 22:11:49 +0000 | [diff] [blame] | 447 | USE_PYTHON3=True |
| 448 | fi |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 449 | setup_develop $bindep $dir |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 450 | } |
| 451 | |
| 452 | # this should be used if you want to install globally, all libraries should |
| 453 | # use this, especially *oslo* ones |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 454 | # |
| 455 | # setup_install project_dir [extras] |
| 456 | # project_dir: directory of project repo (e.g., /opt/stack/keystone) |
| 457 | # extras: comma-separated list of optional dependencies to install |
| 458 | # (e.g., ldap,memcache). |
Takashi NATSUME | fa00777 | 2017-07-22 08:59:43 +0900 | [diff] [blame] | 459 | # See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 460 | # bindep: Set "-bindep" as first argument to install bindep.txt packages |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 461 | # The command is like "pip install <project_dir>[<extras>]" |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 462 | function setup_install { |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 463 | local bindep |
| 464 | if [[ $1 == -bindep* ]]; then |
| 465 | bindep="${1}" |
| 466 | shift |
| 467 | fi |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 468 | local project_dir=$1 |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 469 | local extras=$2 |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 470 | _setup_package_with_constraints_edit $bindep $project_dir "" $extras |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 471 | } |
| 472 | |
| 473 | # this should be used for projects which run services, like all services |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 474 | # |
| 475 | # setup_develop project_dir [extras] |
| 476 | # project_dir: directory of project repo (e.g., /opt/stack/keystone) |
| 477 | # extras: comma-separated list of optional dependencies to install |
| 478 | # (e.g., ldap,memcache). |
Takashi NATSUME | fa00777 | 2017-07-22 08:59:43 +0900 | [diff] [blame] | 479 | # See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 480 | # The command is like "pip install -e <project_dir>[<extras>]" |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 481 | function setup_develop { |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 482 | local bindep |
| 483 | if [[ $1 == -bindep* ]]; then |
| 484 | bindep="${1}" |
| 485 | shift |
| 486 | fi |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 487 | local project_dir=$1 |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 488 | local extras=$2 |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 489 | _setup_package_with_constraints_edit $bindep $project_dir -e $extras |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 490 | } |
| 491 | |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 492 | # ``pip install -e`` the package, which processes the dependencies |
| 493 | # using pip before running `setup.py develop` |
| 494 | # |
Clark Boylan | 05aa384 | 2015-08-03 11:14:13 -0700 | [diff] [blame] | 495 | # Updates the constraints from REQUIREMENTS_DIR to reflect the |
| 496 | # future installed state of this package. This ensures when we |
| 497 | # install this package we get the from source version. |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 498 | # |
Clark Boylan | 05aa384 | 2015-08-03 11:14:13 -0700 | [diff] [blame] | 499 | # Uses globals ``REQUIREMENTS_DIR`` |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 500 | # _setup_package_with_constraints_edit project_dir flags [extras] |
| 501 | # project_dir: directory of project repo (e.g., /opt/stack/keystone) |
| 502 | # flags: pip CLI options/flags |
| 503 | # extras: comma-separated list of optional dependencies to install |
| 504 | # (e.g., ldap,memcache). |
Takashi NATSUME | fa00777 | 2017-07-22 08:59:43 +0900 | [diff] [blame] | 505 | # See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 506 | # The command is like "pip install <flags> <project_dir>[<extras>]" |
| 507 | function _setup_package_with_constraints_edit { |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 508 | local bindep |
| 509 | if [[ $1 == -bindep* ]]; then |
| 510 | bindep="${1}" |
| 511 | shift |
| 512 | fi |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 513 | local project_dir=$1 |
| 514 | local flags=$2 |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 515 | local extras=$3 |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 516 | |
YAMAMOTO Takashi | c8c1c61 | 2016-03-22 14:29:47 +0900 | [diff] [blame] | 517 | # Normalize the directory name to avoid |
| 518 | # "installation from path or url cannot be constrained to a version" |
| 519 | # error. |
| 520 | # REVISIT(yamamoto): Remove this when fixed in pip. |
| 521 | # https://github.com/pypa/pip/pull/3582 |
| 522 | project_dir=$(cd $project_dir && pwd) |
| 523 | |
Robert Collins | 635a5ba | 2015-06-10 08:48:06 +1200 | [diff] [blame] | 524 | if [ -n "$REQUIREMENTS_DIR" ]; then |
| 525 | # Constrain this package to this project directory from here on out. |
Ian Wienand | ada886d | 2015-10-07 14:06:26 +1100 | [diff] [blame] | 526 | local name |
| 527 | name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg) |
Robert Collins | 7c83861 | 2015-07-03 13:28:09 +1200 | [diff] [blame] | 528 | $REQUIREMENTS_DIR/.venv/bin/edit-constraints \ |
| 529 | $REQUIREMENTS_DIR/upper-constraints.txt -- $name \ |
| 530 | "$flags file://$project_dir#egg=$name" |
Robert Collins | 635a5ba | 2015-06-10 08:48:06 +1200 | [diff] [blame] | 531 | fi |
| 532 | |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 533 | setup_package $bindep $project_dir "$flags" $extras |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 534 | |
James E. Blair | e1edde3 | 2018-03-02 15:05:14 +0000 | [diff] [blame] | 535 | # If this project is in LIBS_FROM_GIT, verify it was actually installed |
| 536 | # correctly. This helps catch errors caused by constraints mismatches. |
| 537 | if use_library_from_git "$project_dir"; then |
| 538 | if ! lib_installed_from_git "$project_dir"; then |
| 539 | die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir" |
| 540 | fi |
| 541 | fi |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 542 | } |
| 543 | |
| 544 | # ``pip install -e`` the package, which processes the dependencies |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 545 | # using pip before running `setup.py develop`. The command is like |
| 546 | # "pip install <flags> <project_dir>[<extras>]" |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 547 | # |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 548 | # Uses globals ``STACK_USER`` |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 549 | # |
| 550 | # Usage: |
| 551 | # setup_package [-bindep[=profile,profile]] <project_dir> <flags> [extras] |
| 552 | # |
| 553 | # -bindep : Use bindep to install dependencies; select extra profiles |
| 554 | # as comma separated arguments after "=" |
| 555 | # project_dir : directory of project repo (e.g., /opt/stack/keystone) |
| 556 | # flags : pip CLI options/flags |
| 557 | # extras : comma-separated list of optional dependencies to install |
| 558 | # (e.g., ldap,memcache). |
| 559 | # See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 560 | function setup_package { |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 561 | local bindep=0 |
| 562 | local bindep_flag="" |
| 563 | local bindep_profiles="" |
| 564 | if [[ $1 == -bindep* ]]; then |
| 565 | bindep=1 |
| 566 | IFS="=" read bindep_flag bindep_profiles <<< ${1} |
| 567 | shift |
| 568 | fi |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 569 | local project_dir=$1 |
| 570 | local flags=$2 |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 571 | local extras=$3 |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 572 | |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 573 | # if the flags variable exists, and it doesn't look like a flag, |
| 574 | # assume it's actually the extras list. |
| 575 | if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then |
| 576 | extras=$flags |
| 577 | flags="" |
| 578 | fi |
| 579 | |
| 580 | if [[ ! -z "$extras" ]]; then |
| 581 | extras="[$extras]" |
| 582 | fi |
| 583 | |
Ian Wienand | 58243f6 | 2018-12-13 14:05:53 +1100 | [diff] [blame] | 584 | # install any bindep packages |
| 585 | if [[ $bindep == 1 ]]; then |
| 586 | install_bindep $project_dir/bindep.txt $bindep_profiles |
| 587 | fi |
| 588 | |
Brant Knudson | 0842b81 | 2015-08-03 13:31:25 -0500 | [diff] [blame] | 589 | pip_install $flags "$project_dir$extras" |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 590 | # ensure that further actions can do things like setup.py sdist |
| 591 | if [[ "$flags" == "-e" ]]; then |
| 592 | safe_chown -R $STACK_USER $1/*.egg-info |
| 593 | fi |
| 594 | } |
| 595 | |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 596 | # Report whether python 3 should be used |
| 597 | function python3_enabled { |
| 598 | if [[ $USE_PYTHON3 == "True" ]]; then |
| 599 | return 0 |
| 600 | else |
| 601 | return 1 |
| 602 | fi |
| 603 | } |
| 604 | |
| 605 | # Install python3 packages |
| 606 | function install_python3 { |
| 607 | if is_ubuntu; then |
Lubosz "diltram" Kosnik | 0a09976 | 2016-08-03 10:21:41 -0500 | [diff] [blame] | 608 | apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev |
Armando Migliaccio | bacfb94 | 2017-03-20 22:27:20 -0700 | [diff] [blame] | 609 | elif is_suse; then |
| 610 | install_package python3-devel python3-dbm |
Doug Hellmann | ddc3839 | 2015-05-07 21:06:24 +0000 | [diff] [blame] | 611 | fi |
| 612 | } |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 613 | |
Sean Dague | f80e2cf | 2017-01-18 15:42:32 -0500 | [diff] [blame] | 614 | function install_devstack_tools { |
| 615 | # intentionally old to ensure devstack-gate has control |
| 616 | local dstools_version=${DSTOOLS_VERSION:-0.1.2} |
| 617 | install_python3 |
| 618 | sudo pip3 install -U devstack-tools==${dstools_version} |
| 619 | } |
| 620 | |
Dean Troyer | 490430d | 2015-01-30 14:38:35 -0600 | [diff] [blame] | 621 | # Restore xtrace |
| 622 | $INC_PY_TRACE |
| 623 | |
| 624 | # Local variables: |
| 625 | # mode: shell-script |
| 626 | # End: |