blob: 5a9a9ed588e21c569060a4d7e26b363f146b457a [file] [log] [blame]
Dean Troyer490430d2015-01-30 14:38:35 -06001#!/bin/bash
2#
3# **inc/python** - Python-related functions
4#
5# Support for pip/setuptools interfaces and virtual environments
6#
7# External functions used:
8# - GetOSVersion
9# - is_fedora
10# - is_suse
11# - safe_chown
12
13# Save trace setting
14INC_PY_TRACE=$(set +o | grep xtrace)
15set +o xtrace
16
17
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060018# Global Config Variables
19
Atsushi SAKAI5509ed52015-11-30 20:20:21 +090020# PROJECT_VENV contains the name of the virtual environment for each
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060021# project. A null value installs to the system Python directories.
22declare -A PROJECT_VENV
23
24
Dean Troyer490430d2015-01-30 14:38:35 -060025# Python Functions
26# ================
27
28# Get the path to the pip command.
29# get_pip_command
30function get_pip_command {
Doug Hellmannddc38392015-05-07 21:06:24 +000031 local version="$1"
32 # NOTE(dhellmann): I don't know if we actually get a pip3.4-python
33 # under any circumstances.
34 which pip${version} || which pip${version}-python
Dean Troyer490430d2015-01-30 14:38:35 -060035
36 if [ $? -ne 0 ]; then
Doug Hellmannddc38392015-05-07 21:06:24 +000037 die $LINENO "Unable to find pip${version}; cannot continue"
Dean Troyer490430d2015-01-30 14:38:35 -060038 fi
39}
40
Atsushi SAKAI5509ed52015-11-30 20:20:21 +090041# Get the path to the directory where python executables are installed.
Dean Troyer490430d2015-01-30 14:38:35 -060042# get_python_exec_prefix
43function get_python_exec_prefix {
Ian Wienand433a9b12015-10-07 13:29:31 +110044 local xtrace
45 xtrace=$(set +o | grep xtrace)
Dean Troyer2b564762015-02-11 17:01:02 -060046 set +o xtrace
47 if [[ -z "$os_PACKAGE" ]]; then
48 GetOSVersion
49 fi
50 $xtrace
51
Dean Troyer490430d2015-01-30 14:38:35 -060052 if is_fedora || is_suse; then
53 echo "/usr/bin"
54 else
55 echo "/usr/local/bin"
56 fi
57}
58
Sean Dague60996b12015-04-08 09:06:49 -040059# Wrapper for ``pip install`` that only installs versions of libraries
60# from the global-requirements specification.
61#
62# Uses globals ``REQUIREMENTS_DIR``
63#
64# pip_install_gr packagename
65function pip_install_gr {
66 local name=$1
Ian Wienandada886d2015-10-07 14:06:26 +110067 local clean_name
68 clean_name=$(get_from_global_requirements $name)
Sean Dague60996b12015-04-08 09:06:49 -040069 pip_install $clean_name
70}
71
Doug Hellmannddc38392015-05-07 21:06:24 +000072# Determine the python versions supported by a package
73function get_python_versions_for_package {
74 local name=$1
75 cd $name && python setup.py --classifiers \
76 | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
77}
78
Davanum Srinivasafa8a002016-12-19 09:51:01 -050079# Check for python3 classifier in local directory
80function check_python3_support_for_package_local {
81 local name=$1
82 cd $name
83 set +e
84 classifier=$(python setup.py --classifiers \
85 | grep 'Programming Language :: Python :: 3$')
86 set -e
87 echo $classifier
88}
89
90# Check for python3 classifier on pypi
91function check_python3_support_for_package_remote {
92 local name=$1
93 set +e
94 classifier=$(curl -s -L "https://pypi.python.org/pypi/$name/json" \
95 | grep '"Programming Language :: Python :: 3"')
96 set -e
97 echo $classifier
98}
99
Dean Troyer490430d2015-01-30 14:38:35 -0600100# Wrapper for ``pip install`` to set cache and proxy environment variables
Dean Troyer41d6f852015-03-25 22:42:46 -0500101# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
Robert Collins635a5ba2015-06-10 08:48:06 +1200102# ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
Dean Troyer490430d2015-01-30 14:38:35 -0600103# pip_install package [package ...]
104function pip_install {
Federico Ressie208d062015-11-21 11:15:39 +0000105 local xtrace result
Ian Wienand433a9b12015-10-07 13:29:31 +1100106 xtrace=$(set +o | grep xtrace)
Dean Troyer490430d2015-01-30 14:38:35 -0600107 set +o xtrace
Chris Dentebdd9ac2015-03-04 12:35:14 +0000108 local upgrade=""
Dean Troyer490430d2015-01-30 14:38:35 -0600109 local offline=${OFFLINE:-False}
110 if [[ "$offline" == "True" || -z "$@" ]]; then
111 $xtrace
112 return
113 fi
114
Sean Daguecb658fa2015-10-08 17:12:03 -0400115 time_start "pip_install"
116
Chris Dentebdd9ac2015-03-04 12:35:14 +0000117 PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
118 if [[ "$PIP_UPGRADE" = "True" ]] ; then
119 upgrade="--upgrade"
120 fi
121
Dean Troyer490430d2015-01-30 14:38:35 -0600122 if [[ -z "$os_PACKAGE" ]]; then
123 GetOSVersion
124 fi
125 if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then
126 # TRACK_DEPENDS=True installation creates a circular dependency when
Atsushi SAKAI5509ed52015-11-30 20:20:21 +0900127 # we attempt to install virtualenv into a virtualenv, so we must global
Dean Troyer490430d2015-01-30 14:38:35 -0600128 # that installation.
129 source $DEST/.venv/bin/activate
130 local cmd_pip=$DEST/.venv/bin/pip
131 local sudo_pip="env"
132 else
Dean Troyer2b564762015-02-11 17:01:02 -0600133 if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
134 local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
135 local sudo_pip="env"
136 else
Ian Wienandada886d2015-10-07 14:06:26 +1100137 local cmd_pip
Doug Hellmannddc38392015-05-07 21:06:24 +0000138 cmd_pip=$(get_pip_command $PYTHON2_VERSION)
Dean Troyer2b564762015-02-11 17:01:02 -0600139 local sudo_pip="sudo -H"
Doug Hellmannddc38392015-05-07 21:06:24 +0000140 if python3_enabled; then
141 # Look at the package classifiers to find the python
142 # versions supported, and if we find the version of
143 # python3 we've been told to use, use that instead of the
144 # default pip
145 local package_dir=${!#}
146 local python_versions
Davanum Srinivasafa8a002016-12-19 09:51:01 -0500147
148 # Special case some services that have experimental
149 # support for python3 in progress, but don't claim support
150 # in their classifier
151 echo "Check python version for : $package_dir"
Davanum Srinivas0c0d8482017-01-03 08:52:25 -0500152 if [[ ${package_dir##*/} == "nova" || ${package_dir##*/} == "glance" || \
153 ${package_dir##*/} == "cinder" || ${package_dir##*/} == "swift" || \
154 ${package_dir##*/} == "uwsgi" ]]; then
Davanum Srinivasafa8a002016-12-19 09:51:01 -0500155 echo "Using $PYTHON3_VERSION version to install $package_dir"
156 sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
157 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
158 elif [[ -d "$package_dir" ]]; then
Doug Hellmannddc38392015-05-07 21:06:24 +0000159 python_versions=$(get_python_versions_for_package $package_dir)
160 if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
Davanum Srinivasafa8a002016-12-19 09:51:01 -0500161 echo "Using $PYTHON3_VERSION version to install $package_dir"
162 sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
163 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
164 else
165 # The package may not have yet advertised python3.5
166 # support so check for just python3 classifier and log
167 # a warning.
168 python3_classifier=$(check_python3_support_for_package_local $package_dir)
169 if [[ ! -z "$python3_classifier" ]]; then
170 echo "Using $PYTHON3_VERSION version to install $package_dir"
171 sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
172 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
173 fi
174 fi
175 else
176 # Check pypi as we don't have the package on disk
177 package=$(echo $package_dir | grep -o '^[.a-zA-Z0-9_-]*')
178 python3_classifier=$(check_python3_support_for_package_remote $package)
179 if [[ ! -z "$python3_classifier" ]]; then
180 echo "Using $PYTHON3_VERSION version to install $package"
181 sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
Doug Hellmannddc38392015-05-07 21:06:24 +0000182 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
183 fi
184 fi
185 fi
Dean Troyer2b564762015-02-11 17:01:02 -0600186 fi
Dean Troyer490430d2015-01-30 14:38:35 -0600187 fi
188
Robert Collins635a5ba2015-06-10 08:48:06 +1200189 cmd_pip="$cmd_pip install"
Clark Boylan05aa3842015-08-03 11:14:13 -0700190 # Always apply constraints
191 cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
Robert Collins635a5ba2015-06-10 08:48:06 +1200192
Doug Hellmannddc38392015-05-07 21:06:24 +0000193 # FIXME(dhellmann): Need to force multiple versions of pip for
194 # packages like setuptools?
Ian Wienandada886d2015-10-07 14:06:26 +1100195 local pip_version
196 pip_version=$(python -c "import pip; \
Dean Troyer490430d2015-01-30 14:38:35 -0600197 print(pip.__version__.strip('.')[0])")
198 if (( pip_version<6 )); then
199 die $LINENO "Currently installed pip version ${pip_version} does not" \
200 "meet minimum requirements (>=6)."
201 fi
202
203 $xtrace
Spyros Trigazis88ccd472016-07-24 22:13:57 +0200204 # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
205 # the same behaviour of setuptools before version 25.0.0.
206 # related issue: https://github.com/pypa/pip/issues/3874
Dean Troyer490430d2015-01-30 14:38:35 -0600207 $sudo_pip \
Eli Qiao6a83c422015-03-17 16:54:16 +0800208 http_proxy="${http_proxy:-}" \
209 https_proxy="${https_proxy:-}" \
210 no_proxy="${no_proxy:-}" \
Joe Gordoncd8824a2015-03-04 16:40:19 -0800211 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Spyros Trigazis88ccd472016-07-24 22:13:57 +0200212 SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
Robert Collins635a5ba2015-06-10 08:48:06 +1200213 $cmd_pip $upgrade \
Dean Troyer490430d2015-01-30 14:38:35 -0600214 $@
Federico Ressie208d062015-11-21 11:15:39 +0000215 result=$?
Dean Troyer490430d2015-01-30 14:38:35 -0600216
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400217 # Also install test requirements
Sirushti Murugesan713fd2f2015-09-30 15:12:50 +0530218 local test_req="${!#}/test-requirements.txt"
Federico Ressie208d062015-11-21 11:15:39 +0000219 if [[ $result == 0 ]] && [[ -e "$test_req" ]]; then
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400220 echo "Installing test-requirements for $test_req"
221 $sudo_pip \
222 http_proxy=${http_proxy:-} \
223 https_proxy=${https_proxy:-} \
224 no_proxy=${no_proxy:-} \
225 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Robert Collins635a5ba2015-06-10 08:48:06 +1200226 $cmd_pip $upgrade \
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400227 -r $test_req
Federico Ressie208d062015-11-21 11:15:39 +0000228 result=$?
Dean Troyer490430d2015-01-30 14:38:35 -0600229 fi
Sean Daguecb658fa2015-10-08 17:12:03 -0400230
231 time_stop "pip_install"
Federico Ressie208d062015-11-21 11:15:39 +0000232 return $result
Dean Troyer490430d2015-01-30 14:38:35 -0600233}
234
Joe Gordond5ac7852015-02-06 19:29:23 -0800235# get version of a package from global requirements file
236# get_from_global_requirements <package>
237function get_from_global_requirements {
238 local package=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100239 local required_pkg
240 required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
Joe Gordond5ac7852015-02-06 19:29:23 -0800241 if [[ $required_pkg == "" ]]; then
242 die $LINENO "Can't find package $package in requirements"
243 fi
244 echo $required_pkg
245}
246
Dean Troyer490430d2015-01-30 14:38:35 -0600247# should we use this library from their git repo, or should we let it
248# get pulled in via pip dependencies.
249function use_library_from_git {
250 local name=$1
251 local enabled=1
Marc Koderer46f8cb72016-05-13 09:08:16 +0200252 [[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
Dean Troyer490430d2015-01-30 14:38:35 -0600253 return $enabled
254}
255
Sean Daguec71973e2015-09-08 07:12:48 -0400256# determine if a package was installed from git
257function lib_installed_from_git {
258 local name=$1
259 pip freeze 2>/dev/null | grep -- "$name" | grep -q -- '-e git'
260}
261
262# check that everything that's in LIBS_FROM_GIT was actually installed
263# correctly, this helps double check issues with library fat fingering.
264function check_libs_from_git {
265 local lib=""
266 local not_installed=""
267 for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
268 if ! lib_installed_from_git "$lib"; then
269 not_installed+=" $lib"
270 fi
271 done
272 # if anything is not installed, say what it is.
273 if [[ -n "$not_installed" ]]; then
274 die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
275 fi
276}
277
Dean Troyer490430d2015-01-30 14:38:35 -0600278# setup a library by name. If we are trying to use the library from
279# git, we'll do a git based install, otherwise we'll punt and the
280# library should be installed by a requirements pull from another
281# project.
282function setup_lib {
283 local name=$1
284 local dir=${GITDIR[$name]}
285 setup_install $dir
286}
287
Atsushi SAKAI5509ed52015-11-30 20:20:21 +0900288# setup a library by name in editable mode. If we are trying to use
Dean Troyer490430d2015-01-30 14:38:35 -0600289# the library from git, we'll do a git based install, otherwise we'll
290# punt and the library should be installed by a requirements pull from
291# another project.
292#
293# use this for non namespaced libraries
294function setup_dev_lib {
295 local name=$1
296 local dir=${GITDIR[$name]}
297 setup_develop $dir
298}
299
300# this should be used if you want to install globally, all libraries should
301# use this, especially *oslo* ones
Brant Knudson0842b812015-08-03 13:31:25 -0500302#
303# setup_install project_dir [extras]
304# project_dir: directory of project repo (e.g., /opt/stack/keystone)
305# extras: comma-separated list of optional dependencies to install
306# (e.g., ldap,memcache).
307# See http://docs.openstack.org/developer/pbr/#extra-requirements
308# The command is like "pip install <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600309function setup_install {
310 local project_dir=$1
Brant Knudson0842b812015-08-03 13:31:25 -0500311 local extras=$2
312 _setup_package_with_constraints_edit $project_dir "" $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600313}
314
315# this should be used for projects which run services, like all services
Brant Knudson0842b812015-08-03 13:31:25 -0500316#
317# setup_develop project_dir [extras]
318# project_dir: directory of project repo (e.g., /opt/stack/keystone)
319# extras: comma-separated list of optional dependencies to install
320# (e.g., ldap,memcache).
321# See http://docs.openstack.org/developer/pbr/#extra-requirements
322# The command is like "pip install -e <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600323function setup_develop {
324 local project_dir=$1
Brant Knudson0842b812015-08-03 13:31:25 -0500325 local extras=$2
326 _setup_package_with_constraints_edit $project_dir -e $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600327}
328
329# determine if a project as specified by directory is in
330# projects.txt. This will not be an exact match because we throw away
331# the namespacing when we clone, but it should be good enough in all
332# practical ways.
333function is_in_projects_txt {
334 local project_dir=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100335 local project_name
336 project_name=$(basename $project_dir)
Ihar Hrachyshka2ba4a722015-06-26 10:45:44 +0200337 grep -q "/$project_name\$" $REQUIREMENTS_DIR/projects.txt
Dean Troyer490430d2015-01-30 14:38:35 -0600338}
339
340# ``pip install -e`` the package, which processes the dependencies
341# using pip before running `setup.py develop`
342#
Clark Boylan05aa3842015-08-03 11:14:13 -0700343# Updates the constraints from REQUIREMENTS_DIR to reflect the
344# future installed state of this package. This ensures when we
345# install this package we get the from source version.
Dean Troyer490430d2015-01-30 14:38:35 -0600346#
Clark Boylan05aa3842015-08-03 11:14:13 -0700347# Uses globals ``REQUIREMENTS_DIR``
Brant Knudson0842b812015-08-03 13:31:25 -0500348# _setup_package_with_constraints_edit project_dir flags [extras]
349# project_dir: directory of project repo (e.g., /opt/stack/keystone)
350# flags: pip CLI options/flags
351# extras: comma-separated list of optional dependencies to install
352# (e.g., ldap,memcache).
353# See http://docs.openstack.org/developer/pbr/#extra-requirements
354# The command is like "pip install <flags> <project_dir>[<extras>]"
355function _setup_package_with_constraints_edit {
Dean Troyer490430d2015-01-30 14:38:35 -0600356 local project_dir=$1
357 local flags=$2
Brant Knudson0842b812015-08-03 13:31:25 -0500358 local extras=$3
Dean Troyer490430d2015-01-30 14:38:35 -0600359
YAMAMOTO Takashic8c1c612016-03-22 14:29:47 +0900360 # Normalize the directory name to avoid
361 # "installation from path or url cannot be constrained to a version"
362 # error.
363 # REVISIT(yamamoto): Remove this when fixed in pip.
364 # https://github.com/pypa/pip/pull/3582
365 project_dir=$(cd $project_dir && pwd)
366
Robert Collins635a5ba2015-06-10 08:48:06 +1200367 if [ -n "$REQUIREMENTS_DIR" ]; then
368 # Constrain this package to this project directory from here on out.
Ian Wienandada886d2015-10-07 14:06:26 +1100369 local name
370 name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
Robert Collins7c838612015-07-03 13:28:09 +1200371 $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
372 $REQUIREMENTS_DIR/upper-constraints.txt -- $name \
373 "$flags file://$project_dir#egg=$name"
Robert Collins635a5ba2015-06-10 08:48:06 +1200374 fi
375
Brant Knudson0842b812015-08-03 13:31:25 -0500376 setup_package $project_dir "$flags" $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600377
Dean Troyer490430d2015-01-30 14:38:35 -0600378}
379
380# ``pip install -e`` the package, which processes the dependencies
381# using pip before running `setup.py develop`
Brant Knudson0842b812015-08-03 13:31:25 -0500382#
Dean Troyer490430d2015-01-30 14:38:35 -0600383# Uses globals ``STACK_USER``
Brant Knudson0842b812015-08-03 13:31:25 -0500384# setup_package project_dir [flags] [extras]
385# project_dir: directory of project repo (e.g., /opt/stack/keystone)
386# flags: pip CLI options/flags
387# extras: comma-separated list of optional dependencies to install
388# (e.g., ldap,memcache).
389# See http://docs.openstack.org/developer/pbr/#extra-requirements
390# The command is like "pip install <flags> <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600391function setup_package {
392 local project_dir=$1
393 local flags=$2
Brant Knudson0842b812015-08-03 13:31:25 -0500394 local extras=$3
Dean Troyer490430d2015-01-30 14:38:35 -0600395
Brant Knudson0842b812015-08-03 13:31:25 -0500396 # if the flags variable exists, and it doesn't look like a flag,
397 # assume it's actually the extras list.
398 if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
399 extras=$flags
400 flags=""
401 fi
402
403 if [[ ! -z "$extras" ]]; then
404 extras="[$extras]"
405 fi
406
407 pip_install $flags "$project_dir$extras"
Dean Troyer490430d2015-01-30 14:38:35 -0600408 # ensure that further actions can do things like setup.py sdist
409 if [[ "$flags" == "-e" ]]; then
410 safe_chown -R $STACK_USER $1/*.egg-info
411 fi
412}
413
Doug Hellmannddc38392015-05-07 21:06:24 +0000414# Report whether python 3 should be used
415function python3_enabled {
416 if [[ $USE_PYTHON3 == "True" ]]; then
417 return 0
418 else
419 return 1
420 fi
421}
422
423# Install python3 packages
424function install_python3 {
425 if is_ubuntu; then
Lubosz "diltram" Kosnik0a099762016-08-03 10:21:41 -0500426 apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
Doug Hellmannddc38392015-05-07 21:06:24 +0000427 fi
428}
Dean Troyer490430d2015-01-30 14:38:35 -0600429
430# Restore xtrace
431$INC_PY_TRACE
432
433# Local variables:
434# mode: shell-script
435# End: