blob: 54fd90533aedc17d70b46707d4abf90aded3b70b [file] [log] [blame]
Dean Troyer490430d2015-01-30 14:38:35 -06001#!/bin/bash
2#
3# **inc/python** - Python-related functions
4#
5# Support for pip/setuptools interfaces and virtual environments
6#
7# External functions used:
8# - GetOSVersion
9# - is_fedora
10# - is_suse
11# - safe_chown
12
13# Save trace setting
14INC_PY_TRACE=$(set +o | grep xtrace)
15set +o xtrace
16
17
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060018# Global Config Variables
19
Atsushi SAKAI5509ed52015-11-30 20:20:21 +090020# PROJECT_VENV contains the name of the virtual environment for each
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060021# project. A null value installs to the system Python directories.
22declare -A PROJECT_VENV
23
24
Dean Troyer490430d2015-01-30 14:38:35 -060025# Python Functions
26# ================
27
28# Get the path to the pip command.
29# get_pip_command
30function get_pip_command {
Doug Hellmannddc38392015-05-07 21:06:24 +000031 local version="$1"
32 # NOTE(dhellmann): I don't know if we actually get a pip3.4-python
33 # under any circumstances.
34 which pip${version} || which pip${version}-python
Dean Troyer490430d2015-01-30 14:38:35 -060035
36 if [ $? -ne 0 ]; then
Doug Hellmannddc38392015-05-07 21:06:24 +000037 die $LINENO "Unable to find pip${version}; cannot continue"
Dean Troyer490430d2015-01-30 14:38:35 -060038 fi
39}
40
Atsushi SAKAI5509ed52015-11-30 20:20:21 +090041# Get the path to the directory where python executables are installed.
Dean Troyer490430d2015-01-30 14:38:35 -060042# get_python_exec_prefix
43function get_python_exec_prefix {
Ian Wienand433a9b12015-10-07 13:29:31 +110044 local xtrace
45 xtrace=$(set +o | grep xtrace)
Dean Troyer2b564762015-02-11 17:01:02 -060046 set +o xtrace
47 if [[ -z "$os_PACKAGE" ]]; then
48 GetOSVersion
49 fi
50 $xtrace
51
Dean Troyer490430d2015-01-30 14:38:35 -060052 if is_fedora || is_suse; then
53 echo "/usr/bin"
54 else
55 echo "/usr/local/bin"
56 fi
57}
58
Sean Dague60996b12015-04-08 09:06:49 -040059# Wrapper for ``pip install`` that only installs versions of libraries
60# from the global-requirements specification.
61#
62# Uses globals ``REQUIREMENTS_DIR``
63#
64# pip_install_gr packagename
65function pip_install_gr {
66 local name=$1
Ian Wienandada886d2015-10-07 14:06:26 +110067 local clean_name
68 clean_name=$(get_from_global_requirements $name)
Sean Dague60996b12015-04-08 09:06:49 -040069 pip_install $clean_name
70}
71
Doug Hellmannddc38392015-05-07 21:06:24 +000072# Determine the python versions supported by a package
73function get_python_versions_for_package {
74 local name=$1
75 cd $name && python setup.py --classifiers \
76 | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
77}
78
Davanum Srinivasafa8a002016-12-19 09:51:01 -050079# Check for python3 classifier in local directory
80function check_python3_support_for_package_local {
81 local name=$1
82 cd $name
83 set +e
84 classifier=$(python setup.py --classifiers \
85 | grep 'Programming Language :: Python :: 3$')
86 set -e
87 echo $classifier
88}
89
90# Check for python3 classifier on pypi
91function check_python3_support_for_package_remote {
92 local name=$1
93 set +e
94 classifier=$(curl -s -L "https://pypi.python.org/pypi/$name/json" \
95 | grep '"Programming Language :: Python :: 3"')
96 set -e
97 echo $classifier
98}
99
Dean Troyer490430d2015-01-30 14:38:35 -0600100# Wrapper for ``pip install`` to set cache and proxy environment variables
Dean Troyer41d6f852015-03-25 22:42:46 -0500101# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
Robert Collins635a5ba2015-06-10 08:48:06 +1200102# ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
Dean Troyer490430d2015-01-30 14:38:35 -0600103# pip_install package [package ...]
104function pip_install {
Federico Ressie208d062015-11-21 11:15:39 +0000105 local xtrace result
Ian Wienand433a9b12015-10-07 13:29:31 +1100106 xtrace=$(set +o | grep xtrace)
Dean Troyer490430d2015-01-30 14:38:35 -0600107 set +o xtrace
Chris Dentebdd9ac2015-03-04 12:35:14 +0000108 local upgrade=""
Dean Troyer490430d2015-01-30 14:38:35 -0600109 local offline=${OFFLINE:-False}
110 if [[ "$offline" == "True" || -z "$@" ]]; then
111 $xtrace
112 return
113 fi
114
Sean Daguecb658fa2015-10-08 17:12:03 -0400115 time_start "pip_install"
116
Chris Dentebdd9ac2015-03-04 12:35:14 +0000117 PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
118 if [[ "$PIP_UPGRADE" = "True" ]] ; then
119 upgrade="--upgrade"
120 fi
121
Dean Troyer490430d2015-01-30 14:38:35 -0600122 if [[ -z "$os_PACKAGE" ]]; then
123 GetOSVersion
124 fi
125 if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then
126 # TRACK_DEPENDS=True installation creates a circular dependency when
Atsushi SAKAI5509ed52015-11-30 20:20:21 +0900127 # we attempt to install virtualenv into a virtualenv, so we must global
Dean Troyer490430d2015-01-30 14:38:35 -0600128 # that installation.
129 source $DEST/.venv/bin/activate
130 local cmd_pip=$DEST/.venv/bin/pip
131 local sudo_pip="env"
132 else
Dean Troyer2b564762015-02-11 17:01:02 -0600133 if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
134 local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
135 local sudo_pip="env"
136 else
Ian Wienandada886d2015-10-07 14:06:26 +1100137 local cmd_pip
Doug Hellmannddc38392015-05-07 21:06:24 +0000138 cmd_pip=$(get_pip_command $PYTHON2_VERSION)
Dean Troyer2b564762015-02-11 17:01:02 -0600139 local sudo_pip="sudo -H"
Doug Hellmannddc38392015-05-07 21:06:24 +0000140 if python3_enabled; then
141 # Look at the package classifiers to find the python
142 # versions supported, and if we find the version of
143 # python3 we've been told to use, use that instead of the
144 # default pip
145 local package_dir=${!#}
146 local python_versions
Davanum Srinivasafa8a002016-12-19 09:51:01 -0500147
148 # Special case some services that have experimental
149 # support for python3 in progress, but don't claim support
150 # in their classifier
151 echo "Check python version for : $package_dir"
152 if [[ ${package_dir##*/} == "nova" || ${package_dir##*/} == "glance" || ${package_dir##*/} == "cinder" ]]; then
153 echo "Using $PYTHON3_VERSION version to install $package_dir"
154 sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
155 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
156 elif [[ -d "$package_dir" ]]; then
Doug Hellmannddc38392015-05-07 21:06:24 +0000157 python_versions=$(get_python_versions_for_package $package_dir)
158 if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
Davanum Srinivasafa8a002016-12-19 09:51:01 -0500159 echo "Using $PYTHON3_VERSION version to install $package_dir"
160 sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
161 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
162 else
163 # The package may not have yet advertised python3.5
164 # support so check for just python3 classifier and log
165 # a warning.
166 python3_classifier=$(check_python3_support_for_package_local $package_dir)
167 if [[ ! -z "$python3_classifier" ]]; then
168 echo "Using $PYTHON3_VERSION version to install $package_dir"
169 sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
170 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
171 fi
172 fi
173 else
174 # Check pypi as we don't have the package on disk
175 package=$(echo $package_dir | grep -o '^[.a-zA-Z0-9_-]*')
176 python3_classifier=$(check_python3_support_for_package_remote $package)
177 if [[ ! -z "$python3_classifier" ]]; then
178 echo "Using $PYTHON3_VERSION version to install $package"
179 sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
Doug Hellmannddc38392015-05-07 21:06:24 +0000180 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
181 fi
182 fi
183 fi
Dean Troyer2b564762015-02-11 17:01:02 -0600184 fi
Dean Troyer490430d2015-01-30 14:38:35 -0600185 fi
186
Robert Collins635a5ba2015-06-10 08:48:06 +1200187 cmd_pip="$cmd_pip install"
Clark Boylan05aa3842015-08-03 11:14:13 -0700188 # Always apply constraints
189 cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
Robert Collins635a5ba2015-06-10 08:48:06 +1200190
Doug Hellmannddc38392015-05-07 21:06:24 +0000191 # FIXME(dhellmann): Need to force multiple versions of pip for
192 # packages like setuptools?
Ian Wienandada886d2015-10-07 14:06:26 +1100193 local pip_version
194 pip_version=$(python -c "import pip; \
Dean Troyer490430d2015-01-30 14:38:35 -0600195 print(pip.__version__.strip('.')[0])")
196 if (( pip_version<6 )); then
197 die $LINENO "Currently installed pip version ${pip_version} does not" \
198 "meet minimum requirements (>=6)."
199 fi
200
201 $xtrace
Spyros Trigazis88ccd472016-07-24 22:13:57 +0200202 # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
203 # the same behaviour of setuptools before version 25.0.0.
204 # related issue: https://github.com/pypa/pip/issues/3874
Dean Troyer490430d2015-01-30 14:38:35 -0600205 $sudo_pip \
Eli Qiao6a83c422015-03-17 16:54:16 +0800206 http_proxy="${http_proxy:-}" \
207 https_proxy="${https_proxy:-}" \
208 no_proxy="${no_proxy:-}" \
Joe Gordoncd8824a2015-03-04 16:40:19 -0800209 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Spyros Trigazis88ccd472016-07-24 22:13:57 +0200210 SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
Robert Collins635a5ba2015-06-10 08:48:06 +1200211 $cmd_pip $upgrade \
Dean Troyer490430d2015-01-30 14:38:35 -0600212 $@
Federico Ressie208d062015-11-21 11:15:39 +0000213 result=$?
Dean Troyer490430d2015-01-30 14:38:35 -0600214
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400215 # Also install test requirements
Sirushti Murugesan713fd2f2015-09-30 15:12:50 +0530216 local test_req="${!#}/test-requirements.txt"
Federico Ressie208d062015-11-21 11:15:39 +0000217 if [[ $result == 0 ]] && [[ -e "$test_req" ]]; then
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400218 echo "Installing test-requirements for $test_req"
219 $sudo_pip \
220 http_proxy=${http_proxy:-} \
221 https_proxy=${https_proxy:-} \
222 no_proxy=${no_proxy:-} \
223 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Robert Collins635a5ba2015-06-10 08:48:06 +1200224 $cmd_pip $upgrade \
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400225 -r $test_req
Federico Ressie208d062015-11-21 11:15:39 +0000226 result=$?
Dean Troyer490430d2015-01-30 14:38:35 -0600227 fi
Sean Daguecb658fa2015-10-08 17:12:03 -0400228
229 time_stop "pip_install"
Federico Ressie208d062015-11-21 11:15:39 +0000230 return $result
Dean Troyer490430d2015-01-30 14:38:35 -0600231}
232
Joe Gordond5ac7852015-02-06 19:29:23 -0800233# get version of a package from global requirements file
234# get_from_global_requirements <package>
235function get_from_global_requirements {
236 local package=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100237 local required_pkg
238 required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
Joe Gordond5ac7852015-02-06 19:29:23 -0800239 if [[ $required_pkg == "" ]]; then
240 die $LINENO "Can't find package $package in requirements"
241 fi
242 echo $required_pkg
243}
244
Dean Troyer490430d2015-01-30 14:38:35 -0600245# should we use this library from their git repo, or should we let it
246# get pulled in via pip dependencies.
247function use_library_from_git {
248 local name=$1
249 local enabled=1
Marc Koderer46f8cb72016-05-13 09:08:16 +0200250 [[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
Dean Troyer490430d2015-01-30 14:38:35 -0600251 return $enabled
252}
253
Sean Daguec71973e2015-09-08 07:12:48 -0400254# determine if a package was installed from git
255function lib_installed_from_git {
256 local name=$1
257 pip freeze 2>/dev/null | grep -- "$name" | grep -q -- '-e git'
258}
259
260# check that everything that's in LIBS_FROM_GIT was actually installed
261# correctly, this helps double check issues with library fat fingering.
262function check_libs_from_git {
263 local lib=""
264 local not_installed=""
265 for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
266 if ! lib_installed_from_git "$lib"; then
267 not_installed+=" $lib"
268 fi
269 done
270 # if anything is not installed, say what it is.
271 if [[ -n "$not_installed" ]]; then
272 die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
273 fi
274}
275
Dean Troyer490430d2015-01-30 14:38:35 -0600276# setup a library by name. If we are trying to use the library from
277# git, we'll do a git based install, otherwise we'll punt and the
278# library should be installed by a requirements pull from another
279# project.
280function setup_lib {
281 local name=$1
282 local dir=${GITDIR[$name]}
283 setup_install $dir
284}
285
Atsushi SAKAI5509ed52015-11-30 20:20:21 +0900286# setup a library by name in editable mode. If we are trying to use
Dean Troyer490430d2015-01-30 14:38:35 -0600287# the library from git, we'll do a git based install, otherwise we'll
288# punt and the library should be installed by a requirements pull from
289# another project.
290#
291# use this for non namespaced libraries
292function setup_dev_lib {
293 local name=$1
294 local dir=${GITDIR[$name]}
295 setup_develop $dir
296}
297
298# this should be used if you want to install globally, all libraries should
299# use this, especially *oslo* ones
Brant Knudson0842b812015-08-03 13:31:25 -0500300#
301# setup_install project_dir [extras]
302# project_dir: directory of project repo (e.g., /opt/stack/keystone)
303# extras: comma-separated list of optional dependencies to install
304# (e.g., ldap,memcache).
305# See http://docs.openstack.org/developer/pbr/#extra-requirements
306# The command is like "pip install <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600307function setup_install {
308 local project_dir=$1
Brant Knudson0842b812015-08-03 13:31:25 -0500309 local extras=$2
310 _setup_package_with_constraints_edit $project_dir "" $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600311}
312
313# this should be used for projects which run services, like all services
Brant Knudson0842b812015-08-03 13:31:25 -0500314#
315# setup_develop project_dir [extras]
316# project_dir: directory of project repo (e.g., /opt/stack/keystone)
317# extras: comma-separated list of optional dependencies to install
318# (e.g., ldap,memcache).
319# See http://docs.openstack.org/developer/pbr/#extra-requirements
320# The command is like "pip install -e <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600321function setup_develop {
322 local project_dir=$1
Brant Knudson0842b812015-08-03 13:31:25 -0500323 local extras=$2
324 _setup_package_with_constraints_edit $project_dir -e $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600325}
326
327# determine if a project as specified by directory is in
328# projects.txt. This will not be an exact match because we throw away
329# the namespacing when we clone, but it should be good enough in all
330# practical ways.
331function is_in_projects_txt {
332 local project_dir=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100333 local project_name
334 project_name=$(basename $project_dir)
Ihar Hrachyshka2ba4a722015-06-26 10:45:44 +0200335 grep -q "/$project_name\$" $REQUIREMENTS_DIR/projects.txt
Dean Troyer490430d2015-01-30 14:38:35 -0600336}
337
338# ``pip install -e`` the package, which processes the dependencies
339# using pip before running `setup.py develop`
340#
Clark Boylan05aa3842015-08-03 11:14:13 -0700341# Updates the constraints from REQUIREMENTS_DIR to reflect the
342# future installed state of this package. This ensures when we
343# install this package we get the from source version.
Dean Troyer490430d2015-01-30 14:38:35 -0600344#
Clark Boylan05aa3842015-08-03 11:14:13 -0700345# Uses globals ``REQUIREMENTS_DIR``
Brant Knudson0842b812015-08-03 13:31:25 -0500346# _setup_package_with_constraints_edit project_dir flags [extras]
347# project_dir: directory of project repo (e.g., /opt/stack/keystone)
348# flags: pip CLI options/flags
349# extras: comma-separated list of optional dependencies to install
350# (e.g., ldap,memcache).
351# See http://docs.openstack.org/developer/pbr/#extra-requirements
352# The command is like "pip install <flags> <project_dir>[<extras>]"
353function _setup_package_with_constraints_edit {
Dean Troyer490430d2015-01-30 14:38:35 -0600354 local project_dir=$1
355 local flags=$2
Brant Knudson0842b812015-08-03 13:31:25 -0500356 local extras=$3
Dean Troyer490430d2015-01-30 14:38:35 -0600357
YAMAMOTO Takashic8c1c612016-03-22 14:29:47 +0900358 # Normalize the directory name to avoid
359 # "installation from path or url cannot be constrained to a version"
360 # error.
361 # REVISIT(yamamoto): Remove this when fixed in pip.
362 # https://github.com/pypa/pip/pull/3582
363 project_dir=$(cd $project_dir && pwd)
364
Robert Collins635a5ba2015-06-10 08:48:06 +1200365 if [ -n "$REQUIREMENTS_DIR" ]; then
366 # Constrain this package to this project directory from here on out.
Ian Wienandada886d2015-10-07 14:06:26 +1100367 local name
368 name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
Robert Collins7c838612015-07-03 13:28:09 +1200369 $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
370 $REQUIREMENTS_DIR/upper-constraints.txt -- $name \
371 "$flags file://$project_dir#egg=$name"
Robert Collins635a5ba2015-06-10 08:48:06 +1200372 fi
373
Brant Knudson0842b812015-08-03 13:31:25 -0500374 setup_package $project_dir "$flags" $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600375
Dean Troyer490430d2015-01-30 14:38:35 -0600376}
377
378# ``pip install -e`` the package, which processes the dependencies
379# using pip before running `setup.py develop`
Brant Knudson0842b812015-08-03 13:31:25 -0500380#
Dean Troyer490430d2015-01-30 14:38:35 -0600381# Uses globals ``STACK_USER``
Brant Knudson0842b812015-08-03 13:31:25 -0500382# setup_package project_dir [flags] [extras]
383# project_dir: directory of project repo (e.g., /opt/stack/keystone)
384# flags: pip CLI options/flags
385# extras: comma-separated list of optional dependencies to install
386# (e.g., ldap,memcache).
387# See http://docs.openstack.org/developer/pbr/#extra-requirements
388# The command is like "pip install <flags> <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600389function setup_package {
390 local project_dir=$1
391 local flags=$2
Brant Knudson0842b812015-08-03 13:31:25 -0500392 local extras=$3
Dean Troyer490430d2015-01-30 14:38:35 -0600393
Brant Knudson0842b812015-08-03 13:31:25 -0500394 # if the flags variable exists, and it doesn't look like a flag,
395 # assume it's actually the extras list.
396 if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
397 extras=$flags
398 flags=""
399 fi
400
401 if [[ ! -z "$extras" ]]; then
402 extras="[$extras]"
403 fi
404
405 pip_install $flags "$project_dir$extras"
Dean Troyer490430d2015-01-30 14:38:35 -0600406 # ensure that further actions can do things like setup.py sdist
407 if [[ "$flags" == "-e" ]]; then
408 safe_chown -R $STACK_USER $1/*.egg-info
409 fi
410}
411
Doug Hellmannddc38392015-05-07 21:06:24 +0000412# Report whether python 3 should be used
413function python3_enabled {
414 if [[ $USE_PYTHON3 == "True" ]]; then
415 return 0
416 else
417 return 1
418 fi
419}
420
421# Install python3 packages
422function install_python3 {
423 if is_ubuntu; then
Lubosz "diltram" Kosnik0a099762016-08-03 10:21:41 -0500424 apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
Doug Hellmannddc38392015-05-07 21:06:24 +0000425 fi
426}
Dean Troyer490430d2015-01-30 14:38:35 -0600427
428# Restore xtrace
429$INC_PY_TRACE
430
431# Local variables:
432# mode: shell-script
433# End: