blob: e013dfab362c76da419e0e8b291560c564a0b1f9 [file] [log] [blame]
Dean Troyer490430d2015-01-30 14:38:35 -06001#!/bin/bash
2#
3# **inc/python** - Python-related functions
4#
5# Support for pip/setuptools interfaces and virtual environments
6#
7# External functions used:
8# - GetOSVersion
9# - is_fedora
10# - is_suse
11# - safe_chown
12
13# Save trace setting
14INC_PY_TRACE=$(set +o | grep xtrace)
15set +o xtrace
16
17
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060018# Global Config Variables
19
Atsushi SAKAI5509ed52015-11-30 20:20:21 +090020# PROJECT_VENV contains the name of the virtual environment for each
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060021# project. A null value installs to the system Python directories.
22declare -A PROJECT_VENV
23
24
Dean Troyer490430d2015-01-30 14:38:35 -060025# Python Functions
26# ================
27
28# Get the path to the pip command.
29# get_pip_command
30function get_pip_command {
Doug Hellmannddc38392015-05-07 21:06:24 +000031 local version="$1"
32 # NOTE(dhellmann): I don't know if we actually get a pip3.4-python
33 # under any circumstances.
34 which pip${version} || which pip${version}-python
Dean Troyer490430d2015-01-30 14:38:35 -060035
36 if [ $? -ne 0 ]; then
Doug Hellmannddc38392015-05-07 21:06:24 +000037 die $LINENO "Unable to find pip${version}; cannot continue"
Dean Troyer490430d2015-01-30 14:38:35 -060038 fi
39}
40
Atsushi SAKAI5509ed52015-11-30 20:20:21 +090041# Get the path to the directory where python executables are installed.
Dean Troyer490430d2015-01-30 14:38:35 -060042# get_python_exec_prefix
43function get_python_exec_prefix {
Ian Wienand433a9b12015-10-07 13:29:31 +110044 local xtrace
45 xtrace=$(set +o | grep xtrace)
Dean Troyer2b564762015-02-11 17:01:02 -060046 set +o xtrace
47 if [[ -z "$os_PACKAGE" ]]; then
48 GetOSVersion
49 fi
50 $xtrace
51
Dean Troyer490430d2015-01-30 14:38:35 -060052 if is_fedora || is_suse; then
53 echo "/usr/bin"
54 else
55 echo "/usr/local/bin"
56 fi
57}
58
Sean Dague60996b12015-04-08 09:06:49 -040059# Wrapper for ``pip install`` that only installs versions of libraries
60# from the global-requirements specification.
61#
62# Uses globals ``REQUIREMENTS_DIR``
63#
64# pip_install_gr packagename
65function pip_install_gr {
66 local name=$1
Ian Wienandada886d2015-10-07 14:06:26 +110067 local clean_name
68 clean_name=$(get_from_global_requirements $name)
Sean Dague60996b12015-04-08 09:06:49 -040069 pip_install $clean_name
70}
71
Doug Hellmannddc38392015-05-07 21:06:24 +000072# Determine the python versions supported by a package
73function get_python_versions_for_package {
74 local name=$1
75 cd $name && python setup.py --classifiers \
76 | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
77}
78
Dean Troyer490430d2015-01-30 14:38:35 -060079# Wrapper for ``pip install`` to set cache and proxy environment variables
Dean Troyer41d6f852015-03-25 22:42:46 -050080# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
Robert Collins635a5ba2015-06-10 08:48:06 +120081# ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
Dean Troyer490430d2015-01-30 14:38:35 -060082# pip_install package [package ...]
83function pip_install {
Federico Ressie208d062015-11-21 11:15:39 +000084 local xtrace result
Ian Wienand433a9b12015-10-07 13:29:31 +110085 xtrace=$(set +o | grep xtrace)
Dean Troyer490430d2015-01-30 14:38:35 -060086 set +o xtrace
Chris Dentebdd9ac2015-03-04 12:35:14 +000087 local upgrade=""
Dean Troyer490430d2015-01-30 14:38:35 -060088 local offline=${OFFLINE:-False}
89 if [[ "$offline" == "True" || -z "$@" ]]; then
90 $xtrace
91 return
92 fi
93
Sean Daguecb658fa2015-10-08 17:12:03 -040094 time_start "pip_install"
95
Chris Dentebdd9ac2015-03-04 12:35:14 +000096 PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
97 if [[ "$PIP_UPGRADE" = "True" ]] ; then
98 upgrade="--upgrade"
99 fi
100
Dean Troyer490430d2015-01-30 14:38:35 -0600101 if [[ -z "$os_PACKAGE" ]]; then
102 GetOSVersion
103 fi
104 if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then
105 # TRACK_DEPENDS=True installation creates a circular dependency when
Atsushi SAKAI5509ed52015-11-30 20:20:21 +0900106 # we attempt to install virtualenv into a virtualenv, so we must global
Dean Troyer490430d2015-01-30 14:38:35 -0600107 # that installation.
108 source $DEST/.venv/bin/activate
109 local cmd_pip=$DEST/.venv/bin/pip
110 local sudo_pip="env"
111 else
Dean Troyer2b564762015-02-11 17:01:02 -0600112 if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
113 local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
114 local sudo_pip="env"
115 else
Ian Wienandada886d2015-10-07 14:06:26 +1100116 local cmd_pip
Doug Hellmannddc38392015-05-07 21:06:24 +0000117 cmd_pip=$(get_pip_command $PYTHON2_VERSION)
Dean Troyer2b564762015-02-11 17:01:02 -0600118 local sudo_pip="sudo -H"
Doug Hellmannddc38392015-05-07 21:06:24 +0000119 if python3_enabled; then
120 # Look at the package classifiers to find the python
121 # versions supported, and if we find the version of
122 # python3 we've been told to use, use that instead of the
123 # default pip
124 local package_dir=${!#}
125 local python_versions
126 if [[ -d "$package_dir" ]]; then
127 python_versions=$(get_python_versions_for_package $package_dir)
128 if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
129 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
130 fi
131 fi
132 fi
Dean Troyer2b564762015-02-11 17:01:02 -0600133 fi
Dean Troyer490430d2015-01-30 14:38:35 -0600134 fi
135
Robert Collins635a5ba2015-06-10 08:48:06 +1200136 cmd_pip="$cmd_pip install"
Clark Boylan05aa3842015-08-03 11:14:13 -0700137 # Always apply constraints
138 cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
Robert Collins635a5ba2015-06-10 08:48:06 +1200139
Doug Hellmannddc38392015-05-07 21:06:24 +0000140 # FIXME(dhellmann): Need to force multiple versions of pip for
141 # packages like setuptools?
Ian Wienandada886d2015-10-07 14:06:26 +1100142 local pip_version
143 pip_version=$(python -c "import pip; \
Dean Troyer490430d2015-01-30 14:38:35 -0600144 print(pip.__version__.strip('.')[0])")
145 if (( pip_version<6 )); then
146 die $LINENO "Currently installed pip version ${pip_version} does not" \
147 "meet minimum requirements (>=6)."
148 fi
149
150 $xtrace
151 $sudo_pip \
Eli Qiao6a83c422015-03-17 16:54:16 +0800152 http_proxy="${http_proxy:-}" \
153 https_proxy="${https_proxy:-}" \
154 no_proxy="${no_proxy:-}" \
Joe Gordoncd8824a2015-03-04 16:40:19 -0800155 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Robert Collins635a5ba2015-06-10 08:48:06 +1200156 $cmd_pip $upgrade \
Dean Troyer490430d2015-01-30 14:38:35 -0600157 $@
Federico Ressie208d062015-11-21 11:15:39 +0000158 result=$?
Dean Troyer490430d2015-01-30 14:38:35 -0600159
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400160 # Also install test requirements
Sirushti Murugesan713fd2f2015-09-30 15:12:50 +0530161 local test_req="${!#}/test-requirements.txt"
Federico Ressie208d062015-11-21 11:15:39 +0000162 if [[ $result == 0 ]] && [[ -e "$test_req" ]]; then
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400163 echo "Installing test-requirements for $test_req"
164 $sudo_pip \
165 http_proxy=${http_proxy:-} \
166 https_proxy=${https_proxy:-} \
167 no_proxy=${no_proxy:-} \
168 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Robert Collins635a5ba2015-06-10 08:48:06 +1200169 $cmd_pip $upgrade \
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400170 -r $test_req
Federico Ressie208d062015-11-21 11:15:39 +0000171 result=$?
Dean Troyer490430d2015-01-30 14:38:35 -0600172 fi
Sean Daguecb658fa2015-10-08 17:12:03 -0400173
174 time_stop "pip_install"
Federico Ressie208d062015-11-21 11:15:39 +0000175 return $result
Dean Troyer490430d2015-01-30 14:38:35 -0600176}
177
Joe Gordond5ac7852015-02-06 19:29:23 -0800178# get version of a package from global requirements file
179# get_from_global_requirements <package>
180function get_from_global_requirements {
181 local package=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100182 local required_pkg
183 required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
Joe Gordond5ac7852015-02-06 19:29:23 -0800184 if [[ $required_pkg == "" ]]; then
185 die $LINENO "Can't find package $package in requirements"
186 fi
187 echo $required_pkg
188}
189
Dean Troyer490430d2015-01-30 14:38:35 -0600190# should we use this library from their git repo, or should we let it
191# get pulled in via pip dependencies.
192function use_library_from_git {
193 local name=$1
194 local enabled=1
Marc Koderer46f8cb72016-05-13 09:08:16 +0200195 [[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
Dean Troyer490430d2015-01-30 14:38:35 -0600196 return $enabled
197}
198
Sean Daguec71973e2015-09-08 07:12:48 -0400199# determine if a package was installed from git
200function lib_installed_from_git {
201 local name=$1
202 pip freeze 2>/dev/null | grep -- "$name" | grep -q -- '-e git'
203}
204
205# check that everything that's in LIBS_FROM_GIT was actually installed
206# correctly, this helps double check issues with library fat fingering.
207function check_libs_from_git {
208 local lib=""
209 local not_installed=""
210 for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
211 if ! lib_installed_from_git "$lib"; then
212 not_installed+=" $lib"
213 fi
214 done
215 # if anything is not installed, say what it is.
216 if [[ -n "$not_installed" ]]; then
217 die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
218 fi
219}
220
Dean Troyer490430d2015-01-30 14:38:35 -0600221# setup a library by name. If we are trying to use the library from
222# git, we'll do a git based install, otherwise we'll punt and the
223# library should be installed by a requirements pull from another
224# project.
225function setup_lib {
226 local name=$1
227 local dir=${GITDIR[$name]}
228 setup_install $dir
229}
230
Atsushi SAKAI5509ed52015-11-30 20:20:21 +0900231# setup a library by name in editable mode. If we are trying to use
Dean Troyer490430d2015-01-30 14:38:35 -0600232# the library from git, we'll do a git based install, otherwise we'll
233# punt and the library should be installed by a requirements pull from
234# another project.
235#
236# use this for non namespaced libraries
237function setup_dev_lib {
238 local name=$1
239 local dir=${GITDIR[$name]}
240 setup_develop $dir
241}
242
243# this should be used if you want to install globally, all libraries should
244# use this, especially *oslo* ones
Brant Knudson0842b812015-08-03 13:31:25 -0500245#
246# setup_install project_dir [extras]
247# project_dir: directory of project repo (e.g., /opt/stack/keystone)
248# extras: comma-separated list of optional dependencies to install
249# (e.g., ldap,memcache).
250# See http://docs.openstack.org/developer/pbr/#extra-requirements
251# The command is like "pip install <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600252function setup_install {
253 local project_dir=$1
Brant Knudson0842b812015-08-03 13:31:25 -0500254 local extras=$2
255 _setup_package_with_constraints_edit $project_dir "" $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600256}
257
258# this should be used for projects which run services, like all services
Brant Knudson0842b812015-08-03 13:31:25 -0500259#
260# setup_develop project_dir [extras]
261# project_dir: directory of project repo (e.g., /opt/stack/keystone)
262# extras: comma-separated list of optional dependencies to install
263# (e.g., ldap,memcache).
264# See http://docs.openstack.org/developer/pbr/#extra-requirements
265# The command is like "pip install -e <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600266function setup_develop {
267 local project_dir=$1
Brant Knudson0842b812015-08-03 13:31:25 -0500268 local extras=$2
269 _setup_package_with_constraints_edit $project_dir -e $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600270}
271
272# determine if a project as specified by directory is in
273# projects.txt. This will not be an exact match because we throw away
274# the namespacing when we clone, but it should be good enough in all
275# practical ways.
276function is_in_projects_txt {
277 local project_dir=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100278 local project_name
279 project_name=$(basename $project_dir)
Ihar Hrachyshka2ba4a722015-06-26 10:45:44 +0200280 grep -q "/$project_name\$" $REQUIREMENTS_DIR/projects.txt
Dean Troyer490430d2015-01-30 14:38:35 -0600281}
282
283# ``pip install -e`` the package, which processes the dependencies
284# using pip before running `setup.py develop`
285#
Clark Boylan05aa3842015-08-03 11:14:13 -0700286# Updates the constraints from REQUIREMENTS_DIR to reflect the
287# future installed state of this package. This ensures when we
288# install this package we get the from source version.
Dean Troyer490430d2015-01-30 14:38:35 -0600289#
Clark Boylan05aa3842015-08-03 11:14:13 -0700290# Uses globals ``REQUIREMENTS_DIR``
Brant Knudson0842b812015-08-03 13:31:25 -0500291# _setup_package_with_constraints_edit project_dir flags [extras]
292# project_dir: directory of project repo (e.g., /opt/stack/keystone)
293# flags: pip CLI options/flags
294# extras: comma-separated list of optional dependencies to install
295# (e.g., ldap,memcache).
296# See http://docs.openstack.org/developer/pbr/#extra-requirements
297# The command is like "pip install <flags> <project_dir>[<extras>]"
298function _setup_package_with_constraints_edit {
Dean Troyer490430d2015-01-30 14:38:35 -0600299 local project_dir=$1
300 local flags=$2
Brant Knudson0842b812015-08-03 13:31:25 -0500301 local extras=$3
Dean Troyer490430d2015-01-30 14:38:35 -0600302
YAMAMOTO Takashic8c1c612016-03-22 14:29:47 +0900303 # Normalize the directory name to avoid
304 # "installation from path or url cannot be constrained to a version"
305 # error.
306 # REVISIT(yamamoto): Remove this when fixed in pip.
307 # https://github.com/pypa/pip/pull/3582
308 project_dir=$(cd $project_dir && pwd)
309
Robert Collins635a5ba2015-06-10 08:48:06 +1200310 if [ -n "$REQUIREMENTS_DIR" ]; then
311 # Constrain this package to this project directory from here on out.
Ian Wienandada886d2015-10-07 14:06:26 +1100312 local name
313 name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
Robert Collins7c838612015-07-03 13:28:09 +1200314 $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
315 $REQUIREMENTS_DIR/upper-constraints.txt -- $name \
316 "$flags file://$project_dir#egg=$name"
Robert Collins635a5ba2015-06-10 08:48:06 +1200317 fi
318
Brant Knudson0842b812015-08-03 13:31:25 -0500319 setup_package $project_dir "$flags" $extras
Dean Troyer490430d2015-01-30 14:38:35 -0600320
Dean Troyer490430d2015-01-30 14:38:35 -0600321}
322
323# ``pip install -e`` the package, which processes the dependencies
324# using pip before running `setup.py develop`
Brant Knudson0842b812015-08-03 13:31:25 -0500325#
Dean Troyer490430d2015-01-30 14:38:35 -0600326# Uses globals ``STACK_USER``
Brant Knudson0842b812015-08-03 13:31:25 -0500327# setup_package project_dir [flags] [extras]
328# project_dir: directory of project repo (e.g., /opt/stack/keystone)
329# flags: pip CLI options/flags
330# extras: comma-separated list of optional dependencies to install
331# (e.g., ldap,memcache).
332# See http://docs.openstack.org/developer/pbr/#extra-requirements
333# The command is like "pip install <flags> <project_dir>[<extras>]"
Dean Troyer490430d2015-01-30 14:38:35 -0600334function setup_package {
335 local project_dir=$1
336 local flags=$2
Brant Knudson0842b812015-08-03 13:31:25 -0500337 local extras=$3
Dean Troyer490430d2015-01-30 14:38:35 -0600338
Brant Knudson0842b812015-08-03 13:31:25 -0500339 # if the flags variable exists, and it doesn't look like a flag,
340 # assume it's actually the extras list.
341 if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
342 extras=$flags
343 flags=""
344 fi
345
346 if [[ ! -z "$extras" ]]; then
347 extras="[$extras]"
348 fi
349
350 pip_install $flags "$project_dir$extras"
Dean Troyer490430d2015-01-30 14:38:35 -0600351 # ensure that further actions can do things like setup.py sdist
352 if [[ "$flags" == "-e" ]]; then
353 safe_chown -R $STACK_USER $1/*.egg-info
354 fi
355}
356
Doug Hellmannddc38392015-05-07 21:06:24 +0000357# Report whether python 3 should be used
358function python3_enabled {
359 if [[ $USE_PYTHON3 == "True" ]]; then
360 return 0
361 else
362 return 1
363 fi
364}
365
366# Install python3 packages
367function install_python3 {
368 if is_ubuntu; then
369 apt_get install python3.4 python3.4-dev
370 fi
371}
Dean Troyer490430d2015-01-30 14:38:35 -0600372
373# Restore xtrace
374$INC_PY_TRACE
375
376# Local variables:
377# mode: shell-script
378# End: