blob: f4f65fd2e89268157ff29f69a35e2a651bba7443 [file] [log] [blame]
Dean Troyer490430d2015-01-30 14:38:35 -06001#!/bin/bash
2#
3# **inc/python** - Python-related functions
4#
5# Support for pip/setuptools interfaces and virtual environments
6#
7# External functions used:
8# - GetOSVersion
9# - is_fedora
10# - is_suse
11# - safe_chown
12
13# Save trace setting
14INC_PY_TRACE=$(set +o | grep xtrace)
15set +o xtrace
16
17
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060018# Global Config Variables
19
Atsushi SAKAI5509ed52015-11-30 20:20:21 +090020# PROJECT_VENV contains the name of the virtual environment for each
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060021# project. A null value installs to the system Python directories.
22declare -A PROJECT_VENV
23
24
Dean Troyer490430d2015-01-30 14:38:35 -060025# Python Functions
26# ================
27
28# Get the path to the pip command.
29# get_pip_command
30function get_pip_command {
Doug Hellmannddc38392015-05-07 21:06:24 +000031 local version="$1"
32 # NOTE(dhellmann): I don't know if we actually get a pip3.4-python
33 # under any circumstances.
34 which pip${version} || which pip${version}-python
Dean Troyer490430d2015-01-30 14:38:35 -060035
36 if [ $? -ne 0 ]; then
Doug Hellmannddc38392015-05-07 21:06:24 +000037 die $LINENO "Unable to find pip${version}; cannot continue"
Dean Troyer490430d2015-01-30 14:38:35 -060038 fi
39}
40
Atsushi SAKAI5509ed52015-11-30 20:20:21 +090041# Get the path to the directory where python executables are installed.
Dean Troyer490430d2015-01-30 14:38:35 -060042# get_python_exec_prefix
43function get_python_exec_prefix {
Ian Wienand433a9b12015-10-07 13:29:31 +110044 local xtrace
45 xtrace=$(set +o | grep xtrace)
Dean Troyer2b564762015-02-11 17:01:02 -060046 set +o xtrace
47 if [[ -z "$os_PACKAGE" ]]; then
48 GetOSVersion
49 fi
50 $xtrace
51
Dean Troyer490430d2015-01-30 14:38:35 -060052 if is_fedora || is_suse; then
53 echo "/usr/bin"
54 else
55 echo "/usr/local/bin"
56 fi
57}
58
Sean Dague60996b12015-04-08 09:06:49 -040059# Wrapper for ``pip install`` that only installs versions of libraries
60# from the global-requirements specification.
61#
62# Uses globals ``REQUIREMENTS_DIR``
63#
64# pip_install_gr packagename
65function pip_install_gr {
66 local name=$1
Ian Wienandada886d2015-10-07 14:06:26 +110067 local clean_name
68 clean_name=$(get_from_global_requirements $name)
Sean Dague60996b12015-04-08 09:06:49 -040069 pip_install $clean_name
70}
71
Doug Hellmannddc38392015-05-07 21:06:24 +000072# Determine the python versions supported by a package
73function get_python_versions_for_package {
74 local name=$1
75 cd $name && python setup.py --classifiers \
76 | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
77}
78
Dean Troyer490430d2015-01-30 14:38:35 -060079# Wrapper for ``pip install`` to set cache and proxy environment variables
Dean Troyer41d6f852015-03-25 22:42:46 -050080# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
Robert Collins635a5ba2015-06-10 08:48:06 +120081# ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
Dean Troyer490430d2015-01-30 14:38:35 -060082# pip_install package [package ...]
83function pip_install {
Federico Ressie208d062015-11-21 11:15:39 +000084 local xtrace result
Ian Wienand433a9b12015-10-07 13:29:31 +110085 xtrace=$(set +o | grep xtrace)
Dean Troyer490430d2015-01-30 14:38:35 -060086 set +o xtrace
Chris Dentebdd9ac2015-03-04 12:35:14 +000087 local upgrade=""
Dean Troyer490430d2015-01-30 14:38:35 -060088 local offline=${OFFLINE:-False}
89 if [[ "$offline" == "True" || -z "$@" ]]; then
90 $xtrace
91 return
92 fi
93
Sean Daguecb658fa2015-10-08 17:12:03 -040094 time_start "pip_install"
95
Chris Dentebdd9ac2015-03-04 12:35:14 +000096 PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
97 if [[ "$PIP_UPGRADE" = "True" ]] ; then
98 upgrade="--upgrade"
99 fi
100
Dean Troyer490430d2015-01-30 14:38:35 -0600101 if [[ -z "$os_PACKAGE" ]]; then
102 GetOSVersion
103 fi
104 if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then
105 # TRACK_DEPENDS=True installation creates a circular dependency when
Atsushi SAKAI5509ed52015-11-30 20:20:21 +0900106 # we attempt to install virtualenv into a virtualenv, so we must global
Dean Troyer490430d2015-01-30 14:38:35 -0600107 # that installation.
108 source $DEST/.venv/bin/activate
109 local cmd_pip=$DEST/.venv/bin/pip
110 local sudo_pip="env"
111 else
Dean Troyer2b564762015-02-11 17:01:02 -0600112 if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
113 local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
114 local sudo_pip="env"
115 else
Ian Wienandada886d2015-10-07 14:06:26 +1100116 local cmd_pip
Doug Hellmannddc38392015-05-07 21:06:24 +0000117 cmd_pip=$(get_pip_command $PYTHON2_VERSION)
Dean Troyer2b564762015-02-11 17:01:02 -0600118 local sudo_pip="sudo -H"
Doug Hellmannddc38392015-05-07 21:06:24 +0000119 if python3_enabled; then
120 # Look at the package classifiers to find the python
121 # versions supported, and if we find the version of
122 # python3 we've been told to use, use that instead of the
123 # default pip
124 local package_dir=${!#}
125 local python_versions
126 if [[ -d "$package_dir" ]]; then
127 python_versions=$(get_python_versions_for_package $package_dir)
128 if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
129 cmd_pip=$(get_pip_command $PYTHON3_VERSION)
130 fi
131 fi
132 fi
Dean Troyer2b564762015-02-11 17:01:02 -0600133 fi
Dean Troyer490430d2015-01-30 14:38:35 -0600134 fi
135
Robert Collins635a5ba2015-06-10 08:48:06 +1200136 cmd_pip="$cmd_pip install"
Clark Boylan05aa3842015-08-03 11:14:13 -0700137 # Always apply constraints
138 cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
Robert Collins635a5ba2015-06-10 08:48:06 +1200139
Doug Hellmannddc38392015-05-07 21:06:24 +0000140 # FIXME(dhellmann): Need to force multiple versions of pip for
141 # packages like setuptools?
Ian Wienandada886d2015-10-07 14:06:26 +1100142 local pip_version
143 pip_version=$(python -c "import pip; \
Dean Troyer490430d2015-01-30 14:38:35 -0600144 print(pip.__version__.strip('.')[0])")
145 if (( pip_version<6 )); then
146 die $LINENO "Currently installed pip version ${pip_version} does not" \
147 "meet minimum requirements (>=6)."
148 fi
149
150 $xtrace
151 $sudo_pip \
Eli Qiao6a83c422015-03-17 16:54:16 +0800152 http_proxy="${http_proxy:-}" \
153 https_proxy="${https_proxy:-}" \
154 no_proxy="${no_proxy:-}" \
Joe Gordoncd8824a2015-03-04 16:40:19 -0800155 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Robert Collins635a5ba2015-06-10 08:48:06 +1200156 $cmd_pip $upgrade \
Dean Troyer490430d2015-01-30 14:38:35 -0600157 $@
Federico Ressie208d062015-11-21 11:15:39 +0000158 result=$?
Dean Troyer490430d2015-01-30 14:38:35 -0600159
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400160 # Also install test requirements
Sirushti Murugesan713fd2f2015-09-30 15:12:50 +0530161 local test_req="${!#}/test-requirements.txt"
Federico Ressie208d062015-11-21 11:15:39 +0000162 if [[ $result == 0 ]] && [[ -e "$test_req" ]]; then
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400163 echo "Installing test-requirements for $test_req"
164 $sudo_pip \
165 http_proxy=${http_proxy:-} \
166 https_proxy=${https_proxy:-} \
167 no_proxy=${no_proxy:-} \
168 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Robert Collins635a5ba2015-06-10 08:48:06 +1200169 $cmd_pip $upgrade \
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400170 -r $test_req
Federico Ressie208d062015-11-21 11:15:39 +0000171 result=$?
Dean Troyer490430d2015-01-30 14:38:35 -0600172 fi
Sean Daguecb658fa2015-10-08 17:12:03 -0400173
174 time_stop "pip_install"
Federico Ressie208d062015-11-21 11:15:39 +0000175 return $result
Dean Troyer490430d2015-01-30 14:38:35 -0600176}
177
Joe Gordond5ac7852015-02-06 19:29:23 -0800178# get version of a package from global requirements file
179# get_from_global_requirements <package>
180function get_from_global_requirements {
181 local package=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100182 local required_pkg
183 required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
Joe Gordond5ac7852015-02-06 19:29:23 -0800184 if [[ $required_pkg == "" ]]; then
185 die $LINENO "Can't find package $package in requirements"
186 fi
187 echo $required_pkg
188}
189
Dean Troyer490430d2015-01-30 14:38:35 -0600190# should we use this library from their git repo, or should we let it
191# get pulled in via pip dependencies.
192function use_library_from_git {
193 local name=$1
194 local enabled=1
195 [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
196 return $enabled
197}
198
Sean Daguec71973e2015-09-08 07:12:48 -0400199# determine if a package was installed from git
200function lib_installed_from_git {
201 local name=$1
202 pip freeze 2>/dev/null | grep -- "$name" | grep -q -- '-e git'
203}
204
205# check that everything that's in LIBS_FROM_GIT was actually installed
206# correctly, this helps double check issues with library fat fingering.
207function check_libs_from_git {
208 local lib=""
209 local not_installed=""
210 for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
211 if ! lib_installed_from_git "$lib"; then
212 not_installed+=" $lib"
213 fi
214 done
215 # if anything is not installed, say what it is.
216 if [[ -n "$not_installed" ]]; then
217 die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
218 fi
219}
220
Dean Troyer490430d2015-01-30 14:38:35 -0600221# setup a library by name. If we are trying to use the library from
222# git, we'll do a git based install, otherwise we'll punt and the
223# library should be installed by a requirements pull from another
224# project.
225function setup_lib {
226 local name=$1
227 local dir=${GITDIR[$name]}
228 setup_install $dir
229}
230
Atsushi SAKAI5509ed52015-11-30 20:20:21 +0900231# setup a library by name in editable mode. If we are trying to use
Dean Troyer490430d2015-01-30 14:38:35 -0600232# the library from git, we'll do a git based install, otherwise we'll
233# punt and the library should be installed by a requirements pull from
234# another project.
235#
236# use this for non namespaced libraries
237function setup_dev_lib {
238 local name=$1
239 local dir=${GITDIR[$name]}
240 setup_develop $dir
241}
242
243# this should be used if you want to install globally, all libraries should
244# use this, especially *oslo* ones
245function setup_install {
246 local project_dir=$1
Clark Boylan05aa3842015-08-03 11:14:13 -0700247 setup_package_with_constraints_edit $project_dir
Dean Troyer490430d2015-01-30 14:38:35 -0600248}
249
250# this should be used for projects which run services, like all services
251function setup_develop {
252 local project_dir=$1
Clark Boylan05aa3842015-08-03 11:14:13 -0700253 setup_package_with_constraints_edit $project_dir -e
Dean Troyer490430d2015-01-30 14:38:35 -0600254}
255
256# determine if a project as specified by directory is in
257# projects.txt. This will not be an exact match because we throw away
258# the namespacing when we clone, but it should be good enough in all
259# practical ways.
260function is_in_projects_txt {
261 local project_dir=$1
Ian Wienandada886d2015-10-07 14:06:26 +1100262 local project_name
263 project_name=$(basename $project_dir)
Ihar Hrachyshka2ba4a722015-06-26 10:45:44 +0200264 grep -q "/$project_name\$" $REQUIREMENTS_DIR/projects.txt
Dean Troyer490430d2015-01-30 14:38:35 -0600265}
266
267# ``pip install -e`` the package, which processes the dependencies
268# using pip before running `setup.py develop`
269#
Clark Boylan05aa3842015-08-03 11:14:13 -0700270# Updates the constraints from REQUIREMENTS_DIR to reflect the
271# future installed state of this package. This ensures when we
272# install this package we get the from source version.
Dean Troyer490430d2015-01-30 14:38:35 -0600273#
Clark Boylan05aa3842015-08-03 11:14:13 -0700274# Uses globals ``REQUIREMENTS_DIR``
Dean Troyer490430d2015-01-30 14:38:35 -0600275# setup_develop directory
Clark Boylan05aa3842015-08-03 11:14:13 -0700276function setup_package_with_constraints_edit {
Dean Troyer490430d2015-01-30 14:38:35 -0600277 local project_dir=$1
278 local flags=$2
279
Robert Collins635a5ba2015-06-10 08:48:06 +1200280 if [ -n "$REQUIREMENTS_DIR" ]; then
281 # Constrain this package to this project directory from here on out.
Ian Wienandada886d2015-10-07 14:06:26 +1100282 local name
283 name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
Robert Collins7c838612015-07-03 13:28:09 +1200284 $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
285 $REQUIREMENTS_DIR/upper-constraints.txt -- $name \
286 "$flags file://$project_dir#egg=$name"
Robert Collins635a5ba2015-06-10 08:48:06 +1200287 fi
288
Dean Troyer490430d2015-01-30 14:38:35 -0600289 setup_package $project_dir $flags
290
Dean Troyer490430d2015-01-30 14:38:35 -0600291}
292
293# ``pip install -e`` the package, which processes the dependencies
294# using pip before running `setup.py develop`
295# Uses globals ``STACK_USER``
296# setup_develop_no_requirements_update directory
297function setup_package {
298 local project_dir=$1
299 local flags=$2
300
301 pip_install $flags $project_dir
302 # ensure that further actions can do things like setup.py sdist
303 if [[ "$flags" == "-e" ]]; then
304 safe_chown -R $STACK_USER $1/*.egg-info
305 fi
306}
307
Doug Hellmannddc38392015-05-07 21:06:24 +0000308# Report whether python 3 should be used
309function python3_enabled {
310 if [[ $USE_PYTHON3 == "True" ]]; then
311 return 0
312 else
313 return 1
314 fi
315}
316
317# Install python3 packages
318function install_python3 {
319 if is_ubuntu; then
320 apt_get install python3.4 python3.4-dev
321 fi
322}
Dean Troyer490430d2015-01-30 14:38:35 -0600323
324# Restore xtrace
325$INC_PY_TRACE
326
327# Local variables:
328# mode: shell-script
329# End: