blob: 7d026c59337f5760085edfabe6a54d94838b9e64 [file] [log] [blame]
Dean Troyer490430d2015-01-30 14:38:35 -06001#!/bin/bash
2#
3# **inc/python** - Python-related functions
4#
5# Support for pip/setuptools interfaces and virtual environments
6#
7# External functions used:
8# - GetOSVersion
9# - is_fedora
10# - is_suse
11# - safe_chown
12
13# Save trace setting
14INC_PY_TRACE=$(set +o | grep xtrace)
15set +o xtrace
16
17
Dean Troyer8c2ce6e2015-02-18 14:47:54 -060018# Global Config Variables
19
20# PROJECT_VENV contains the name of the virtual enviromnet for each
21# project. A null value installs to the system Python directories.
22declare -A PROJECT_VENV
23
24
Dean Troyer490430d2015-01-30 14:38:35 -060025# Python Functions
26# ================
27
28# Get the path to the pip command.
29# get_pip_command
30function get_pip_command {
31 which pip || which pip-python
32
33 if [ $? -ne 0 ]; then
34 die $LINENO "Unable to find pip; cannot continue"
35 fi
36}
37
38# Get the path to the direcotry where python executables are installed.
39# get_python_exec_prefix
40function get_python_exec_prefix {
Ian Wienand433a9b12015-10-07 13:29:31 +110041 local xtrace
42 xtrace=$(set +o | grep xtrace)
Dean Troyer2b564762015-02-11 17:01:02 -060043 set +o xtrace
44 if [[ -z "$os_PACKAGE" ]]; then
45 GetOSVersion
46 fi
47 $xtrace
48
Dean Troyer490430d2015-01-30 14:38:35 -060049 if is_fedora || is_suse; then
50 echo "/usr/bin"
51 else
52 echo "/usr/local/bin"
53 fi
54}
55
Sean Dague60996b12015-04-08 09:06:49 -040056# Wrapper for ``pip install`` that only installs versions of libraries
57# from the global-requirements specification.
58#
59# Uses globals ``REQUIREMENTS_DIR``
60#
61# pip_install_gr packagename
62function pip_install_gr {
63 local name=$1
64 local clean_name=$(get_from_global_requirements $name)
65 pip_install $clean_name
66}
67
Dean Troyer490430d2015-01-30 14:38:35 -060068# Wrapper for ``pip install`` to set cache and proxy environment variables
Dean Troyer41d6f852015-03-25 22:42:46 -050069# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
Robert Collins635a5ba2015-06-10 08:48:06 +120070# ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
Dean Troyer490430d2015-01-30 14:38:35 -060071# pip_install package [package ...]
72function pip_install {
Ian Wienand433a9b12015-10-07 13:29:31 +110073 local xtrace
74 xtrace=$(set +o | grep xtrace)
Dean Troyer490430d2015-01-30 14:38:35 -060075 set +o xtrace
Chris Dentebdd9ac2015-03-04 12:35:14 +000076 local upgrade=""
Dean Troyer490430d2015-01-30 14:38:35 -060077 local offline=${OFFLINE:-False}
78 if [[ "$offline" == "True" || -z "$@" ]]; then
79 $xtrace
80 return
81 fi
82
Sean Daguecb658fa2015-10-08 17:12:03 -040083 time_start "pip_install"
84
Chris Dentebdd9ac2015-03-04 12:35:14 +000085 PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
86 if [[ "$PIP_UPGRADE" = "True" ]] ; then
87 upgrade="--upgrade"
88 fi
89
Dean Troyer490430d2015-01-30 14:38:35 -060090 if [[ -z "$os_PACKAGE" ]]; then
91 GetOSVersion
92 fi
93 if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then
94 # TRACK_DEPENDS=True installation creates a circular dependency when
95 # we attempt to install virtualenv into a virualenv, so we must global
96 # that installation.
97 source $DEST/.venv/bin/activate
98 local cmd_pip=$DEST/.venv/bin/pip
99 local sudo_pip="env"
100 else
Dean Troyer2b564762015-02-11 17:01:02 -0600101 if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
102 local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
103 local sudo_pip="env"
104 else
105 local cmd_pip=$(get_pip_command)
106 local sudo_pip="sudo -H"
107 fi
Dean Troyer490430d2015-01-30 14:38:35 -0600108 fi
109
Robert Collins635a5ba2015-06-10 08:48:06 +1200110 cmd_pip="$cmd_pip install"
Clark Boylan05aa3842015-08-03 11:14:13 -0700111 # Always apply constraints
112 cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
Robert Collins635a5ba2015-06-10 08:48:06 +1200113
Dean Troyer490430d2015-01-30 14:38:35 -0600114 local pip_version=$(python -c "import pip; \
115 print(pip.__version__.strip('.')[0])")
116 if (( pip_version<6 )); then
117 die $LINENO "Currently installed pip version ${pip_version} does not" \
118 "meet minimum requirements (>=6)."
119 fi
120
121 $xtrace
122 $sudo_pip \
Eli Qiao6a83c422015-03-17 16:54:16 +0800123 http_proxy="${http_proxy:-}" \
124 https_proxy="${https_proxy:-}" \
125 no_proxy="${no_proxy:-}" \
Joe Gordoncd8824a2015-03-04 16:40:19 -0800126 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Robert Collins635a5ba2015-06-10 08:48:06 +1200127 $cmd_pip $upgrade \
Dean Troyer490430d2015-01-30 14:38:35 -0600128 $@
129
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400130 # Also install test requirements
Sirushti Murugesan713fd2f2015-09-30 15:12:50 +0530131 local test_req="${!#}/test-requirements.txt"
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400132 if [[ -e "$test_req" ]]; then
133 echo "Installing test-requirements for $test_req"
134 $sudo_pip \
135 http_proxy=${http_proxy:-} \
136 https_proxy=${https_proxy:-} \
137 no_proxy=${no_proxy:-} \
138 PIP_FIND_LINKS=$PIP_FIND_LINKS \
Robert Collins635a5ba2015-06-10 08:48:06 +1200139 $cmd_pip $upgrade \
Sean Dagueeeb7bda2015-03-25 11:55:32 -0400140 -r $test_req
Dean Troyer490430d2015-01-30 14:38:35 -0600141 fi
Sean Daguecb658fa2015-10-08 17:12:03 -0400142
143 time_stop "pip_install"
Dean Troyer490430d2015-01-30 14:38:35 -0600144}
145
Joe Gordond5ac7852015-02-06 19:29:23 -0800146# get version of a package from global requirements file
147# get_from_global_requirements <package>
148function get_from_global_requirements {
149 local package=$1
Amrith Kumar98608762015-04-08 15:37:58 -0400150 local required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
Joe Gordond5ac7852015-02-06 19:29:23 -0800151 if [[ $required_pkg == "" ]]; then
152 die $LINENO "Can't find package $package in requirements"
153 fi
154 echo $required_pkg
155}
156
Dean Troyer490430d2015-01-30 14:38:35 -0600157# should we use this library from their git repo, or should we let it
158# get pulled in via pip dependencies.
159function use_library_from_git {
160 local name=$1
161 local enabled=1
162 [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
163 return $enabled
164}
165
Sean Daguec71973e2015-09-08 07:12:48 -0400166# determine if a package was installed from git
167function lib_installed_from_git {
168 local name=$1
169 pip freeze 2>/dev/null | grep -- "$name" | grep -q -- '-e git'
170}
171
172# check that everything that's in LIBS_FROM_GIT was actually installed
173# correctly, this helps double check issues with library fat fingering.
174function check_libs_from_git {
175 local lib=""
176 local not_installed=""
177 for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
178 if ! lib_installed_from_git "$lib"; then
179 not_installed+=" $lib"
180 fi
181 done
182 # if anything is not installed, say what it is.
183 if [[ -n "$not_installed" ]]; then
184 die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
185 fi
186}
187
Dean Troyer490430d2015-01-30 14:38:35 -0600188# setup a library by name. If we are trying to use the library from
189# git, we'll do a git based install, otherwise we'll punt and the
190# library should be installed by a requirements pull from another
191# project.
192function setup_lib {
193 local name=$1
194 local dir=${GITDIR[$name]}
195 setup_install $dir
196}
197
198# setup a library by name in editiable mode. If we are trying to use
199# the library from git, we'll do a git based install, otherwise we'll
200# punt and the library should be installed by a requirements pull from
201# another project.
202#
203# use this for non namespaced libraries
204function setup_dev_lib {
205 local name=$1
206 local dir=${GITDIR[$name]}
207 setup_develop $dir
208}
209
210# this should be used if you want to install globally, all libraries should
211# use this, especially *oslo* ones
212function setup_install {
213 local project_dir=$1
Clark Boylan05aa3842015-08-03 11:14:13 -0700214 setup_package_with_constraints_edit $project_dir
Dean Troyer490430d2015-01-30 14:38:35 -0600215}
216
217# this should be used for projects which run services, like all services
218function setup_develop {
219 local project_dir=$1
Clark Boylan05aa3842015-08-03 11:14:13 -0700220 setup_package_with_constraints_edit $project_dir -e
Dean Troyer490430d2015-01-30 14:38:35 -0600221}
222
223# determine if a project as specified by directory is in
224# projects.txt. This will not be an exact match because we throw away
225# the namespacing when we clone, but it should be good enough in all
226# practical ways.
227function is_in_projects_txt {
228 local project_dir=$1
229 local project_name=$(basename $project_dir)
Ihar Hrachyshka2ba4a722015-06-26 10:45:44 +0200230 grep -q "/$project_name\$" $REQUIREMENTS_DIR/projects.txt
Dean Troyer490430d2015-01-30 14:38:35 -0600231}
232
233# ``pip install -e`` the package, which processes the dependencies
234# using pip before running `setup.py develop`
235#
Clark Boylan05aa3842015-08-03 11:14:13 -0700236# Updates the constraints from REQUIREMENTS_DIR to reflect the
237# future installed state of this package. This ensures when we
238# install this package we get the from source version.
Dean Troyer490430d2015-01-30 14:38:35 -0600239#
Clark Boylan05aa3842015-08-03 11:14:13 -0700240# Uses globals ``REQUIREMENTS_DIR``
Dean Troyer490430d2015-01-30 14:38:35 -0600241# setup_develop directory
Clark Boylan05aa3842015-08-03 11:14:13 -0700242function setup_package_with_constraints_edit {
Dean Troyer490430d2015-01-30 14:38:35 -0600243 local project_dir=$1
244 local flags=$2
245
Robert Collins635a5ba2015-06-10 08:48:06 +1200246 if [ -n "$REQUIREMENTS_DIR" ]; then
247 # Constrain this package to this project directory from here on out.
248 local name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
Robert Collins7c838612015-07-03 13:28:09 +1200249 $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
250 $REQUIREMENTS_DIR/upper-constraints.txt -- $name \
251 "$flags file://$project_dir#egg=$name"
Robert Collins635a5ba2015-06-10 08:48:06 +1200252 fi
253
Dean Troyer490430d2015-01-30 14:38:35 -0600254 setup_package $project_dir $flags
255
Dean Troyer490430d2015-01-30 14:38:35 -0600256}
257
258# ``pip install -e`` the package, which processes the dependencies
259# using pip before running `setup.py develop`
260# Uses globals ``STACK_USER``
261# setup_develop_no_requirements_update directory
262function setup_package {
263 local project_dir=$1
264 local flags=$2
265
266 pip_install $flags $project_dir
267 # ensure that further actions can do things like setup.py sdist
268 if [[ "$flags" == "-e" ]]; then
269 safe_chown -R $STACK_USER $1/*.egg-info
270 fi
271}
272
273
274# Restore xtrace
275$INC_PY_TRACE
276
277# Local variables:
278# mode: shell-script
279# End: