Skip to content

Commit b824ee8

Browse files
MatthijsBurghMatthijsBurgh
committed
WIP
1 parent 9ff9f5f commit b824ee8

File tree

4 files changed

+165
-40
lines changed

4 files changed

+165
-40
lines changed

‎installer/bootstrap.bash‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ function main
5858
TUE_ROS_DISTRO=
5959
TUE_ROS_VERSION=
6060

61+
# Python configuration
62+
create_virtualenv="true"
63+
virtualenv_include_system_site_packages="false"
64+
TUE_PPM="pip" # Default to pip
65+
6166
for i in "$@"
6267
do
6368
case $i in
@@ -73,13 +78,29 @@ function main
7378
--create-virtualenv=* )
7479
create_virtualenv="${i#*=}"
7580
;;
81+
--virtualenv-include-system-site-packages=* )
82+
virtualenv_include_system_site_packages="${i#*=}"
83+
;;
84+
--pip )
85+
TUE_PPM="pip"
86+
;;
87+
--poetry )
88+
TUE_PPM="poetry"
89+
;;
7690
* )
7791
echo "[tue-env](bootstrap) Error! Unknown argument '${i}' provided to bootstrap script."
78-
return 1
92+
exit 1
7993
;;
8094
esac
8195
done
8296

97+
# Poetry should only be used in combination with a virtualenv
98+
if [[ "${TUE_PPM}" == "poetry" && "${create_virtualenv}" == "false" ]]
99+
then
100+
echo "[tue-env](bootstrap) Error! Poetry is not installed."
101+
return 1
102+
fi
103+
83104
case ${DISTRIB_RELEASE} in
84105
"20.04")
85106
if [[ "${ros_version}" -eq 2 ]]
@@ -200,6 +221,13 @@ function main
200221
git clone "${env_url}" "${env_dir}"
201222
fi
202223

224+
# Install Poetry when needed
225+
if [[ "${TUE_PPM}" == "poetry" ]]
226+
then
227+
echo "[tue-env](bootstrap) Installing Poetry"
228+
curl -sSL https://install.python-poetry.org | python3 -
229+
fi
230+
203231
# Source the installer commands
204232
# No need to follow to a file which is already checked by CI
205233
# shellcheck disable=SC1090
@@ -209,11 +237,15 @@ function main
209237
mkdir -p "${workspace_dir}"
210238

211239
# Initialize ros environment directory incl. targets
212-
tue-env init "${workspace}" "${workspace_dir}" "--create-virtualenv=${create_virtualenv}" "--targets-url=${env_targets_url}"
240+
tue-env init "${workspace}" "${workspace_dir}" \
241+
"--create-virtualenv=${create_virtualenv}" \
242+
"--virtualenv-include-system-site-packages=${virtualenv_include_system_site_packages}" \
243+
"--targets-url=${env_targets_url}"
213244

214245
# Configure environment
215246
tue-env config "${workspace}" set "TUE_ROS_DISTRO" "${TUE_ROS_DISTRO}"
216247
tue-env config "${workspace}" set "TUE_ROS_VERSION" "${TUE_ROS_VERSION}"
248+
tue-env config "${workspace}" set "TUE_PPM" "${TUE_PPM}"
217249

218250
# Add loading of TU/e tools (tue-env, tue-get, etc) to bashrc
219251
# shellcheck disable=SC2088

‎installer/check-pip-pkg-installed-version.py‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,20 @@
11
#! /usr/bin/env python3
22

3-
import sys
3+
import site
4+
from typing import List
45
from pip._internal.req.constructors import install_req_from_line
56
from pip._internal.utils.virtualenv import running_under_virtualenv
67

78

8-
def main() -> int:
9-
if len(sys.argv) < 2:
10-
print("Usage: check-pip-pkg-installed-version.py requirement [requirements]")
11-
return 2
12-
9+
def main(req_strs: List[str]) -> int:
1310
return_code = 0
1411
pkg_installed = []
1512

1613
try:
17-
for arg in sys.argv[1:]:
18-
req = install_req_from_line(arg)
14+
for req_str in req_strs:
15+
req = install_req_from_line(req_str)
1916

20-
req.check_if_exists(not running_under_virtualenv())
17+
req.check_if_exists(not running_under_virtualenv)
2118

2219
if req.satisfied_by:
2320
pkg_installed.append(str(req.satisfied_by).replace(" ", "^"))
@@ -34,4 +31,14 @@ def main() -> int:
3431

3532

3633
if __name__ == "__main__":
37-
sys.exit(main())
34+
import argparse
35+
import sys
36+
37+
parser = argparse.ArgumentParser(
38+
description="Check if a set of pip package is installed, meeting a requirement string."
39+
)
40+
parser.add_argument("req_strs", nargs="+")
41+
42+
args = parser.parse_args()
43+
44+
sys.exit(main(**vars(args)))

‎installer/tue-install-impl.bash‎

Lines changed: 111 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ function _remove_old_target_dep_recursively
168168
old_dep_dep_file="${TUE_INSTALL_DEPENDENCIES_DIR}"/"${old_dep_target}"
169169
if [[ -f "${old_dep_dep_file}" ]]
170170
then
171-
# Iterate over all depencies of old_dep_target, which is removed.
171+
# Iterate over all dependencies of old_dep_target, which is removed.
172172
while read -r dep_of_old_dep
173173
do
174174
# Actually remove the deps
@@ -183,11 +183,11 @@ function _remove_old_target_dep_recursively
183183
done < "${old_dep_dep_file}"
184184
rm -f "${old_dep_dep_file}"
185185
else
186-
tue-install-debug "[remove_old_dep] No depencies file exist for target: ${old_dep_target}"
186+
tue-install-debug "[remove_old_dep] No dependencies file exist for target: ${old_dep_target}"
187187
fi
188188

189189
tue-install-debug "[remove_old_dep] Uninstalled '${old_dep_target}' as a dependency of '${parent_target}'"
190-
tue-install-info "[remove_old_dep] '${old_dep_target}' has been uninstalled, you can remove it from the workspace or deinstall it in another way"
190+
tue-install-info "[remove_old_dep] '${old_dep_target}' has been uninstalled, you can remove it from the workspace or uninstall it in another way"
191191
return ${error_code}
192192
}
193193

@@ -309,7 +309,7 @@ function tue-install-target
309309
[ "$now" == "true" ] && now_cmd="--now"
310310
# Do not use 'local cmds=' because it does not preserve command output status ($?)
311311
local cmds
312-
if cmds=$("$TUE_INSTALL_SCRIPTS_DIR"/parse_install_yaml.py "$install_file".yaml $now_cmd)
312+
if cmds=$(/usr/bin/python "${TUE_INSTALL_SCRIPTS_DIR}"/parse_install_yaml.py "${install_file}".yaml ${now_cmd})
313313
then
314314
for cmd in $cmds
315315
do
@@ -352,7 +352,7 @@ function tue-install-target
352352
old_deps_removed=$(comm -23 <(echo "${old_deps}") <(echo "${new_deps}"))
353353
if [[ -n ${old_deps_removed} ]]
354354
then
355-
tue-install-debug "Following dropped depedencies need to be removed:\n${old_deps_removed}"
355+
tue-install-debug "Following dropped dependencies need to be removed:\n${old_deps_removed}"
356356
else
357357
tue-install-debug "No dropped dependencies to be removed"
358358
fi
@@ -361,7 +361,7 @@ function tue-install-target
361361
do
362362
# Remove this target from dep-on file of dep
363363
# When the dep-on file is now empty, remove it
364-
# Recurisvely -> Remove it from the dep-on files of its deps
364+
# Recursively -> Remove it from the dep-on files of its deps
365365
tue-install-debug "Going to remove '${dep}' as a dependency"
366366
_remove_old_target_dep_recursively "${target}" "${dep}" || tue-install-error "Something went wrong while removing '${dep}' as a dependency"
367367
done
@@ -1093,6 +1093,49 @@ function tue-install-pip3
10931093

10941094
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
10951095

1096+
function _handle_python_sites
1097+
{
1098+
local pv
1099+
pv=$1
1100+
shift
1101+
# requires the parent function to have declared the following local variables:
1102+
# python_exec, site_arg, sudo_cmd, user_arg
1103+
tue-install-debug "_handle_python${pv}_sites $*"
1104+
1105+
local site
1106+
site=$1
1107+
1108+
case ${site} in
1109+
"system" )
1110+
python_exec=/usr/bin/python"${pv}"
1111+
site_arg="-s"
1112+
sudo_cmd="sudo -H"
1113+
user_arg=""
1114+
;;
1115+
"user" )
1116+
python_exec=/usr/bin/python"${pv}"
1117+
site_arg=""
1118+
sudo_cmd=""
1119+
user_arg="--user"
1120+
;;
1121+
"venv" )
1122+
python_exec=python"${pv}"
1123+
site_arg="-s"
1124+
sudo_cmd=""
1125+
user_arg=""
1126+
;;
1127+
* )
1128+
tue-install-error "_handle_python${pv}_sites: Unknown input variable ${i}"
1129+
;;
1130+
esac
1131+
1132+
tue-install-debug "python_site: ${python_site}"
1133+
tue-install-debug "python_exec: ${python_exec}"
1134+
tue-install-debug "site_arg: ${site_arg}"
1135+
tue-install-debug "sudo_cmd: ${sudo_cmd}"
1136+
tue-install-debug "user_arg: ${user_arg}"
1137+
}
1138+
10961139
function _tue-install-pip-now
10971140
{
10981141
local pv
@@ -1105,25 +1148,42 @@ function _tue-install-pip-now
11051148
tue-install-error "Invalid tue-install-pip${pv}-now call: needs package as argument."
11061149
fi
11071150

1108-
local user_arg
1109-
[[ -z "${VIRTUAL_ENV}" ]] && user_arg="--user"
1151+
local pips python_site
1152+
{ [[ -n ${VIRTUAL_ENV} ]] && python_site="venv"; } || python_site="user"
1153+
if [[ -n $1 ]]
1154+
then
1155+
for i in "$@"
1156+
do
1157+
case $i in
1158+
--python-site=* )
1159+
python_site="${i#*=}"
1160+
;;
1161+
--* )
1162+
tue-install-error "tue-install-pip${pv}-now: Unknown input variable ${i}" ;;
1163+
* )
1164+
pips+=("$i") ;;
1165+
esac
1166+
done
1167+
fi
1168+
1169+
local python_exec site_arg sudo_cmd user_arg
1170+
_handle_python_sites "${pv}" "${python_site}"
11101171

11111172
# Make sure pip is up-to-date before checking version and installing
11121173
local pip_version desired_pip_version
1113-
pip_version=$(python"${pv}" -m pip --version | awk '{print $2}')
1174+
pip_version=$(python"${pv}" ${site_arg} -m pip --version | awk '{print $2}')
11141175
desired_pip_version="23"
11151176
if version_gt "$desired_pip_version" "$pip_version"
11161177
then
11171178
tue-install-debug "pip${pv} not yet version >=$desired_pip_version, but $pip_version"
1118-
tue-install-pipe python"${pv}" -m pip install ${user_arg} --upgrade pip
1179+
tue-install-pipe python"${pv}" ${site_arg} -m pip install "${user_arg}" --upgrade pip
11191180
hash -r
11201181
else
11211182
tue-install-debug "Already pip${pv}>=$desired_pip_version"
11221183
fi
11231184

11241185
local pips_to_check pips_to_check_w_options pips_to_install pips_to_install_w_options git_pips_to_install
1125-
# shellcheck disable=SC2048
1126-
for pkg in $*
1186+
for pkg in "${pips[@]}"
11271187
do
11281188
if [[ "$pkg" == "git+"* ]]
11291189
then
@@ -1143,7 +1203,7 @@ function _tue-install-pip-now
11431203
then
11441204
local indexes_to_install
11451205
# shellcheck disable=SC2086
1146-
_tue-install-pip-check "$pv" $pips_to_check
1206+
_tue-install-pip-check "${pv}" --python-site=${python_site} ${pips_to_check}
11471207

11481208
read -r -a pips_to_check <<< "$pips_to_check"
11491209

@@ -1165,7 +1225,7 @@ function _tue-install-pip-now
11651225
pips_to_check_options_removed="$pips_to_check_options_removed ${pkg_split[0]}"
11661226
done
11671227
# shellcheck disable=SC2086
1168-
_tue-install-pip-check "$pv" $pips_to_check_options_removed
1228+
_tue-install-pip-check "${pv}" --python-site=${python_site} ${pips_to_check_options_removed}
11691229

11701230
read -r -a pips_to_check_w_options <<< "$pips_to_check_w_options"
11711231

@@ -1181,15 +1241,15 @@ function _tue-install-pip-now
11811241
if [ -n "$pips_to_install" ]
11821242
then
11831243
# shellcheck disable=SC2048,SC2086
1184-
tue-install-pipe python"${pv}" -m pip install ${user_arg} $pips_to_install <<< yes || tue-install-error "An error occurred while installing pip${pv} packages."
1244+
tue-install-pipe ${sudo_cmd} ${python_exec} ${site_arg} -m pip install ${user_arg} $pips_to_install <<< yes || tue-install-error "An error occurred while installing pip${pv} packages."
11851245
fi
11861246

11871247
if [ -n "$pips_to_install_w_options" ]
11881248
then
11891249
for pkg in $pips_to_install_w_options
11901250
do
11911251
# shellcheck disable=SC2048,SC2086
1192-
tue-install-pipe python"${pv}" -m pip install ${user_arg} ${pkg//^/ } <<< yes || tue-install-error "An error occurred while installing pip${pv} packages with options."
1252+
tue-install-pipe ${sudo_cmd} ${python_exec} ${site_arg} -m pip install ${user_arg} ${pkg//^/ } <<< yes || tue-install-error "An error occurred while installing pip${pv} packages with options."
11931253
done
11941254
fi
11951255

@@ -1198,7 +1258,7 @@ function _tue-install-pip-now
11981258
for pkg in $git_pips_to_install
11991259
do
12001260
# shellcheck disable=SC2048,SC2086
1201-
tue-install-pipe python"${pv}" -m pip install ${user_arg} ${pkg} <<< yes || tue-install-error "An error occurred while installing pip${pv} git packages."
1261+
tue-install-pipe ${sudo_cmd} ${python_exec} ${site_arg} -m pip install ${user_arg} ${pkg} <<< yes || tue-install-error "An error occurred while installing pip${pv} git packages."
12021262
done
12031263
fi
12041264
}
@@ -1210,12 +1270,36 @@ function _tue-install-pip-check
12101270
pv=$1
12111271
shift
12121272

1213-
local pips_to_check installed_versions
1214-
pips_to_check=("$@")
1273+
local installed_versions python_site pips_to_check
1274+
if [[ -n $1 ]]
1275+
then
1276+
for i in "$@"
1277+
do
1278+
case $i in
1279+
--python-site=* )
1280+
python_site="${i#*=}"
1281+
;;
1282+
--* )
1283+
tue-install-error "Unknown input variable ${i}" ;;
1284+
* )
1285+
pips_to_check+=("$i") ;;
1286+
esac
1287+
done
1288+
fi
1289+
1290+
if [[ -z "${VIRTUAL_ENV}" && "${site}" == "venv" ]]
1291+
then
1292+
tue-install-error "Trying to check pip packages in a virtualenv, but no virtualenv is activated"
1293+
fi
1294+
1295+
# Set by _handle_python_sites
1296+
local python_exec sudo_cmd site_arg user_arg
1297+
_handle_python_sites "${pv}" "${python_site}"
1298+
12151299
if [ ${#pips_to_check[@]} -gt 0 ]
12161300
then
12171301
local error_code
1218-
installed_versions=$(python"${pv}" "$TUE_INSTALL_SCRIPTS_DIR"/check-pip-pkg-installed-version.py "${pips_to_check[@]}")
1302+
installed_versions=$(${sudo_cmd} "${python_exec}" ${site_arg} "${TUE_INSTALL_SCRIPTS_DIR}"/check-pip-pkg-installed-version.py "${pips_to_check[@]}")
12191303
error_code=$?
12201304
if [ "$error_code" -gt 1 ]
12211305
then
@@ -1493,10 +1577,11 @@ function tue-install-ros
14931577
local pkg_xml="$ros_pkg_dir"/package.xml
14941578
if [ -f "$pkg_xml" ]
14951579
then
1496-
# Catkin
1580+
# catkin/ament/colcon
14971581
tue-install-debug "Parsing $pkg_xml"
1582+
14981583
local deps
1499-
deps=$("$TUE_INSTALL_SCRIPTS_DIR"/parse_package_xml.py "$pkg_xml")
1584+
deps=$(/usr/bin/python "${TUE_INSTALL_SCRIPTS_DIR}"/parse_package_xml.py "${pkg_xml}")
15001585
tue-install-debug "Parsed package.xml\n$deps"
15011586

15021587
for dep in $deps
@@ -1636,7 +1721,8 @@ TUE_INSTALL_INFOS=
16361721
# Make sure tools used by this installer are installed
16371722
tue-install-system-now curl git jq python-is-python3 python3-pip wget
16381723

1639-
tue-install-pip3-now catkin-pkg PyYAML
1724+
# Install in user or system site-packages
1725+
tue-install-pip3-now --python-site="user" catkin-pkg PyYAML
16401726

16411727

16421728
# Handling of targets
@@ -1674,7 +1760,7 @@ do
16741760
tue-install-debug "[$target] marked as installed after a successful install"
16751761
touch "$TUE_INSTALL_INSTALLED_DIR"/"$target"
16761762
else
1677-
tue-install-debug "[$target] succesfully updated"
1763+
tue-install-debug "[$target] successfully updated"
16781764
fi
16791765
done
16801766

@@ -1746,6 +1832,6 @@ fi
17461832

17471833
TUE_INSTALL_CURRENT_TARGET="main-loop"
17481834

1749-
tue-install-echo "Installer completed succesfully"
1835+
tue-install-echo "Installer completed successfully"
17501836

17511837
return 0

0 commit comments

Comments
 (0)