Skip to content

Commit 2ad2f05

Browse files
committed
Update configure scripts
Drop using Python etc/configure.py. Instead use a plain bash and plain bat scripts. Improve the way the Python interpreter is picked using either the PYTHON_EXECUTABLE environment variable or the PYTHON_EXECUTABLE file and fallback to use the default Python foudn in the PATH. Update release, other scripts and CI scripts accordingly Signed-off-by: Philippe Ombredanne <[email protected]>
1 parent bb04420 commit 2ad2f05

15 files changed

+363
-554
lines changed

CHANGELOG.rst

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
Changelog
22
=========
33

4-
v21.4.x (next)
4+
v21.x.x (next)
55
--------------
66

77
Breaking API changes:
@@ -24,6 +24,19 @@ Breaking API changes:
2424
multiple manifests for a single package instance.
2525

2626

27+
v21.5.11
28+
--------
29+
30+
Breaking API changes:
31+
~~~~~~~~~~~~~~~~~~~~~
32+
33+
- The configure scripts for Linux, macOS and Windows are new. These are now
34+
only native scripts (.bat on Windows and .sh on POSIX) and the Python script
35+
etc/configure.py has been removed. Use the PYTHON_EXECUTABLE enviroment
36+
varibale to point to alternative non-default Python executable. An this on
37+
all OSes.
38+
39+
2740
Ouputs:
2841
~~~~~~~
2942

@@ -43,6 +56,7 @@ License scanning:
4356
- Till Jaeger @LeChasseur
4457

4558

59+
4660
v21.3.31
4761
--------
4862

MANIFEST.in

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ include *.toml
1616
include extractcode*
1717
include scancode*
1818

19+
# path to executable used by a built app
20+
include PYTHON_EXECUTABLE
1921

2022
include *.rst
2123
include Dockerfile

appveyor.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ build: off
1616

1717
test_script:
1818
- python -c "import sys;print(sys.getdefaultencoding())"
19-
- cmd: "set PYTHON_EXE=%PYTHON%\\python.exe && configure --dev && Scripts\\pytest -vvs --reruns 2 tests\\scancode"
19+
- cmd: "set PYTHON_EXECUTABLE=%PYTHON%\\python.exe && configure --dev && Scripts\\pytest -vvs --reruns 2 tests\\scancode"

azure-pipelines.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
# These jobs are using VMs and Azure-provided Pythons 3.6
1111
################################################################################
1212

13-
- template: etc/ci/azure-linux.yml
13+
- template: etc/ci/azure-posix.yml
1414
parameters:
1515
job_name: vm_ubuntu16_py36
1616
image_name: ubuntu-16.04
@@ -95,7 +95,7 @@ jobs:
9595
bin/pytest -n 3 -vvs --test-suite=all \
9696
tests/licensedcode/test_zzzz_cache.py
9797
98-
- template: etc/ci/azure-mac.yml
98+
- template: etc/ci/azure-posix.yml
9999
parameters:
100100
job_name: macos_1014_py3x
101101
image_name: macos-10.14
@@ -104,7 +104,7 @@ jobs:
104104
all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py
105105

106106

107-
- template: etc/ci/azure-mac.yml
107+
- template: etc/ci/azure-posix.yml
108108
parameters:
109109
job_name: macos_1015_py3x
110110
image_name: macos-10.15
@@ -113,7 +113,7 @@ jobs:
113113
all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py
114114

115115

116-
# - template: etc/ci/azure-mac.yml
116+
# - template: etc/ci/azure-posix.yml
117117
# parameters:
118118
# job_name: macos_110_py3x
119119
# image_name: macos-11.0
@@ -142,7 +142,7 @@ jobs:
142142
all: Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py
143143

144144

145-
- template: etc/ci/azure-linux.yml
145+
- template: etc/ci/azure-posix.yml
146146
parameters:
147147
job_name: vm_ubuntu16_py3x
148148
image_name: ubuntu-16.04
@@ -151,7 +151,7 @@ jobs:
151151
all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py
152152

153153

154-
- template: etc/ci/azure-linux.yml
154+
- template: etc/ci/azure-posix.yml
155155
parameters:
156156
job_name: vm_ubuntu18_py3x
157157
image_name: ubuntu-18.04
@@ -160,7 +160,7 @@ jobs:
160160
all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py
161161

162162

163-
- template: etc/ci/azure-linux.yml
163+
- template: etc/ci/azure-posix.yml
164164
parameters:
165165
job_name: vm_ubuntu20_py3x
166166
image_name: ubuntu-20.04

configure

Lines changed: 159 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -3,100 +3,199 @@
33
# Copyright (c) nexB Inc. and others. All rights reserved.
44
# SPDX-License-Identifier: Apache-2.0
55
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6-
# ScanCode is a trademark of nexB Inc.
7-
# See https://github.com/nexB/scancode-toolkit for support or download.
6+
# See https://github.com/nexB/ for support or download.
87
# See https://aboutcode.org for more information about nexB OSS projects.
98
#
9+
1010
set -e
11+
#set -x
12+
13+
################################
14+
# A configuration script to set things up:
15+
# create a virtualenv and install or update thirdparty packages.
16+
# Source this script for initial configuration
17+
# Use configure --help for details
18+
#
19+
# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz
20+
# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
21+
################################
22+
CLI_ARGS=$1
23+
24+
################################
25+
# Defaults. Change these variables to customize this script
26+
################################
27+
28+
# Requirement arguments passed to pip and used by default or with --dev.
29+
REQUIREMENTS="--editable . --constraint requirements.txt"
30+
DEV_REQUIREMENTS="--editable .[dev] --editable .[packages] --constraint requirements.txt --constraint requirements-dev.txt"
31+
32+
# where we create a virtualenv
33+
VIRTUALENV_DIR=.
34+
35+
# Cleanable files and directories to delete with the --clean option
36+
CLEANABLE="
37+
build
38+
bin
39+
lib
40+
lib64
41+
include
42+
tcl
43+
local
44+
.Python
45+
.eggs
46+
pip-selfcheck.json
47+
src/scancode_toolkit.egg-info
48+
SCANCODE_DEV_MODE
49+
man
50+
Scripts"
51+
52+
# extra arguments passed to pip
53+
PIP_EXTRA_ARGS=" "
54+
55+
# the URL to download virtualenv.pyz if needed
56+
VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
57+
################################
1158

12-
if [[ "$OS" == "Windows_NT" ]]; then
13-
echo "You seem to be running on Windows under Cygwin / MSYS(2),"
14-
echo "like e.g. Git for Windows Bash. This script does not properly work in"
15-
echo "this scenario. As a Windows user, please run 'configure.bat' from a "
16-
echo "regular command prompt instead."
17-
exit 1
59+
60+
################################
61+
# Current directory where this script lives
62+
CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
63+
CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
64+
65+
66+
################################
67+
# scancode-specific: Thirdparty package locations and index handling
68+
# Do we have thirdparty/files? use the mit.LICENSE as a proxy
69+
if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then
70+
LINKS=$CFG_ROOT_DIR/thirdparty
71+
else
72+
LINKS=https://thirdparty.aboutcode.org/pypi
1873
fi
74+
PIP_EXTRA_ARGS="--no-index --find-links $LINKS"
75+
################################
1976

2077

2178
################################
22-
# A configuration script to set things up: create a virtualenv and install
23-
# update thirdparty packages
79+
# Set the quiet flag to empty if not defined
80+
if [[ "$CFG_QUIET" == "" ]]; then
81+
CFG_QUIET=" "
82+
fi
83+
84+
85+
################################
86+
# find a proper Python to run
87+
# Use environment variables or a file if available.
88+
# Otherwise the latest Python by default.
89+
if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
90+
# check for a file named PYTHON_EXECUTABLE
91+
if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
92+
PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
93+
else
94+
PYTHON_EXECUTABLE=python3
95+
fi
96+
fi
2497

25-
function cli_help {
98+
99+
################################
100+
cli_help() {
26101
echo An initial configuration script
27102
echo " usage: ./configure [options]"
28103
echo
29-
echo The default is to configure for regular use.
30-
echo The script will attempt to find a suitable Python executable.
31-
echo Set the PYTHON_EXECUTABLE environment variable to provide your own
32-
echo Python executable path.
104+
echo The default is to configure for regular use. Use --dev for development.
33105
echo
34106
echo The options are:
35-
echo "--clean: clean built and installed files and exit."
36-
echo "--dev: configure the environment for development."
37-
echo "--help: display these help messages and exit."
107+
echo " --clean: clean built and installed files and exit."
108+
echo " --dev: configure the environment for development."
109+
echo " --help: display this help message and exit."
38110
echo
111+
echo By default, the python interpreter version found in the path is used.
112+
echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
113+
echo configure another Python executable interpreter to use. If this is not
114+
echo set, a file named PYTHON_EXECUTABLE containing a single line with the
115+
echo path of the Python executable to use will be checked last.
39116
set +e
40117
exit
41118
}
42119

43120

44-
################################
45-
# Defaults. Change these variables to customize this script locally
46-
################################
121+
clean() {
122+
# Remove cleanable file and directories and files from the root dir.
123+
echo "* Cleaning ..."
124+
for cln in $CLEANABLE;
125+
do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
126+
done
127+
set +e
128+
exit
129+
}
47130

48-
# thirdparty package locations
49-
export THIRDPARTY_DIR="thirdparty"
50-
export THIRDPARTY_LINKS="https://thirdparty.aboutcode.org/pypi"
51131

52-
# requirements used by default or with --dev.
53-
# note the use of constraints with -c
54-
REQUIREMENTS="--editable . --constraint requirements.txt"
55-
DEV_REQUIREMENTS="--editable .[dev] --editable .[packages] --constraint requirements.txt --constraint requirements-dev.txt"
132+
create_virtualenv() {
133+
# create a virtualenv for Python
134+
# Note: we do not use the bundled Python 3 "venv" because its behavior and
135+
# presence is not consistent across Linux distro and sometimes pip is not
136+
# included either by default. The virtualenv.pyz app cures all these issues.
137+
138+
VENV_DIR="$1"
139+
if [ ! -f "$CFG_BIN_DIR/python" ]; then
140+
141+
mkdir -p "$CFG_ROOT_DIR/$VENV_DIR"
142+
143+
if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then
144+
VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz"
145+
else
146+
VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz"
147+
wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL"
148+
fi
149+
150+
$PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
151+
--wheel embed --pip embed --setuptools embed \
152+
--seeder pip \
153+
--never-download \
154+
--no-periodic-update \
155+
--no-vcs-ignore \
156+
$CFG_QUIET \
157+
"$CFG_ROOT_DIR/$VENV_DIR"
158+
fi
159+
}
56160

57-
# default supported Python version
58-
if [[ "$CONFIGURE_SUPPORTED_PYTHON" == "" ]]; then
59-
CONFIGURE_SUPPORTED_PYTHON=3.6
60-
fi
61161

62-
################################
162+
install_packages() {
163+
# install requirements in virtualenv
164+
# note: --no-build-isolation means that pip/wheel/setuptools will not
165+
# be reinstalled a second time and reused from the virtualenv and this
166+
# speeds up the installation.
167+
# We always have the PEP517 build dependencies installed already.
168+
169+
"$CFG_BIN_DIR/pip" install \
170+
--upgrade \
171+
--no-build-isolation \
172+
$CFG_QUIET \
173+
$PIP_EXTRA_ARGS \
174+
$1
175+
}
63176

64-
# Current directory where this script lives
65-
PROJECT_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
66177

178+
################################
179+
# Main command line entry point
180+
CFG_DEV_MODE=0
181+
CFG_REQUIREMENTS=$REQUIREMENTS
67182

68-
# parse command line options
69-
CLI_ARGS="$REQUIREMENTS"
70-
CONFIGURE_DEV_MODE=0
71-
case "$1" in
183+
case "$CLI_ARGS" in
72184
--help) cli_help;;
73-
--clean) CLI_ARGS=--clean;;
74-
--dev) CLI_ARGS="$DEV_REQUIREMENTS" && CONFIGURE_DEV_MODE=1;;
185+
--clean) clean;;
186+
--dev) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
75187
esac
76188

77-
78-
# find a proper Python to run
79-
80-
if [[ "$CONFIGURE_PYTHON_EXECUTABLE" == "" ]]; then
81-
CONFIGURE_PYTHON_EXECUTABLE=python3
82-
fi
189+
create_virtualenv "$VIRTUALENV_DIR"
190+
install_packages "$CFG_REQUIREMENTS"
191+
. "$CFG_BIN_DIR/activate"
83192

84193
################################
194+
# scancode-specific: Setup development mode
85195

86-
# Setup development mode
87-
88-
if [[ "$CONFIGURE_DEV_MODE" == 1 ]]; then
196+
if [[ "$CFG_DEV_MODE" == 1 ]]; then
89197
# Add development tag file to auto-regen license index on file changes
90-
touch $PROJECT_ROOT_DIR/SCANCODE_DEV_MODE
91-
fi
92-
93-
# Run configure scripts proper and activate
94-
95-
$CONFIGURE_PYTHON_EXECUTABLE "$PROJECT_ROOT_DIR/etc/configure.py" $CLI_ARGS
96-
97-
# Activate the virtualenv if it exists
98-
if [[ -f "$PROJECT_ROOT_DIR/bin/activate" ]]; then
99-
source "$PROJECT_ROOT_DIR/bin/activate"
198+
touch "$CFG_ROOT_DIR/SCANCODE_DEV_MODE"
100199
fi
101200

102201
set +e

0 commit comments

Comments
 (0)