Merge pull request #355 from nexB/update-deps

Update deps

Author: pombredanne

.gitignore

@@ -15,6 +15,7 @@
 /include
 /Include
 /local
+/man/
 
 # Installer logs
 pip-log.txt

configure

@@ -6,28 +6,22 @@
 # change these variables to customize this script locally
 ################################
 # you can define one or more thirdparty dirs, each prefixed with TPP_DIR
-export TPP_DIR_BASE="thirdparty/base"
-export TPP_DIR_PROD="thirdparty/prod"
-export TPP_DIR_DEV="thirdparty/dev"
+export TPP_DIR="thirdparty"
 
 # default configurations
 CONF_DEFAULT="etc/conf"
 #################################
 
 CFG_CMD_LINE_ARGS="$@"
 
-if [[ "$1" == "--init" ]]; then
-    CFG_CMD_LINE_ARGS=$CONF_INIT
-fi
-
 if [[ "$1" == "" ]]; then
-    # default conf if not argument is provided
+    # default for dev conf if not argument is provided
     CFG_CMD_LINE_ARGS=$CONF_DEFAULT
 fi
 
 CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-python $CONFIGURE_ROOT_DIR/etc/configure.py $CFG_CMD_LINE_ARGS
+python "$CONFIGURE_ROOT_DIR/etc/configure.py" $CFG_CMD_LINE_ARGS
 if [ -f "$CONFIGURE_ROOT_DIR/bin/activate" ]; then
     source $CONFIGURE_ROOT_DIR/bin/activate
 fi

configure.bat

@@ -1,21 +1,24 @@
 @echo OFF
 
-@rem Copyright (c) 2015 nexB Inc. http://www.nexb.com/ - All rights reserved.
+@rem Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved.
 
 @rem ################################
 @rem # change these variables to customize this script locally
 @rem ################################
 @rem # you can define one or more thirdparty dirs, each prefixed with TPP_DIR
-set TPP_DIR_BASE=thirdparty/base
-set TPP_DIR_PROD=thirdparty/prod
-set TPP_DIR_DEV=thirdparty/dev
+set TPP_DIR=thirdparty
 
 
 @rem # default configurations
 set CONF_DEFAULT="etc/conf"
 @rem #################################
 
 set ABOUT_ROOT_DIR=%~dp0
+@rem !!!!!!!!!!! ATTENTION !!!!!
+@rem there is a space at the end of the set SCANCODE_CLI_ARGS=  line ... 
+@rem NEVER remove this!
+@rem otherwise, this script and scancode do not work.  
+
 set ABOUT_CLI_ARGS= 
 @rem Collect/Slurp all command line arguments in a variable
 :collectarg
@@ -44,17 +47,17 @@ if not exist "c:\python27\python.exe" (
     echo Do NOT install Python v3 or any 64 bits edition.
     echo Instead download Python from this url and see the README.rst file for more details:
     echo(
-    echo    https://www.python.org/ftp/python/2.7.10/python-2.7.10.msi
+    echo    https://www.python.org/ftp/python/2.7.15/python-2.7.15.msi
     echo(
     exit /b 1
 )
 
-call c:\python27\python.exe %ABOUT_ROOT_DIR%etc\configure.py %ABOUT_CLI_ARGS%
+call c:\python27\python.exe "%ABOUT_ROOT_DIR%etc\configure.py" %ABOUT_CLI_ARGS%
 if %errorlevel% neq 0 (
     exit /b %errorlevel%
 )
-if exist %SCANCODE_ROOT_DIR%bin\activate (
-    %SCANCODE_ROOT_DIR%bin\activate
+if exist "%SCANCODE_ROOT_DIR%bin\activate" (
+    "%SCANCODE_ROOT_DIR%bin\activate"
 )
 goto EOS
 

etc/conf/base.txt

@@ -6,8 +6,8 @@ pip
 wincertstore
 
 # used for templating
-jinja2==2.9.6
-MarkupSafe==1.0.0
+jinja2>=2.9.6
+MarkupSafe>=1.0.0
 
 click==6.7
 

etc/configure.py

@@ -1,9 +1,9 @@
 #!/usr/bin/python
 
-# Copyright (c) 2015 nexB Inc. http://www.nexb.com/ - All rights reserved.
+# Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved.
 
 """
-This script a configuration helper to select pip requirement files to install
+This script is a configuration helper to select pip requirement files to install
 and python and shell configuration scripts to execute based on provided config
 directories paths arguments and the operating system platform. To use, create
 a configuration directory tree that contains any of these:
@@ -21,8 +21,9 @@
  - posix.sh, linux.sh, mac.sh are os-specific scripts to execute.
 
 The config directory structure contains one or more directories paths. This
-way you can have a main configuration and additional sub-configurations of a
-product such as for prod, test, ci, dev, or anything else.
+way you can have a main configuration (that is always used) and additional
+sub-configurations of a product such as for prod, test, ci, dev, or anything
+else.
 
 All scripts and requirements are optional and only used if presents. Scripts
 are executed in sequence, one after the other after all requirements are
@@ -67,15 +68,15 @@
 # platform-specific file base names
 sys_platform = str(sys.platform).lower()
 on_win = False
-if 'linux' in sys_platform:
+if sys_platform.startswith('linux'):
     platform_names = ('posix', 'linux',)
-elif'win32' in sys_platform:
+elif 'win32' in sys_platform:
     platform_names = ('win',)
     on_win = True
 elif 'darwin' in sys_platform:
     platform_names = ('posix', 'mac',)
 else:
-    raise Exception('Unsupported OS/platform')
+    raise Exception('Unsupported OS/platform %r' % sys_platform)
     platform_names = tuple()
 
 
@@ -157,7 +158,7 @@ def build_pip_dirs_args(paths, root_dir, option='--extra-search-dir='):
         if not os.path.isabs(path):
             path = os.path.join(root_dir, path)
         if os.path.exists(path):
-            yield option + path
+            yield option + '"' + path + '"'
 
 
 def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False):
@@ -176,13 +177,14 @@ def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False):
     vendored Python distributions that pip will use to find required
     components.
     """
-    print("* Configuring Python ...")
+    if not quiet:
+        print("* Configuring Python ...")
     # search virtualenv.py in the tpp_dirs. keep the first found
     venv_py = None
     for tpd in tpp_dirs:
         venv = os.path.join(root_dir, tpd, 'virtualenv.py')
         if os.path.exists(venv):
-            venv_py = venv
+            venv_py = '"' + venv + '"'
             break
 
     # error out if venv_py not found
@@ -196,12 +198,13 @@ def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False):
     # third parties may be in more than one directory
     vcmd.extend(build_pip_dirs_args(tpp_dirs, root_dir))
     # we create the virtualenv in the root_dir
-    vcmd.append(root_dir)
+    vcmd.append('"' + root_dir + '"')
     call(vcmd, root_dir)
 
 
 def activate(root_dir):
     """ Activate a virtualenv in the current process."""
+    bin_dir = os.path.join(root_dir, 'bin')
     activate_this = os.path.join(bin_dir, 'activate_this.py')
     with open(activate_this) as f:
         code = compile(f.read(), activate_this, 'exec')
@@ -213,37 +216,45 @@ def install_3pp(configs, root_dir, tpp_dirs, quiet=False):
     Install requirements from requirement files found in `configs` with pip,
     using the vendored components in `tpp_dirs`.
     """
-    print("* Installing components ...")
-    requirement_files = get_conf_files(configs, root_dir, requirements)
+    if not quiet:
+        print("* Installing components ...")
+    requirement_files = get_conf_files(configs, root_dir, requirements, quiet)
+    if on_win:
+        bin_dir = os.path.join(root_dir, 'bin')
+        configured_python = os.path.join(bin_dir, 'python.exe')
+        base_cmd = [configured_python, '-m', 'pip']
+    else:
+        base_cmd = ['pip']
     for req_file in requirement_files:
-        pcmd = ['pip', 'install', '--no-allow-external',
-                '--use-wheel', '--no-index', '--no-cache-dir']
+        pcmd = base_cmd + ['install', '--upgrade', '--no-index', '--no-cache-dir']
         if quiet:
             pcmd += ['--quiet']
         pip_dir_args = list(build_pip_dirs_args(tpp_dirs, root_dir, '--find-links='))
         pcmd.extend(pip_dir_args)
         req_loc = os.path.join(root_dir, req_file)
-        pcmd.extend(['-r' , req_loc])
+        pcmd.extend(['-r' , '"' + req_loc + '"'])
         call(pcmd, root_dir)
 
 
-def run_scripts(configs, root_dir, configured_python):
+def run_scripts(configs, root_dir, configured_python, quiet=False):
     """
     Run Python scripts and shell scripts found in `configs`.
     """
-    print("* Configuring ...")
+    if not quiet:
+        print("* Configuring ...")
     # Run Python scripts for each configurations
     for py_script in get_conf_files(configs, root_dir, python_scripts):
-        cmd = [configured_python, os.path.join(root_dir, py_script)]
+        cmd = [configured_python, '"' + os.path.join(root_dir, py_script) + '"']
         call(cmd, root_dir)
 
     # Run sh_script scripts for each configurations
     for sh_script in get_conf_files(configs, root_dir, shell_scripts):
-        # we source the scripts on posix
-        cmd = ['.']
         if on_win:
             cmd = []
-        cmd = cmd + [os.path.join(root_dir, sh_script)]
+        else:
+            # we source the scripts on posix
+            cmd = ['.']
+        cmd.extend([os.path.join(root_dir, sh_script)])
         call(cmd, root_dir)
 
 
@@ -258,7 +269,7 @@ def chmod_bin(directory):
             os.chmod(os.path.join(path, f), rwx)
 
 
-def get_conf_files(config_dir_paths, root_dir, file_names=requirements):
+def get_conf_files(config_dir_paths, root_dir, file_names=requirements, quiet=False):
     """
     Return a list of collected path-prefixed file paths matching names in a
     file_names tuple, based on config_dir_paths, root_dir and the types of
@@ -286,8 +297,9 @@ def get_conf_files(config_dir_paths, root_dir, file_names=requirements):
     for config_dir_path in config_dir_paths:
         abs_config_dir_path = os.path.join(root_dir, config_dir_path)
         if not os.path.exists(abs_config_dir_path):
-            print('Configuration directory %(config_dir_path)s '
-                  'does not exists. Skipping.' % locals())
+            if not quiet:
+                print('Configuration directory %(config_dir_path)s '
+                      'does not exists. Skipping.' % locals())
             continue
         # Support args like enterprise or enterprise/dev
         paths = config_dir_path.strip('/').replace('\\', '/').split('/')
@@ -313,31 +325,42 @@ def get_conf_files(config_dir_paths, root_dir, file_names=requirements):
     return collected
 
 
+usage = '\nUsage: configure [--clean] <path/to/configuration/directory> ...\n'
+
+
 if __name__ == '__main__':
+
+    # you must create a CONFIGURE_QUIET env var if you want to run quietly
+    quiet = 'CONFIGURE_QUIET' in os.environ
+
     # define/setup common directories
     etc_dir = os.path.abspath(os.path.dirname(__file__))
     root_dir = os.path.dirname(etc_dir)
 
     args = sys.argv[1:]
-    if args[0] == '--clean':
-        clean(root_dir)
-        sys.exit(0)
+    if args:
+        arg0 = args[0]
+        if arg0 == '--clean':
+            clean(root_dir)
+            sys.exit(0)
+        elif arg0.startswith('-'):
+            print()
+            print('ERROR: unknown option: %(arg0)s' % locals())
+            print(usage)
+            sys.exit(1)
 
     sys.path.insert(0, root_dir)
     bin_dir = os.path.join(root_dir, 'bin')
     standard_python = sys.executable
 
-    # you must create a CONFIGURE_QUIET env var if you want to run quietly
-    run_quiet = 'CONFIGURE_QUIET' in os.environ
-
     if on_win:
         configured_python = os.path.join(bin_dir, 'python.exe')
         scripts_dir = os.path.join(root_dir, 'Scripts')
         bin_dir = os.path.join(root_dir, 'bin')
         if not os.path.exists(scripts_dir):
             os.makedirs(scripts_dir)
         if not os.path.exists(bin_dir):
-            cmd = ('mklink /J %(bin_dir)s %(scripts_dir)s' % locals()).split()
+            cmd = ('mklink /J "%(bin_dir)s" "%(scripts_dir)s"' % locals()).split()
             call(cmd, root_dir)
     else:
         configured_python = os.path.join(bin_dir, 'python')
@@ -346,41 +369,47 @@ def get_conf_files(config_dir_paths, root_dir, file_names=requirements):
     # Get requested configuration paths to collect components and scripts later
     configs = []
     for path in args[:]:
+        abs_path = path
         if not os.path.isabs(path):
             abs_path = os.path.join(root_dir, path)
-            if os.path.exists(abs_path):
-                configs.append(path)
-        else:
-            print()
-            print('WARNING: Skipping missing Configuration directory:\n'
-                  '  %(path)s does not exist.' % locals())
+        if not os.path.exists(abs_path):
             print()
+            print('ERROR: Configuration directory does not exists:\n'
+                  '  %(path)s: %(abs_path)r'
+                  % locals())
+            print(usage)
+            sys.exit(1)
+
+        configs.append(path)
 
     # Collect vendor directories from environment variables: one or more third-
     # party directories may exist as environment variables prefixed with TPP_DIR
     thirdparty_dirs = []
     for envvar, path in os.environ.items():
         if not envvar.startswith('TPP_DIR'):
             continue
+        abs_path = path
         if not os.path.isabs(path):
             abs_path = os.path.join(root_dir, path)
-            if os.path.exists(abs_path):
-                thirdparty_dirs.append(path)
+        if not os.path.exists(abs_path):
+            if not quiet:
+                print()
+                print('WARNING: Third-party Python libraries directory does not exists:\n'
+                      '  %(path)r: %(abs_path)r\n'
+                      '  Provided by environment variable:\n'
+                      '  set %(envvar)s=%(path)r' % locals())
+                print()
         else:
-            print()
-            print('WARNING: Skipping missing Python thirdparty directory:\n'
-                  '  %(path)s does not exist.\n'
-                  '  Provided by environment variable:\n'
-                  '  set %(envvar)s=%(path)s' % locals())
-            print()
+            thirdparty_dirs.append(path)
 
     # Finally execute our three steps: venv, install and scripts
     if not os.path.exists(configured_python):
-        create_virtualenv(standard_python, root_dir, thirdparty_dirs, quiet=run_quiet)
+        create_virtualenv(standard_python, root_dir, thirdparty_dirs, quiet=quiet)
     activate(root_dir)
 
-    install_3pp(configs, root_dir, thirdparty_dirs, quiet=run_quiet)
-    run_scripts(configs, root_dir, configured_python)
+    install_3pp(configs, root_dir, thirdparty_dirs, quiet=quiet)
+    run_scripts(configs, root_dir, configured_python, quiet=quiet)
     chmod_bin(bin_dir)
-    print("* Configuration completed.")
-    print()
+    if not quiet:
+        print("* Configuration completed.")
+        print()

man/man1/.gitignore

@@ -24,7 +24,7 @@
 try:
     basestring  # Python 2
 except NameError:
-    basestring = str  # Python 3
+    basestring = str  # Python 3 #NOQA
 
 
 __version__ = '3.2.2'