|
| 1 | +#!/usr/bin/env python |
| 2 | +""" |
| 3 | +This script can be used to run all things dev. Environment setup, Style-checks, Testing etc. |
| 4 | +""" |
| 5 | + |
| 6 | +from __future__ import print_function |
| 7 | + |
| 8 | +import os |
| 9 | +import subprocess |
| 10 | +import sys |
| 11 | + |
| 12 | +import argcomplete |
| 13 | +import argh |
| 14 | +import six |
| 15 | + |
| 16 | +import pylint as _pylint |
| 17 | +import prospector as _prospector |
| 18 | +import yapf as _yapf |
| 19 | + |
| 20 | + |
| 21 | +# path of directory this file resides in |
| 22 | +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) |
| 23 | +HOME = os.getenv('HOME', '') # HOME environment variable |
| 24 | + |
| 25 | + |
| 26 | +def call_subprocess(cmd, add_env={}, verbose=True): |
| 27 | + """Wrapper function for subprocess.call that prints additional environment and command run""" |
| 28 | + print_if(verbose and add_env, "updating environment...\n" + _env2shellcode(add_env)) |
| 29 | + print_if(verbose and cmd, "running command...\n" + " ".join(cmd)) |
| 30 | + env = os.environ.copy() |
| 31 | + env.update(add_env) |
| 32 | + return subprocess.call(cmd, env=env) |
| 33 | + |
| 34 | + |
| 35 | +def _list_files_with_extension(dir_name, ext): |
| 36 | + return [os.path.join(dir_name, f) for f in os.listdir(dir_name) if f.endswith(ext)] |
| 37 | + |
| 38 | + |
| 39 | +def _get_configured_env(_required_env): |
| 40 | + spark_home = _required_env["SPARK_HOME"] |
| 41 | + scala_version = _required_env["SCALA_VERSION"] |
| 42 | + |
| 43 | + if not spark_home: |
| 44 | + raise ValueError("set SPARK_HOME environment variable to configure environment") |
| 45 | + if not scala_version: |
| 46 | + raise ValueError("set SCALA_VERSION environment variable to configure environment") |
| 47 | + |
| 48 | + configured_env = {} |
| 49 | + |
| 50 | + spark_lib_path = os.path.join(spark_home, "python/lib") |
| 51 | + configured_env["LIBS"] = ":".join(_list_files_with_extension(spark_lib_path, ".zip")) |
| 52 | + |
| 53 | + scala_version_major_minor = ".".join(scala_version.split(".")[0:2]) |
| 54 | + assembly_path = os.path.join(PROJECT_DIR, "target/scala-" + scala_version_major_minor) |
| 55 | + jar_path = ":".join(_list_files_with_extension(assembly_path, ".jar")) |
| 56 | + configured_env["JAR_PATH"] = jar_path |
| 57 | + |
| 58 | + python_paths = [ |
| 59 | + os.getenv("PYTHONPATH", ""), |
| 60 | + os.path.join(PROJECT_DIR, "python"), |
| 61 | + _list_files_with_extension(assembly_path, ".jar")[-1], |
| 62 | + os.path.join(spark_home, "python"), |
| 63 | + configured_env["LIBS"] |
| 64 | + ] |
| 65 | + configured_env["PYTHONPATH"] = ":".join(python_paths) |
| 66 | + |
| 67 | + return configured_env |
| 68 | + |
| 69 | + |
| 70 | +def pylint(rcfile="./python/.pylint/accepted.rc", reports="y", persistent="n", *args): |
| 71 | + """ |
| 72 | + Wraps `pylint` and provides defaults. Run `prospector --help` for more details. Trailing |
| 73 | + arguments are a list of files, packages or modules. if nothing is specified, default value is |
| 74 | + ./python/sparkdl |
| 75 | + """ |
| 76 | + if not args: |
| 77 | + args = ("./python/sparkdl", ) |
| 78 | + cmd = ("python", "-mpylint", "--rcfile=" + rcfile, "--reports=" + reports, "--persistent=" + |
| 79 | + persistent) |
| 80 | + return call_subprocess(cmd + args, add_env=_get_configured_env(dict(os.environ))) |
| 81 | + |
| 82 | + |
| 83 | +def pylint_suggested(*args): |
| 84 | + """Wrapper for pylint that is used to generate suggestions""" |
| 85 | + return pylint(rcfile="./python/.pylint/suggested.rc", reports="n", persistent="n", *args) |
| 86 | + |
| 87 | + |
| 88 | +def prospector(without_tool="pylint", output_format="pylint", *args): |
| 89 | + """ |
| 90 | + Wraps `prospector` and provides defaults. Run `prospector --help` for more details. Trailing |
| 91 | + arguments are a list of files, packages or modules. if nothing is specified, default value is |
| 92 | + ./python/sparkdl |
| 93 | + """ |
| 94 | + if not args: |
| 95 | + args = ("./python/sparkdl", ) |
| 96 | + cmd = ("python", "-mprospector", "--without-tool=" + without_tool, "--output-format=" + |
| 97 | + output_format) |
| 98 | + return call_subprocess(cmd + args, add_env=_get_configured_env(dict(os.environ))) |
| 99 | + |
| 100 | + |
| 101 | +def yapf(style="{based_on_style=pep8, COLUMN_LIMIT=100}", in_place=False, recursive=False, *args): |
| 102 | + """Wraps `yapf` and provides some defaults. Run `yapf --help` for more details.""" |
| 103 | + if in_place: |
| 104 | + args = ("-i",) + args |
| 105 | + if recursive: |
| 106 | + args = ("-r",) + args |
| 107 | + cmd = ("python", "-myapf", "--style=" + style) |
| 108 | + return call_subprocess(cmd + args, add_env=_get_configured_env(dict(os.environ))) |
| 109 | + |
| 110 | + |
| 111 | +def _safe_input_prompt(k, default): |
| 112 | + try: |
| 113 | + current = input("{}=".format(k)) |
| 114 | + except SyntaxError: |
| 115 | + # this error is thrown in python 2.7 |
| 116 | + # SyntaxError: unexpected EOF while parsing |
| 117 | + current = "" |
| 118 | + return current if current else default |
| 119 | + |
| 120 | + |
| 121 | +def print_if(cond, *args): |
| 122 | + if cond: |
| 123 | + print(*args) |
| 124 | + |
| 125 | + |
| 126 | +def _get_required_env(default=False, interactive=False, override=False, verbose=False): |
| 127 | + default_env = {'PYSPARK_PYTHON': 'python', |
| 128 | + 'SPARK_VERSION': '2.3.0', |
| 129 | + 'SPARK_HOME': os.path.join(HOME, 'bin/spark-2.3.0-bin-hadoop2.7/'), |
| 130 | + 'SCALA_VERSION': '2.11.8'} |
| 131 | + |
| 132 | + if override and not (interactive or default): |
| 133 | + raise ValueError("override mode requires to use default or interactive mode") |
| 134 | + |
| 135 | + # identify which required variables are given and which are missing |
| 136 | + given_env = {} |
| 137 | + missing_env = {} |
| 138 | + for k in default_env: |
| 139 | + v = os.getenv(k, "") |
| 140 | + if v and not override: |
| 141 | + given_env[k] = v |
| 142 | + else: |
| 143 | + missing_env[k] = "" |
| 144 | + print_if(given_env and verbose, 'given environment variables: {}'.format(given_env)) |
| 145 | + |
| 146 | + # set the missing variables interactively or from defaults |
| 147 | + if missing_env: |
| 148 | + print_if(verbose, 'missing environment variables: {}'.format(missing_env)) |
| 149 | + |
| 150 | + if not (default or interactive): |
| 151 | + raise ValueError( |
| 152 | + """Use default or interactive mode to set required environment variables: |
| 153 | + {}""".format(",".join(missing_env))) |
| 154 | + |
| 155 | + if default: |
| 156 | + print_if(verbose, 'using default values') |
| 157 | + missing_env = {k: default_env[k] for k in missing_env} |
| 158 | + |
| 159 | + if interactive: |
| 160 | + print_if(verbose, 'enter values for the following') |
| 161 | + print_if(default and verbose, 'if left blank, default values will be used') |
| 162 | + missing_env = {k: _safe_input_prompt(k, v) for k, v in missing_env.items()} |
| 163 | + |
| 164 | + return given_env, missing_env |
| 165 | + |
| 166 | + |
| 167 | +def _env2shellcode(env): |
| 168 | + # Dict[str, str] -> str |
| 169 | + return "\n".join("export {}={}".format(k, v) for k, v in env.items()) |
| 170 | + |
| 171 | + |
| 172 | +def env_setup(default=False, interactive=False, missing_only=False, override=False, configure=False, |
| 173 | + auto_completion=False, verbose=False): |
| 174 | + """ |
| 175 | + Prints out shell commands that can be used in terminal to setup the environment. |
| 176 | +
|
| 177 | + This tool inspects the current environment, and/or adds default values, and/or interactively |
| 178 | + asks user for values and prints all or the missing variables that need to be set. It can also |
| 179 | + provide completion for this script. You can source the setup as follows: |
| 180 | +
|
| 181 | + ``` |
| 182 | + python/run.py envsetup -d -c > ./python/.setup.sh && source ./python/.setup.sh |
| 183 | + ``` |
| 184 | +
|
| 185 | + :param default: if default values should be set in this script or not |
| 186 | + :param interactive: if user should be prompted for values or not |
| 187 | + :param missing_only: if only missing variable should be printed |
| 188 | + :param override: if current environment variables should be overridden |
| 189 | + :param configure: if configuration should be printed or not |
| 190 | + :param auto_completion: if auto complete code should be printed |
| 191 | + :param verbose: if user should be guided or not |
| 192 | + """ |
| 193 | + given_env, missing_env = _get_required_env( |
| 194 | + default=default, interactive=interactive, override=override, verbose=verbose) |
| 195 | + |
| 196 | + # print according to options |
| 197 | + output_env = {} |
| 198 | + output_env.update(missing_env) |
| 199 | + |
| 200 | + if not missing_only: |
| 201 | + output_env.update(given_env) |
| 202 | + |
| 203 | + if configure: |
| 204 | + required_env = dict(given_env) |
| 205 | + required_env.update(missing_env) |
| 206 | + configured_env = _get_configured_env(required_env) |
| 207 | + output_env.update(configured_env) |
| 208 | + |
| 209 | + env_str = "#!/bin/bash\n" |
| 210 | + env_str += _env2shellcode(output_env) |
| 211 | + |
| 212 | + if auto_completion: |
| 213 | + env_str += argcomplete.shellcode(sys.argv[0]) |
| 214 | + |
| 215 | + print(env_str) |
| 216 | + return 0 |
| 217 | + |
| 218 | + |
| 219 | +def python_tests(*args): |
| 220 | + """Wrapper for python/run-tests.sh""" |
| 221 | + return call_subprocess(("./python/run-tests.sh",) + args) |
| 222 | + |
| 223 | + |
| 224 | +def sbt(*args): |
| 225 | + """Wrapper for build/sbt""" |
| 226 | + required_env, missing_env = _get_required_env() |
| 227 | + assert(not missing_env) |
| 228 | + cmd = ("./build/sbt", "-Dspark.version=" + required_env.get("SPARK_VERSION"), |
| 229 | + "-Dscala.version=" + required_env.get("SCALA_VERSION")) |
| 230 | + call_subprocess(cmd + args) |
| 231 | + |
| 232 | + |
| 233 | +def assembly(): |
| 234 | + """Wrapper for build/sbt assembly""" |
| 235 | + return sbt("set test in assembly := {}", "assembly") |
| 236 | + |
| 237 | + |
| 238 | +def scala_tests(): |
| 239 | + """Wrapper for build/sbt coverage test coverageReport""" |
| 240 | + return sbt("coverage", "test", "coverageReport") |
| 241 | + |
| 242 | + |
| 243 | +parser = argh.ArghParser() |
| 244 | +parser.add_commands([pylint, prospector, yapf, env_setup, python_tests, pylint_suggested, |
| 245 | + scala_tests, assembly, sbt]) |
| 246 | + |
| 247 | +if __name__ == '__main__': |
| 248 | + dispatch_result = parser.dispatch(output_file=None) |
| 249 | + # argh.dispatch formats the returns of functions as strings |
| 250 | + try: |
| 251 | + return_code = int(dispatch_result) |
| 252 | + except ValueError: |
| 253 | + print(dispatch_result) |
| 254 | + return_code = 0 |
| 255 | + exit(return_code) |
| 256 | + |
0 commit comments