Generalize Skylark parser for Bazel and Buck #1678

    * Update tests

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

src/packagedcode/bazel.py

@@ -25,98 +25,16 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from collections import defaultdict
-from collections import OrderedDict
-import ast
 import os
 
 import attr
 
-from commoncode import fileutils
-from packagedcode.build import BaseBuildManifestPackage
+from packagedcode.build import StarlarkManifestPackage
 from packagedcode.utils import combine_expressions
 from scancode.api import get_licenses
 
 
 @attr.s()
-class BazelPackage(BaseBuildManifestPackage):
+class BazelPackage(StarlarkManifestPackage):
     metafiles = ('BUILD',)
     default_type = 'bazel'
-
-    @classmethod
-    def recognize(cls, location):
-        if not cls._is_build_manifest(location):
-            return
-        for package in bazel_parse(location):
-            yield package
-
-    def compute_normalized_license(self):
-        return compute_normalized_license(
-            self.declared_license,
-            manifest_parent_path=self.root_path
-        )
-
-
-def bazel_parse(location):
-    build_rules = defaultdict(list)
-    # Thanks to the Skylark language being a Python DSL, we can use the `ast`
-    # library to parse Bazel BUILD files
-    with open(location, 'rb') as f:
-        tree = ast.parse(f.read())
-    for statement in tree.body:
-        if (isinstance(statement, ast.Expr)
-                or isinstance(statement, ast.Call)
-                or isinstance(statement, ast.Assign)
-                and isinstance(statement.value, ast.Call)
-                and isinstance(statement.value.func, ast.Name)):
-            rule_name = statement.value.func.id
-            # Process the rule arguments
-            args = OrderedDict()
-            for kw in statement.value.keywords:
-                arg_name = kw.arg
-                if isinstance(kw.value, ast.Str):
-                    args[arg_name] = kw.value.s
-                if isinstance(kw.value, ast.List):
-                     # We collect the elements of a list if the element is not a function call
-                    args[arg_name] = [elt.s for elt in kw.value.elts if not isinstance(elt, ast.Call)]
-            if args:
-                build_rules[rule_name].append(args)
-
-    if build_rules:
-        for rule_name, rule_instances_args in build_rules.items():
-            for args in rule_instances_args:
-                name = args.get('name')
-                if not name:
-                    continue
-                license_files = args.get('licenses')
-                yield BazelPackage(
-                    name=name,
-                    declared_license=license_files,
-                    root_path=fileutils.parent_directory(location)
-                )
-    else:
-        # If we don't find anything in the BUCK file, we yield a Package with
-        # the parent directory as the name, like the default implementation of
-        # `recognize()` for `BaseBuildManifestPackage`
-        yield BazelPackage(
-            # we use the parent directory as a name
-            name=fileutils.file_name(fileutils.parent_directory(location))
-        )
-
-
-def compute_normalized_license(declared_license, manifest_parent_path):
-    """
-    Return a normalized license expression string detected from a list of
-    declared license items.
-    """
-    if not declared_license or not manifest_parent_path:
-        return
-
-    license_expressions = []
-    for license_file in declared_license:
-        license_file_path = os.path.join(manifest_parent_path, license_file)
-        if os.path.exists(license_file_path) and os.path.isfile(license_file_path):
-            licenses = get_licenses(license_file_path)
-            license_expressions.extend(licenses.get('license_expressions', []))
-
-    return combine_expressions(license_expressions)

src/packagedcode/buck.py

@@ -25,185 +25,14 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from collections import defaultdict
-from collections import OrderedDict
-import ast
 import os
 
 import attr
 
-from commoncode import fileutils
-from packagedcode.build import BaseBuildManifestPackage
-from packagedcode.utils import combine_expressions
-from scancode.api import get_licenses
+from packagedcode.build import StarlarkManifestPackage
 
 
 @attr.s()
-class BuckPackage(BaseBuildManifestPackage):
+class BuckPackage(StarlarkManifestPackage):
     metafiles = ('BUCK',)
     default_type = 'buck'
-
-    @classmethod
-    def recognize(cls, location):
-        if not cls._is_build_manifest(location):
-            return
-        for package in buck_parse(location):
-            yield package
-
-    def compute_normalized_license(self):
-        return compute_normalized_license(
-            self.declared_license,
-            manifest_parent_path=self.root_path
-        )
-
-
-# TODO: Prune rule names that do not create things we do not care about like
-# `robolectric_test` or `genrule`
-buck_rule_names = [
-    'command_alias',
-    'export_file',
-    'filegroup',
-    'genrule',
-    'http_archive',
-    'http_file',
-    'remote_file',
-    'test_suite',
-    'worker_tool',
-    'zip_file',
-    'android_aar',
-    'android_binary',
-    'android_build_config',
-    'android_instrumentation_apk',
-    'android_instrumentation_test',
-    'android_library',
-    'android_manifest',
-    'android_prebuilt_aar',
-    'android_resource',
-    'apk_genrule',
-    'gen_aidl',
-    'keystore',
-    'ndk_library',
-    'prebuilt_jar',
-    'prebuilt_native_library',
-    'robolectric_test',
-    'cxx_binary',
-    'cxx_library',
-    'cxx_genrule',
-    'cxx_precompiled_header',
-    'cxx_test',
-    'prebuilt_cxx_library',
-    'prebuilt_cxx_library_group',
-    'd_binary',
-    'd_library',
-    'd_test',
-    'go_binary',
-    'go_library',
-    'go_test',
-    'cgo_library',
-    'groovy_library',
-    'halide_library',
-    'haskell_binary',
-    'haskell_library',
-    'prebuilt_haskell_library',
-    'apple_asset_catalog',
-    'apple_binary',
-    'apple_bundle',
-    'apple_library',
-    'apple_package',
-    'apple_resource',
-    'apple_test',
-    'core_data_model',
-    'prebuilt_apple_framework',
-    'java_binary',
-    'java_library',
-    'java_test',
-    'prebuilt_jar',
-    'prebuilt_native_library',
-    'kotlin_library',
-    'kotlin_test',
-    'cxx_lua_extension',
-    'lua_binary',
-    'lua_library',
-    'ocaml_binary',
-    'ocaml_library',
-    'prebuilt_python_library',
-    'python_binary',
-    'python_library',
-    'python_test',
-    'rust_binary',
-    'rust_library',
-    'rust_test',
-    'prebuilt_rust_library',
-    'sh_binary',
-    'sh_test',
-    'csharp_library',
-    'prebuilt_dotnet_library'
-]
-
-
-def buck_parse(location):
-    build_rules = defaultdict(list)
-    # Thanks to the BUCK language being a Python DSL, we can use the `ast`
-    # library to parse BUCK files
-    with open(location, 'rb') as f:
-        tree = ast.parse(f.read())
-    for statement in tree.body:
-        # We only care about function calls or assignments to functions whose
-        # names are in `buck_rule_names`
-        if (isinstance(statement, ast.Expr)
-                or isinstance(statement, ast.Call)
-                or isinstance(statement, ast.Assign)
-                and isinstance(statement.value, ast.Call)
-                and isinstance(statement.value.func, ast.Name)
-                and statement.value.func.id in buck_rule_names):
-            rule_name = statement.value.func.id
-            # Process the rule arguments
-            args = OrderedDict()
-            for kw in statement.value.keywords:
-                arg_name = kw.arg
-                if isinstance(kw.value, ast.Str):
-                    args[arg_name] = kw.value.s
-                if isinstance(kw.value, ast.List):
-                     # We collect the elements of a list if the element is not a function call
-                    args[arg_name] = [elt.s for elt in kw.value.elts if not isinstance(elt, ast.Call)]
-            if args:
-                build_rules[rule_name].append(args)
-
-    if build_rules:
-        for rule_name, rule_instances_args in build_rules.items():
-            for args in rule_instances_args:
-                name = args.get('name')
-                if not name:
-                    continue
-                license_files = args.get('licenses')
-                yield BuckPackage(
-                    name=name,
-                    declared_license=license_files,
-                    root_path=fileutils.parent_directory(location)
-                )
-    else:
-        # If we don't find anything in the BUCK file, we yield a Package with
-        # the parent directory as the name, like the default implementation of
-        # `recognize()` for `BaseBuildManifestPackage`
-        yield BuckPackage(
-            # we use the parent directory as a name
-            name=fileutils.file_name(fileutils.parent_directory(location))
-        )
-
-
-def compute_normalized_license(declared_license, manifest_parent_path):
-    """
-    Return a normalized license expression string detected from a list of
-    declared license items.
-    """
-    if not declared_license or not manifest_parent_path:
-        return
-
-    license_expressions = []
-    for license_file in declared_license:
-        license_file_path = os.path.join(manifest_parent_path, license_file)
-        if os.path.exists(license_file_path) and os.path.isfile(license_file_path):
-            licenses = get_licenses(license_file_path)
-            license_expressions.extend(licenses.get('license_expressions', []))
-
-    return combine_expressions(license_expressions)

src/packagedcode/build.py

@@ -26,13 +26,19 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+from collections import defaultdict
+from collections import OrderedDict
+import ast
 import logging
+import os
 
 import attr
 
 from commoncode import filetype
 from commoncode import fileutils
 from packagedcode import models
+from packagedcode.utils import combine_expressions
+from scancode.api import get_licenses
 
 
 TRACE = False
@@ -89,3 +95,82 @@ def _is_build_manifest(cls, location):
 class AutotoolsPackage(BaseBuildManifestPackage):
     metafiles = ('configure', 'configure.ac',)
     default_type = 'autotools'
+
+
+starlark_rule_types = [
+    'binary',
+    'library'
+]
+
+
+@attr.s()
+class StarlarkManifestPackage(BaseBuildManifestPackage):
+    @classmethod
+    def recognize(cls, location):
+        if not cls._is_build_manifest(location):
+            return
+        build_rules = defaultdict(list)
+        # Thanks to Starlark being a Python dialect, we can use the `ast`
+        # library to parse it
+        with open(location, 'rb') as f:
+            tree = ast.parse(f.read())
+        for statement in tree.body:
+            # We only care about function calls or assignments to functions whose
+            # names ends with one of the strings in `rule_types`
+            if (isinstance(statement, ast.Expr)
+                    or isinstance(statement, ast.Call)
+                    or isinstance(statement, ast.Assign)
+                    and isinstance(statement.value, ast.Call)
+                    and isinstance(statement.value.func, ast.Name)
+                    and statement.value.func.id.endswith(starlark_rule_types)):
+                rule_name = statement.value.func.id
+                # Process the rule arguments
+                args = OrderedDict()
+                for kw in statement.value.keywords:
+                    arg_name = kw.arg
+                    if isinstance(kw.value, ast.Str):
+                        args[arg_name] = kw.value.s
+                    if isinstance(kw.value, ast.List):
+                        # We collect the elements of a list if the element is not a function call
+                        args[arg_name] = [elt.s for elt in kw.value.elts if not isinstance(elt, ast.Call)]
+                if args:
+                    build_rules[rule_name].append(args)
+
+        if build_rules:
+            for rule_name, rule_instances_args in build_rules.items():
+                for args in rule_instances_args:
+                    name = args.get('name')
+                    if not name:
+                        continue
+                    license_files = args.get('licenses')
+                    yield cls(
+                        name=name,
+                        declared_license=license_files,
+                        root_path=fileutils.parent_directory(location)
+                    )
+        else:
+            # If we don't find anything in the manifest file, we yield a Package with
+            # the parent directory as the name
+            yield cls(
+                name=fileutils.file_name(fileutils.parent_directory(location))
+            )
+
+    def compute_normalized_license(self):
+        """
+        Return a normalized license expression string detected from a list of
+        declared license items.
+        """
+        declared_license = self.declared_license
+        manifest_parent_path = self.root_path
+
+        if not declared_license or not manifest_parent_path:
+            return
+
+        license_expressions = []
+        for license_file in declared_license:
+            license_file_path = os.path.join(manifest_parent_path, license_file)
+            if os.path.exists(license_file_path) and os.path.isfile(license_file_path):
+                licenses = get_licenses(license_file_path)
+                license_expressions.extend(licenses.get('license_expressions', []))
+
+        return combine_expressions(license_expressions)