Rename get_key_files to get_top_level_resources

    * Add test for get_top_level_resources in maven tests
    * Add test in test_classify

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

src/packagedcode/maven.py

@@ -153,16 +153,17 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
         )
 
     @classmethod
-    def get_key_files(cls, manifest_resource, codebase):
+    def get_top_level_resources(cls, manifest_resource, codebase):
         """
-        Yield Resources that are key files based on a JAR's directory structure
+        Yield Resources that are top-level based on a JAR's directory structure
         """
         if 'META-INF' in manifest_resource.path:
             path_segments = manifest_resource.path.split('META-INF')
             leading_segment = path_segments[0].strip()
             meta_inf_dir_path = f'{leading_segment}/META-INF'
             meta_inf_resource = codebase.get_resource(meta_inf_dir_path)
             if meta_inf_resource:
+                yield meta_inf_resource
                 yield from meta_inf_resource.walk(codebase)
 
 

src/packagedcode/models.py

@@ -1305,9 +1305,9 @@ def populate_license_fields(cls, package_data):
             package_data.extracted_license_statement = repr(package_data.extracted_license_statement)
 
     @classmethod
-    def get_key_files(cls, manifest_resource, codebase):
+    def get_top_level_resources(cls, manifest_resource, codebase):
         """
-        Yield Resources that are considered key files for a Package type.
+        Yield Resources that are considered top-level for a Package type.
         """
         pass
 

src/summarycode/classify_plugin.py

@@ -117,7 +117,11 @@ def process_codebase(self, codebase, classify, **kwargs):
             real_root = codebase.root
         real_root_dist = real_root.distance(codebase)
 
+        seen_resources = set()
         for resource in codebase.walk(topdown=True):
+            if resource.path in seen_resources:
+                continue
+
             has_package_data = bool(getattr(resource, 'package_data', False))
             if not has_package_data:
                 real_dist = resource.distance(codebase) - real_root_dist
@@ -127,14 +131,17 @@ def process_codebase(self, codebase, classify, **kwargs):
                     # TODO: should we do something about directories? for now we only consider files
                     set_classification_flags(resource)
                 resource.save(codebase)
+                seen_resources.add(resource.path)
             else:
                 for package_data in resource.package_data:
                     pd = PackageData.from_dict(package_data)
                     package_handler = get_package_handler(pd)
-                    key_files = package_handler.get_key_files(resource, codebase)
-                    if not key_files:
+                    top_level_resources = package_handler.get_top_level_resources(resource, codebase)
+                    if not top_level_resources:
                         break
-                    for key_file in key_files:
-                        key_file.is_top_level = True
-                        set_classification_flags(key_file)
-                        key_file.save(codebase)
+                    for r in top_level_resources:
+                        r.is_top_level = True
+                        set_classification_flags(r)
+                        r.save(codebase)
+                        seen_resources.add(r.path)
+                seen_resources.add(resource.path)

tests/packagedcode/test_maven.py

@@ -20,6 +20,7 @@
 from packagedcode import maven
 from packagedcode import models
 from packagedcode.licensing import get_license_detections_and_expression
+from scancode.cli import run_scan
 from scancode.cli_test_utils import check_json_scan
 from scancode.cli_test_utils import run_scan_click
 from scancode_config import REGEN_TEST_FIXTURES
@@ -222,6 +223,37 @@ def test_package_dependency_populate_is_resolved_field(self):
         test_file = self.get_test_loc('maven_misc/parse/swagger-java-sample-app_2.10-1.3.1.pom')
         self.check_parse_to_package(test_file, regen=REGEN_TEST_FIXTURES)
 
+    def test_get_top_level_resources(self):
+        processes = 2
+        test_dir = self.get_test_loc('maven_misc/extracted-jar/activiti-image-generator-7-201802-EA-sources.jar-extract')
+        _, codebase = run_scan(
+            input=test_dir,
+            processes=processes,
+            quiet=True,
+            verbose=False,
+            max_in_memory=0,
+            return_results=False,
+            return_codebase=True,
+            system_package=True,
+        )
+        pom_resource = codebase.get_resource(
+            'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven/org.activiti/activiti-image-generator/pom.xml'
+        )
+        self.assertTrue(pom_resource)
+        top_level_resources_paths = [
+            r.path for r in maven.MavenPomXmlHandler.get_top_level_resources(pom_resource, codebase)
+        ]
+        expected_resource_paths = [
+            'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF',
+            'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/MANIFEST.MF',
+            'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven',
+            'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven/org.activiti',
+            'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven/org.activiti/activiti-image-generator',
+            'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven/org.activiti/activiti-image-generator/pom.properties',
+            'activiti-image-generator-7-201802-EA-sources.jar-extract/META-INF/maven/org.activiti/activiti-image-generator/pom.xml',
+        ]
+        self.assertEquals(expected_resource_paths, top_level_resources_paths)
+
 
 class TestPomProperties(testcase.FileBasedTesting):
     test_data_dir = os.path.join(os.path.dirname(__file__), 'data')