Enable a paths argument in Codebase/VirtualCodebase

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

CHANGELOG.rst

@@ -4,17 +4,45 @@ Release notes
 Version (next) 
 ------------------------------
 
+TBD.
+
 Version 31.0.0 - (2022-05-16)
 ------------------------------
 
 This is a major version with API-breaking behavious changes in the reosurce
 module.
 
+- The Resource has no rid (resource id) and no pid (parent id). Instead
+  we now use internally a simpler mapping of {path: Resource} object.
+  As a result the iteration on a Codebase is faster but this requires more
+  memory.
+
+- The Codebase and VirtualCodebase accepts a new "paths" argument that is list
+  of paths. When provided, the Codebase will only contain Resources with these
+  paths and no other resources. This handy to create a Codebase with only a
+  subset of paths of interest. When we create a Codebase or VirtualCodebase
+  with paths, we also always create any internediate directories. So if you
+  ask for a path of "root/dir/file", we create three resources: "root",
+  "root/dir" and "root/dir/file". We accumulate codebase errors if the paths
+  does not exists in the Codebase or VirtualCodebase. The paths must start with
+  the root path segment and must be POSIX paths.
+
 - When you create a VirtualCodebase with multiple scans, we now prefix each
   scan path with a codebase-1/, codebase-2/, etc. directory in addition to the
   "virtual_root" shared root directory. Otherwise files data was overwritten
   and inconsistent when each location "files" were sharing leading path
-  segments.
+  segments. So if you provide to JSON inputs with that each contain the path
+  "root/dir/file", the VirtualCodebase will contain these paths:
+    - "virtual_root/codebase-1/root/dir/file"
+    - "virtual_root/codebase-2/root/dir/file"
+  It is otherwise practically impossble to correctly merge file data from
+  multiple codebases reliably, so adding this prefix ensures that we are doing
+  the right thing
+
+- The Resource.path now never contains leading or trailing slash. We also
+  normalize the path everywhere. In particular this behaviour is visible when
+  you create a Codebase with a "full_root" argument. Previously, the paths of a
+  "full_root" Codebase were prefixed with a slash "/".
 
 - When you create a VirtualCodebase with more than one Resource, we now recreate
   the directory tree for any intermediary directory used in a path that is
@@ -24,24 +52,9 @@ module.
   missing paths of a "full_root" Codebase were kept unchanged. 
   Noet that the VirtualCodebase has always ignored the "full_root" argument.
 
-- The Resource has no rid (resource id) and no pid (parent id). Instead
-  we now use internally a simpler mapping of {path: Resource} object.
-
-- The Codebase and VirtualCodebase accepts a new "paths" argument that is list
-  of paths. When provided, the Codebase will only contain Resources with these
-  paths and no other resources. has no rid (resource id) and no pid (parent id).
-  Instead we now use internally a simpler mapping of {path: Resource} object.
-  As a result the iteration on a Codebase is faster but this requires more
-  memory.
-
 - The Codebase and VirtualCodebase are now iterable. Iterating on a codebase
   is the same as a top-down walk.
 
-- The Resource.path now never contains leading or trailing slash. We also
-  normalize the path everywhere. In particular this behaviour is visible when
-  you create a Codebase with a "full_root" argument. Previously, the paths of a
-  "full_root" Codebase were prefixed with a slash "/".
-
 
 Other changes:
 

src/commoncode/resource.py

@@ -17,6 +17,7 @@
 from os import walk as os_walk
 from os.path import abspath
 from os.path import exists
+from os.path import isfile
 from os.path import expanduser
 from os.path import join
 from os.path import normpath
@@ -33,7 +34,6 @@
     temp_dir = tempfile.mkdtemp(prefix='scancode-resource-cache')
 
 from commoncode import ignore
-from commoncode import paths
 from commoncode.datautils import List
 from commoncode.datautils import Mapping
 from commoncode.datautils import String
@@ -45,6 +45,7 @@
 from commoncode.fileutils import file_name
 from commoncode.fileutils import parent_directory
 from commoncode.fileutils import splitext_name
+from commoncode.paths import split as paths_split
 
 """
 This module provides Codebase and Resource objects as an abstraction for files
@@ -59,7 +60,7 @@
 """
 
 # Tracing flags
-TRACE = False
+TRACE = True
 TRACE_DEEP = False
 
 
@@ -308,16 +309,18 @@ def __init__(
         self._setup_essentials(temp_dir, max_in_memory)
 
         # finally populate
-        self.paths = self._clean_paths(paths)
+        self.paths = self._prepare_clean_paths(paths)
         self._populate()
 
-    def _clean_paths(self, paths=tuple()):
+    def _prepare_clean_paths(self, paths=tuple()):
         """
-        Return a new list cleaned ``paths``.
+        Return a new set of cleaned ``paths`` possibly empty.
+        We convert to POSIX and ensure we have no slash at both ends.
         """
-        paths = paths or []
-        # convert to posix and ensure we have no slash at both ends
-        return [clean_path(p) for p in paths]
+        paths = (clean_path(p) for p in (paths or []) if p)
+        # we sort by path segments (e.g. essentially a topo sort)
+        _sorter = lambda p: p.split('/')
+        return sorted(paths, key=_sorter)
 
     def _setup_essentials(self, temp_dir=temp_dir, max_in_memory=10000):
         """
@@ -465,10 +468,79 @@ def _populate(self):
             # childless directory
             return
 
+        if self.paths:
+            return self._create_resources_from_paths(root=root, paths=self.paths)
+        else:
+            return self._create_resources_from_root(root=root)
+
+    def _create_resources_from_paths(self, root, paths):
+        # without paths we iterate the provided paths. We report an error
+        # if a path is missing on disk.
+
+        # !!!NOTE: WE DO NOT skip_ignored in this case!!!!!
+
+        base_location = parent_directory(root.location)
+
+        # track resources parents by path during construction to avoid
+        # recreating all ancestor directories
+        parents_by_path = {root.path: root}
+
+        for path in paths:
+            res_loc = join(base_location, path)
+            if not exists(res_loc):
+                msg = (
+                    f'ERROR: cannot populate codebase: '
+                    f'path: {path!r} not found in {res_loc!r}'
+                )
+                self.errors.append(msg)
+                raise Exception(path, join(base_location, path))
+                continue
+
+            # create all parents. The last parent is the one we want to use
+            parent = root
+            if TRACE:
+                logger_debug('Codebase._create_resources_from_paths: parent', parent)
+            for parent_path in get_ancestor_paths(path, include_self=False):
+                if TRACE:
+                    logger_debug('  Codebase._create_resources_from_paths: parent_path', repr(parent_path))
+                if not parent_path:
+                    continue
+                newpar = parents_by_path.get(parent_path)
+                if TRACE:
+                    logger_debug('  Codebase._create_resources_from_paths: newpar', repr(newpar))
+
+                if not newpar:
+                    newpar = self._get_or_create_resource(
+                        name=file_name(parent_path),
+                        parent=parent,
+                        path=parent_path,
+                        is_file=False,
+                    )
+                    if not newpar:
+                        raise Exception(f'ERROR: Codebase._create_resources_from_paths: cannot create parent for: {parent_path}')
+                    parent = newpar
+
+                    parents_by_path[parent_path] = parent
+
+                    if TRACE:
+                        logger_debug('Codebase._create_resources_from_paths: created newpar:', repr(newpar))
+
+            res = self._get_or_create_resource(
+                name=file_name(path),
+                parent=parent,
+                path=path,
+                is_file=isfile(res_loc),
+            )
+            if TRACE:
+                logger_debug('Codebase._create_resources_from_paths: resource', res)
+
+    def _create_resources_from_root(self, root):
+        # without paths we walks the root location top-down
+
         # track resources parents by location during construction.
         # NOTE: this cannot exhaust memory on a large codebase, because we do
         # not keep parents already walked and we walk topdown.
-        parent_by_loc = {root.location: root}
+        parents_by_loc = {root.location: root}
 
         def err(_error):
             """os.walk error handler"""
@@ -483,7 +555,7 @@ def err(_error):
             max_depth=self.max_depth,
             error_handler=err,
         ):
-            parent = parent_by_loc.pop(top)
+            parent = parents_by_loc.pop(top)
             for created in self._create_resources(
                 parent=parent,
                 top=top,
@@ -492,7 +564,7 @@ def err(_error):
             ):
                 # on the plain, bare FS, files cannot be parents
                 if not created.is_file:
-                    parent_by_loc[created.location] = created
+                    parents_by_loc[created.location] = created
 
     def _create_resources(self, parent, top, dirs, files, skip_ignored=skip_ignored):
         """
@@ -575,7 +647,7 @@ def _get_or_create_resource(
         Create and return a new codebase Resource with ``path`` and ``location``.
         """
         if not parent:
-            raise TypeError('Cannot create resource without parent.')
+            raise TypeError(f'Cannot create resource without parent: name: {name!r}, path: {path!r}')
 
         # If the codebase is virtual, we provide the path
         if not path:
@@ -1461,7 +1533,7 @@ def strip_first_path_segment(path):
         >>> strip_first_path_segment('foo/')
         'foo/'
     """
-    segments = paths.split(path)
+    segments = paths_split(path)
     if not segments or len(segments) == 1:
         return path
     stripped = segments[1:]
@@ -1566,7 +1638,7 @@ def __init__(
         self.location = location
 
         scan_data = self._get_scan_data(location)
-        self.paths = self._clean_paths(paths)
+        self.paths = self._prepare_clean_paths(paths)
         self._populate(scan_data)
 
     def _get_scan_data_helper(self, location):
@@ -1754,7 +1826,7 @@ def _populate(self, scan_data):
 
         for fdata in files_data:
             sample_resource_data_update(fdata)
-            segments =  fdata['path'].split('/')
+            segments = fdata['path'].split('/')
             root_names_add(segments[0])
             fdata['path_segments'] = segments
 
@@ -1816,12 +1888,21 @@ def _populate(self, scan_data):
                 setattr(root, name, value)
 
         if TRACE: logger_debug('VirtualCodebase.populate: root:', root)
+
+        # TODO: report error if filtering the root with a paths?
         self.save_resource(root)
 
         if self.has_single_resource:
             if TRACE: logger_debug('VirtualCodebase.populate: with single resource.')
             return
 
+        all_paths = None
+        if self.paths:
+            # build a set of all all paths and all their ancestors
+            all_paths = set()
+            for path in self.paths:
+                all_paths.update(get_ancestor_paths(path, include_self=True))
+
         # Create other Resources from scan info
 
         # Note that we do not know the ordering there.
@@ -1833,10 +1914,15 @@ def _populate(self, scan_data):
         duplicated_paths = set()
         last_path = None
         for fdata in files_data:
+            path = fdata.get('path')
+
+            # skip the ones we did not request
+            if all_paths and path not in all_paths:
+                continue
+
             # these are no longer needed
             path_segments = fdata.pop('path_segments')
 
-            path = fdata.get('path')
             if not last_path:
                 last_path = path
             elif last_path == path:
@@ -1935,3 +2021,27 @@ def remove_properties_and_basics(resource_data):
     mapping with the known properties removed.
     """
     return {k: v for k, v in resource_data.items() if k not in KNOW_PROPS}
+
+
+def get_ancestor_paths(path, include_self=False):
+    """
+    Yield all subpaths from a POSIX path.
+
+    For example::
+    >>> path = 'foo/bar/baz'
+    >>> results = list(get_ancestor_paths(path))
+    >>> assert results == ['foo', 'foo/bar'], results
+    >>> results = list(get_ancestor_paths(path, include_self=True))
+    >>> assert results == ['foo', 'foo/bar', 'foo/bar/baz'], results
+    >>> results = list(get_ancestor_paths('foo', include_self=False))
+    >>> assert results == [], results
+    """
+    assert path
+    segments = path.split('/')
+    if not include_self:
+        segments = segments[:-1]
+    subpath = []
+    for segment in segments:
+        subpath.append(segment)
+        yield '/'.join(subpath)
+

tests/data/resource/with_path/codebase-expected.json

@@ -0,0 +1,17 @@
+[
+  {
+    "path": "codebase",
+    "type": "directory",
+    "scan_errors": []
+  },
+  {
+    "path": "codebase/other dir",
+    "type": "directory",
+    "scan_errors": []
+  },
+  {
+    "path": "codebase/other dir/file",
+    "type": "file",
+    "scan_errors": []
+  }
+]

tests/data/resource/with_path/codebase/abc

@@ -0,0 +1,47 @@
+/* Copyright © 2005 Agere Systems Inc.
+ * All rights reserved.
+ *   http://www.agere.com
+ *
+ * SOFTWARE LICENSE
+ *
+ * This software is provided subject to the following terms and conditions,
+ * which you should read carefully before using the software.  Using this
+ * software indicates your acceptance of these terms and conditions.  If you do
+ * not agree with these terms and conditions, do not use the software.
+ *
+ * Copyright © 2005 Agere Systems Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source or binary forms, with or without
+ * modifications, are permitted provided that the following conditions are met:
+ *
+ * . Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following Disclaimer as comments in the code as
+ *    well as in the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * . Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following Disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * . Neither the name of Agere Systems Inc. nor the names of the contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * Disclaimer
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, INFRINGEMENT AND THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  ANY
+ * USE, MODIFICATION OR DISTRIBUTION OF THIS SOFTWARE IS SOLELY AT THE USERS OWN
+ * RISK. IN NO EVENT SHALL AGERE SYSTEMS INC. OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, INCLUDING, BUT NOT LIMITED TO, CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ */
+