Fixed #22 - Add support to collect redistributable sources

Author: chinyeungli

docs/CHANGELOG.rst

@@ -2,6 +2,7 @@
     Release 5.1.0
 
     * Add support for `package_url` #396
+    * Add support to collect redistributable sources #22
 
 2020-08-11
     Release 5.0.0

src/attributecode/model.py

@@ -935,8 +935,9 @@ def process(self, fields, about_file_path, running_inventory=False,
         errors = self.hydrate(fields)
         # We want to copy the license_files before the validation
         if reference_dir:
-            copy_license_notice_files(
+            copy_err = copy_license_notice_files(
                 fields, base_dir, reference_dir, afp)
+            errors.extend(copy_err)
 
         # TODO: why? we validate all fields, not only these hydrated
         validation_errors = validate_fields(
@@ -1278,21 +1279,26 @@ def copy_redist_src(abouts, location, output):
     errors = []
     for about in abouts:
         if about.redistribute.value:
+            file_exist = True
             for e in about.errors:
                 if 'Field about_resource' in e.message and 'not found' in e.message:
                     msg = e.message + u' and cannot be copied.'
                     errors.append(Error(CRITICAL, msg))
+                    file_exist = False
                     continue
-            for k in about.about_resource.value:
-                from_path = about.about_resource.value.get(k)
-                norm_from_path = norm(from_path)
-                relative_from_path = norm_from_path.partition(util.norm(location))[2]
-                # Need to strip the '/' to use the join
-                if relative_from_path.startswith('/'):
-                    relative_from_path = relative_from_path.partition('/')[2]
-                # Get the directory name of the output path
-                output_dir = os.path.dirname(os.path.join(output, util.norm(relative_from_path)))
-                copy_file(from_path, output_dir)
+            if file_exist:
+                for k in about.about_resource.value:
+                    from_path = about.about_resource.value.get(k)
+                    norm_from_path = norm(from_path)
+                    relative_from_path = norm_from_path.partition(util.norm(location))[2]
+                    # Need to strip the '/' to use the join
+                    if relative_from_path.startswith('/'):
+                        relative_from_path = relative_from_path.partition('/')[2]
+                    # Get the directory name of the output path
+                    output_dir = os.path.dirname(os.path.join(output, util.norm(relative_from_path)))
+                    err = copy_file(from_path, output_dir)
+                    if err:
+                        errors.extend(err)
     return errors
 
 

src/attributecode/util.py

@@ -436,6 +436,7 @@ def copy_license_notice_files(fields, base_dir, reference_dir, afp):
     about_file_path value, this function will copy to the base_dir the
     license_file or notice_file if found in the reference_dir
     """
+    errors = []
     copy_file_name = ''
     for key, value in fields:
         if key == 'license_file' or key == 'notice_file':
@@ -470,13 +471,16 @@ def copy_license_notice_files(fields, base_dir, reference_dir, afp):
                 about_file_dir = os.path.dirname(to_posix(afp)).lstrip('/')
                 to_lic_path = posixpath.join(to_posix(base_dir), about_file_dir)
 
-                copy_file(from_lic_path, to_lic_path)
+                err = copy_file(from_lic_path, to_lic_path)
+                if err:
+                    errors.append(err)
+    return errors
 
 
 def copy_file(from_path, to_path):
     # Return if the from_path is empty or None.
     if not from_path:
-        return
+        return []
 
     if on_windows:
         if not from_path.startswith(UNC_PREFIXES):
@@ -490,7 +494,7 @@ def copy_file(from_path, to_path):
 
     # Errors will be captured when doing the validation
     if not posixpath.exists(from_path):
-        return
+        return []
 
     if not posixpath.exists(to_path):
         os.makedirs(to_path)
@@ -501,14 +505,17 @@ def copy_file(from_path, to_path):
             to_path = os.path.join(to_path, folder_name)
             # Since we need to copy everything along with the directory structure,
             # making sure the directory does not exist will not hurt.
-            shutil.rmtree(to_path)
+            if os.path.exists(to_path):
+                shutil.rmtree(to_path)
             # Copy the directory recursively along with its structure 
             shutil.copytree(from_path, to_path)
         else:
             shutil.copy2(from_path, to_path)
+        return []
     except Exception as e:
-        print(repr(e))
-        print('Cannot copy file at %(from_path)r.' % locals())
+        msg = 'Cannot copy file at %(from_path)r.' % locals()
+        error = [Error(CRITICAL, msg)]
+        return error
 
 
 # FIXME: we should use a license object instead

tests/test_model.py

@@ -1210,6 +1210,21 @@ def test_collect_inventory_does_not_convert_lf_to_crlf_from_directory(self):
         expected = get_test_loc('test_model/crlf/expected.csv')
         check_csv(expected, result, fix_cell_linesep=True, regen=False)
 
+    def test_copy_redist_src(self):
+        test_loc = get_test_loc('test_model/redistribution')
+        output = get_temp_dir()
+        errors, abouts = model.collect_inventory(test_loc)
+        expected_file = ['this.c']
+
+        err = model.copy_redist_src(abouts, test_loc, output)
+        assert err == []
+
+        from os import listdir
+        copied_files = listdir(output)
+        assert len(expected_file) == len(copied_files)
+        assert err == []
+        for file in expected_file:
+            assert file in copied_files
 
 class FetchLicenseTest(unittest.TestCase):
 

tests/test_util.py

@@ -628,3 +628,37 @@ def test_copy_license_notice_files(self):
         assert len(licenses) == len(copied_files)
         for license in licenses:
             assert license in copied_files
+
+    def test_copy_file(self):
+        des = get_temp_dir()
+        test_file = get_test_loc('test_util/licenses/mit.LICENSE')
+        licenses = ['mit.LICENSE']
+        err = util.copy_file(test_file, des)
+        from os import listdir
+        copied_files = listdir(des)
+        assert len(licenses) == len(copied_files)
+        assert err == []
+        for license in licenses:
+            assert license in copied_files
+
+    def test_copy_file_with_dir(self):
+        des = get_temp_dir()
+        test_dir = get_test_loc('test_util/licenses/')
+        licenses = ['mit.LICENSE', 'mit2.LICENSE', 'public-domain.LICENSE']
+        err = util.copy_file(test_dir, des)
+        assert err == []
+
+        import os
+        files_list = []
+        dir_list = []
+        # Get the directories and files in the 'des' recursively
+        for root, dir, files in os.walk(des):
+            for d in dir:
+                dir_list.append(d)
+            for f in files:
+                files_list.append(f)
+
+        assert dir_list == [u'licenses']
+        assert len(licenses) == len(files_list)
+        for license in licenses:
+            assert license in files_list

tests/testdata/test_cmd/help/about_help.txt

@@ -13,9 +13,12 @@ Options:
   -h, --help  Show this message and exit.
 
 Commands:
-  attrib     Generate an attribution document from .ABOUT files.
-  check      Validate that the format of .ABOUT files is correct and report
-             errors and warnings.
-  gen        Generate .ABOUT files from an inventory as CSV or JSON.
-  inventory  Collect the inventory of .ABOUT files to a CSV or JSON file.
-  transform  Transform a CSV/JSON by applying renamings, filters and checks.
+  attrib              Generate an attribution document from .ABOUT files.
+  check               Validate that the format of .ABOUT files is correct and
+                      report errors and warnings.
+  collect_redist_src  Collect redistributable sources.
+  gen                 Generate .ABOUT files from an inventory as CSV or JSON.
+  inventory           Collect the inventory of .ABOUT files to a CSV or JSON
+                      file.
+  transform           Transform a CSV/JSON by applying renamings, filters and
+                      checks.

tests/testdata/test_model/redistribution/this.ABOUT

@@ -0,0 +1,4 @@
+about_resource: this.c
+name: this.c
+version: 0.11.0
+redistribute: x

tests/testdata/test_model/redistribution/this.c

@@ -0,0 +1,4 @@
+about_resource: this.c
+name: this.c
+version: 0.11.0
+redistribute: N