Merge branch '108_options_to_include_non_supported_keys' into develop

chinyeungli · chinyeungli · commit f5a0e855fe18 · 2014-08-20T14:40:07.000-07:00
Conflicts:
	about_code_tool/tests/test_about.py
	about_code_tool/tests/test_genabout.py

Fixed all the test cases conflicts
diff --git a/about_code_tool.egg-info/PKG-INFO b/about_code_tool.egg-info/PKG-INFO
@@ -0,0 +1,22 @@
+Metadata-Version: 1.1
+Name: about-code-tool
+Version: 0.9.0
+Summary: Document the provenance (origin and license) of third-party software. Collect inventories, generate attribution docs.
+Home-page: http://dejacode.org
+Author: Jillian Daguil, Thomas Druez, Chin-Yeung Li, Philippe Ombredanne and others.
+Author-email: info@nexb.com
+License: Apache License 2.0
+Description: AboutCode provides a simple way to document theprovenance (i.e. origin and license) and other important orinteresting information about third-party software components thatyou use in your project. This documentation is stored in *.ABOUTfiles, side-by-side with the code they document.
+Platform: any
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Natural Language :: English
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Documentation
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: System :: Software Distribution
+Classifier: Topic :: Utilities
diff --git a/about_code_tool.egg-info/SOURCES.txt b/about_code_tool.egg-info/SOURCES.txt
@@ -0,0 +1,15 @@
+README.rst
+setup.py
+about_code_tool/__init__.py
+about_code_tool/about.py
+about_code_tool/genabout.py
+about_code_tool/genattrib.py
+about_code_tool.egg-info/PKG-INFO
+about_code_tool.egg-info/SOURCES.txt
+about_code_tool.egg-info/dependency_links.txt
+about_code_tool.egg-info/not-zip-safe
+about_code_tool.egg-info/top_level.txt
+about_code_tool/tests/__init__.py
+about_code_tool/tests/test_about.py
+about_code_tool/tests/test_genabout.py
+about_code_tool/tests/test_genattrib.py
diff --git a/about_code_tool.egg-info/dependency_links.txt b/about_code_tool.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/about_code_tool.egg-info/not-zip-safe b/about_code_tool.egg-info/not-zip-safe
@@ -0,0 +1 @@
+
diff --git a/about_code_tool.egg-info/top_level.txt b/about_code_tool.egg-info/top_level.txt
@@ -0,0 +1 @@
+about_code_tool
diff --git a/about_code_tool/MAPPING.CONFIG b/about_code_tool/MAPPING.CONFIG
@@ -6,15 +6,10 @@
 #   i.e.
 #   python genabout.py --mapping [Input File] [Generated Location]
 #
-#
-# The current convention is to append the '.ABOUT' after the filename, OR
-# replace the last '/' with the '.ABOUT' for directories.
-#
-#
 # Below are the fields that are needed in the genabout.py
-#   - Essential Fields: Fields that are essential for the genabout.py to execute.
+#   - Essential Fields: Fields that are essential for the ABOUT Code.
 #   - Mandatory Fields: These are the mandatory fields for the ABOUT specification. 
-#       Although the genabout.py will still be able to generate ABOUT files if these fields are blank,
+#       Although ABOUT Code will still be able to generate ABOUT files if these fields are blank,
 #       it's highly recommend to have these fields filled in.
 #
 # How to example:
@@ -25,7 +20,15 @@
 #   name: Component
 #   version: file_version
 #
+# In addition, if there are fields that you want to put in the ABOUT
+# files, please use the 'Custom Fields' to include these fields.
+#
+#   Note: All the Custom Field's keys will be converted to lower case and 
+#         all the spaces will be replaced by '_'
+#
+#
 #   See http://www.dejacode.org/about_spec_v0.8.1.html for more information
+#
 
 # Essential Fields
 about_file: Directory/Filename
@@ -35,4 +38,8 @@ name: Component
 version: Confirmed Version
 
 # Optional Fields
-copyright: Confirmed Copyright
+copyright: Confirmed Copyright
+
+
+# Custom Fields
+confirmed_license: Confirmed License
diff --git a/about_code_tool/about.py b/about_code_tool/about.py
@@ -223,8 +223,7 @@ def repr_problem(obj):
 
 HEADER_ROW_FIELDS = (('about_file',)
                      + MANDATORY_FIELDS
-                     + OPTIONAL_FIELDS
-                     + ERROR_WARN_FIELDS)
+                     + OPTIONAL_FIELDS)
 
 
 # SPDX License Identifiers from http://spdx.org/licenses/
@@ -720,10 +719,9 @@ def validate(self):
 
         for field_name, value in self.validated_fields.items():
             self.check_is_ascii(self.validated_fields.get(field_name))
-            self.validate_known_optional_fields(field_name)
+            #self.validate_known_optional_fields(field_name)
             self.validate_file_field_exists(field_name, value)
             self.validate_url_field(field_name, network_check=False)
-
             self.validate_spdx_license(field_name, value)
             self.check_date_format(field_name)
 
@@ -953,17 +951,32 @@ def check_url_reachable(host, path):
             # http://en.wikipedia.org/wiki/List_of_HTTP_status_codes
             return conn.getresponse().status
 
-    def get_row_data(self, updated_path):
+    def get_custom_field_keys(self):
+        custom_key = []
+        for key in self.validated_fields:
+            if key not in MANDATORY_FIELDS + OPTIONAL_FIELDS:
+                custom_key.append(key)
+        return custom_key
+
+    def get_row_data(self, updated_path, custom_keys):
         """
         Create a csv compatible row of data for this object.
         """
         row = [updated_path]
+        custom_field = []
         for field in MANDATORY_FIELDS + OPTIONAL_FIELDS:
             if field in self.validated_fields:
                 row += [self.validated_fields[field]]
             else:
                 row += ['']
 
+        # Add custom field value
+        for key in custom_keys:
+            try:
+                row += [self.validated_fields[key]]
+            except:
+                row += ['']
+
         warnings = [repr(w) for w in self.warnings]
         errors = [repr(e) for e in self.errors]
         row += ['\n'.join(warnings), '\n'.join(errors)]
@@ -1180,18 +1193,33 @@ def get_relative_path2(self, about_object_location):
         else:
             return user_provided_path.replace('\\', '/')
 
+    def custom_keys(self):
+        custom_keys = []
+        for about_object in self:
+            keys = about_object.get_custom_field_keys()
+            for key in keys:
+                if key not in custom_keys:
+                    custom_keys.append(key)
+        return custom_keys
+
     def write_to_csv(self, output_path):
         """
         Build a row for each about instance and write results in CSV file
         located at `output_path`.
         """
+        custom_keys = self.custom_keys()
         with open(output_path, 'wb') as output_file:
             csv_writer = csv.writer(output_file)
-            csv_writer.writerow(HEADER_ROW_FIELDS)
+            header_row = HEADER_ROW_FIELDS
+            # Add the non-supported fields if exist
+            for key in custom_keys:
+                header_row += (key, )
+            header_row += ERROR_WARN_FIELDS
+            csv_writer.writerow(header_row)
 
             for about_object in self:
                 relative_path = self.get_relative_path(about_object.location)
-                row_data = about_object.get_row_data(relative_path)
+                row_data = about_object.get_row_data(relative_path, custom_keys)
                 csv_writer.writerow(row_data)
 
     def generate_attribution(self,
diff --git a/about_code_tool/about_code_tool.egg-info/PKG-INFO b/about_code_tool/about_code_tool.egg-info/PKG-INFO
@@ -0,0 +1,25 @@
+Metadata-Version: 1.1
+Name: about-code-tool
+Version: 0.9.0
+Summary: Document the origin of third-party software.
+Home-page: http://dejacode.org
+Author: Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez
+Author-email: info@nexb.com
+License: Apache License 2.0
+Description: The ABOUT tool and ABOUT files provide a simple way
+            to document the provenance (origin and license) and other important or
+            interesting information about third-party software components that you use
+            in your project.
+Platform: any
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Natural Language :: English
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Documentation
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: System :: Software Distribution
+Classifier: Topic :: Utilities
diff --git a/about_code_tool/about_code_tool.egg-info/SOURCES.txt b/about_code_tool/about_code_tool.egg-info/SOURCES.txt
diff --git a/about_code_tool/about_code_tool.egg-info/dependency_links.txt b/about_code_tool/about_code_tool.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/about_code_tool/about_code_tool.egg-info/not-zip-safe b/about_code_tool/about_code_tool.egg-info/not-zip-safe
@@ -0,0 +1 @@
+
diff --git a/about_code_tool/about_code_tool.egg-info/top_level.txt b/about_code_tool/about_code_tool.egg-info/top_level.txt
@@ -0,0 +1 @@
+about_code_tool
diff --git a/about_code_tool/genabout.py b/about_code_tool/genabout.py
@@ -132,6 +132,10 @@ def get_mapping_list():
                     if not line.startswith('#') and ':' in line:
                         about_spec_key = line.partition(':')[0]
                         user_spec_key = line.partition(':')[2].strip()
+                        # Handle cases which keys contain spaces
+                        if about_spec_key.endswith(' '):
+                            about_spec_key = about_spec_key.strip()
+                        about_spec_key = about_spec_key.replace(' ', '_')
                         mapping_list[about_spec_key.lower()] = user_spec_key.lower()
         except Exception as e:
             print(repr(e))
@@ -208,12 +212,12 @@ def get_only_supported_fields(self, input_list, ignored_keys_list):
         return copied_list
 
     @staticmethod
-    def get_non_supported_fields(input_list):
+    def get_non_supported_fields(input_list, mapping_keys):
         """
         Returns a list of the non-supported fields in a given line.
         """
         first_line = input_list[0]
-        return [field for field in first_line.keys() if not field in SUPPORTED_FIELDS]
+        return [field for field in first_line.keys() if not field in SUPPORTED_FIELDS and not field in mapping_keys]
 
     def verify_files_existence(self, input_list, project_dir, file_in_project):
         """
@@ -716,6 +720,7 @@ def main(parser, options, args):
     api_key = ''
     gen_license = False
     dje_license_dict = {}
+    mapping_keys = []
 
     if options.version:
         print('ABOUT tool {0}\n{1}'.format(__version__, __copyright__))
@@ -807,10 +812,11 @@ def main(parser, options, args):
     if mapping_config:
         mapping_list = gen.get_mapping_list()
         input_list = gen.convert_input_list(input_list, mapping_list)
+        mapping_keys = mapping_list.keys()
 
     gen.validate(input_list)
 
-    ignored_fields_list = gen.get_non_supported_fields(input_list)
+    ignored_fields_list = gen.get_non_supported_fields(input_list, mapping_keys)
     if ignored_fields_list:
         input_list = gen.get_only_supported_fields(input_list, ignored_fields_list)
 
diff --git a/about_code_tool/tests/test_about.py b/about_code_tool/tests/test_about.py
@@ -95,7 +95,9 @@ def test_header_row_in_csv_output(self):
         'license_spdx,redistribute,attribute,track_changes,vcs_tool,'\
         'vcs_repository,vcs_path,vcs_tag,vcs_branch,vcs_revision,'\
         'checksum_sha1,checksum_md5,checksum_sha256,dje_component,'\
-        'dje_license,dje_organization,dje_license_name,warnings,errors'
+        'dje_license,dje_organization,dje_license_name,scm_branch,'\
+        'scm_repository,signature_gpg_file,redistribute_sources,about_format,'\
+        'usage,scm_tool,scm_path,scm_tag,scm_rev,organization,warnings,errors'
 
         test_file = 'about_code_tool/tests/testdata/basic'
         temp_file = tempfile.NamedTemporaryFile(suffix='.csv', delete=True)
@@ -132,8 +134,9 @@ def test_collector_errors_encapsulation(self):
     def test_collector_warnings_encapsulation(self):
         test_file = 'about_code_tool/tests/testdata/allAboutInOneDir'
         collector = about.Collector(test_file)
-        self.assertEqual(4, len(collector.warnings))
-
+        #self.assertEqual(4, len(collector.warnings))
+        # No warning is thrown as all fields from ABOUT files are accepted.
+        self.assertEqual(0, len(collector.warnings))
 
 class ParserTest(unittest.TestCase):
     def test_valid_chars_in_field_name(self):
@@ -597,3 +600,16 @@ def test_notice_text_returns_empty_string_when_ref_file_doesnt_exist(self):
         about_file = about.AboutFile(join(TESTDATA_DIR, 'attrib/missing_notice_license_files.ABOUT'))
         notice_text = about_file.notice_text()
         self.assertEqual(notice_text, expected)
+
+
+class OtherTest(unittest.TestCase):
+    def test_get_custom_field_keys(self):
+        about_file = about.AboutFile(join(TESTDATA_DIR, 'basic/basic.about'))
+        custom_keys = about_file.get_custom_field_keys()
+        expected_keys = ['scm_branch', 'scm_repository', 'signature_gpg_file',
+                         'redistribute_sources', 'about_format', 'usage',
+                         'scm_tool', 'scm_path', 'scm_tag', 'scm_rev',
+                         'organization']
+        self.assertEqual(custom_keys, expected_keys)
+
+
diff --git a/about_code_tool/tests/test_genabout.py b/about_code_tool/tests/test_genabout.py
@@ -58,13 +58,13 @@ def test_get_non_empty_rows_list(self):
         output = gen.get_non_empty_rows_list(input_list)
         self.assertEquals(output, expected_list)
 
-
     def test_get_mapping_list(self):
         gen = genabout.GenAbout()
         expected_list = {'about_file': 'directory/filename',
                           'version': 'confirmed version',
-                           'name': 'component',
-                           'copyright': 'confirmed copyright'}
+                          'name': 'component',
+                          'copyright': 'confirmed copyright',
+                          'confirmed_license': 'confirmed license'}
         output = gen.get_mapping_list()
         self.assertEquals(output, expected_list)
 
@@ -156,17 +156,28 @@ def test_validate_mandatory_fields_missing_about_resource(self):
         result = gen.validate_mandatory_fields(input_list)
         self.assertTrue(result)
 
-    def test_get_non_supported_fields(self):
+    def test_get_non_supported_fields_no_mapping(self):
         gen = genabout.GenAbout()
         test_fields = [{'about_file': '',
                         'name': 'OpenSans Fonts',
                         'non_supported field': 'TEST',
                         'version': '1',
                         'about_resource': 'opensans'}]
-        result = gen.get_non_supported_fields(test_fields)
+        mapping_keys = []
+        result = gen.get_non_supported_fields(test_fields, mapping_keys)
         expected = ['non_supported field']
         self.assertEquals(expected, result)
 
+    def test_get_non_supported_fields_with_mapping(self):
+            gen = genabout.GenAbout()
+            input = [{'about_file': '', 'name': 'OpenSans Fonts',
+                     'non_supported field': 'TEST', 'version': '1',
+                     'about_resource': 'opensans'}]
+            mapping_keys = ['non_supported field']
+            non_supported_list = gen.get_non_supported_fields(input, mapping_keys)
+            expected_list = []
+            self.assertEquals(non_supported_list, expected_list)
+
     def test_get_only_supported_fields(self):
         gen = genabout.GenAbout()
         input_list = [{'about_file': '/about.ABOUT',
