Merge branch '86_genattrib_text_update' into develop

chinyeungli · chinyeungli · commit fb614c07e3c2 · 2014-05-19T14:55:46.000-07:00
Conflicts:
	about_code_tool/about.py
	about_code_tool/templates/default.html

The common_license should use the DJE license name instead of the name that we used for the confirmed license.

In addition, we should use the dje_license_name instead of the dje_license.
diff --git a/about_code_tool/about.py b/about_code_tool/about.py
@@ -156,6 +156,7 @@ def repr_problem(obj):
     'dje_component',
     'dje_license',
     'dje_organization',
+    'dje_license_name'
 )
 
 OPTIONAL_FIELDS = BASIC_FIELDS + OWNERSHIP_FIELDS + LICENSE_FIELDS + \
@@ -1097,7 +1098,8 @@ def generate_attribution(self, template_path='templates/default.html',
         license_key = []
         license_text = []
         license_dict = {}
-        common_license = ['GPL 2.0','OpenSSL/SSLeay License', 'Apache 2.0', 'BSD-Modified', 'CC-BY-SA-3.0']
+        common_license = ['GNU General Public License 2.0','OpenSSL/SSLeay License', 'Apache License 2.0', 'BSD-Modified']
+
         for about_object in self:
             about_relative_path = '/'+ about_object.location.partition(self.user_provided_path)[2]
             if not limit_to or about_relative_path in limit_to:
diff --git a/about_code_tool/genabout.py b/about_code_tool/genabout.py
@@ -57,7 +57,7 @@
 
 # The 'dje_license_key' will be removed and will use the 'dje_license' instead.
 SUPPORTED_FIELDS = about.OPTIONAL_FIELDS + about.MANDATORY_FIELDS + \
-    ('about_file', 'dje_license_key',)
+    ('about_file',)
 
 Warn = namedtuple('Warn', 'field_name field_value message',)
 Error = namedtuple('Error', 'field_name field_value message',)
@@ -232,6 +232,8 @@ def request_license_data(self, url, username, api_key, license_key):
         full_url = '{0}{1}/?{2}'.format(
             url if url.endswith('/') else url + '/',
             license_key, urllib.urlencode(payload))
+        # The following is to handle special characters in URL such as space etc. 
+        full_url = urllib.quote(full_url, safe="%/:=&?~#+!$,;'@()*[]")
 
         try:
             request = urllib2.Request(full_url)
@@ -247,7 +249,7 @@ def request_license_data(self, url, username, api_key, license_key):
                 self.extract_dje_license_error = True
                 self.errors.append(Error('username/key', username + '/' + api_key, error_msg))
             else:
-                self.errors.append(Error('dje_license_key', license_key, "Invalid 'dje_license_key'"))
+                self.errors.append(Error('dje_license', license_key, "Invalid 'dje_license_key'"))
             return {}
         except urllib2.URLError:
             if about.check_network_connection():
@@ -281,19 +283,22 @@ def copy_license_files(gen_location, license_list):
     def write_licenses(self, license_context_list):
         for gen_license_path, license_context in license_context_list:
             try:
+                if not _exists(dirname(gen_license_path)):
+                    makedirs(dirname(gen_license_path))
                 with open(gen_license_path, 'wb') as output:
                     output.write(license_context)
             except Exception as e:
                 self.errors.append(Error('Unknown', gen_license_path, "Something is wrong."))
 
-    def get_license_text_from_api(self, url, username, api_key, license_key):
+    def get_license_details_from_api(self, url, username, api_key, license_key):
         """
         Returns the license_text of a given license_key using an API request.
         Returns an empty string if the text is not available.
         """
         data = self.request_license_data(url, username, api_key, license_key)
+        license_name = data.get('name', '')
         license_text = data.get('full_text', '')
-        return license_text
+        return [license_name, license_text]
 
     def get_dje_license_list(self, gen_location, input_list, gen_license):
         license_output_list = []
@@ -311,22 +316,49 @@ def get_dje_license_list(self, gen_location, input_list, gen_license):
                         self.errors.append(Error('license_text_file', license_file, "The 'license_text_file' does not exist."))
                 else:
                     if gen_license:
-                        if line['dje_license_key']:
+                        if line['dje_license']:
                             license_output_list.append(self.gen_license_list(line))
                         else:
-                            self.warnings.append(Warn('dje_license_key', '',
-                                                      "Missing 'dje_license_key' for " + line['about_file']))
+                            self.warnings.append(Warn('dje_license', '',
+                                                      "Missing 'dje_license' for " + line['about_file']))
             # This except condition will force the tool to create the
             # 'license_text_file' key column from the self.gen_license_list(line)
             except Exception as e:
                 if gen_license:
-                    if line['dje_license_key']:
+                    if line['dje_license']:
                         license_output_list.append(self.gen_license_list(line))
                     else:
-                        self.warnings.append(Warn('dje_license_key', '',
-                                                  "Missing 'dje_license_key' for " + line['about_file']))
+                        self.warnings.append(Warn('dje_license', '',
+                                                  "Missing 'dje_license' for " + line['about_file']))
         return license_output_list
 
+    def pre_process_and_dje_license_dict(self, input_list, api_url, api_username, api_key):
+        key_text_dict = {}
+        for line in input_list:
+            try:
+                if line['dje_license']:
+                    detail = self.get_license_details_from_api(api_url, api_username, api_key, line['dje_license'])
+                    line['dje_license_name'], key_text_dict[line['dje_license']] = detail
+            except Exception as e:
+                self.warnings.append(Warn('dje_license', '',
+                                                  "Missing 'dje_license' for " + line['about_file']))
+        return key_text_dict
+
+    def process_dje_licenses(self, dje_license_list, dje_license_dict, output_path):
+        license_list_context = []
+        for gen_path, license_key in dje_license_list:
+            if gen_path.startswith('/'):
+                gen_path = gen_path.partition('/')[2]
+            gen_license_path = join(output_path, gen_path, license_key) + '.LICENSE'
+            if not _exists(gen_license_path) and not self.extract_dje_license_error:
+                context = dje_license_dict[license_key]
+                if context:
+                    gen_path_context = []
+                    gen_path_context.append(gen_license_path)
+                    gen_path_context.append(context.encode('utf8'))
+                    license_list_context.append(gen_path_context)
+        return license_list_context
+
     def pre_generation(self, gen_location, input_list, action_num, all_in_one):
         output_list = []
         for line in input_list:
@@ -372,7 +404,7 @@ def pre_generation(self, gen_location, input_list, action_num, all_in_one):
 
     @staticmethod
     def gen_license_list(line):
-        dje_key = line['dje_license_key']
+        dje_key = line['dje_license']
         file_location = line['about_file']
         if file_location.endswith('/'):
             file_location = file_location.rpartition('/')[0]
@@ -636,38 +668,34 @@ def main(parser, options, args):
             sys.exit(errno.EINVAL)
         for line in input_list:
             try:
-                if line['dje_license_key']:
+                if line['dje_license']:
                     break
             except Exception as e:
                 print(repr(e))
-                print("The input does not have the 'dje_license_key' key which is required.")
+                print("The input does not have the 'dje_license' key which is required.")
                 sys.exit(errno.EINVAL)
 
     dje_license_list = gen.get_dje_license_list(output_path, input_list, gen_license)
+
+    # The dje_license_list is an empty list if gen_license is 'False'
+    if gen_license:
+        dje_license_dict = gen.pre_process_and_dje_license_dict(input_list,
+                                                                    api_url,
+                                                                    api_username,
+                                                                    api_key)
+        license_list_context = gen.process_dje_licenses(dje_license_list,
+                                                        dje_license_dict,
+                                                        output_path)
+        gen.write_licenses(license_list_context)
+
     components_list = gen.pre_generation(output_path, input_list, action_num, all_in_one)
     formatted_output = gen.format_output(components_list)
     gen.write_output(formatted_output)
 
-    if dje_license_list:
-        license_list_context = []
-        for gen_path, license_key in dje_license_list:
-            if gen_path.startswith('/'):
-                gen_path = gen_path.partition('/')[2]
-            gen_license_path = join(output_path, gen_path, license_key) + '.LICENSE'
-            if not _exists(gen_license_path) and not gen.extract_dje_license_error:
-                context = gen.get_license_text_from_api(api_url, api_username, api_key, license_key)
-                if context:
-                    gen_path_context = []
-                    gen_path_context.append(gen_license_path)
-                    gen_path_context.append(context.encode('utf8'))
-                    license_list_context.append(gen_path_context)
-        gen.write_licenses(license_list_context)
-
     gen.warnings_errors_summary()
     print('Warnings: %s' % len(gen.warnings))
     print('Errors: %s' % len(gen.errors))
 
-
 def get_parser():
     class MyFormatter(optparse.IndentedHelpFormatter):
         def _format_text(self, text):
diff --git a/about_code_tool/templates/default.html b/about_code_tool/templates/default.html
@@ -51,8 +51,8 @@ <h1>OPEN SOURCE SOFTWARE INFORMATION</h1>
                 <h3 class="component-name">{{ about_object.name }}
                     {% if about_object.version %}{{ about_object.version }}{% endif %}
                 </h3>
-                {% if about_object.dje_license in license_dicts.keys() %} 
-                    <p>This component is licensed under {{about_object.dje_license }}.
+                {% if about_object.dje_license_name in license_dicts.keys() %} 
+                    <p>This component is licensed under {{about_object.dje_license_name }}.
                 {% endif %}
                 {% if about_object.copyright %}
                     <pre>{{about_object.copyright}}</pre>
@@ -62,18 +62,21 @@ <h3 class="component-name">{{ about_object.name }}
                 {% elif about_object.notice_file %}
                     <pre class="component-notice">{{ notice_texts[loop.index0] }}</pre>
                 {% endif %}
-                {% if about_object.dje_license in license_dicts.keys() %} 
-                    {% if about_object.dje_license in common_licenses %}
+                {% if about_object.dje_license_name in license_dicts.keys() %}
+                    {% if about_object.dje_license_name in common_licenses %}
                         <p>Full text of
-                            <a class="{{ about_object.dje_license }}" href="#component-license-{{ about_object.dje_license }}">
-                             {{ about_object.dje_license}}
-                            </a>
+                            <a class="{{ about_object.dje_license_name }}" href="#component-license-{{ about_object.dje_license_name }}">
+                             {{ about_object.dje_license_name }}
+                             </a>
                                 is available at the end of this document.</p>
                     {% else %}
-                        <pre>{{ license_dicts[about_object.dje_license]|e }}</pre>
+                        <pre>{{ license_dicts[about_object.dje_license_name]|e }}</pre>
                     {% endif %}
                 {% elif about_object.license_text_file in license_dicts.keys() %}
-                    {% if about_object.license_text_file in common_licenses %}
+                    {% if about_object.dje_license_name in common_licenses %}
+                        <p>
+                            This component is licensed under {{ about_object.dje_license_name }}.
+                        </p>
                         <p>Full text of
                             <a class="{{ about_object.license_text_file }}" href="#component-license-{{ about_object.license_text_file }}">
                              {{ about_object.license_text_file }}
diff --git a/about_code_tool/tests/test_about.py b/about_code_tool/tests/test_about.py
@@ -94,7 +94,7 @@ def test_header_row_in_csv_output(self):
         'license_spdx,redistribute,attribute,track_changes,vcs_tool,' \
         'vcs_repository,vcs_path,vcs_tag,vcs_branch,vcs_revision,' \
         'checksum_sha1,checksum_md5,checksum_sha256,dje_component,' \
-        'dje_license,dje_organization,warnings,errors'
+        'dje_license,dje_organization,dje_license_name,warnings,errors'
 
         input = "about_code_tool/tests/testdata/basic"
         temp_file = tempfile.NamedTemporaryFile(suffix='.csv', delete=True)
diff --git a/about_code_tool/tests/test_genabout.py b/about_code_tool/tests/test_genabout.py
@@ -231,7 +231,7 @@ def test_get_dje_license_list_gen_license_with_dje_license_key_empty_license_tex
         gen_location = join(TESTDATA_PATH, "test_files_for_genabout/")
         input_list = [{'about_file': '/about.py.ABOUT', 'version': '0.8.1',
                         'about_resource': '.', 'name': 'ABOUT tool',
-                        'license_text_file': '', 'dje_license_key': 'apache-2.0'}]
+                        'license_text_file': '', 'dje_license': 'apache-2.0'}]
         expected_output_list = [('/', 'apache-2.0')]
         gen_license = True
         lic_output_list = gen.get_dje_license_list(gen_location, input_list, gen_license)
@@ -244,7 +244,7 @@ def test_get_dje_license_list_gen_license_with_empty_dje_license_key_empty_licen
         gen_location = join(TESTDATA_PATH, "test_files_for_genabout/")
         input_list = [{'about_file': '/about.py.ABOUT', 'version': '0.8.1',
                         'about_resource': '.', 'name': 'ABOUT tool',
-                        'license_text_file': '', 'dje_license_key': ''}]
+                        'license_text_file': '', 'dje_license': ''}]
         expected_output_list = []
         gen_license = True
         lic_output_list = gen.get_dje_license_list(gen_location, input_list, gen_license)
@@ -257,7 +257,7 @@ def test_get_dje_license_list_gen_license_with_dje_license_key_no_license_text_f
         gen_location = join(TESTDATA_PATH, "test_files_for_genabout/")
         input_list = [{'about_file': '/about.py.ABOUT', 'version': '0.8.1',
                         'about_resource': '.', 'name': 'ABOUT tool',
-                        'dje_license_key': 'apache-2.0'}]
+                        'dje_license': 'apache-2.0'}]
         expected_output_list = [('/', 'apache-2.0')]
         gen_license = True
         lic_output_list = gen.get_dje_license_list(gen_location, input_list, gen_license)
@@ -433,7 +433,7 @@ def test_verify_license_files_no_key(self):
     def test_gen_license_list_license_text_file_no_value(self):
         gen = genabout.GenAbout()
         input_list = {'about_file': '/tmp/3pp/opensans/', 'name': 'OpenSans Fonts',
-                       'version': '1', 'dje_license_key': 'apache-2.0',
+                       'version': '1', 'dje_license': 'apache-2.0',
                        'license_text_file': '', 'about_resource': 'opensans'}
         expected_list = ('/tmp/3pp', 'apache-2.0')
         output = gen.gen_license_list(input_list)
@@ -443,7 +443,7 @@ def test_gen_license_list_license_text_file_no_value(self):
     def test_gen_license_list_no_license_text_file_key(self):
         gen = genabout.GenAbout()
         input_list = {'about_file': '/tmp/3pp/opensans/', 'name': 'OpenSans Fonts',
-                       'version': '1', 'dje_license_key': 'apache-2.0',
+                       'version': '1', 'dje_license': 'apache-2.0',
                        'about_resource': 'opensans'}
         expected_list = ('/tmp/3pp', 'apache-2.0')
         output = gen.gen_license_list(input_list)
@@ -487,3 +487,11 @@ def test_write_licenses(self):
                 context = line
         self.assertTrue(context == tmp_license_context)
 
+    def test_process_dje_licenses(self):
+        gen = genabout.GenAbout()
+        test_license_list = [('/', 'test')]
+        test_license_dict = {'test': u'This is a test license.'}
+        test_path = '/test'
+        expected_output = [['/test/test.LICENSE', 'This is a test license.']]
+        output = gen.process_dje_licenses(test_license_list, test_license_dict, test_path)
+        self.assertTrue(output == expected_output)
