Merge pull request #430 from sschuberth/spdx-tv

Add basic SPDX tag/value (and also RDF) output for #338

Author: pombredanne

setup.py

@@ -100,6 +100,7 @@ def read(*names, **kwargs):
         'MarkupSafe >= 0.23',
         'colorama',
         'simplejson',
+        'spdx-tools',
 
         # packagedcode
         'requests >= 2.7.0, < 3.0.0',

src/scancode/cli.py

@@ -62,7 +62,6 @@ def wrap(self, timeout=None):
 
 import click
 from click.termui import style
-import simplejson as json
 
 from commoncode import filetype
 from commoncode import fileutils
@@ -132,8 +131,6 @@ def wrap(self, timeout=None):
   Visit https://github.com/nexB/scancode-toolkit/ for support and download.
 '''
 
-acknowledgment_text_json = acknowledgment_text.strip().replace('  ', '')
-
 
 extra_notice_text = '''
 
@@ -248,7 +245,7 @@ class ScanCommand(utils.BaseCommand):
 Try 'scancode --help' for help on options and arguments.'''
 
 
-formats = ('json', 'html', 'html-app',)
+formats = ('json', 'html', 'html-app', 'spdx-tv', 'spdx-rdf')
 
 def validate_formats(ctx, param, value):
     value_lower = value.lower()
@@ -640,13 +637,64 @@ def save_results(files_count, scanned_files, format, input, output_file):
             echo_stderr('\nFailed to create HTML app.', fg='red')
 
     elif format == 'json':
+        import simplejson as json
+
         meta = OrderedDict()
-        meta['scancode_notice'] = acknowledgment_text_json
+        meta['scancode_notice'] = acknowledgment_text.strip().replace('  ', '')
         meta['scancode_version'] = version
         meta['files_count'] = files_count
         # TODO: add scanning options to meta
         meta['files'] = scanned_files
         output_file.write(unicode(json.dumps(meta, indent=2 * ' ', iterable_as_array=True, encoding='utf-8')))
         output_file.write('\n')
+
+    elif format == 'spdx-tv' or format == 'spdx-rdf':
+        from spdx.checksum import Algorithm
+        from spdx.creationinfo import Tool
+        from spdx.document import Document, License
+        from spdx.file import File
+        from spdx.package import Package
+        from spdx.utils import NoAssert
+        from spdx.version import Version
+
+        doc = Document(Version(2, 1), License.from_identifier('CC0-1.0'))
+
+        doc.creation_info.add_creator(Tool('ScanCode ' + version))
+        doc.creation_info.set_created_now()
+
+        doc.package = Package(input, NoAssert())
+
+        for file_data in scanned_files:
+            file_entry = File(file_data['path'])
+            # FIXME: should we really compue the checcksum here rather than get it from the scan?
+            file_entry.chk_sum = Algorithm('SHA1', file_entry.calc_chksum())
+            for file_license in file_data['licenses']:
+                spdx_id = file_license.get('spdx_license_key')
+                # TODO: we should create a "LicenseRef:xxx" identifier 
+                # if the license is not known to SPDX
+                if spdx_id:
+                    spdx_license = License.from_identifier(spdx_id)
+                    file_entry.add_lics(spdx_license)
+                    doc.package.add_lics_from_file(spdx_license)
+
+            file_entry.conc_lics = NoAssert()
+            file_entry.copyright = NoAssert()
+            doc.package.add_file(file_entry)
+
+        # Remove duplicate licenses from the list.
+        doc.package.licenses_from_files = list(set(doc.package.licenses_from_files))
+
+        doc.package.verif_code = doc.package.calc_verif_code()
+        doc.package.cr_text = NoAssert()
+        doc.package.license_declared = NoAssert()
+        doc.package.conc_lics = NoAssert()
+
+        if format == 'spdx-tv':
+            from spdx.writers.tagvalue import write_document
+            write_document(doc, output_file)
+        else:
+            from spdx.writers.rdf import write_document
+            write_document(doc, output_file)
+
     else:
         raise Exception('Unknown format')

tests/textcode/data/markup_expected/Label.html

@@ -72,7 +72,7 @@
   //-->
 
 
-  <A HREF="../../allclasses-noframe.html"><B>All Classes</B></A>
+  All Classes
 
 
 
@@ -622,7 +622,7 @@
   //-->
 
 
-  <A HREF="../../allclasses-noframe.html"><B>All Classes</B></A>
+  All Classes
 
 
 
@@ -639,4 +639,6 @@
 
 
 
-Submit a bug or featureFor further API reference and developer documentation, see Java 2 SDK SE Developer Documentation. That documentation contains more detailed, developer-targeted descriptions, with conceptual overviews, definitions of terms, workarounds, and working code examples. 
+Submit a bug or featureFor further API reference and developer documentation, see Java 2 SDK SE Developer Documentation. That documentation contains more detailed, developer-targeted descriptions, with conceptual overviews, definitions of terms, workarounds, and working code examples. Copyright 2004 Sun Microsystems, Inc. All rights reserved. Use is subject to license terms. Also see the documentation redistribution policy.
+
+

tests/textcode/data/markup_expected/chinese.htm

@@ -29,7 +29,7 @@
 
   var dmtrack_c='{-}'; var dmtrack_pageid='bc69c0cd0a93681e1429438175';
 (function(d){var t=d.createElement('script');t.type='text/javascript';t.async=true;t.src='http://style.c.aliimg.com/sys/js/beacon/cnb.js';d.getElementsByTagName('head')[0].appendChild(t);}(document)); 
-<img src="http://dmtracking.1688.com/b.jpg?cD0yJnU9e3d3dy4xNjg4LmNvbVwvfSZtPXtHRVR9JnM9ezIwMH0mcj17LX0mYT17LX0mYj0tJmM9ey19Cg==&ver=40&pageid=bc69c0cd0a93681e1429438175&time=1429438175" width="1" height="1" style="display:none"> 
+ 
        var _SPM_a='{-}'; var _SPM_b='{-}';var _SPM_app_name='{-}';var _SPM_template_path='{-}';
 
 with(document)with(body)with(insertBefore(createElement("script"),firstChild))setAttribute("exparams","category=&userid=&aplus&&asid=AADffjNVzesW9Lwhfic=&aat=&abi=141%2e62%2e60%2e13%2e1427127568230%2e412828%2e7&abb=&",id="tb-beacon-aplus",src=(location>"https"?"//g":"//g")+".alicdn.com/alilog/mlog/aplus_b2b.js")

tests/textcode/data/markup_expected/services.htm

@@ -17,7 +17,7 @@
 
 
 
-	<iframe src="//www.googletagmanager.com/ns.html?id=GTM-TWT88B" height="0" width="0" style="display:none;visibility:hidden"></iframe>
+	
 
 
 

…e/setuptools-25.2.0-py2.py3-none-any.whl …e/setuptools-32.3.1-py2.py3-none-any.whlthirdparty/base/setuptools-25.2.0-py2.py3-none-any.whl renamed to thirdparty/base/setuptools-32.3.1-py2.py3-none-any.whl thirdparty/base/setuptools-25.2.0-py2.py3-none-any.whl renamed to thirdparty/base/setuptools-32.3.1-py2.py3-none-any.whl

@@ -1,8 +1,8 @@
-about_resource: setuptools-25.2.0-py2.py3-none-any.whl
+about_resource: setuptools-32.3.1-py2.py3-none-any.whl
 name: setuptools
-version: 25.2.0
+version: 32.3.1
 
-download_url: setuptools-25.2.0-py2.py3-none-any.whl
+download_url: https://pypi.python.org/packages/69/19/b1dff551058ce79d88b1e3688f1c735590d7ddf44d10681512133b35019f/setuptools-32.3.1-py2.py3-none-any.whl#md5=9fe4e32f20a9b13c206c1bdc4c9feaf4
 
 home_url: https://pypi.python.org/pypi/setuptools
 owner: Python Packaging Authority

thirdparty/base/setuptools.ABOUT

@@ -0,0 +1,19 @@
+Copyright (C) 2016 Jason R Coombs <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

thirdparty/base/setuptools.LICENSE

@@ -0,0 +1,14 @@
+about_resource: SPARQLWrapper-1.8.0-cp27-none-any.whl
+name: SPARQLWrapper
+version: 1.8.0
+home_url: http://rdflib.github.io/sparqlwrapper/
+download_url: https://pypi.python.org/packages/03/69/3a625d5315a85174c4a0ab9b9afc0018c21a618117de141dc53e14d5de97/SPARQLWrapper-1.8.0.tar.gz#md5=9541c7d6f01ed2a98efb4b86bbc982e8
+authors: Ivan Herman, Sergio Fernández, Carlos Tejo Alonso, Alexey Zakhlestin
+owner: World Wide Web Consortium <http://www.w3.org> Foundation CTIC <http://www.fundacionctic.org/>
+copyright: Copyright (c) 2002-2015, RDFLib Team
+
+vcs_tool: git
+vcs_repository: https://github.com/rdflib/sparqlwrapper.git
+
+dje_license: w3c
+license_text_file: SPARQLWrapper.LICENSE