#387 Initial support to extract XAR archives and Apple pkg or mpkg

	* added minimal tests
	
Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/extractcode/archive.py

@@ -51,11 +51,13 @@
 
 
 logger = logging.getLogger(__name__)
-DEBUG = False
-DEBUG_DEEP = False
-# import sys
-# logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
-# logger.setLevel(logging.DEBUG)
+TRACE = False
+TRACE_DEEP = False
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
 
 
 
@@ -176,15 +178,16 @@ def get_handlers(location):
             mime_matched = handler.mimetypes and any(m in mtype for m in handler.mimetypes)
             extension_matched = handler.extensions and location.lower().endswith(handler.extensions)
 
-            if DEBUG_DEEP:
-                logger.debug('get_handlers: %(location)s: ftype: %(ftype)s, mtype: %(mtype)s ' % locals())
+            if TRACE_DEEP:
+                handler_name = handler.name
+                logger.debug('get_handlers: considering %(handler_name)r  handler for %(location)s: ftype: %(ftype)s, mtype: %(mtype)s ' % locals())
                 logger.debug('get_handlers: %(location)s: matched type: %(type_matched)s, mime: %(mime_matched)s, ext: %(extension_matched)s' % locals())
 
             if handler.strict and not all([type_matched, mime_matched, extension_matched]):
                 continue
 
             if type_matched or mime_matched or extension_matched:
-                if DEBUG_DEEP:
+                if TRACE_DEEP:
                     logger.debug('get_handlers: %(location)s: matched type: %(type_matched)s, mime: %(mime_matched)s, ext: %(extension_matched)s' % locals())
                     logger.debug('get_handlers: %(location)s: handler: %(handler)r' % locals())
                 yield handler, type_matched, mime_matched, extension_matched
@@ -300,7 +303,7 @@ def extract_twice(location, target_dir, extractor1, extractor2):
     # extract first the intermediate payload to a temp dir
     temp_target = unicode(fileutils.get_temp_dir('extract'))
     warnings = extractor1(abs_location, temp_target)
-    if DEBUG:
+    if TRACE:
         logger.debug('extract_twice: temp_target: %(temp_target)r' % locals())
 
     # extract this intermediate payload to the final target_dir
@@ -310,7 +313,7 @@ def extract_twice(location, target_dir, extractor1, extractor2):
             warnings.append(location + ': No files found in archive.')
         else:
             for extracted1_loc in inner_archives:
-                if DEBUG:
+                if TRACE:
                     logger.debug('extract_twice: extractor2: %(extracted1_loc)r' % locals())
                 warnings.extend(extractor2(extracted1_loc, abs_target_dir))
     finally:
@@ -334,14 +337,14 @@ def extract_with_fallback(location, target_dir, extractor1, extractor2):
     temp_target1 = unicode(fileutils.get_temp_dir('extract1'))
     try:
         warnings = extractor1(abs_location, temp_target1)
-        if DEBUG:
+        if TRACE:
             logger.debug('extract_with_fallback: temp_target1: %(temp_target1)r' % locals())
         fileutils.copytree(temp_target1, abs_target_dir)
     except:
         try:
             temp_target2 = unicode(fileutils.get_temp_dir('extract2'))
             warnings = extractor2(abs_location, temp_target2)
-            if DEBUG:
+            if TRACE:
                 logger.debug('extract_with_fallback: temp_target2: %(temp_target2)r' % locals())
             fileutils.copytree(temp_target2, abs_target_dir)
         finally:
@@ -375,6 +378,7 @@ def extract_with_fallback(location, target_dir, extractor1, extractor2):
 extract_nsis = sevenzip.extract
 extract_ishield = sevenzip.extract
 extract_Z = sevenzip.extract
+extract_xarpkg = sevenzip.extract
 
 
 # Archive handlers.
@@ -799,6 +803,26 @@ def extract_with_fallback(location, target_dir, extractor1, extractor2):
     strict=True
 )
 
+ApplePkgHandler = Handler(
+    name='Apple pkg or mpkg package installer',
+    filetypes=('xar archive',),
+    mimetypes=('application/octet-stream',),
+    extensions=('.pkg', '.mpkg',),
+    kind=package,
+    extractors=[extract_xarpkg],
+    strict=True
+)
+
+XarHandler = Handler(
+    name='Xar archive v1',
+    filetypes=('xar archive',),
+    mimetypes=('application/octet-stream',),
+    extensions=('.xar',),
+    kind=package,
+    extractors=[extract_xarpkg],
+    strict=True
+)
+
 IsoImageHandler = Handler(
     name='ISO CD image',
     filetypes=('iso 9660 cd-rom', 'high sierra cd-rom',),
@@ -858,6 +882,8 @@ def extract_with_fallback(location, target_dir, extractor1, extractor2):
     RarHandler,
     CabHandler,
     MsiInstallerHandler,
+    ApplePkgHandler,
+    XarHandler,
     # notes: this may catch all exe and fails too often
     InstallShieldHandler,
     NSISInstallerHandler,

src/extractcode/extract.py

@@ -37,10 +37,12 @@
 from extractcode import archive
 
 logger = logging.getLogger(__name__)
-DEBUG = False
-# import sys
-# logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
-# logger.setLevel(logging.DEBUG)
+TRACE = False
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
 
 
 """
@@ -118,46 +120,46 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False):
     first before resuming the file system walk.
     """
     ignored = partial(ignore.is_ignored, ignores=ignore.default_ignores, unignores={})
-    if DEBUG:
+    if TRACE:
         logger.debug('extract:start: %(location)r  recurse: %(recurse)r\n' % locals())
     abs_location = abspath(expanduser(location))
     for top, dirs, files in fileutils.walk(abs_location, ignored):
-        if DEBUG:
+        if TRACE:
             logger.debug('extract:walk: top:  %(top)r dirs: %(dirs)r files: r(files)r' % locals())
 
         if not recurse:
-            if DEBUG:
+            if TRACE:
                 drs = set(dirs)
             for d in dirs[:]:
                 if extractcode.is_extraction_path(d):
                     dirs.remove(d)
-            if DEBUG:
+            if TRACE:
                 logger.debug('extract:walk: not recurse: removed dirs:' + repr(drs.symmetric_difference(set(dirs))))
         for f in files:
             loc = join(top, f)
             if not recurse and extractcode.is_extraction_path(loc):
-                if DEBUG:
+                if TRACE:
                     logger.debug('extract:walk not recurse: skipped  file: %(loc)r' % locals())
                 continue
 
             if not archive.should_extract(loc, kinds):
-                if DEBUG:
+                if TRACE:
                     logger.debug('extract:walk: skipped file: not should_extract: %(loc)r' % locals())
                 continue
 
             target = join(abspath(top), extractcode.get_extraction_path(loc))
-            if DEBUG:
+            if TRACE:
                 logger.debug('extract:target: %(target)r' % locals())
             for xevent in extract_file(loc, target, kinds):
-                if DEBUG:
+                if TRACE:
                     logger.debug('extract:walk:extraction event: %(xevent)r' % locals())
                 yield xevent
 
             if recurse:
-                if DEBUG:
+                if TRACE:
                     logger.debug('extract:walk: recursing on target: %(target)r' % locals())
                 for xevent in extract(target, kinds, recurse):
-                    if DEBUG:
+                    if TRACE:
                         logger.debug('extract:walk:recurse:extraction event: %(xevent)r' % locals())
                     yield xevent
 
@@ -170,8 +172,8 @@ def extract_file(location, target, kinds=extractcode.default_kinds):
     warnings = []
     errors = []
     extractor = archive.get_extractor(location, kinds)
-    if DEBUG:
-        logger.debug('extract_file: extractor: for: %(location)r with kinds: r(kinds)r : ' % locals()
+    if TRACE:
+        logger.debug('extract_file: extractor: for: %(location)r with kinds: %(kinds)r : ' % locals()
                      + getattr(extractor, '__module__', '')
                      + '.' + getattr(extractor, '__name__', ''))
     if extractor:
@@ -181,12 +183,12 @@ def extract_file(location, target, kinds=extractcode.default_kinds):
             # if there is an error,  the extracted files will not be moved
             # to target
             tmp_tgt = fileutils.get_temp_dir('extract')
-            abs_location= abspath(expanduser(location))
+            abs_location = abspath(expanduser(location))
             warnings.extend(extractor(abs_location, tmp_tgt))
             fileutils.copytree(tmp_tgt, target)
             fileutils.delete(tmp_tgt)
         except Exception, e:
-            if DEBUG:
+            if TRACE:
                 logger.debug('extract_file: ERROR: %(location)r: %(errors)r, %(e)r.\n' % locals())
             errors = [str(e).strip(' \'"')]
         finally:

tests/extractcode/data/archive/xar/origin.ABOUT

@@ -0,0 +1,2 @@
+date: 2016-12-03
+download_url: https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/xar/xar-1.4.xar

tests/extractcode/data/archive/xar/xar-1.4.xar

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
+# Copyright (c) 2016 nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/scancode-toolkit/
 # The ScanCode software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode require an acknowledgment.
@@ -91,6 +91,7 @@ def test_get_extractors(self):
             ('archive/7z/z.7z', [archive.extract_7z]),
             ('archive/Z/tr2tex.Z', [archive.extract_Z, ]),
             ('archive/Z/tkWWW-0.11.tar.Z', [archive.extract_Z, archive.extract_tar]),
+            ('archive/xar/xar-1.4.xar', [archive.extract_xarpkg]),
         ]
 
         for test_file, expected in test_data:
@@ -1863,3 +1864,14 @@ def test_extract_z_compress_basic(self):
         archive.extract_Z(test_file, test_dir)
         result = os.path.join(test_dir, 'tr2tex')
         assert os.path.exists(result)
+
+
+class TestXar(BaseArchiveTestCase):
+    def test_extract_xar_basic(self):
+        test_file = self.get_test_loc('archive/xar/xar-1.4.xar')
+        test_dir = self.get_temp_dir()
+        archive.extract_Z(test_file, test_dir)
+        result = os.path.join(test_dir, '[TOC].xml')
+        assert os.path.exists(result)
+        result = os.path.join(test_dir, 'xar-1.4', 'Makefile.in')
+        assert os.path.exists(result)