xattrs: monkey patch, instead of vendor

Docker-DCO-1.1-Signed-off-by: Vincent Batts <[email protected]> (github: vbatts)

Author: vbatts

docker_registry/images.py

@@ -19,7 +19,10 @@
 from .lib import checksums
 from .lib import layers
 from .lib import mirroring
-from .vendor import xtarfile
+from .lib import xtarfile
+
+
+tarfile = xtarfile.tarfile
 
 
 store = storage.load()
@@ -244,13 +247,13 @@ def put_image_layer(image_id):
         tarsum = checksums.TarSum(json_data)
         try:
             tmp.seek(0)
-            tar = xtarfile.open(mode='r|*', fileobj=tmp)
+            tar = tarfile.open(mode='r|*', fileobj=tmp)
             tarfilesinfo = layers.TarFilesInfo()
             for member in tar:
                 tarsum.append(member, tar)
                 tarfilesinfo.append(member)
             layers.set_image_files_cache(image_id, tarfilesinfo.json())
-        except (IOError, xtarfile.TarError) as e:
+        except (IOError, tarfile.TarError) as e:
             logger.debug('put_image_layer: Error when reading Tar stream '
                          'tarsum. Disabling TarSum, TarFilesInfo. '
                          'Error: {0}'.format(e))
@@ -433,7 +436,7 @@ def get_private_image_files(image_id, headers):
         return toolkit.response(data, headers=headers, raw=True)
     except exceptions.FileNotFoundError:
         return toolkit.api_error('Image not found', 404)
-    except xtarfile.TarError:
+    except tarfile.TarError:
         return toolkit.api_error('Layer format not supported', 400)
 
 
@@ -452,7 +455,7 @@ def get_image_files(image_id, headers):
         return toolkit.response(data, headers=headers, raw=True)
     except exceptions.FileNotFoundError:
         return toolkit.api_error('Image not found', 404)
-    except xtarfile.TarError:
+    except tarfile.TarError:
         return toolkit.api_error('Layer format not supported', 400)
 
 
@@ -479,5 +482,5 @@ def get_image_diff(image_id, headers):
         return toolkit.response(diff_json, headers=headers, raw=True)
     except exceptions.FileNotFoundError:
         return toolkit.api_error('Image not found', 404)
-    except xtarfile.TarError:
+    except tarfile.TarError:
         return toolkit.api_error('Layer format not supported', 400)

docker_registry/lib/layers.py

@@ -10,25 +10,28 @@
 from .. import storage
 from . import cache
 from . import rqueue
-# this is our vendored 'tarfile' from python v2.7.6, with xattr support
-from ..vendor import xtarfile
+# this is our monkey patched 'tarfile' from python v2.7.6, with xattr support
+from . import xtarfile
+
+
+tarfile = xtarfile.tarfile
 
 
 store = storage.load()
 
 FILE_TYPES = {
-    xtarfile.REGTYPE: 'f',
-    xtarfile.AREGTYPE: 'f',
-    xtarfile.LNKTYPE: 'l',
-    xtarfile.SYMTYPE: 's',
-    xtarfile.CHRTYPE: 'c',
-    xtarfile.BLKTYPE: 'b',
-    xtarfile.DIRTYPE: 'd',
-    xtarfile.FIFOTYPE: 'i',
-    xtarfile.CONTTYPE: 't',
-    xtarfile.GNUTYPE_LONGNAME: 'L',
-    xtarfile.GNUTYPE_LONGLINK: 'K',
-    xtarfile.GNUTYPE_SPARSE: 'S',
+    tarfile.REGTYPE: 'f',
+    tarfile.AREGTYPE: 'f',
+    tarfile.LNKTYPE: 'l',
+    tarfile.SYMTYPE: 's',
+    tarfile.CHRTYPE: 'c',
+    tarfile.BLKTYPE: 'b',
+    tarfile.DIRTYPE: 'd',
+    tarfile.FIFOTYPE: 'i',
+    tarfile.CONTTYPE: 't',
+    tarfile.GNUTYPE_LONGNAME: 'L',
+    tarfile.GNUTYPE_LONGLINK: 'K',
+    tarfile.GNUTYPE_SPARSE: 'S',
 }
 
 logger = logging.getLogger(__name__)
@@ -182,7 +185,7 @@ def get_image_files_from_fobj(layer_file):
     '''
     layer_file.seek(0)
     archive_file = Archive(layer_file)
-    tar_file = xtarfile.open(fileobj=archive_file)
+    tar_file = tarfile.open(fileobj=archive_file)
     files = read_tarfile(tar_file)
     return files
 

docker_registry/lib/xtarfile.py

@@ -0,0 +1,86 @@
+'''
+This is __proc_pax from ./Lib/tarfile.py from v2.7.6
+catching raw (non-utf8) bytes to support some xattr headers in tar archives
+
+This is for the use-case of reading the tar archive, not for the use case of
+interacting with inodes on the filesystem that have xattr's.
+ -- vbatts
+'''
+
+import re
+import tarfile
+
+tarfile.PAX_FIELDS = tarfile.PAX_FIELDS + ("SCHILY.xattr.",)
+tarfile.PAX_NUMBER_FIELDS["SCHILY.xattr."] = dict
+
+# class XattrTarInfo(tarfile.TarInfo):
+
+
+def _proc_pax(self, filetar):
+    """Process an extended or global header as described in
+      POSIX.1-2001.
+   """
+    # Read the header information.
+    buf = filetar.fileobj.read(self._block(self.size))
+
+    # A pax header stores supplemental information for either
+    # the following file (extended) or all following files
+    # (global).
+    if self.type == tarfile.XGLTYPE:
+        pax_headers = filetar.pax_headers
+    else:
+        pax_headers = filetar.pax_headers.copy()
+
+    # Parse pax header information. A record looks like that:
+    # "%d %s=%s\n" % (length, keyword, value). length is the size
+    # of the complete record including the length field itself and
+    # the newline. keyword and value are both UTF-8 encoded strings.
+    regex = re.compile(r"(\d+) ([^=]+)=", re.U)
+    pos = 0
+    while True:
+        match = regex.match(buf, pos)
+        if not match:
+            break
+
+        length, keyword = match.groups()
+        length = int(length)
+        value = buf[match.end(2) + 1:match.start(1) + length - 1]
+
+        try:
+            keyword = keyword.decode("utf8")
+        except Exception:
+            # just leave the raw bytes
+            pass
+
+        try:
+            value = value.decode("utf8")
+        except Exception:
+            # just leave the raw bytes
+            pass
+
+        pax_headers[keyword] = value
+        pos += length
+
+    # Fetch the next header.
+    try:
+        next = self.fromtarfile(filetar)
+    except tarfile.HeaderError:
+        raise tarfile.SubsequentHeaderError("missing or bad subsequent header")
+
+    if self.type in (tarfile.XHDTYPE, tarfile.SOLARIS_XHDTYPE):
+        # Patch the TarInfo object with the extended header info.
+        next._apply_pax_info(pax_headers, filetar.encoding, filetar.errors)
+        next.offset = self.offset
+
+        if "size" in pax_headers:
+            # If the extended header replaces the size field,
+            # we need to recalculate the offset where the next
+            # header starts.
+            offset = next.offset_data
+            if next.isreg() or next.type not in tarfile.SUPPORTED_TYPES:
+                offset += next._block(next.size)
+            filetar.offset = offset
+
+    return next
+
+tarfile.TarInfo._proc_pax = _proc_pax