Merge pull request #381 from vbatts/vbatts-xattr_support

tarfile: pax header and xattr support

Author: dmp42

docker_registry/images.py

@@ -3,7 +3,6 @@
 import datetime
 import functools
 import logging
-import tarfile
 import time
 
 import flask
@@ -20,6 +19,9 @@
 from .lib import checksums
 from .lib import layers
 from .lib import mirroring
+# this is our monkey patched snippet from python v2.7.6 'tarfile'
+# with xattr support
+from .lib.xtarfile import tarfile
 
 
 store = storage.load()

docker_registry/lib/layers.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import logging
-import tarfile
 import tempfile
 
 import backports.lzma as lzma
@@ -11,6 +10,10 @@
 from .. import storage
 from . import cache
 from . import rqueue
+# this is our monkey patched snippet from python v2.7.6 'tarfile'
+# with xattr support
+from .xtarfile import tarfile
+
 
 store = storage.load()
 

docker_registry/lib/xtarfile.py

@@ -0,0 +1,81 @@
+'''
+This is __proc_pax from ./Lib/tarfile.py from v2.7.6
+catching raw (non-utf8) bytes to support some xattr headers in tar archives
+
+This is for the use-case of reading the tar archive, not for the use case of
+interacting with inodes on the filesystem that have xattr's.
+ -- vbatts
+'''
+
+import re
+import tarfile
+
+
+def _proc_pax(self, filetar):
+    """Process an extended or global header as described in
+      POSIX.1-2001.
+   """
+    # Read the header information.
+    buf = filetar.fileobj.read(self._block(self.size))
+
+    # A pax header stores supplemental information for either
+    # the following file (extended) or all following files
+    # (global).
+    if self.type == tarfile.XGLTYPE:
+        pax_headers = filetar.pax_headers
+    else:
+        pax_headers = filetar.pax_headers.copy()
+
+    # Parse pax header information. A record looks like that:
+    # "%d %s=%s\n" % (length, keyword, value). length is the size
+    # of the complete record including the length field itself and
+    # the newline. keyword and value are both UTF-8 encoded strings.
+    regex = re.compile(r"(\d+) ([^=]+)=", re.U)
+    pos = 0
+    while True:
+        match = regex.match(buf, pos)
+        if not match:
+            break
+
+        length, keyword = match.groups()
+        length = int(length)
+        value = buf[match.end(2) + 1:match.start(1) + length - 1]
+
+        try:
+            keyword = keyword.decode("utf8")
+        except Exception:
+            # just leave the raw bytes
+            pass
+
+        try:
+            value = value.decode("utf8")
+        except Exception:
+            # just leave the raw bytes
+            pass
+
+        pax_headers[keyword] = value
+        pos += length
+
+    # Fetch the next header.
+    try:
+        next = self.fromtarfile(filetar)
+    except tarfile.HeaderError:
+        raise tarfile.SubsequentHeaderError("missing or bad subsequent header")
+
+    if self.type in (tarfile.XHDTYPE, tarfile.SOLARIS_XHDTYPE):
+        # Patch the TarInfo object with the extended header info.
+        next._apply_pax_info(pax_headers, filetar.encoding, filetar.errors)
+        next.offset = self.offset
+
+        if "size" in pax_headers:
+            # If the extended header replaces the size field,
+            # we need to recalculate the offset where the next
+            # header starts.
+            offset = next.offset_data
+            if next.isreg() or next.type not in tarfile.SUPPORTED_TYPES:
+                offset += next._block(next.size)
+            filetar.offset = offset
+
+    return next
+
+tarfile.TarInfo._proc_pax = _proc_pax

tests/base.py

@@ -3,12 +3,15 @@
 import docker_registry.run as run
 
 import hashlib
+import os
 import random
 import string
 import unittest
 
 from docker_registry.core import compat
 
+data_dir = os.path.join(os.path.dirname(__file__), "data")
+
 
 class TestCase(unittest.TestCase):
 

tests/data/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json

@@ -0,0 +1 @@
+{"id":"46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457","parent":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","created":"2014-04-07T02:45:52.610504484Z","container":"e0f07f8d72cae171a3dcc35859960e7e956e0628bce6fedc4122bf55b2c287c7","container_config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","sed -ri 's/^(%wheel.*)(ALL)$/\\1NOPASSWD: \\2/' /etc/sudoers"],"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.9.1-dev","config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":3425}

tests/data/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar

@@ -0,0 +1 @@
+{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}

tests/data/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json

@@ -0,0 +1 @@
+{"id":"4439c3c7f847954100b42b267e7e5529cac1d6934db082f65795c5ca2e594d93","parent":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","created":"2014-05-16T17:19:44.091534414Z","container":"5f92fb06cc58f357f0cde41394e2bbbb664e663974b2ac1693ab07b7a306749b","container_config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","setcap 'cap_setgid,cap_setuid+ep' ./file \u0026\u0026 getcap ./file"],"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.11.1-dev","config":{"Hostname":"9565c6517a0e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"73b164f4437db87e96e90083c73a6592f549646ae2ec00ed33c6b9b49a5c4470","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":0}