tarfile: pax header and xattr support

tar archive have additional headers that store file data like security
capabilities and ACLs. These values can be packed structures, not utf8
encoded.
Python's ( < v3.3) tarfile does not support these xattr headers, and has
encoding issues on the values stored. This change includes using a
vendored version of python-2.7.6's tarfile.py, with changes to support
this.

Docker-DCO-1.1-Signed-off-by: Vincent Batts <[email protected]> (github: vbatts)

Author: vbatts

docker_registry/images.py

@@ -3,7 +3,7 @@
 import datetime
 import functools
 import logging
-import tarfile
+import xtarfile
 import time
 
 import flask
@@ -244,13 +244,13 @@ def put_image_layer(image_id):
         tarsum = checksums.TarSum(json_data)
         try:
             tmp.seek(0)
-            tar = tarfile.open(mode='r|*', fileobj=tmp)
+            tar = xtarfile.open(mode='r|*', fileobj=tmp)
             tarfilesinfo = layers.TarFilesInfo()
             for member in tar:
                 tarsum.append(member, tar)
                 tarfilesinfo.append(member)
             layers.set_image_files_cache(image_id, tarfilesinfo.json())
-        except (IOError, tarfile.TarError) as e:
+        except (IOError, xtarfile.TarError) as e:
             logger.debug('put_image_layer: Error when reading Tar stream '
                          'tarsum. Disabling TarSum, TarFilesInfo. '
                          'Error: {0}'.format(e))
@@ -433,7 +433,7 @@ def get_private_image_files(image_id, headers):
         return toolkit.response(data, headers=headers, raw=True)
     except exceptions.FileNotFoundError:
         return toolkit.api_error('Image not found', 404)
-    except tarfile.TarError:
+    except xtarfile.TarError:
         return toolkit.api_error('Layer format not supported', 400)
 
 
@@ -452,7 +452,7 @@ def get_image_files(image_id, headers):
         return toolkit.response(data, headers=headers, raw=True)
     except exceptions.FileNotFoundError:
         return toolkit.api_error('Image not found', 404)
-    except tarfile.TarError:
+    except xtarfile.TarError:
         return toolkit.api_error('Layer format not supported', 400)
 
 
@@ -479,5 +479,5 @@ def get_image_diff(image_id, headers):
         return toolkit.response(diff_json, headers=headers, raw=True)
     except exceptions.FileNotFoundError:
         return toolkit.api_error('Image not found', 404)
-    except tarfile.TarError:
+    except xtarfile.TarError:
         return toolkit.api_error('Layer format not supported', 400)

docker_registry/lib/layers.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import logging
-import tarfile
 import tempfile
 
 import backports.lzma as lzma
@@ -12,21 +11,25 @@
 from . import cache
 from . import rqueue
 
+# this is our vendored 'tarfile' from python v2.7.6, with xattr support
+import xtarfile
+
+
 store = storage.load()
 
 FILE_TYPES = {
-    tarfile.REGTYPE: 'f',
-    tarfile.AREGTYPE: 'f',
-    tarfile.LNKTYPE: 'l',
-    tarfile.SYMTYPE: 's',
-    tarfile.CHRTYPE: 'c',
-    tarfile.BLKTYPE: 'b',
-    tarfile.DIRTYPE: 'd',
-    tarfile.FIFOTYPE: 'i',
-    tarfile.CONTTYPE: 't',
-    tarfile.GNUTYPE_LONGNAME: 'L',
-    tarfile.GNUTYPE_LONGLINK: 'K',
-    tarfile.GNUTYPE_SPARSE: 'S',
+    xtarfile.REGTYPE: 'f',
+    xtarfile.AREGTYPE: 'f',
+    xtarfile.LNKTYPE: 'l',
+    xtarfile.SYMTYPE: 's',
+    xtarfile.CHRTYPE: 'c',
+    xtarfile.BLKTYPE: 'b',
+    xtarfile.DIRTYPE: 'd',
+    xtarfile.FIFOTYPE: 'i',
+    xtarfile.CONTTYPE: 't',
+    xtarfile.GNUTYPE_LONGNAME: 'L',
+    xtarfile.GNUTYPE_LONGLINK: 'K',
+    xtarfile.GNUTYPE_SPARSE: 'S',
 }
 
 logger = logging.getLogger(__name__)
@@ -180,7 +183,7 @@ def get_image_files_from_fobj(layer_file):
     '''
     layer_file.seek(0)
     archive_file = Archive(layer_file)
-    tar_file = tarfile.open(fileobj=archive_file)
+    tar_file = xtarfile.open(fileobj=archive_file)
     files = read_tarfile(tar_file)
     return files