Merge pull request #380 from dotcloud/0.7-fixes

0.7 fixes

Author: dmp42

CHANGELOG.md

@@ -10,6 +10,7 @@ Major architecture rehaul, with potentially breaking changes:
  * updated and cleaned-up Dockerfile now uses latest Ubuntu LTS
  * largely enhanced configuration mechanism (setup-configs.sh is no more)
  * cookies are no longer used
+ * CORS is now enabled on a number of endpoints
 
  * [BUGFIX] unicode issues. Depending on the storage driver you are using, you may encounter unicode issues on already published content (likely garbled content).
  * [BUGFIX] content-length fix for bytes ranges

CONTRIBUTE.md

@@ -99,7 +99,10 @@ Pretty much:
  * you have to use the provided test suite
  * all the tests must pass
 
+### Word of warning
 
+Drivers are expected to receive bytes and to return bytes.
+Don't try to decode or encode content.
 
 ## API documentation
 

config/config_mirror.yml

@@ -1,7 +1,6 @@
 # The `common' part is automatically included (and possibly overriden by all
 # other flavors)
 common:
-    secret_key: REPLACEME
     mirroring:
         source: https://registry-1.docker.io
         source_index: https://index.docker.io

config/config_sample.yml

@@ -100,7 +100,7 @@ openstack:
     <<: *glance
 
 # This flavor stores the images in Glance (to integrate with openstack)
-# and tags in Swift. 
+# and tags in Swift.
 glance-swift: &glance-swift
     <<: *swift
     storage: glance
@@ -128,8 +128,6 @@ elliptics:
 dev: &dev
     <<: *local
     search_backend: _env:SEARCH_BACKEND:sqlalchemy
-    # gunicorn will set this automatically if unset in dev mode
-    secret_key: _env:SECRET_KEY:secret
 
 # This flavor is automatically used by unit tests
 test:

depends/docker-registry-core/docker_registry/core/driver.py

@@ -235,7 +235,7 @@ def fetch(name):
     except ImportError:
         raise NotImplementedError(
             """You requested storage driver docker_registry.drivers.%s
-which is not installed. Try `pip install docker_registry-drivers-%s`
+which is not installed. Try `pip install docker-registry-driver-%s`
 or check your configuration. The following are currently
 available on your system: %s"""
             % (name, name, available())

docker_registry/app.py

@@ -15,7 +15,7 @@
 from . import toolkit
 from .lib import config
 
-VERSION = '0.6.9'
+VERSION = '0.7.0'
 app = flask.Flask('docker-registry')
 cfg = config.load()
 loglevel = getattr(logging, cfg.get('loglevel', 'INFO').upper())

setup.py

@@ -19,6 +19,12 @@
         requirements.insert(0, 'argparse>=1.2.1')
         requirements.insert(0, 'importlib>=1.0.3')
 
+packages = ['docker_registry',
+            'docker_registry.drivers',
+            'docker_registry.lib',
+            'docker_registry.storage',
+            'docker_registry.lib.index']
+
 setuptools.setup(
     name='docker-registry',
     # TODO: Load the version programatically, which is currently available in
@@ -28,7 +34,7 @@
     description='Registry server for Docker',
     long_description=open('README.md').read(),
     namespace_packages=['docker_registry', 'docker_registry.drivers'],
-    packages=setuptools.find_packages(),
+    packages=packages,
     license=open('LICENSE').read(),
     author='Docker Registry Contributors',
     author_email='[email protected]',

tests/requirements-tox.txt

@@ -1,10 +1,10 @@
-Flask==0.9
+Flask==0.10.1
 Flask-cors==1.3.0
-requests==1.2.0
-rsa==3.1.2
-PyYAML==3.10
+requests==2.3.0
+rsa==3.1.4
+PyYAML==3.11
 blinker==1.3
-gevent==0.13.8
-sqlalchemy==0.9.2
-bugsnag==1.3.1
+gevent==1.0.1
+sqlalchemy==0.9.4
+bugsnag==2.0.1
 gunicorn==18.0

tests/test_layers.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+
+import backports.lzma as lzma
+import os
+import random
+import string
+import tarfile
+
+import base
+from docker_registry.lib import layers
+from docker_registry import storage
+
+from docker_registry.core import compat
+json = compat.json
+StringIO = compat.StringIO
+
+
+# from mock import patch
+# from mockredis import mock_strict_redis_client
+
+
+def comp(n, f, *args, **kwargs):
+    return (f(*args, **kwargs) for i in xrange(n))
+
+
+def rndstr(length=5):
+    palette = string.ascii_uppercase + string.digits
+    return ''.join(comp(length, random.choice, palette))
+
+
+def _get_tarfile(filenames):
+    tfobj = StringIO()
+    tar = tarfile.TarFile(fileobj=tfobj, mode='w')
+    data = rndstr(512)
+    for filename in filenames:
+        tarinfo = tarfile.TarInfo(filename)
+        tarinfo.size = len(data)
+        io = StringIO()
+        io.write(data)
+        io.seek(0)
+        tar.addfile(tarinfo, io)
+    tfobj.seek(0)
+    return tfobj
+
+
+def _get_xzfile(filenames):
+    tar_data = _get_tarfile(filenames)
+    lzma_fobj = StringIO()
+    xz_file = lzma.open(lzma_fobj, 'w')
+    xz_file.write(tar_data.read())
+    xz_file.close()
+    lzma_fobj.seek(0)
+    return lzma_fobj
+
+
+class TestLayers(base.TestCase):
+
+    def setUp(self):
+        self.store = storage.load(kind='file')
+        self.filenames = list(comp(5, rndstr))
+
+    def test_tar_archive(self):
+        tfobj = _get_tarfile(self.filenames)
+
+        archive = layers.Archive(tfobj)
+        tar = tarfile.open(fileobj=archive)
+        members = tar.getmembers()
+        for tarinfo in members:
+            assert tarinfo.name in self.filenames
+
+    def test_xz_archive(self):
+        tfobj = _get_xzfile(self.filenames)
+        archive = layers.Archive(tfobj)
+        tar = tarfile.open(fileobj=archive)
+        members = tar.getmembers()
+        for tarinfo in members:
+            assert tarinfo.name in self.filenames
+
+    def test_info_serialization(self):
+        tfobj = _get_tarfile(self.filenames)
+        archive = layers.Archive(tfobj)
+        tar = tarfile.open(fileobj=archive)
+        members = tar.getmembers()
+        for tarinfo in members:
+            sinfo = layers.serialize_tar_info(tarinfo)
+            assert sinfo[0] in self.filenames
+            assert sinfo[1:] == ('f', False, 512, 0, 420, 0, 0)
+
+    def test_tar_serialization(self):
+        tfobj = _get_tarfile(self.filenames)
+        archive = layers.Archive(tfobj)
+        tar = tarfile.open(fileobj=archive)
+        infos = layers.read_tarfile(tar)
+        for tarinfo in infos:
+            assert tarinfo[0] in self.filenames
+            assert tarinfo[1:] == ('f', False, 512, 0, 420, 0, 0)
+
+    def test_layer_cache(self):
+        layer_id = rndstr(16)
+        layers.set_image_files_cache(layer_id, "{}")
+        fetched_json = layers.get_image_files_cache(layer_id)
+        assert fetched_json == "{}"
+
+    def test_tar_from_fobj(self):
+        tfobj = _get_tarfile(self.filenames)
+        files = layers.get_image_files_from_fobj(tfobj)
+        for file in files:
+            assert file[0] in self.filenames
+            assert file[1:] == ('f', False, 512, 0, 420, 0, 0)
+
+    def test_get_image_files_json_cached(self):
+        layer_id = rndstr(16)
+        layers.set_image_files_cache(layer_id, "{}")
+        files_json = layers.get_image_files_json(layer_id)
+        assert files_json == "{}"
+
+    def test_get_image_files_json(self):
+        layer_id = rndstr(16)
+        tfobj = _get_tarfile(self.filenames)
+
+        layer_path = self.store.image_layer_path(layer_id)
+        layer_path = os.path.join(self.store._root_path, layer_path)
+        path_parts = layer_path.split(os.sep)
+        layer_parent = os.path.join(*path_parts[:-1])
+        os.makedirs(layer_parent)
+
+        with open(layer_path, 'w') as fobj:
+            fobj.write(tfobj.read())
+
+        files_json = layers.get_image_files_json(layer_id)
+        file_infos = json.loads(files_json)
+        for info in file_infos:
+            assert info[0] in self.filenames
+            assert info[1:] == [u"f", False, 512, 0, 420, 0, 0]
+
+    def test_get_file_info_map(self):
+        files = (
+            ("test", "f", False, 512, 0, 420, 0, 0),
+        )
+        map = layers.get_file_info_map(files)
+        assert "test" in map
+        assert map['test'] == ("f", False, 512, 0, 420, 0, 0)
+
+    def test_image_diff_cache(self):
+        layer_id = rndstr(16)
+        layers.set_image_diff_cache(layer_id, layer_id)
+        diff_json = layers.get_image_diff_cache(layer_id)
+        assert layer_id == diff_json
+
+    def test_image_diff_json(self):
+        layer_1 = (
+            ("deleted", "f", False, 512, 0, 420, 0, 0),
+            ("changed", "f", False, 512, 0, 420, 0, 0),
+        )
+
+        layer_2 = (
+            ("deleted", "f", True, 512, 0, 420, 0, 0),
+            ("changed", "f", False, 512, 0, 420, 0, 0),
+            ("created", "f", False, 512, 0, 420, 0, 0),
+        )
+        layer_1_id = rndstr(16)
+        layer_2_id = rndstr(16)
+
+        ancestry = json.dumps([layer_2_id, layer_1_id])
+        ancestry_path = self.store.image_ancestry_path(layer_2_id)
+        self.store.put_content(ancestry_path, ancestry)
+
+        layer_1_files_path = self.store.image_files_path(layer_1_id)
+        self.store.put_content(layer_1_files_path, json.dumps(layer_1))
+
+        layer_2_files_path = self.store.image_files_path(layer_2_id)
+        self.store.put_content(layer_2_files_path, json.dumps(layer_2))
+
+        diff_json = layers.get_image_diff_json(layer_2_id)
+        diff = json.loads(diff_json)
+
+        for type in ("deleted", "changed", "created"):
+            assert type in diff
+            assert type in diff[type]

tests/workflow.py

@@ -2,6 +2,10 @@
 import os
 
 import requests
+sess = requests.Session()
+adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)
+sess.mount('https://', adapter)
+requests = sess
 
 from docker_registry.lib import checksums
 from docker_registry.lib import config
@@ -15,6 +19,8 @@
 
 cfg = config.load()
 
+ua = 'docker/1.0.0 registry test pretending to be docker'
+
 
 class TestWorkflow(base.TestCase):
 
@@ -32,7 +38,7 @@ class TestWorkflow(base.TestCase):
 
     def generate_chunk(self, data):
         bufsize = 1024
-        io = StringIO.StringIO(data)
+        io = StringIO(data)
         while True:
             buf = io.read(bufsize)
             if not buf:
@@ -60,14 +66,16 @@ def upload_image(self, image_id, parent_id, token):
             self.registry_endpoint, image_id),
             data=json_data,
             headers={'Authorization': 'Token ' + token,
+                     'User-Agent': ua,
                      'X-Docker-Checksum': layer_checksum},
             cookies=self.cookies)
         self.assertEqual(resp.status_code, 200, resp.text)
         self.update_cookies(resp)
         resp = requests.put('{0}/v1/images/{1}/layer'.format(
             self.registry_endpoint, image_id),
             data=self.generate_chunk(layer),
-            headers={'Authorization': 'Token ' + token},
+            headers={'Authorization': 'Token ' + token,
+                     'User-Agent': ua},
             cookies=self.cookies)
         self.assertEqual(resp.status_code, 200, resp.text)
         self.update_cookies(resp)
@@ -94,7 +102,8 @@ def docker_push(self):
         resp = requests.put('{0}/v1/repositories/{1}/{2}/'.format(
             self.index_endpoint, namespace, repos),
             auth=tuple(self.user_credentials),
-            headers={'X-Docker-Token': 'true'},
+            headers={'X-Docker-Token': 'true',
+                     'User-Agent': ua},
             data=images_json)
         self.assertEqual(resp.status_code, 200, resp.text)
         token = resp.headers.get('x-docker-token')
@@ -108,7 +117,8 @@ def docker_push(self):
         resp = requests.put('{0}/v1/repositories/{1}/{2}/images'.format(
             self.index_endpoint, namespace, repos),
             auth=tuple(self.user_credentials),
-            headers={'X-Endpoints': self.registry_endpoint},
+            headers={'X-Endpoints': self.registry_endpoint,
+                     'User-Agent': ua},
             data=json.dumps(images_json))
         self.assertEqual(resp.status_code, 204)
         return (namespace, repos)
@@ -160,7 +170,7 @@ def docker_pull(self, namespace, repos):
             json_data, checksum, blob = self.fetch_image(image_id)
             # check queried checksum and local computed checksum from the image
             # are the same
-            tmpfile = StringIO.StringIO()
+            tmpfile = StringIO()
             tmpfile.write(blob)
             tmpfile.seek(0)
             computed_checksum = checksums.compute_simple(tmpfile, json_data)