Formatting URL to work with Quay and adding support for fat manifests (#19)

monkey1016 · web-flow · commit 09d7dde2e7ea · 2021-05-29T00:31:20.000+08:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,9 @@
+venv/
+build/
+claircli.egg-info/
+dist/
+__pycache__/
+.coverage
+htmlcov/
+.eggs/
+.tox/
diff --git a/claircli/cli.py b/claircli/cli.py
@@ -177,46 +177,56 @@ def analyze_image(self):
         stats = defaultdict(list)
         for index, image in enumerate(args.images, start=1):
             logger.info('{:*^60}'.format(index))
-            try:
-                clair.analyze_image(image)
-                report = clair.get_report(image)
-                if not report:
-                    stats['IMAGES WERE NOT SUPPORTED'].append(image.name)
-                    continue
-                report.process_data(args.threshold, args.white_list)
-                report.to_console()
-                for format_ in args.formats:
-                    report.to(format_)
-                if report.ok:
-                    stats['IMAGES WITHOUT DETECTED VULNERABILITIES'].append(
-                        image.name)
-                else:
-                    stats['IMAGES WITH DETECTED VULNERABILITIES'].append(
-                        image.name)
-            except exceptions.HTTPError as exp:
-                if exp.response.status_code in [400, 404] and \
-                    ('Not Found for url' in str(exp) or
-                     'no such image' in str(exp)):
-                    logger.warning('%s was not found', image)
-                    stats['IMAGES COULD NOT BE FOUND'].append(image.name)
-                else:
-                    logger.warning('Could not analyze %s: Got response %d '
-                                   'from clair with message: %s',
-                                   image.name, exp.response.status_code,
-                                   exp.response.text)
-                    stats['IMAGES COULD NOT BE ANALYZED'].append(
-                        image.name)
-            except KeyboardInterrupt:
-                logger.warning('Keyboard interrupted')
-                return 2
-            except Exception as exp:
-                stats['IMAGES WERE ANALYZED WITH ERROR'].append(image.name)
-                logger.warning(str(exp))
-                logger.debug('Underlying problem:', exc_info=exp)
-            finally:
-                image.clean()
+            # Check whether we're examining a "fat manifest" or a regular image
+            if image.images:
+                logger.info('Analyzing manifest list ("fat manifest")...')
+                for sub_image in image.images:
+                    self._analyze_single_image(args, clair, sub_image, stats)
+            else:
+                self._analyze_single_image(args, clair, image, stats)
+
         return self.print_stats(stats)
 
+    def _analyze_single_image(self, args, clair, image, stats):
+        try:
+            clair.analyze_image(image)
+            report = clair.get_report(image)
+            if not report:
+                stats['IMAGES WERE NOT SUPPORTED'].append(image.name)
+                return
+            report.process_data(args.threshold, args.white_list)
+            report.to_console()
+            for format_ in args.formats:
+                report.to(format_)
+            if report.ok:
+                stats['IMAGES WITHOUT DETECTED VULNERABILITIES'].append(
+                    image.name)
+            else:
+                stats['IMAGES WITH DETECTED VULNERABILITIES'].append(
+                    image.name)
+        except exceptions.HTTPError as exp:
+            if exp.response.status_code in [400, 404] and \
+                ('Not Found for url' in str(exp) or
+                    'no such image' in str(exp)):
+                logger.warning('%s was not found', image)
+                stats['IMAGES COULD NOT BE FOUND'].append(image.name)
+            else:
+                logger.warning('Could not analyze %s: Got response %d '
+                               'from clair with message: %s',
+                               image.name, exp.response.status_code,
+                               exp.response.text)
+                stats['IMAGES COULD NOT BE ANALYZED'].append(
+                    image.name)
+        except KeyboardInterrupt:
+            logger.warning('Keyboard interrupted')
+            return 2
+        except Exception as exp:
+            stats['IMAGES WERE ANALYZED WITH ERROR'].append(image.name)
+            logger.warning(str(exp))
+            logger.debug('Underlying problem:', exc_info=exp)
+        finally:
+            image.clean()
+
     def print_stats(self, stats):
         total = sum(map(len, stats.values()))
         logger.info('=' * 60)
diff --git a/claircli/docker_image.py b/claircli/docker_image.py
@@ -9,12 +9,16 @@
 
 logger = logging.getLogger(__name__)
 
+MANIFEST_LIST_V2 = 'application/vnd.docker.distribution.manifest.list.v2+json'
+MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json'
+
 
 class Image(object):
 
     def __init__(self, name, registry=None):
         self.name = name
         self._layers = []
+        self._images = []
         self._manifest = None
         reg, repo, tag = self.parse_id(name)
         self.repository = repo
@@ -48,6 +52,22 @@ def manifest(self):
             self._manifest = self.registry.get_manifest(self)
         return self._manifest
 
+    @property
+    def images(self):
+        if not self._images:
+            images_list = []
+            manifest = self.manifest
+            if isinstance(self.registry, LocalRegistry):
+                pass
+            elif manifest['schemaVersion'] == 2 and manifest['mediaType'] \
+                    == MANIFEST_LIST_V2:
+                for single_manifest in manifest['manifests']:
+                    reg, repo, _tag = self.parse_id(self.name)
+                    images_list.append(Image('{}/{}@{}'.format(
+                        reg, repo, single_manifest['digest'])))
+            self._images = images_list
+        return self._images
+
     @property
     def layers(self):
         if not self._layers:
@@ -58,8 +78,12 @@ def layers(self):
             elif manifest['schemaVersion'] == 1:
                 self._layers = [e['blobSum']
                                 for e in manifest['fsLayers']][::-1]
-            elif manifest['schemaVersion'] == 2:
+            elif manifest['schemaVersion'] == 2 and manifest['mediaType'] \
+                    == MANIFEST_V2:
                 self._layers = [e['digest'] for e in manifest['layers']]
+            elif manifest['schemaVersion'] == 2 and manifest['mediaType'] \
+                    == MANIFEST_LIST_V2:
+                self._layers = []
             else:
                 raise ValueError(
                     'Wrong schemaVersion [%s]' % manifest['schemaVersion'])
diff --git a/claircli/docker_registry.py b/claircli/docker_registry.py
@@ -111,13 +111,16 @@ def get_manifest(self, image):
             self.url, image=image)
         headers = {'Accept':
                    'application/vnd.docker.distribution.manifest.v2+json,'
-                   'application/vnd.docker.distribution.manifest.v1+json',
+                   'application/vnd.docker.distribution.manifest.v1+json,'
+                   'application/vnd.docker.distribution.manifest.list.v2+json',
                    'Authorization': self.get_auth(image.repository)}
         resp = request_and_check('GET', url, headers=headers)
         return resp.json()
 
     def get_blobs_url(self, image, layer):
-        return '/'.join([self.url, image.repository, 'blobs', layer])
+        return '/'.join(f.strip('/') for f in [
+            self.url, image.repository, 'blobs', layer
+        ])
 
     def find_images(self, repository, tag):
         if self.domain == DOCKER_HUP_REGISTRY:
diff --git a/setup.py b/setup.py
@@ -12,6 +12,10 @@
     'colorlog',
 ]
 
+test_requirements = [
+    'responses'
+]
+
 here = os.path.abspath(os.path.dirname(__file__))
 about = {}
 with open(os.path.join(here, 'claircli', '__version__.py')) as f:
@@ -31,6 +35,7 @@
     author_email=about['__author_email__'],
     packages=['claircli'],
     install_requires=requires,
+    tests_require=test_requirements,
     license=about['__license__'],
     package_data={'': ['LICENSE'], 'claircli': ['templates/html-report.j2']},
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*,'
diff --git a/tests/test_claircli.py b/tests/test_claircli.py
@@ -64,8 +64,8 @@ def setUp(self):
 
     def tearDown(self):
         RemoteRegistry.tokens = defaultdict(dict)
-        if isfile(self.html):
-            os.remove(self.html)
+        # if isfile(self.html):
+        #     os.remove(self.html)
 
     def assert_called_with_url(self):
         self.assertEqual(responses.calls[0].request.url, self.reg_url)
@@ -100,6 +100,26 @@ def test_manifest(self):
         self.assertEqual(image.manifest, self.manifest)
         self.assert_called_with_url()
 
+    @responses.activate
+    def test_list_manifest(self):
+        with open('tests/test_data/manifest.list.v2.json') as f:
+            list_manifest = json.load(f)
+        responses.replace(responses.GET, self.manifest_url,
+                          json=list_manifest, status=200)
+        image = Image(self.name)
+        self.assertEqual(image.manifest, list_manifest)
+        self.assert_called_with_url()
+
+    @responses.activate
+    def test_unsupported_manifest(self):
+        with open('tests/test_data/manifest.unsupported.json') as f:
+            manifest = json.load(f)
+        responses.replace(responses.GET, self.manifest_url,
+                          json=manifest, status=200)
+        with self.assertRaises(ValueError):
+            image = Image(self.name)
+            image.layers
+
     @patch('docker.from_env')
     def test_manifest_local(self, mock_docker):
         mock_docker_client(mock_docker)
@@ -137,6 +157,27 @@ def test_layers_v2(self):
                          [e['digest'] for e in self.manifest['layers']])
         self.assert_called_with_url()
 
+    @responses.activate
+    def test_layers_list_v2(self):
+        list_image_manifest_url = self.reg_url + \
+            'org/image-name/manifests/sha256:d0fec089e611891a03f3282f10115bb186ed46093c3f083eceb250cee64b63eb'
+
+        with open('tests/test_data/manifest.list.v2.json') as f:
+            list_manifest = json.load(f)
+        with open('tests/test_data/manifest.list.v2-image.json') as f:
+            list_image_manifest = json.load(f)
+        responses.replace(responses.GET, self.manifest_url,
+                          json=list_manifest, status=200)
+        responses.add(responses.GET, list_image_manifest_url,
+                      json=list_image_manifest, status=200)
+        image = Image(self.name)
+        self.assertEqual(image.images[0].layers, [e['digest']
+                         for e in list_image_manifest['layers']])
+        self.assertEqual(image.layers, [])
+        self.assert_called_with_url()
+        self.assertEqual(
+            responses.calls[3].request.url, list_image_manifest_url)
+
 
 class TestClair(ClairCmdTestBase):
 
@@ -196,7 +237,7 @@ def test_read_white_list(self):
 
     @responses.activate
     def test_analyze_images(self):
-        with patch('sys.argv', ['claircli',  '-c',
+        with patch('sys.argv', ['claircli', '-d', '-c',
                                 self.clair_url, self.name]):
             cli = ClairCli()
             cli.run()
@@ -316,3 +357,41 @@ def test_analyze_local_images(self, mock_docker):
             req_body = json.loads(responses.calls[index].request.body)
             self.assertEqual(req_body['Layer']['Name'], layer)
         self.assertTrue(isfile(self.html))
+    
+
+    @responses.activate
+    def test_analyze_manifest_list(self):
+        list_image_manifest_url = self.reg_url + \
+            'org/image-name/manifests/sha256:d0fec089e611891a03f3282f10115bb186ed46093c3f083eceb250cee64b63eb'
+        with open('tests/test_data/manifest.list.v2.json') as f:
+            list_manifest = json.load(f)
+        with open('tests/test_data/manifest.list.v2-image.json') as f:
+            list_image_manifest = json.load(f)
+        with open('tests/test_data/origin_vulnerabilities_list.json') as f:
+            list_origin_data = json.load(f)
+        responses.add(responses.GET, '%s/%s?features&vulnerabilities' %
+                      (self.v1_analyze_url, list_origin_data['Layer']['Name']),
+                      json=list_origin_data)
+        responses.replace(responses.GET, self.manifest_url,
+                          json=list_manifest, status=200)
+        responses.add(responses.GET, list_image_manifest_url,
+                      json=list_image_manifest, status=200)
+        layers = [e['digest'] for e in list_image_manifest['layers']]
+        responses.add(responses.DELETE, '%s/%s' %
+                      (self.v1_analyze_url, layers[0]))
+        for layer in layers:
+            responses.add(responses.GET, '%s/%s' %
+                          (self.v1_analyze_url, layer))
+        with patch('sys.argv', ['claircli', '-d', '-c',
+                                self.clair_url, self.name]):
+            cli = ClairCli()
+            cli.run()
+        image = Image(self.name)
+        self.assert_called_with_url()
+        for index, layer in enumerate(image.images[0].layers, start=5):
+            self.assertEqual(
+                responses.calls[index].request.url, self.v1_analyze_url)
+            req_body = json.loads(responses.calls[index].request.body)
+            self.assertEqual(req_body['Layer']['Name'], layer)
+        self.html = Report.get_report_path('{}/{}@{}'.format(self.reg, self.repo, image.manifest['manifests'][0]['digest']), '.html')
+        self.assertTrue(isfile(self.html))
diff --git a/tests/test_data/manifest.list.v2-image.json b/tests/test_data/manifest.list.v2-image.json
@@ -0,0 +1,26 @@
+{
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+    "config": {
+       "mediaType": "application/vnd.docker.container.image.v1+json",
+       "size": 3689,
+       "digest": "sha256:79e85ca394595673010a283b31e2f237a046c987ca15f15fb51bb23a8b98cce7"
+    },
+    "layers": [
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 73903098,
+          "digest": "sha256:b565332d1d45150165f73227d2ec4e6ef0127e7b55fac73a5131468b25bc4bfb"
+       },
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 1412,
+          "digest": "sha256:04ef0e69dcba4088b009d14d21e86cda00ddbf8e84a4d9746c5d8ec9d61803af"
+       },
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 15199242,
+          "digest": "sha256:debe3b1a97504955e5ff561633738d979a1e58861c3dffda3bd9dd59bf7b4d50"
+       }
+    ]
+ }
diff --git a/tests/test_data/manifest.list.v2.json b/tests/test_data/manifest.list.v2.json
@@ -0,0 +1,15 @@
+{
+    "manifests": [
+        {
+            "digest": "sha256:d0fec089e611891a03f3282f10115bb186ed46093c3f083eceb250cee64b63eb",
+            "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+            "platform": {
+                "architecture": "amd64",
+                "os": "linux"
+            },
+            "size": 949
+        }
+    ],
+    "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
+    "schemaVersion": 2
+}
diff --git a/tests/test_data/manifest.unsupported.json b/tests/test_data/manifest.unsupported.json
@@ -0,0 +1,26 @@
+{
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.docker.distribution.manifest.unsupported+json",
+    "config": {
+       "mediaType": "application/vnd.docker.container.image.v1+json",
+       "size": 3689,
+       "digest": "sha256:79e85ca394595673010a283b31e2f237a046c987ca15f15fb51bb23a8b98cce7"
+    },
+    "layers": [
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 73903098,
+          "digest": "sha256:b565332d1d45150165f73227d2ec4e6ef0127e7b55fac73a5131468b25bc4bfb"
+       },
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 1412,
+          "digest": "sha256:04ef0e69dcba4088b009d14d21e86cda00ddbf8e84a4d9746c5d8ec9d61803af"
+       },
+       {
+          "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+          "size": 15199242,
+          "digest": "sha256:debe3b1a97504955e5ff561633738d979a1e58861c3dffda3bd9dd59bf7b4d50"
+       }
+    ]
+ }
diff --git a/tests/test_data/origin_vulnerabilities_list.json b/tests/test_data/origin_vulnerabilities_list.json
