Insecure registry (#13)

* support insecure registry

Author: joelee2012

README.md

@@ -1,9 +1,12 @@
 [![Build Status](https://travis-ci.com/joelee2012/claircli.svg?branch=master)](https://travis-ci.com/joelee2012/claircli)
 [![Coverage Status](https://coveralls.io/repos/github/joelee2012/claircli/badge.svg?branch=master)](https://coveralls.io/github/joelee2012/claircli?branch=master)
 # claircli
-## claircli is a command line tool to interact with [CoreOS Clair](https://github.com/quay/clair)
-- analyze loacl/remote docker image with [Clair](https://github.com/quay/clair)
-- generate HTML/JSON report, the html report template is from [analysis-template.html](https://github.com/jgsqware/clairctl/blob/master/clair/templates/analysis-template.html)
+## claircli is a command line tool to interact with [Quay Clair](https://github.com/quay/clair), which has following functionalities:
+- analyze docker images in local host
+- analyze docker images in remote host
+- analyze docker images in secure/insecure registry
+- support threshold/whitelist for vulnerabilities
+- report to HTML/JSON, the html report is based on [template](https://github.com/jgsqware/clairctl/blob/master/clair/templates/analysis-template.html)
 
 # Installation
 
@@ -15,45 +18,51 @@ pip install claircli
 
 ```
 claircli -h
-usage: claircli [-h] [-V] [-c CLAIR] [-w WHITE_LIST] [-T THRESHOLD]
-                [-f {html,json}] [-L LOG_FILE] [-d] [-l LOCAL_IP | -r]
-                images [images ...]
+usage: claircli [-h] [-c CLAIR] [-f {html,json}] [-T THRESHOLD]
+                [-w WHITE_LIST] [-l LOCAL_IP | -r] [-i REGISTRY] [-L LOG_FILE]
+                [-d] [-V]
+                IMAGE [IMAGE ...]
 
-Command line tool to interact with CoreOS Clair, analyze docker image with
-clair in different ways
+Command line tool to interact with Quay Clair to analyze docker image in different ways
 
 positional arguments:
-  images                docker images or regular expression
+  IMAGE                 docker images or regular expression
 
 optional arguments:
   -h, --help            show this help message and exit
-  -V, --version         show program's version number and exit
   -c CLAIR, --clair CLAIR
                         clair url, default: http://localhost:6060
-  -w WHITE_LIST, --white-list WHITE_LIST
-                        path to the whitelist file
+  -f {html,json}, --formats {html,json}
+                        output report file with give format, default: ['html']
   -T THRESHOLD, --threshold THRESHOLD
                         cvd severity threshold, if any servity of
                         vulnerability above of threshold, will return non-
                         zero, default: Unknown, choices are: ['Defcon1',
                         'Critical', 'High', 'Medium', 'Low', 'Negligible',
                         'Unknown']
-  -f {html,json}, --formats {html,json}
-                        output report file with give format, default: ['html']
-  -L LOG_FILE, --log-file LOG_FILE
-                        save log to file
-  -d, --debug           print more logs
+  -w WHITE_LIST, --white-list WHITE_LIST
+                        path to the whitelist file
   -l LOCAL_IP, --local-ip LOCAL_IP
                         ip address of local host
   -r, --regex           if set, repository and tag of images will be treated
                         as regular expression
+  -i REGISTRY, --insecure-registry REGISTRY
+                        domain of insecure registry
+  -L LOG_FILE, --log-file LOG_FILE
+                        save log to file
+  -d, --debug           print more logs
+  -V, --version         show program's version number and exit
 
 Examples:
 
     # analyze and output report to html
     # clair is running at http://localhost:6060
     claircli example.reg.com/myimage1:latest example.reg.com/myimage2:latest
 
+    # analyze image in insecure registry
+    # clair is running at http://localhost:6060
+    claircli -i example.reg.com example.reg.com/myimage1:latest
+
     # analyze and output report to html
     # clair is running at https://example.clair.com:6060
     claircli -c https://example.clair.com:6060 example.reg.com/myimage1:latest

claircli/__version__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = '1.0'
+__version__ = '1.1'
 __title__ = 'claircli'
 __description__ = 'Command line tool to interact with Clair'
 __url__ = 'https://github.com/joelee2012/claircli'

claircli/cli.py

@@ -22,15 +22,18 @@
 
 
 class ClairCli(object):
-    description = textwrap.dedent('''
-    Command line tool to interact with CoreOS Clair, analyze docker image with
-    clair in different ways''')
+    description = 'Command line tool to interact with Quay Clair' \
+                  ' to analyze docker image in different ways'
     epilog = '''Examples:
 
     # analyze and output report to html
     # clair is running at http://localhost:6060
     claircli example.reg.com/myimage1:latest example.reg.com/myimage2:latest
 
+    # analyze image in insecure registry
+    # clair is running at http://localhost:6060
+    claircli -i example.reg.com example.reg.com/myimage1:latest
+
     # analyze and output report to html
     # clair is running at https://example.clair.com:6060
     claircli -c https://example.clair.com:6060 example.reg.com/myimage1:latest
@@ -64,26 +67,22 @@ def __init__(self):
             description=self.description,
             formatter_class=argparse.RawDescriptionHelpFormatter,
             epilog=self.epilog)
-        parser.add_argument(
-            '-V', '--version', action='version', version=__version__)
+
         parser.add_argument(
             '-c', '--clair', default='http://localhost:6060',
             help='clair url, default: %(default)s')
         parser.add_argument(
-            '-w', '--white-list', help='path to the whitelist file')
+            '-f', '--formats', choices=['html', 'json'],
+            action='append', default=['html'],
+            help='output report file with give format, default: %(default)s')
         parser.add_argument(
             '-T', '--threshold', choices=SEVERITIES,
             default='Unknown', metavar='THRESHOLD',
             help='cvd severity threshold, if any servity of vulnerability'
             ' above of threshold, will return non-zero, default: %(default)s'
             ', choices are: {}'.format(SEVERITIES))
         parser.add_argument(
-            '-f', '--formats', choices=['html', 'json'],
-            action='append', default=['html'],
-            help='output report file with give format, default: %(default)s')
-        parser.add_argument('-L', '--log-file', help='save log to file')
-        parser.add_argument(
-            '-d', '--debug', action='store_true', help='print more logs')
+            '-w', '--white-list', help='path to the whitelist file')
         group = parser.add_mutually_exclusive_group()
         group.add_argument(
             '-l', '--local-ip', help='ip address of local host')
@@ -92,9 +91,22 @@ def __init__(self):
             help='if set, repository and tag of images will be '
             'treated as regular expression')
         parser.add_argument(
-            'images', nargs='+', help='docker images or regular expression')
+            '-i', '--insecure-registry', action='append',
+            dest='insec_regs', metavar='REGISTRY', default=[],
+            help='domain of insecure registry')
+        parser.add_argument('-L', '--log-file', help='save log to file')
+        parser.add_argument(
+            '-d', '--debug', action='store_true', help='print more logs')
+        parser.add_argument(
+            '-V', '--version', action='version', version=__version__)
+        parser.add_argument(
+            'images', nargs='+', metavar='IMAGE',
+            help='docker images or regular expression')
         parser.set_defaults(func=self.analyze_image)
         self.args = parser.parse_args()
+        if self.args.local_ip and self.args.insec_regs:
+            parser.error('argument --local-ip: not allowed with'
+                         ' argument --insecure-registry')
         self.setup_logging()
 
     def setup_logging(self):
@@ -126,6 +138,7 @@ def resolve_images(self, images):
     def analyze_image(self):
         args = self.args
         registry = None
+        RemoteRegistry.insec_regs = set(args.insec_regs)
         if args.local_ip:
             registry = LocalRegistry(args.local_ip)
         elif args.regex:

claircli/docker_registry.py

@@ -71,10 +71,12 @@ class RemoteRegistry(object):
     tokens = defaultdict(dict)
     token_pattern = re.compile(r'Bearer realm="(?P<realm>[^"]+)".*'
                                r'service="(?P<service>[^"]+).*')
+    insec_regs = set()
 
     def __init__(self, domain):
         self.domain = domain
-        self.url = 'https://{}/v2/'.format(self.domain)
+        schema = 'http' if domain in self.insec_regs else 'https'
+        self.url = '{}://{}/v2/'.format(schema, domain)
 
     def __str__(self):
         return self.domain

tests/test_claircli.py

@@ -16,7 +16,10 @@
 from claircli.docker_image import Image
 from claircli.docker_registry import LocalRegistry, RemoteRegistry
 from claircli.report import Report, WhiteList
-from mock import patch
+try:
+    from unittest.mock import patch
+except:
+    from mock import patch
 
 logger = logging.getLogger(__name__)
 
@@ -193,7 +196,7 @@ def test_read_white_list(self):
 
     @responses.activate
     def test_analyze_images(self):
-        with patch('sys.argv', ['claircli.py',  '-c',
+        with patch('sys.argv', ['claircli',  '-c',
                                 self.clair_url, self.name]):
             cli = ClairCli()
             cli.run()
@@ -205,6 +208,50 @@ def test_analyze_images(self):
             self.assertEqual(req_body['Layer']['Name'], layer)
         self.assertTrue(isfile(self.html))
 
+    @responses.activate
+    def test_analyze_images_in_insecure_registry(self):
+
+        reg_url = 'http://%s/v2/' % self.reg
+        token_url = reg_url + 'token'
+        auth = 'Bearer realm="%s",service="%s"' % (token_url, self.reg)
+        headers = {'WWW-Authenticate': auth}
+        params = {'service': self.reg,
+                  'client_id': 'claircli',
+                  'scope': 'repository:%s:pull' % self.repo}
+        token_url = token_url + '?' + urlencode(params)
+        manifest_url = reg_url + 'org/image-name/manifests/version'
+        responses.reset()
+        responses.add(responses.GET, reg_url,
+                      json={'message': 'authentication required'},
+                      status=401, headers=headers)
+        responses.add(responses.GET, token_url,
+                      json={'token': 'test-token'}, status=200)
+
+        responses.add(responses.GET, manifest_url,
+                      json=self.manifest, status=200)
+        self.layers = [e['digest'] for e in self.manifest['layers']]
+        responses.add(responses.DELETE, '%s/%s' %
+                      (self.v1_analyze_url, self.layers[0]))
+        responses.add(responses.POST, self.v1_analyze_url)
+        responses.add(responses.GET, '%s/%s?features&vulnerabilities' %
+                      (self.v1_analyze_url, self.layers[-1]),
+                      json=self.origin_data)
+
+        with patch('sys.argv', ['claircli',  '-c',
+                                self.clair_url, '-i', self.reg, self.name]):
+            cli = ClairCli()
+            cli.run()
+        for index, url in enumerate([reg_url, token_url, manifest_url]):
+            self.assertEqual(responses.calls[index].request.url, url)
+
+        for index, layer in enumerate(self.layers, start=4):
+            self.assertEqual(
+                responses.calls[index].request.url, self.v1_analyze_url)
+            req_body = json.loads(responses.calls[index].request.body)
+            self.assertEqual(req_body['Layer']['Name'], layer)
+        self.assertTrue(isfile(self.html))
+        self.assertIn(self.reg, RemoteRegistry.insec_regs)
+
     @patch('docker.from_env')
     @responses.activate
     def test_analyze_local_images(self, mock_docker):
@@ -216,7 +263,7 @@ def test_analyze_local_images(self, mock_docker):
                       (self.v1_analyze_url, layers[0]))
         responses.add(responses.GET, '%s/%s?features&vulnerabilities' %
                       (self.v1_analyze_url, layers[-1]), json=self.origin_data)
-        with patch('sys.argv', ['claircli.py', '-l', 'localhost',
+        with patch('sys.argv', ['claircli', '-l', 'localhost',
                                 '-c', self.clair_url, self.name]):
             cli = ClairCli()
             cli.run()