updated oci image support

Author: snps-kumykov

examples/client/multi-image/generate-clone.sh

@@ -15,6 +15,6 @@ ls -l $SPECFILE
 
 COMMAND="python3 examples/client/multi-image/manage_project_structure.py"
 
-$COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@
-# $COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@
+# $COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@
+$COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@
 

examples/client/multi-image/manage_project_structure.py

@@ -360,9 +360,12 @@ def scan_container_images(scan_params, hub):
                 params['project'], 
                 params['version'],
                 detect_options,
-                hub=hub
+                hub=hub,
+                binary=False
             )
         except Exception:
+            import traceback
+            traceback.print_exc()
             logging.error(f"Scanning of {params['image']} failed, skipping")
             skipped_scans.append(params)
 

examples/client/multi-image/scan_docker_image_lite.py

@@ -94,6 +94,7 @@
 from pprint import pprint
 from sys import argv
 import json
+import logging
 import os
 import requests
 import shutil
@@ -136,7 +137,6 @@ def locate_docker(self):
         proc = subprocess.Popen(['which','docker'], stdout=subprocess.PIPE)
         out, err = proc.communicate()
         lines = out.decode().split('\n')
-        print(lines)
         if 'docker' in lines[0]:
             return lines[0]
         else:
@@ -147,7 +147,7 @@ def pull_container_image(self, image_name):
         args.append(self.docker_path)
         args.append('pull')
         args.append(image_name)
-        return subprocess.run(args)
+        return subprocess.run(args, capture_output=True)
 
     def get_container_image_history(self, image_name):
         args = []
@@ -173,7 +173,7 @@ def unravel_container(self):
         args.append(self.imagefile)
         args.append('-C')
         args.append(self.imagedir)
-        return subprocess.run(args)
+        return subprocess.run(args, capture_output=True)
 
     def read_manifest(self):
         filename = self.imagedir + "/manifest.json"
@@ -222,7 +222,7 @@ def detect_run(self, options=['--help']):
         cmd.append('--blackduck.api.token=' + self.token)
         cmd.append('--blackduck.trust.cert=true')
         cmd.extend(options)
-        subprocess.run(cmd)
+        return subprocess.run(cmd, capture_output=True)
 
 class ContainerImageScanner():
 
@@ -249,7 +249,6 @@ def __init__(
         self.extra_options = []
         if detect_options:
             self.extra_options = detect_options.split(" ")
-        print ("<--{}-->".format(self.grouping))
         self.binary = False
 
     def prepare_container_image(self):
@@ -272,12 +271,11 @@ def prepare_container_image(self):
                 history_grouping += str(layer_count) + ":" + found
         if len(history_grouping) and self.grouping == '1024:everything':
             self.grouping = history_grouping
+        self.oci_layout = self.docker.read_oci_layout()
 
     def process_container_image_by_user_defined_groups(self):
         self.manifest = self.docker.read_manifest()
-        print(self.manifest)
         self.config = self.docker.read_config()
-        print (json.dumps(self.config, indent=4))
 
         if self.grouping:
             self.groups = dict(x.split(":") for x in self.grouping.split(","))
@@ -309,14 +307,12 @@ def process_container_image_by_user_defined_groups(self):
             layer['shaid'] = self.config['rootfs']['diff_ids'][num - 1]
             self.layers.append(layer)
             num = num + 1
-        print (json.dumps(self.layers, indent=4))
+        # print (json.dumps(self.layers, indent=4))
 
     def process_container_image_by_base_image_info(self):
         self.manifest = self.docker.read_manifest()
-        print(self.manifest)
         self.config = self.docker.read_config()
-        print (json.dumps(self.config, indent=4))
-
+        
         self.layers = []
         num = 1
         offset = 0
@@ -342,27 +338,92 @@ def process_container_image_by_base_image_info(self):
                 layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(num)
             self.layers.append(layer)
             num = num + 1
-        print (json.dumps(self.layers, indent=4))
+        # print (json.dumps(self.layers, indent=4))
+
+    def process_oci_container_image_by_user_defined_groups(self):
+        self.manifest = self.docker.read_manifest()
+        self.config = self.docker.read_config()
+
+        self.layers = self.config['history']
+        tagged_layers = [x for x in self.layers if '_group_end' in x.get('created_by')]
+        groups =  {re.search('echo (.+?)_group_end', str(x['created_by'])).group(1): self.layers.index(x) for x in tagged_layers}
+        logging.debug(f"Container configuration defines following groups {groups}")
+        layer_paths = self.manifest[0]['Layers'].copy()
+        empty_layers = [x for x in self.layers if x.get('empty_layer', False)]
+        logging.debug(f"Total layers: {len(self.layers)} total paths: {len(layer_paths)}  empty layers: {len(empty_layers)}")
+
+        assert len(self.layers) == len(layer_paths) + len(empty_layers), "Something is wrong with this image, Layer math does not add up."
+
+        for layer in self.layers:
+            layer['index'] = self.layers.index(layer)
+            if self.grouping:
+                layer['group_name'] = self.get_group_name(groups, layer['index'])
+                layer['project_name'] = "{}_{}".format(self.project_name,layer['group_name'])
+                layer['project_version'] = self.project_version
+                layer['name'] = "{}_{}_{}_layer_{}".format(self.project_name,self.project_version,layer['group_name'],str(layer['index']))
+            else:
+                layer['project_name'] = self.project_name
+                layer['project_version'] = self.project_version
+                layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(layer['index'])
+            if not layer.get('empty_layer', False):
+                layer['path'] = layer_paths.pop(0)
+        # print (json.dumps(self.layers, indent=4))
+
+    def get_group_name(self, groups, index):
+        group_name  = 'undefined'
+        for group, value in groups.items():
+            if index <= value:
+                group_name = group
+                break
+        return group_name
+
+    def process_oci_container_image_by_base_image_info(self):
+        print ("Processing by BAse Image not supported for OCI images")
+        sys.exit(1)
+        pass
 
     def process_container_image(self):
+        if self.oci_layout:
+            self.process_oci_container_image()
+        else:
+            self.process_docker_container_image()
+
+    def process_docker_container_image(self):
         if self.grouping:
             self.process_container_image_by_user_defined_groups()
         else:
             self.process_container_image_by_base_image_info()
 
+    def process_oci_container_image(self):
+        if self.grouping:
+            self.process_oci_container_image_by_user_defined_groups()
+        else:
+            self.process_oci_container_image_by_base_image_info()
+
     def submit_layer_scans(self):
         for layer in self.layers:
-            options = []
-            options.append('--detect.project.name={}'.format(layer['project_name']))
-            options.append('--detect.project.version.name="{}"'.format(layer['project_version']))
-            # options.append('--detect.blackduck.signature.scanner.disabled=false')
-            options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path']))
-            options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0]))
-            if self.base_image or self.grouping or self.dockerfile:
-                options.extend(self.adorn_extra_options(layer))
-            else:
-                options.extend(self.extra_options)
-            self.hub_detect.detect_run(options)
+            if not layer.get('empty_layer', False):
+                options = []
+                options.append('--detect.project.name={}'.format(layer['project_name']))
+                options.append('--detect.project.version.name="{}"'.format(layer['project_version']))
+                options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path']))
+                if self.binary:
+                    options.append('--detect.tools=BINARY_SCAN')
+                    options.append('--detect.binary.scan.file.path={}/{}'.format(self.docker.imagedir, layer['path']))
+                else:
+                    options.append('--detect.tools=SIGNATURE_SCAN')
+                    if self.oci_layout:
+                       options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path']))
+                    else:
+                       options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0]))
+                if self.base_image or self.grouping or self.dockerfile:
+                    options.extend(self.adorn_extra_options(layer))
+                else:
+                    options.extend(self.extra_options)
+                logging.info(f"Submitting scan for {layer['name']}")
+                completed = self.hub_detect.detect_run(options)
+                logging.info(f"Detect run for {layer['name']} completed with returncode {completed.returncode}")
+                
 
     def adorn_extra_options(self, layer):
         result = list()
@@ -396,24 +457,24 @@ def get_base_layers(self):
         if self.base_image:
             imagelist.append(self.base_image)
 
-        print (imagelist)
+        # print (imagelist)
         base_layers = []
         for image in imagelist:
             self.docker.initdir()
             self.docker.pull_container_image(image)
             self.docker.save_container_image(image)
             self.docker.unravel_container()
             manifest = self.docker.read_manifest()
-            print(manifest)
+            # print(manifest)
             config = self.docker.read_config()
-            print(config)
+            # print(config)
             base_layers.extend(config['rootfs']['diff_ids'])
         return base_layers  
 
 
 def scan_container_image(
     imagespec, grouping=None, base_image=None, dockerfile=None, 
-    project_name=None, project_version=None, detect_options=None, hub=None):
+    project_name=None, project_version=None, detect_options=None, hub=None, binary=False):
 
     if hub:
         hub = hub
@@ -431,6 +492,8 @@ def scan_container_image(
             scanner.grouping = '1024:everything'
         else:
             scanner.base_layers = scanner.get_base_layers()
+    if binary:
+        scanner.binary = True
     scanner.prepare_container_image()
     scanner.process_container_image()
     scanner.submit_layer_scans()
@@ -450,10 +513,11 @@ def main(argv=None):
     parser.add_argument('--project-name',default=None, type=str, help="Specify project name (default is container image spec)")
     parser.add_argument('--project-version',default=None, type=str, help="Specify project version (default is container image tag/version)")
     parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directly to the detect")
-    
+    parser.add_argument('--binary', action='store_true', help="Use Binary Scan instead of signature scan")
+
     args = parser.parse_args()
 
-    print (args);
+    logging.debug(args);
 
     if not args.imagespec:
         parser.print_help(sys.stdout)
@@ -474,7 +538,8 @@ def main(argv=None):
         args.dockerfile, 
         args.project_name, 
         args.project_version,
-        args.detect_options)
+        args.detect_options,
+        args.binary)
 
 
 if __name__ == "__main__":