Additional qemu safety checking on base images

There is an additional way we can be fooled into using a qcow2 file
with a data-file, which is uploading it as raw to glance and then
booting an instance from it. Because when we go to create the
ephemeral disk from a cached base image, we've lost the information
about the original source's format, we probe the image's file type
without a strict format specified. If a qcow2 file is listed in
glance as a raw, we won't notice it until it is too late.

This brings over another piece of code (proposed against) glance's
format inspector which provides a safe format detection routine. This
patch uses that to detect the format of and run a safety check on the
base image each time we go to use it to create an ephemeral disk
image from it.

This also detects QED files and always marks them as unsafe as we do
not support that format at all. Since we could be fooled into
downloading one and passing it to qemu-img if we don't recognize it,
we need to detect and reject it as unsafe.

Conflicts:
  nova/tests/unit/virt/libvirt/test_utils.py
  nova/virt/libvirt/utils.py

NOTE(elod.illes): conflicts are due to patch to consolidate image
creation functions (I111cfc8a5eae27b15c6312957255fcf973038ddf) is only
introduced in zed.

Change-Id: I4881c8cbceb30c1ff2d2b859c554e0d02043f1f5
(cherry picked from commit b1b88bf)
(cherry picked from commit 8a0d5f2)
(cherry picked from commit 0269234)
(cherry picked from commit 9e10ac2)
(cherry picked from commit 303c2c9)
(cherry picked from commit e7bdaac)

Author: kk7ds

nova/image/format_inspector.py

@@ -368,6 +368,23 @@ def safety_check(self):
                 not self.has_unknown_features)
 
 
+class QEDInspector(FileInspector):
+    def __init__(self, tracing=False):
+        super().__init__(tracing)
+        self.new_region('header', CaptureRegion(0, 512))
+
+    @property
+    def format_match(self):
+        if not self.region('header').complete:
+            return False
+        return self.region('header').data.startswith(b'QED\x00')
+
+    def safety_check(self):
+        # QED format is not supported by anyone, but we want to detect it
+        # and mark it as just always unsafe.
+        return False
+
+
 # The VHD (or VPC as QEMU calls it) format consists of a big-endian
 # 512-byte "footer" at the beginning of the file with various
 # information, most of which does not matter to us:
@@ -871,19 +888,52 @@ def close(self):
             self._source.close()
 
 
+ALL_FORMATS = {
+    'raw': FileInspector,
+    'qcow2': QcowInspector,
+    'vhd': VHDInspector,
+    'vhdx': VHDXInspector,
+    'vmdk': VMDKInspector,
+    'vdi': VDIInspector,
+    'qed': QEDInspector,
+}
+
+
 def get_inspector(format_name):
     """Returns a FormatInspector class based on the given name.
 
     :param format_name: The name of the disk_format (raw, qcow2, etc).
     :returns: A FormatInspector or None if unsupported.
     """
-    formats = {
-        'raw': FileInspector,
-        'qcow2': QcowInspector,
-        'vhd': VHDInspector,
-        'vhdx': VHDXInspector,
-        'vmdk': VMDKInspector,
-        'vdi': VDIInspector,
-    }
-
-    return formats.get(format_name)
+
+    return ALL_FORMATS.get(format_name)
+
+
+def detect_file_format(filename):
+    """Attempts to detect the format of a file.
+
+    This runs through a file one time, running all the known inspectors in
+    parallel. It stops reading the file once one of them matches or all of
+    them are sure they don't match.
+
+    Returns the FileInspector that matched, if any. None if 'raw'.
+    """
+    inspectors = {k: v() for k, v in ALL_FORMATS.items()}
+    with open(filename, 'rb') as f:
+        for chunk in chunked_reader(f):
+            for format, inspector in list(inspectors.items()):
+                try:
+                    inspector.eat_chunk(chunk)
+                except ImageFormatError:
+                    # No match, so stop considering this format
+                    inspectors.pop(format)
+                    continue
+                if (inspector.format_match and inspector.complete and
+                        format != 'raw'):
+                    # First complete match (other than raw) wins
+                    return inspector
+            if all(i.complete for i in inspectors.values()):
+                # If all the inspectors are sure they are not a match, avoid
+                # reading to the end of the file to settle on 'raw'.
+                break
+    return inspectors['raw']

nova/tests/unit/virt/libvirt/test_utils.py

@@ -118,14 +118,26 @@ def test_create_image(self, mock_execute):
     @mock.patch('oslo_concurrency.processutils.execute')
     @mock.patch('nova.virt.images.qemu_img_info')
     @mock.patch('nova.image.format_inspector.detect_file_format')
-    def test_create_cow_image(self, mock_detect, mock_info, mock_execute,
-                              mock_exists):
+    def _test_create_cow_image(
+        self, mock_detect, mock_info, mock_execute,
+        mock_exists, backing_file=None, safety_check=True
+    ):
+        if isinstance(backing_file, dict):
+            backing_info = backing_file
+            backing_file = backing_info.pop('file', None)
+        else:
+            backing_info = {}
+        backing_backing_file = backing_info.pop('backing_file', None)
+
         mock_execute.return_value = ('stdout', None)
         mock_info.return_value = mock.Mock(
             file_format=mock.sentinel.backing_fmt,
             cluster_size=mock.sentinel.cluster_size,
-            backing_file=None)
-        mock_detect.return_value.safety_check.return_value = True
+            backing_file=backing_backing_file,
+            format_specific=backing_info)
+
+        mock_detect.return_value.safety_check.return_value = safety_check
+
         libvirt_utils.create_cow_image(mock.sentinel.backing_path,
                                        mock.sentinel.new_path)
         mock_info.assert_called_once_with(mock.sentinel.backing_path)
@@ -135,7 +147,33 @@ def test_create_cow_image(self, mock_detect, mock_info, mock_execute,
                 mock.sentinel.backing_path, mock.sentinel.backing_fmt,
                 mock.sentinel.cluster_size),
              mock.sentinel.new_path)])
-        mock_detect.return_value.safety_check.assert_called_once_with()
+        if backing_file:
+            mock_detect.return_value.safety_check.assert_called_once_with()
+
+    def test_create_image_qcow2(self):
+        self._test_create_cow_image()
+
+    def test_create_image_backing_file(self):
+        self._test_create_cow_image(
+            backing_file=mock.sentinel.backing_file
+        )
+
+    def test_create_image_base_has_backing_file(self):
+        self.assertRaises(
+            exception.InvalidDiskInfo,
+            self._test_create_cow_image,
+            backing_file={'file': mock.sentinel.backing_file,
+                          'backing_file': mock.sentinel.backing_backing_file},
+        )
+
+    def test_create_image_base_has_data_file(self):
+        self.assertRaises(
+            exception.InvalidDiskInfo,
+            self._test_create_cow_image,
+            backing_file={'file': mock.sentinel.backing_file,
+                          'backing_file': mock.sentinel.backing_backing_file,
+                          'data': {'data-file': mock.sentinel.data_file}},
+        )
 
     @ddt.unpack
     @ddt.data({'fs_type': 'some_fs_type',

nova/virt/libvirt/utils.py

@@ -155,8 +155,7 @@ def create_cow_image(
                     reason=_('Base image failed safety check'))
 
         base_details = images.qemu_img_info(backing_file)
-        if base_details.file_format == 'vmdk':
-            images.check_vmdk_image('base', base_details)
+
         if base_details.backing_file is not None:
             LOG.warning('Base image %s failed safety check', backing_file)
             raise exception.InvalidDiskInfo(