Merge pull request #83 from stackhpc/yoga-format-inspector

Apply unmerged patches from upstream unmaintained/yoga

Author: priteau

.zuul.yaml

@@ -213,6 +213,8 @@
       The regular tempest-integrated-storage job but with glance metadata injection
     post-run: playbooks/post-check-metadata-injection.yaml
     vars:
+      configure_swap_size: 8192
+      tempest_concurrency: 3
       zuul_copy_output:
         /etc/glance-remote: logs
       devstack_localrc:
@@ -236,6 +238,11 @@
                 "glance_devstack_test":"doyouseeme?"
             image_conversion:
               output_format: raw
+        test-config:
+          "$TEMPEST_CONFIG":
+            image:
+              image_caching_enabled: True
+              disk_formats: qcow2,ari,aki,vhd,vmdk,raw,ami,vdi,iso,vhdx
 
 - job:
     name: glance-multistore-cinder-import
@@ -249,27 +256,6 @@
         CINDER_ENABLED_BACKENDS: lvm:lvmdriver-1,lvm:lvmdriver-2
         GLANCE_CINDER_DEFAULT_BACKEND: lvmdriver-1
 
-- job:
-    name: glance-multistore-cinder-import-fips
-    parent: tempest-integrated-storage-import
-    description: |
-      The regular import workflow job to test with multiple cinder stores with
-      fips enabled
-    nodeset: devstack-single-node-centos-8-stream
-    pre-run: playbooks/enable-fips.yaml
-    vars:
-      configure_swap_size: 4096
-      devstack_localrc:
-        USE_CINDER_FOR_GLANCE: True
-        GLANCE_ENABLE_MULTIPLE_STORES: True
-        CINDER_ENABLED_BACKENDS: lvm:lvmdriver-1,lvm:lvmdriver-2
-        GLANCE_CINDER_DEFAULT_BACKEND: lvmdriver-1
-      devstack_local_conf:
-        test-config:
-          "$TEMPEST_CONFIG":
-            validation:
-              ssh_key_type: 'ecdsa'
-
 - job:
     name: tempest-integrated-storage-import-standalone
     parent: tempest-integrated-storage-import
@@ -306,7 +292,6 @@
       - release-notes-jobs-python3
     check:
       jobs:
-        - openstack-tox-functional-py36-fips
         - openstack-tox-functional-py39
         - glance-tox-functional-py39-rbac-defaults
         - glance-ceph-thin-provisioning:
@@ -331,16 +316,18 @@
             irrelevant-files: *tempest-irrelevant-files
         - glance-multistore-cinder-import:
             irrelevant-files: *tempest-irrelevant-files
-        - glance-multistore-cinder-import-fips:
-            voting: false
-            irrelevant-files: *tempest-irrelevant-files
         - grenade:
             irrelevant-files: *tempest-irrelevant-files
         - tempest-ipv6-only:
             irrelevant-files: *tempest-irrelevant-files
         - nova-ceph-multistore:
             irrelevant-files: *tempest-irrelevant-files
-        - glance-secure-rbac-protection-functional
+        - glance-secure-rbac-protection-functional:
+            required-projects:
+              - name: openstack/glance
+                override-checkout: yoga-last
+              - name: openstack/glance-tempest-plugin
+                override-checkout: yoga-last
 
     gate:
       jobs:
@@ -358,7 +345,12 @@
             irrelevant-files: *tempest-irrelevant-files
         - nova-ceph-multistore:
             irrelevant-files: *tempest-irrelevant-files
-        - glance-secure-rbac-protection-functional
+        - glance-secure-rbac-protection-functional:
+            required-projects:
+              - name: openstack/glance
+                override-checkout: yoga-last
+              - name: openstack/glance-tempest-plugin
+                override-checkout: yoga-last
     experimental:
       jobs:
         - glance-tox-py39-glance_store-tips

bindep.txt

@@ -23,7 +23,7 @@ postgresql-server [platform:rpm]
 libpython3-dev [platform:dpkg]
 python3-devel [platform:rpm]
 qemu [platform:dpkg devstack build-image-dib]
-qemu-utils [platform:dpkg devstack build-image-dib]
+qemu-utils [platform:dpkg]
 qemu-img [platform:rpm]
 libpq-dev [platform:dpkg]
 libpcre3-dev [platform:dpkg]

glance/async_/flows/api_image_import.py

@@ -207,6 +207,10 @@ def image_locations(self):
         # should have moderated access like all the other things here.
         return copy.deepcopy(self._image.locations)
 
+    @property
+    def image_disk_format(self):
+        return self._image.disk_format
+
     @property
     def image_status(self):
         return self._image.status

glance/async_/flows/plugins/image_conversion.py

@@ -89,10 +89,7 @@ def _execute(self, action, file_path, **kwargs):
                                              'target': target_format}
         self.dest_path = dest_path
 
-        # Backport fixup due to lack of
-        # Ic51c5fd87caf04d38aeaf758ad2d0e2f28098e4d in Yoga:
-        #source_format = action.image_disk_format
-        source_format = action._image.disk_format
+        source_format = action.image_disk_format
         inspector_cls = format_inspector.get_inspector(source_format)
         if not inspector_cls:
             # We cannot convert from disk_format types that qemu-img doesn't

glance/common/format_inspector.py

@@ -488,6 +488,7 @@ class VHDXInspector(FileInspector):
     """
     METAREGION = '8B7CA206-4790-4B9A-B8FE-575F050F886E'
     VIRTUAL_DISK_SIZE = '2FA54224-CD1B-4876-B211-5DBED83BF4B8'
+    VHDX_METADATA_TABLE_MAX_SIZE = 32 * 2048  # From qemu
 
     def __init__(self, *a, **k):
         super(VHDXInspector, self).__init__(*a, **k)
@@ -602,6 +603,8 @@ def _find_meta_entry(self, desired_guid):
                 item_offset, item_length, _reserved = struct.unpack(
                     '<III',
                     meta_buffer[entry_offset + 16:entry_offset + 28])
+                item_length = min(item_length,
+                                  self.VHDX_METADATA_TABLE_MAX_SIZE)
                 self.region('metadata').length = len(meta_buffer)
                 self._log.debug('Found entry at offset %x', item_offset)
                 # Metadata item offset is from the beginning of the metadata
@@ -652,7 +655,7 @@ def __str__(self):
 #
 # https://www.vmware.com/app/vmdk/?src=vmdk
 class VMDKInspector(FileInspector):
-    """vmware VMDK format (monolithicSparse variant only)
+    """vmware VMDK format (monolithicSparse and streamOptimized variants only)
 
     This needs to store the 512 byte header and the descriptor region
     which should be just after that. The descriptor region is some
@@ -683,7 +686,6 @@ def post_process(self):
 
         if sig != b'KDMV':
             raise ImageFormatError('Signature KDMV not found: %r' % sig)
-            return
 
         if ver not in (1, 2, 3):
             raise ImageFormatError('Unsupported format version %i' % ver)
@@ -693,9 +695,18 @@ def post_process(self):
             # header, which we cannot support since we stream.
             raise ImageFormatError('Unsupported VMDK footer')
 
+        # Since we parse both desc_sec and desc_num (the location of the
+        # VMDK's descriptor, expressed in 512 bytes sectors) we enforce a
+        # check on the bounds to create a reasonable CaptureRegion. This
+        # is similar to how it's done in qemu.
+        desc_offset = desc_sec * 512
+        desc_size = min(desc_num * 512, self.DESC_MAX_SIZE)
+        if desc_offset != self.DESC_OFFSET:
+            raise ImageFormatError("Wrong descriptor location")
+
         if not self.has_region('descriptor'):
             self.new_region('descriptor', CaptureRegion(
-                desc_sec * 512, desc_num * 512))
+                desc_offset, desc_size))
 
     @property
     def format_match(self):
@@ -721,7 +732,7 @@ def virtual_size(self):
             vmdktype = descriptor[type_idx:type_end]
         else:
             vmdktype = b'formatnotfound'
-        if vmdktype != b'monolithicSparse':
+        if vmdktype not in (b'monolithicSparse', b'streamOptimized'):
             LOG.warning('Unsupported VMDK format %s', vmdktype)
             return 0