refactor: use VHD header to fetch block size

Signed-off-by: Mathieu Labourier <mathieu.labourier@vates.tech>

Author: Millefeuille42

libs/sm/VDI.py

@@ -101,6 +101,7 @@ def __init__(self, sr, uuid):
         self.description = ''
         self.vbds = []
         self.size = 0
+        self._block_size = -1
         self.utilisation = 0
         self.vdi_type = ''
         self.has_child = 0
@@ -120,6 +121,12 @@ def __init__(self, sr, uuid):
 
         self.load(uuid)
 
+    @property
+    def block_size(self):
+        if self._block_size < 0:
+            self._block_size = vhdutil.getBlockSize(self.path)
+        return self._block_size
+
     @staticmethod
     def from_uuid(session, vdi_uuid):
 

libs/sm/cleanup.py

@@ -535,12 +535,19 @@ def __init__(self, sr, uuid, raw):
         self.sizeVirt = -1
         self._sizeVHD = -1
         self._sizeAllocated = -1
+        self._block_size = -1
         self._hidden = False
         self.parent = None
         self.children = []
         self._vdiRef = None
         self._clearRef()
 
+    @property
+    def block_size(self):
+        if self._block_size < 0:
+            self._block_size = vhdutil.getBlockSize(self.path)
+        return self._block_size
+
     @staticmethod
     def extractUuid(path):
         raise NotImplementedError("Implement in sub class")
@@ -1075,14 +1082,16 @@ def _getCoalescedSizeData(self):
         blocksParent = self.parent.getVHDBlocks()
         numBlocks = Util.countBits(blocksChild, blocksParent)
         Util.log("Num combined blocks = %d" % numBlocks)
-        sizeData = numBlocks * vhdutil.VHD_BLOCK_SIZE
+        sizeData = numBlocks * self.block_size
         assert(sizeData <= self.sizeVirt)
         return sizeData
 
     def _calcExtraSpaceForCoalescing(self):
         sizeData = self._getCoalescedSizeData()
-        sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(sizeData) + \
-                vhdutil.calcOverheadEmpty(self.sizeVirt)
+        sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(
+            sizeData,
+            self.block_size
+        ) + vhdutil.calcOverheadEmpty(self.sizeVirt)
         Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced))
         return sizeCoalesced - self.parent.getSizeVHD()
 
@@ -1238,7 +1247,7 @@ def deflate(self):
         self._sizeAllocated = -1
 
     def inflateFully(self):
-        self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt))
+        self.inflate(lvhdutil.calcSizeVHDLV(self.sizeVirt, self.block_size))
 
     def inflateParentForCoalesce(self):
         """Inflate the parent only as much as needed for the purposes of
@@ -1461,7 +1470,10 @@ def _queryVHDBlocks(self):
     def _calcExtraSpaceForCoalescing(self):
         if self.parent.raw:
             return 0  # raw parents are never deflated in the first place
-        sizeCoalesced = lvhdutil.calcSizeVHDLV(self._getCoalescedSizeData())
+        sizeCoalesced = lvhdutil.calcSizeVHDLV(
+            self._getCoalescedSizeData(),
+            self.block_size
+        )
         Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced))
         return sizeCoalesced - self.parent.sizeLV
 
@@ -2775,7 +2787,7 @@ def _finishCoalesceLeaf(self, parent):
             parent.deflate()
 
     def _calcExtraSpaceNeeded(self, child, parent):
-        return lvhdutil.calcSizeVHDLV(parent.sizeVirt) - parent.sizeLV
+        return lvhdutil.calcSizeVHDLV(parent.sizeVirt, parent.block_size) - parent.sizeLV
 
     def _handleInterruptedCoalesceLeaf(self):
         entries = self.journaler.getAll(VDI.JRN_LEAF)

libs/sm/drivers/FileSR.py

@@ -656,7 +656,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
             return VDI.VDI.get_params(self)
 
         # We already checked it is a VDI_TYPE_VHD
-        size = vhdutil.validate_and_round_vhd_size(int(size))
+        size = vhdutil.validate_and_round_vhd_size(int(size), self.block_size)
 
         jFile = JOURNAL_FILE_PREFIX + self.uuid
         try:

libs/sm/drivers/LVHDSR.py

@@ -716,7 +716,10 @@ def scan(self, uuid):
                                     util.roundup(lvutil.LVM_SIZE_INCREMENT,
                                       vhdutil.calcOverheadEmpty(lvhdutil.MSIZE))
                             else:
-                                utilisation = lvhdutil.calcSizeVHDLV(int(size))
+                                utilisation = lvhdutil.calcSizeVHDLV(
+                                    int(size),
+                                    vhdutil.getBlockSize(lvPath)
+                                )
 
                         vdi_ref = self.session.xenapi.VDI.db_introduce(
                                         vdi_uuid,
@@ -984,7 +987,10 @@ def _undoCloneOp(self, lvs, origUuid, baseUuid, clonUuid):
 
         # inflate the parent to fully-allocated size
         if base.vdiType == vhdutil.VDI_TYPE_VHD:
-            fullSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt)
+            fullSize = lvhdutil.calcSizeVHDLV(
+                vhdInfo.sizeVirt,
+                vhdutil.getBlockSize(basePath)
+            )
             lvhdutil.inflate(self.journaler, self.uuid, baseUuid, fullSize)
 
         # rename back
@@ -1173,7 +1179,7 @@ def _undoAllVHDJournals(self):
             util.SMlog("Found VHD journal %s, reverting %s" % (uuid, vdi.path))
             self.lvActivator.activate(uuid, vdi.lvname, False)
             self.lvmCache.activateNoRefcount(jlvName)
-            fullSize = lvhdutil.calcSizeVHDLV(vdi.size)
+            fullSize = lvhdutil.calcSizeVHDLV(vdi.size, vdi.block_size)
             lvhdutil.inflate(self.journaler, self.uuid, vdi.uuid, fullSize)
             try:
                 jFile = os.path.join(self.path, jlvName)
@@ -1184,7 +1190,7 @@ def _undoAllVHDJournals(self):
                 util.SMlog("VHD revert failed but VHD ok: removing journal")
             # Attempt to reclaim unused space
             vhdInfo = vhdutil.getVHDInfo(vdi.path, lvhdutil.extractUuid, False)
-            NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt)
+            NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt, vdi.block_size)
             if NewSize < fullSize:
                 lvhdutil.deflate(self.lvmCache, vdi.lvname, int(NewSize))
             lvhdutil.lvRefreshOnAllSlaves(self.session, self.uuid,
@@ -1459,7 +1465,10 @@ def attach(self, sr_uuid, vdi_uuid):
             needInflate = False
         else:
             self._loadThis()
-            if self.utilisation >= lvhdutil.calcSizeVHDLV(self.size):
+            if (
+                self.utilisation >=
+                lvhdutil.calcSizeVHDLV(self.size, self.block_size)
+            ):
                 needInflate = False
 
         if needInflate:
@@ -1479,7 +1488,7 @@ def detach(self, sr_uuid, vdi_uuid):
         util.SMlog("LVHDVDI.detach for %s" % self.uuid)
         self._loadThis()
         already_deflated = (self.utilisation < \
-                lvhdutil.calcSizeVHDLV(self.size))
+                lvhdutil.calcSizeVHDLV(self.size, self.block_size))
         needDeflate = True
         if self.vdi_type == vhdutil.VDI_TYPE_RAW or already_deflated:
             needDeflate = False
@@ -1520,7 +1529,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
                     '(current size: %d, new size: %d)' % (self.size, size))
             raise xs_errors.XenError('VDISize', opterr='shrinking not allowed')
 
-        size = vhdutil.validate_and_round_vhd_size(int(size))
+        size = vhdutil.validate_and_round_vhd_size(int(size), self.block_size)
 
         if size == self.size:
             return VDI.VDI.get_params(self)
@@ -1530,7 +1539,7 @@ def resize(self, sr_uuid, vdi_uuid, size):
             lvSizeNew = util.roundup(lvutil.LVM_SIZE_INCREMENT, size)
         else:
             lvSizeOld = self.utilisation
-            lvSizeNew = lvhdutil.calcSizeVHDLV(size)
+            lvSizeNew = lvhdutil.calcSizeVHDLV(size, self.block_size)
             if self.sr.provision == "thin":
                 # VDI is currently deflated, so keep it deflated
                 lvSizeNew = lvSizeOld
@@ -1696,7 +1705,7 @@ def _snapshot(self, snapType, cloneOp=False, cbtlog=None, cbt_consistency=None):
         self.issnap = self.session.xenapi.VDI.get_is_a_snapshot( \
                                                 self.sr.srcmd.params['vdi_ref'])
 
-        fullpr = lvhdutil.calcSizeVHDLV(self.size)
+        fullpr = lvhdutil.calcSizeVHDLV(self.size, self.block_size)
         thinpr = util.roundup(lvutil.LVM_SIZE_INCREMENT, \
                 vhdutil.calcOverheadEmpty(lvhdutil.MSIZE))
         lvSizeOrig = thinpr

libs/sm/lvhdutil.py

@@ -95,11 +95,11 @@ def calcSizeLV(sizeVHD):
     return util.roundup(LVM_SIZE_INCREMENT, sizeVHD)
 
 
-def calcSizeVHDLV(sizeVirt):
+def calcSizeVHDLV(sizeVirt, block_size=vhdutil.DEFAULT_VHD_BLOCK_SIZE):
     # all LVHD VDIs have the metadata area preallocated for the maximum
     # possible virtual size (for fast online VDI.resize)
     metaOverhead = vhdutil.calcOverheadEmpty(MSIZE)
-    bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt)
+    bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt, block_size)
     return calcSizeLV(sizeVirt + metaOverhead + bitmapOverhead)
 
 
@@ -208,7 +208,12 @@ def setSizeVirt(journaler, srUuid, vdiUuid, size, jFile):
     lvName = LV_PREFIX[vhdutil.VDI_TYPE_VHD] + vdiUuid
     vgName = VG_PREFIX + srUuid
     path = os.path.join(VG_LOCATION, vgName, lvName)
-    inflate(journaler, srUuid, vdiUuid, calcSizeVHDLV(size))
+    inflate(
+        journaler,
+        srUuid,
+        vdiUuid,
+        calcSizeVHDLV(size, vhdutil.getBlockSize(path))
+    )
     vhdutil.setSizeVirt(path, size, jFile)
 
 
@@ -233,7 +238,8 @@ def attachThin(journaler, srUuid, vdiUuid):
     _tryAcquire(lock)
     lvmCache.refresh()
     vhdInfo = vhdutil.getVHDInfoLVM(lvName, extractUuid, vgName)
-    newSize = calcSizeVHDLV(vhdInfo.sizeVirt)
+    path = os.path.join(VG_LOCATION, vgName, lvName)
+    newSize = calcSizeVHDLV(vhdInfo.sizeVirt, vhdutil.getBlockSize(path))
     currSizeLV = lvmCache.getSize(lvName)
     if newSize <= currSizeLV:
         return

libs/sm/vhdutil.py

@@ -30,7 +30,7 @@
 MAX_CHAIN_SIZE = 30  # max VHD parent chain size
 VHD_UTIL = "/usr/bin/vhd-util"
 OPT_LOG_ERR = "--debug"
-VHD_BLOCK_SIZE = 2 * 1024 * 1024
+DEFAULT_VHD_BLOCK_SIZE = 2 * 1024 * 1024
 VHD_FOOTER_SIZE = 512
 
 # lock to lock the entire SR for short ops
@@ -82,9 +82,9 @@ def calcOverheadEmpty(virtual_size):
     return overhead
 
 
-def calcOverheadBitmap(virtual_size):
-    num_blocks = virtual_size // VHD_BLOCK_SIZE
-    if virtual_size % VHD_BLOCK_SIZE:
+def calcOverheadBitmap(virtual_size, block_size=DEFAULT_VHD_BLOCK_SIZE):
+    num_blocks = virtual_size // block_size
+    if virtual_size % block_size:
         num_blocks += 1
     return num_blocks * 4096
 
@@ -93,10 +93,18 @@ def ioretry(cmd, text=True):
     return util.ioretry(lambda: util.pread2(cmd, text=text),
                         errlist=[errno.EIO, errno.EAGAIN])
 
+def getBlockSize(path):
+    cmd = [VHD_UTIL, "read", "-pn", path]
+    ret = ioretry(cmd)
+    for field in ret.split('\n'):
+        field = field.lstrip()
+        if not field.startswith("Block size"): continue
+        return int(field.split(':')[1].lstrip().split()[0])
+    raise util.SMException("Unable to find block size in VHD with path: {}".format(path))
+
 
-def convertAllocatedSizeToBytes(size):
-    # Assume we have standard 2MB allocation blocks
-    return size * 2 * 1024 * 1024
+def convertAllocatedSizeToBytes(size, block_size=DEFAULT_VHD_BLOCK_SIZE):
+    return size * block_size
 
 
 def getVHDInfo(path, extractUuidFunction, includeParent=True):
@@ -120,7 +128,10 @@ def getVHDInfo(path, extractUuidFunction, includeParent=True):
             vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex])
         nextIndex += 1
     vhdInfo.hidden = int(fields[nextIndex].replace("hidden: ", ""))
-    vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(int(fields[nextIndex+1]))
+    vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(
+        int(fields[nextIndex+1]),
+        getBlockSize(path)
+    )
     vhdInfo.path = path
     return vhdInfo
 
@@ -279,7 +290,7 @@ def setSizePhys(path, size, debug=True):
 def getAllocatedSize(path):
     cmd = [VHD_UTIL, "query", OPT_LOG_ERR, '-a', '-n', path]
     ret = ioretry(cmd)
-    return convertAllocatedSizeToBytes(int(ret))
+    return convertAllocatedSizeToBytes(int(ret), getBlockSize(path))
 
 def killData(path):
     "zero out the disk (kill all data inside the VHD file)"
@@ -406,7 +417,7 @@ def repair(path):
     ioretry([VHD_UTIL, 'repair', '-n', path])
 
 
-def validate_and_round_vhd_size(size):
+def validate_and_round_vhd_size(size, block_size=DEFAULT_VHD_BLOCK_SIZE):
     """ Take the supplied vhd size, in bytes, and check it is positive and less
     that the maximum supported size, rounding up to the next block boundary
     """
@@ -419,7 +430,7 @@ def validate_and_round_vhd_size(size):
     if size < MIN_VHD_SIZE:
         size = MIN_VHD_SIZE
 
-    size = util.roundup(VHD_BLOCK_SIZE, size)
+    size = util.roundup(block_size, size)
 
     return size
 

tests/test_FileSR.py

@@ -327,6 +327,7 @@ def test_create_vdi_vhd(self, mock_vhdutil):
         vdi = FakeFileVDI(sr, vdi_uuid)
         vdi.vdi_type = vhdutil.VDI_TYPE_VHD
         mock_vhdutil.validate_and_round_vhd_size.side_effect = vhdutil.validate_and_round_vhd_size
+        mock_vhdutil.DEFAULT_VHD_BLOCK_SIZE = vhdutil.DEFAULT_VHD_BLOCK_SIZE
 
         # Act
         vdi.create(sr_uuid, vdi_uuid, 20 * 1024 * 1024)

tests/test_LVHDSR.py

@@ -307,9 +307,13 @@ def setUp(self):
         self.mock_lvhdutil.LV_PREFIX = lvhdutil.LV_PREFIX
         vhdutil_patcher = mock.patch('sm.drivers.LVHDSR.vhdutil', autospec=True)
         self.mock_vhdutil = vhdutil_patcher.start()
+        self.mock_vhdutil.getBlockSize.return_value = vhdutil.DEFAULT_VHD_BLOCK_SIZE
         self.mock_vhdutil.VDI_TYPE_VHD = vhdutil.VDI_TYPE_VHD
         self.mock_vhdutil.VDI_TYPE_RAW = vhdutil.VDI_TYPE_RAW
         self.mock_vhdutil.MAX_CHAIN_SIZE = vhdutil.MAX_CHAIN_SIZE
+        vdi_vhdutil_patcher = mock.patch('sm.VDI.vhdutil', autospec=True)
+        self.mock_vdi_vhdutil = vdi_vhdutil_patcher.start()
+        self.mock_vdi_vhdutil.getBlockSize.return_value = vhdutil.DEFAULT_VHD_BLOCK_SIZE
         lvutil_patcher = mock.patch('sm.drivers.LVHDSR.lvutil', autospec=True)
         self.mock_lvutil = lvutil_patcher.start()
         vdi_util_patcher = mock.patch('sm.VDI.util', autospec=True)

tests/test_vhdutil.py

@@ -376,10 +376,14 @@ def test_function(args, inp):
 
         context.add_executable(VHD_UTIL, test_function)
         from sm.drivers import FileSR
-        vhdinfo = vhdutil.getVHDInfo(TEST_VHD_PATH, FileSR.FileVDI.extractUuid)
+        with unittest.mock.patch(
+                "sm.vhdutil.getBlockSize",
+                return_value=vhdutil.DEFAULT_VHD_BLOCK_SIZE
+        ):
+            vhdinfo = vhdutil.getVHDInfo(TEST_VHD_PATH, FileSR.FileVDI.extractUuid)
 
-        # Act/Assert
-        self.assertEqual(18856*2*1024*1024 , vhdinfo.sizeAllocated)
+            # Act/Assert
+            self.assertEqual(18856*2*1024*1024 , vhdinfo.sizeAllocated)
 
     @testlib.with_context
     def test_get_allocated_size(self, context):
@@ -396,11 +400,35 @@ def test_function(args, inp):
         context.add_executable(VHD_UTIL, test_function)
 
         # Act
-        result = vhdutil.getAllocatedSize(TEST_VHD_NAME)
+        result = 0
+        with unittest.mock.patch(
+                "sm.vhdutil.getBlockSize",
+                return_value=vhdutil.DEFAULT_VHD_BLOCK_SIZE
+        ):
+            result = vhdutil.getAllocatedSize(TEST_VHD_NAME)
 
         # Assert
         self.assertEqual(18856*2*1024*1024, result)
         self.assertEqual(
             [VHD_UTIL, "query", "--debug", "-a",
              "-n", TEST_VHD_NAME],
             call_args)
+
+    @testlib.with_context
+    def test_get_block_size(self, context):
+        """
+        Test that vhdutil.getBlockSize returns the block size in bytes
+        """
+
+        # Arrange
+        call_args = None
+
+        def test_function(args, inp):
+            nonlocal call_args
+            call_args = args
+            return 0, "Header version: 0x00010000\nBlock size: {} (2 MB)".format(vhdutil.DEFAULT_VHD_BLOCK_SIZE), ""
+
+        context.add_executable(VHD_UTIL, test_function)
+
+        # Act/Assert
+        self.assertEqual(vhdutil.DEFAULT_VHD_BLOCK_SIZE, vhdutil.getBlockSize(TEST_VHD_NAME))