server, plugin: enhance storage stats for IOPS

api/src/main/java/com/cloud/storage/StorageStats.java

@@ -26,4 +26,7 @@ public interface StorageStats {
      * @return bytes capacity of the storage server
      */
     public long getCapacityBytes();
+
+    Long getCapacityIops();
+    Long getUsedIops();
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -509,6 +509,7 @@ public class ApiConstants {
     public static final String URL = "url";
     public static final String USAGE_INTERFACE = "usageinterface";
     public static final String USED_SUBNETS = "usedsubnets";
+    public static final String USED_IOPS = "usediops";
     public static final String USER_DATA = "userdata";
 
     public static final String USER_DATA_NAME = "userdataname";

api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java

@@ -97,6 +97,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
     @Param(description = "total min IOPS currently in use by volumes")
     private Long allocatedIops;
 
+    @SerializedName(ApiConstants.USED_IOPS)
+    @Param(description = "total IOPS currently in use", since = "4.20.1")
+    private Long usedIops;
+
     @SerializedName(ApiConstants.STORAGE_CUSTOM_STATS)
     @Param(description = "the storage pool custom stats", since = "4.18.1")
     private Map<String, String> customStats;
@@ -312,6 +316,14 @@ public void setAllocatedIops(Long allocatedIops) {
        this.allocatedIops = allocatedIops;
     }
 
+    public Long getUsedIops() {
+        return usedIops;
+    }
+
+    public void setUsedIops(Long usedIops) {
+        this.usedIops = usedIops;
+    }
+
     public Map<String, String> getCustomStats() {
         return customStats;
     }

core/src/main/java/com/cloud/agent/api/GetStorageStatsAnswer.java

@@ -27,24 +27,46 @@ public class GetStorageStatsAnswer extends Answer implements StorageStats {
     protected GetStorageStatsAnswer() {
     }
 
-    protected long used;
+    protected long usedBytes;
 
-    protected long capacity;
+    protected long capacityBytes;
+
+    protected Long capacityIops;
+
+    protected Long usedIops;
 
     @Override
     public long getByteUsed() {
-        return used;
+        return usedBytes;
     }
 
     @Override
     public long getCapacityBytes() {
-        return capacity;
+        return capacityBytes;
+    }
+
+    @Override
+    public Long getCapacityIops() {
+        return capacityIops;
+    }
+
+    @Override
+    public Long getUsedIops() {
+        return usedIops;
+    }
+
+    public GetStorageStatsAnswer(GetStorageStatsCommand cmd, long capacityBytes, long usedBytes) {
+        super(cmd, true, null);
+        this.capacityBytes = capacityBytes;
+        this.usedBytes = usedBytes;
     }
 
-    public GetStorageStatsAnswer(GetStorageStatsCommand cmd, long capacity, long used) {
+    public GetStorageStatsAnswer(GetStorageStatsCommand cmd, long capacityBytes, long usedBytes, Long capacityIops, Long usedIops) {
         super(cmd, true, null);
-        this.capacity = capacity;
-        this.used = used;
+        this.capacityBytes = capacityBytes;
+        this.usedBytes = usedBytes;
+        this.capacityIops = capacityIops;
+        this.usedIops = usedIops;
     }
 
     public GetStorageStatsAnswer(GetStorageStatsCommand cmd, String details) {

debian/control

@@ -24,7 +24,7 @@ Description: CloudStack server library
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd
+Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java

@@ -111,6 +111,14 @@ default Map<String, String> getCustomStorageStats(StoragePool pool) {
      */
     Pair<Long, Long> getStorageStats(StoragePool storagePool);
 
+    /**
+     * Intended for managed storage
+     * returns the capacity and used IOPS or null if not supported
+     */
+    default Pair<Long, Long> getStorageIopsStats(StoragePool storagePool) {
+        return null;
+    }
+
     /**
      * intended for managed storage
      * returns true if the storage can provide the volume stats (physical and virtual size)

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41700to41710.java

@@ -23,12 +23,16 @@
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDaoImpl;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.dao.VolumeDaoImpl;
 import com.cloud.upgrade.SystemVmTemplateRegistration;
+import com.cloud.utils.db.GenericSearchBuilder;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 public class Upgrade41700to41710 extends DbUpgradeAbstractImpl implements DbUpgradeSystemVmTemplate {
@@ -95,24 +99,46 @@ public void updateSystemVmTemplates(Connection conn) {
         }
     }
 
+    /*
+    GenericDao.customSearch using GenericSearchBuilder and GenericDao.update using
+    GenericDao.createSearchBuilder used here to prevent any future issues when new fields
+    are added to StoragePoolVO or VolumeVO and this upgrade path starts to fail.
+     */
     private void updateStorPoolStorageType() {
         storageDao = new PrimaryDataStoreDaoImpl();
-        List<StoragePoolVO> storPoolPools = storageDao.findPoolsByProvider("StorPool");
-        for (StoragePoolVO storagePoolVO : storPoolPools) {
-            if (StoragePoolType.SharedMountPoint == storagePoolVO.getPoolType()) {
-                storagePoolVO.setPoolType(StoragePoolType.StorPool);
-                storageDao.update(storagePoolVO.getId(), storagePoolVO);
-            }
-            updateStorageTypeForStorPoolVolumes(storagePoolVO.getId());
+        StoragePoolVO pool = storageDao.createForUpdate();
+        pool.setPoolType(StoragePoolType.StorPool);
+        SearchBuilder<StoragePoolVO> sb = storageDao.createSearchBuilder();
+        sb.and("provider", sb.entity().getStorageProviderName(), SearchCriteria.Op.EQ);
+        sb.and("type", sb.entity().getPoolType(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<StoragePoolVO> sc = sb.create();
+        sc.setParameters("provider", StoragePoolType.StorPool.name());
+        sc.setParameters("type", StoragePoolType.SharedMountPoint.name());
+        storageDao.update(pool, sc);
+
+        GenericSearchBuilder<StoragePoolVO, Long> gSb = storageDao.createSearchBuilder(Long.class);
+        gSb.selectFields(gSb.entity().getId());
+        gSb.and("provider", gSb.entity().getStorageProviderName(), SearchCriteria.Op.EQ);
+        gSb.done();
+        SearchCriteria<Long> gSc = gSb.create();
+        gSc.setParameters("provider", StoragePoolType.StorPool.name());
+        List<Long> poolIds = storageDao.customSearch(gSc, null);
+        if (CollectionUtils.isEmpty(poolIds)) {
+            return;
         }
+        updateStorageTypeForStorPoolVolumes(poolIds);
     }
 
-    private void updateStorageTypeForStorPoolVolumes(long storagePoolId) {
+    private void updateStorageTypeForStorPoolVolumes(List<Long> storagePoolIds) {
         volumeDao = new VolumeDaoImpl();
-        List<VolumeVO> volumes = volumeDao.findByPoolId(storagePoolId, null);
-        for (VolumeVO volumeVO : volumes) {
-            volumeVO.setPoolType(StoragePoolType.StorPool);
-            volumeDao.update(volumeVO.getId(), volumeVO);
-        }
+        VolumeVO volume = volumeDao.createForUpdate();
+        volume.setPoolType(StoragePoolType.StorPool);
+        SearchBuilder<VolumeVO> sb = volumeDao.createSearchBuilder();
+        sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.IN);
+        sb.done();
+        SearchCriteria<VolumeVO> sc = sb.create();
+        sc.setParameters("poolId", storagePoolIds.toArray());
+        volumeDao.update(volume, sc);
     }
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java

@@ -118,6 +118,9 @@ public class StoragePoolVO implements StoragePool {
     @Column(name = "capacity_iops", updatable = true, nullable = true)
     private Long capacityIops;
 
+    @Column(name = "used_iops", updatable = true, nullable = true)
+    private Long usedIops;
+
     @Column(name = "hypervisor")
     @Convert(converter = HypervisorTypeConverter.class)
     private HypervisorType hypervisor;
@@ -255,6 +258,14 @@ public Long getCapacityIops() {
         return capacityIops;
     }
 
+    public Long getUsedIops() {
+        return usedIops;
+    }
+
+    public void setUsedIops(Long usedIops) {
+        this.usedIops = usedIops;
+    }
+
     @Override
     public Long getClusterId() {
         return clusterId;

engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql

@@ -29,3 +29,6 @@ CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__
 CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_mshost_peer__owner_peer_runid','mshost_peer');
 CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.mshost_peer', 'i_mshost_peer__owner_peer', '(owner_mshost, peer_mshost)');
 CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.mshost_peer', 'fk_mshost_peer__owner_mshost', '(owner_mshost)', '`mshost`(`id`)');
+
+-- Add used_iops column to support IOPS data in storage stats
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.storage_pool', 'used_iops', 'bigint unsigned DEFAULT NULL COMMENT "IOPS currently in use for this storage pool" ');

engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql

@@ -32,6 +32,7 @@ SELECT
     `storage_pool`.`removed` AS `removed`,
     `storage_pool`.`capacity_bytes` AS `capacity_bytes`,
     `storage_pool`.`capacity_iops` AS `capacity_iops`,
+    `storage_pool`.`used_iops` AS `used_iops`,
     `storage_pool`.`scope` AS `scope`,
     `storage_pool`.`hypervisor` AS `hypervisor`,
     `storage_pool`.`storage_provider_name` AS `storage_provider_name`,

packaging/el8/cloud.spec

@@ -118,6 +118,7 @@ Requires: cryptsetup
 Requires: rng-tools
 Requires: (libgcrypt > 1.8.3 or libgcrypt20)
 Requires: (selinux-tools if qemu-tools)
+Requires: sysstat
 Provides: cloud-agent
 Group: System Environment/Libraries
 %description agent

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetStorageStatsCommandWrapper.java

@@ -40,7 +40,8 @@ public Answer execute(final GetStorageStatsCommand command, final LibvirtComputi
             if (sp == null) {
                 return new GetStorageStatsAnswer(command, "no storage pool to get statistics from");
             }
-            return new GetStorageStatsAnswer(command, sp.getCapacity(), sp.getUsed());
+            return new GetStorageStatsAnswer(command, sp.getCapacity(), sp.getUsed(), sp.getCapacityIops(),
+                    sp.getUsedIops());
         } catch (final CloudRuntimeException e) {
             return new GetStorageStatsAnswer(command, e.toString());
         }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java

@@ -62,6 +62,14 @@ public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDis
 
     public long getUsed();
 
+    default Long getCapacityIops() {
+        return null;
+    }
+
+    default Long getUsedIops() {
+        return null;
+    }
+
     public long getAvailable();
 
     public boolean refresh();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java

@@ -16,14 +16,21 @@
 // under the License.
 package com.cloud.hypervisor.kvm.storage;
 
+import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
+
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
 
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.utils.cryptsetup.KeyFile;
@@ -33,9 +40,10 @@
 import org.apache.cloudstack.utils.qemu.QemuImgFile;
 import org.apache.cloudstack.utils.qemu.QemuObject;
 import org.apache.commons.codec.binary.Base64;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.libvirt.Connect;
 import org.libvirt.LibvirtException;
 import org.libvirt.Secret;
@@ -69,14 +77,6 @@
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.Script;
 
-import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-
-
 public class LibvirtStorageAdaptor implements StorageAdaptor {
     protected Logger logger = LogManager.getLogger(getClass());
     private StorageLayer _storageLayer;
@@ -521,6 +521,54 @@ public KVMStoragePool getStoragePool(String uuid) {
         return this.getStoragePool(uuid, false);
     }
 
+    protected void updateLocalPoolIops(LibvirtStoragePool pool) {
+        if (!StoragePoolType.Filesystem.equals(pool.getType()) || StringUtils.isBlank(pool.getLocalPath())) {
+            return;
+        }
+        logger.trace("Updating used IOPS for pool: {}", pool.getName());
+
+        // Run script to get data
+        List<String[]> commands = new ArrayList<>();
+        commands.add(new String[]{
+                Script.getExecutableAbsolutePath("bash"),
+                "-c",
+                String.format(
+                        "%s %s | %s 'NR==2 {print $1}'",
+                        Script.getExecutableAbsolutePath("df"),
+                        pool.getLocalPath(),
+                        Script.getExecutableAbsolutePath("awk")
+                )
+        });
+        String result = Script.executePipedCommands(commands, 1000).second();
+        if (StringUtils.isBlank(result)) {
+            return;
+        }
+        result = result.trim();
+        commands.add(new String[]{
+                Script.getExecutableAbsolutePath("bash"),
+                "-c",
+                String.format(
+                        "%s -z %s 1 2 | %s 'NR==7 {read=$4; write=$5; total=read+write; print total}'",
+                        Script.getExecutableAbsolutePath("iostat"),
+                        result,
+                        Script.getExecutableAbsolutePath("awk")
+                )
+        });
+        result = Script.executePipedCommands(commands, 10000).second();
+        logger.trace("Pool used IOPS result: {}", result);
+        if (StringUtils.isBlank(result)) {
+            return;
+        }
+        try {
+            double doubleValue = Double.parseDouble(result);
+            pool.setUsedIops((long) doubleValue);
+            logger.debug("Updated used IOPS: {} for pool: {}", pool.getUsedIops(), pool.getName());
+        } catch (NumberFormatException e) {
+            logger.warn(String.format("Unable to parse retrieved used IOPS: %s for pool: %s", result,
+                    pool.getName()));
+        }
+    }
+
     @Override
     public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
         logger.info("Trying to fetch storage pool " + uuid + " from libvirt");
@@ -591,6 +639,7 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
             }
             pool.setCapacity(storage.getInfo().capacity);
             pool.setUsed(storage.getInfo().allocation);
+            updateLocalPoolIops(pool);
             pool.setAvailable(storage.getInfo().available);
 
             logger.debug("Successfully refreshed pool " + uuid +