Add object storage capacity and alerts.

Author: abh1sar

api/src/main/java/com/cloud/capacity/Capacity.java

@@ -34,13 +34,17 @@ public interface Capacity extends InternalIdentity, Identity {
     public static final short CAPACITY_TYPE_LOCAL_STORAGE = 9;
     public static final short CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET = 10;
     public static final short CAPACITY_TYPE_GPU = 19;
+    public static final short CAPACITY_TYPE_OBJECT_STORAGE = 20;
+    public static final short CAPACITY_TYPE_BACKUP_STORAGE = 21;
 
     public static final short CAPACITY_TYPE_CPU_CORE = 90;
 
     public static final List<Short> STORAGE_CAPACITY_TYPES = List.of(CAPACITY_TYPE_STORAGE,
             CAPACITY_TYPE_STORAGE_ALLOCATED,
             CAPACITY_TYPE_SECONDARY_STORAGE,
-            CAPACITY_TYPE_LOCAL_STORAGE);
+            CAPACITY_TYPE_LOCAL_STORAGE,
+            CAPACITY_TYPE_BACKUP_STORAGE,
+            CAPACITY_TYPE_OBJECT_STORAGE);
 
     public Long getHostOrPoolId();
 

api/src/main/java/com/cloud/storage/StorageService.java

@@ -131,7 +131,7 @@ public interface StorageService {
 
     void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd);
 
-    ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
+    ObjectStore discoverObjectStore(String name, String url, Long size, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
 
     boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd);
 

api/src/main/java/org/apache/cloudstack/alert/AlertService.java

@@ -73,6 +73,8 @@ private AlertType(short type, String name, boolean isDefault) {
         public static final AlertType ALERT_TYPE_VM_SNAPSHOT = new AlertType((short)32, "ALERT.VM.SNAPSHOT", true);
         public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PUBLIC.IFACE.MTU", true);
         public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PRIVATE.IFACE.MTU", true);
+        public static final AlertType ALERT_TYPE_BACKUP_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, "ALERT.STORAGE.BACKUP", true);
+        public static final AlertType ALERT_TYPE_OBJECT_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, "ALERT.STORAGE.OBJECT", true);
 
         public short getType() {
             return type;

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -29,6 +29,7 @@ public class ApiConstants {
     public static final String ADDRESS = "address";
     public static final String ALGORITHM = "algorithm";
     public static final String ALIAS = "alias";
+    public static final String ALLOCATED = "allocated";
     public static final String ALLOCATED_DATE = "allocateddate";
     public static final String ALLOCATED_ONLY = "allocatedonly";
     public static final String ALLOCATED_TIME = "allocated";
@@ -521,6 +522,7 @@ public class ApiConstants {
     public static final String TRUST_STORE_PASSWORD = "truststorepass";
     public static final String URL = "url";
     public static final String USAGE_INTERFACE = "usageinterface";
+    public static final String USED = "used";
     public static final String USED_SUBNETS = "usedsubnets";
     public static final String USED_IOPS = "usediops";
     public static final String USER_DATA = "userdata";

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java

@@ -56,6 +56,9 @@ public class AddObjectStoragePoolCmd extends BaseCmd {
     @Parameter(name = ApiConstants.TAGS, type = CommandType.STRING, description = "the tags for the storage pool")
     private String tags;
 
+    @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, description = "the total size of the object store in GiB. Used for tracking capacity and sending alerts", since = "4.21")
+    private Long size;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -68,6 +71,10 @@ public String getName() {
         return name;
     }
 
+    public Long getTotalSize() {
+        return size;
+    }
+
     public Map<String, String> getDetails() {
         Map<String, String> detailsMap = null;
         if (details != null && !details.isEmpty()) {
@@ -112,7 +119,7 @@ public long getEntityOwnerId() {
     @Override
     public void execute(){
         try{
-            ObjectStore result = _storageService.discoverObjectStore(getName(), getUrl(), getProviderName(), getDetails());
+            ObjectStore result = _storageService.discoverObjectStore(getName(), getUrl(), getTotalSize(), getProviderName(), getDetails());
             ObjectStoreResponse storeResponse = null;
             if (result != null) {
                 storeResponse = _responseGenerator.createObjectStoreResponse(result);

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java

@@ -44,6 +44,8 @@ public class UpdateObjectStoragePoolCmd extends BaseCmd {
     @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the url for the object store")
     private String url;
 
+    @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, description = "the total size of the object store in GiB. Used for tracking capacity and sending alerts", since = "4.21")
+    private Long size;
 
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
@@ -61,6 +63,10 @@ public String getUrl() {
         return url;
     }
 
+    public Long getSize() {
+        return size;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java

@@ -17,34 +17,40 @@
 package org.apache.cloudstack.api.response;
 
 import com.cloud.serializer.Param;
+
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.storage.object.ObjectStore;
 import com.google.gson.annotations.SerializedName;
 import org.apache.cloudstack.api.BaseResponseWithAnnotations;
 import org.apache.cloudstack.api.EntityReference;
 
 @EntityReference(value = ObjectStore.class)
 public class ObjectStoreResponse extends BaseResponseWithAnnotations {
-    @SerializedName("id")
+    @SerializedName(ApiConstants.ID)
     @Param(description = "the ID of the object store")
     private String id;
 
-    @SerializedName("name")
+    @SerializedName(ApiConstants.NAME)
     @Param(description = "the name of the object store")
     private String name;
 
-    @SerializedName("url")
+    @SerializedName(ApiConstants.URL)
     @Param(description = "the url of the object store")
     private String url;
 
-    @SerializedName("providername")
-    @Param(description = "the provider name of the object store")
+    @SerializedName(ApiConstants.PROVIDER)
+    @Param(description = "the name of the object store provider")
     private String providerName;
 
-    @SerializedName("storagetotal")
+    @SerializedName(ApiConstants.SIZE)
     @Param(description = "the total size of the object store")
     private Long storageTotal;
 
-    @SerializedName("storageused")
+    @SerializedName(ApiConstants.ALLOCATED)
+    @Param(description = "the object store currently allocated size")
+    private Long storageAllocated;
+
+    @SerializedName(ApiConstants.USED)
     @Param(description = "the object store currently used size")
     private Long storageUsed;
 
@@ -96,6 +102,14 @@ public void setStorageTotal(Long storageTotal) {
         this.storageTotal = storageTotal;
     }
 
+    public Long getStorageAllocated() {
+        return storageAllocated;
+    }
+
+    public void setStorageAllocated(Long storageAllocated) {
+        this.storageAllocated = storageAllocated;
+    }
+
     public Long getStorageUsed() {
         return storageUsed;
     }

api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java

@@ -61,6 +61,8 @@ public class AddObjectStoragePoolCmdTest {
 
     String provider = "Simulator";
 
+    Long size = 10L;
+
     Map<String, String> details;
 
     private AutoCloseable closeable;
@@ -74,6 +76,7 @@ public void setUp() throws Exception {
         ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "url", url);
         ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "providerName", provider);
         ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "details", details);
+        ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "size", size);
         addObjectStoragePoolCmdSpy._storageService = storageService;
         addObjectStoragePoolCmdSpy._responseGenerator = responseGenerator;
     }
@@ -87,12 +90,12 @@ public void tearDown() throws Exception {
     @Test
     public void testAddObjectStore() throws DiscoveryException {
         Mockito.doReturn(objectStore).when(storageService).discoverObjectStore(Mockito.anyString(),
-                Mockito.anyString(), Mockito.anyString(), any());
+                Mockito.anyString(), Mockito.anyLong(), Mockito.anyString(), any());
         ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse();
         Mockito.doReturn(objectStoreResponse).when(responseGenerator).createObjectStoreResponse(any());
         addObjectStoragePoolCmdSpy.execute();
 
         Mockito.verify(storageService, Mockito.times(1))
-                .discoverObjectStore(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any());
+                .discoverObjectStore(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any());
     }
 }

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -408,4 +408,5 @@ void connectHostsToPool(DataStore primaryStore, List<Long> hostIds, Scope scope,
 
     void validateChildDatastoresToBeAddedInUpState(StoragePoolVO datastoreClusterPool, List<ModifyStoragePoolAnswer> childDatastoreAnswerList);
 
+    CapacityVO getObjectStorageUsedStats(Long zoneId);
 }

engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java

@@ -254,6 +254,8 @@ public String getUuid() {
         capacityNames.put(CAPACITY_TYPE_GPU, "GPU");
         capacityNames.put(CAPACITY_TYPE_CPU_CORE, "CPU_CORE");
         capacityNames.put(CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET, "VIRTUAL_NETWORK_IPV6_SUBNET");
+        capacityNames.put(CAPACITY_TYPE_BACKUP_STORAGE, "BACKUP_STORAGE");
+        capacityNames.put(CAPACITY_TYPE_OBJECT_STORAGE, "OBJECT_STORAGE");
     }
 
     public static String getCapacityName (Short capacityType) {