diff --git a/api/src/main/java/com/cloud/configuration/Resource.java b/api/src/main/java/com/cloud/configuration/Resource.java index bf8fca9d9051..c7bf44de76c6 100644 --- a/api/src/main/java/com/cloud/configuration/Resource.java +++ b/api/src/main/java/com/cloud/configuration/Resource.java @@ -21,7 +21,7 @@ public interface Resource { short RESOURCE_UNLIMITED = -1; String UNLIMITED = "Unlimited"; - enum ResourceType { // Primary and Secondary storage are allocated_storage and not the physical storage. + enum ResourceType { // All storage type resources are allocated_storage and not the physical storage. user_vm("user_vm", 0), public_ip("public_ip", 1), volume("volume", 2), @@ -33,7 +33,11 @@ enum ResourceType { // Primary and Secondary storage are allocated_storage and n cpu("cpu", 8), memory("memory", 9), primary_storage("primary_storage", 10), - secondary_storage("secondary_storage", 11); + secondary_storage("secondary_storage", 11), + backup("backup", 12), + backup_storage("backup_storage", 13), + bucket("bucket", 14), + object_storage("object_storage", 15); private String name; private int ordinal; @@ -62,6 +66,10 @@ public static ResourceType fromOrdinal(int ordinal) { } return null; } + + public static Boolean isStorageType(ResourceType type) { + return (type == primary_storage || type == secondary_storage || type == backup_storage || type == object_storage); + } } public static class ResourceOwnerType { diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 81ed185dae5a..862a6e21fa82 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -785,6 +785,9 @@ public class EventTypes { public static final String EVENT_SHAREDFS_EXPUNGE = "SHAREDFS.EXPUNGE"; public static final String EVENT_SHAREDFS_RECOVER = "SHAREDFS.RECOVER"; + // Resource Limit + public static final String EVENT_RESOURCE_LIMIT_UPDATE = "RESOURCE.LIMIT.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 6f4c7aa09e26..3a72f8bddc4c 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -190,4 +190,6 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId, boolean stateTransitTo(Volume vol, Volume.Event event) throws NoTransitionException; Pair checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException; + + Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 03de07c37da0..6f4780a7b1d4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -51,9 +51,15 @@ public class ApiConstants { public static final String AVAILABLE = "available"; public static final String AVAILABLE_SUBNETS = "availablesubnets"; public static final String AVAILABLE_VIRTUAL_MACHINE_COUNT = "availablevirtualmachinecount"; + public static final String BACKUP_AVAILABLE = "backupavailable"; public static final String BACKUP_ID = "backupid"; + public static final String BACKUP_LIMIT = "backuplimit"; public static final String BACKUP_OFFERING_NAME = "backupofferingname"; public static final String BACKUP_OFFERING_ID = "backupofferingid"; + public static final String BACKUP_STORAGE_AVAILABLE = "backupstorageavailable"; + public static final String BACKUP_STORAGE_LIMIT = "backupstoragelimit"; + public static final String BACKUP_STORAGE_TOTAL = "backupstoragetotal"; + public static final String BACKUP_TOTAL = "backuptotal"; public static final String BASE64_IMAGE = "base64image"; public static final String BGP_PEERS = "bgppeers"; public static final String BGP_PEER_IDS = "bgppeerids"; @@ -322,6 +328,7 @@ public class ApiConstants { public static final String MAC_ADDRESS = "macaddress"; public static final String MAX = "max"; public static final String MAX_SNAPS = "maxsnaps"; + public static final String MAX_BACKUPS = "maxbackups"; public static final String MAX_CPU_NUMBER = "maxcpunumber"; public static final String MAX_MEMORY = "maxmemory"; public static final String MIN_CPU_NUMBER = "mincpunumber"; @@ -436,6 +443,7 @@ public class ApiConstants { public static final String QUALIFIERS = "qualifiers"; public static final String QUERY_FILTER = "queryfilter"; public static final String SCHEDULE = "schedule"; + public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; public static final String SEARCH_BASE = "searchbase"; public static final String SECONDARY_IP = "secondaryip"; @@ -1148,7 +1156,6 @@ public class ApiConstants { public static final String MTU = "mtu"; public static final String AUTO_ENABLE_KVM_HOST = "autoenablekvmhost"; public static final String LIST_APIS = "listApis"; - public static final String OBJECT_STORAGE_ID = "objectstorageid"; public static final String VERSIONING = "versioning"; public static final String OBJECT_LOCKING = "objectlocking"; public static final String ENCRYPTION = "encryption"; @@ -1162,7 +1169,6 @@ public class ApiConstants { public static final String DISK_PATH = "diskpath"; public static final String IMPORT_SOURCE = "importsource"; public static final String TEMP_PATH = "temppath"; - public static final String OBJECT_STORAGE = "objectstore"; public static final String HEURISTIC_RULE = "heuristicrule"; public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME."; public static final String MANAGEMENT = "management"; @@ -1190,11 +1196,20 @@ public class ApiConstants { public static final String SHAREDFSVM_MIN_CPU_COUNT = "sharedfsvmmincpucount"; public static final String SHAREDFSVM_MIN_RAM_SIZE = "sharedfsvmminramsize"; + // Object Storage related + public static final String BUCKET_AVAILABLE = "bucketavailable"; + public static final String BUCKET_LIMIT = "bucketlimit"; + public static final String BUCKET_TOTAL = "buckettotal"; + public static final String OBJECT_STORAGE_ID = "objectstorageid"; + public static final String OBJECT_STORAGE = "objectstore"; + public static final String OBJECT_STORAGE_AVAILABLE = "objectstorageavailable"; + public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit"; + public static final String OBJECT_STORAGE_TOTAL = "objectstoragetotal"; + public static final String PARAMETER_DESCRIPTION_ACTIVATION_RULE = "Quota tariff's activation rule. It can receive a JS script that results in either " + "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " + "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " + "value will be applied."; - public static final String PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS = "The recommended format is \"yyyy-MM-dd'T'HH:mm:ssZ\" (e.g.: \"2023-01-01T12:00:00+0100\"); " + "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " + "added, it will be interpreted as \"00:00:00\"). If the recommended format is not used, the date will be considered in the server timezone."; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index 558f92e4006d..2d3877882430 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -19,6 +19,7 @@ import javax.inject.Inject; +import com.cloud.storage.Snapshot; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -27,6 +28,7 @@ import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.BackupManager; @@ -60,6 +62,13 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { description = "ID of the VM") private Long vmId; + @Parameter(name = ApiConstants.SCHEDULE_ID, + type = CommandType.LONG, + entityType = BackupScheduleResponse.class, + description = "backup schedule ID of the VM, if this is null, it indicates that it is a manual backup.", + since = "4.21.0") + private Long scheduleId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -68,6 +77,14 @@ public Long getVmId() { return vmId; } + public Long getScheduleId() { + if (scheduleId != null) { + return scheduleId; + } else { + return Snapshot.MANUAL_POLICY_ID; + } + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -75,7 +92,7 @@ public Long getVmId() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try { - boolean result = backupManager.createBackup(getVmId()); + boolean result = backupManager.createBackup(getVmId(), getScheduleId()); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index 5dc06af21231..1d0741e6217b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -75,6 +75,12 @@ public class CreateBackupScheduleCmd extends BaseCmd { description = "Specifies a timezone for this command. For more information on the timezone parameter, see TimeZone Format.") private String timezone; + @Parameter(name = ApiConstants.MAX_BACKUPS, + type = CommandType.INTEGER, + description = "maximum number of backups to retain", + since = "4.21.0") + private Integer maxBackups; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -95,6 +101,10 @@ public String getTimezone() { return timezone; } + public Integer getMaxBackups() { + return maxBackups; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java index d2c91e578713..722556b8e2de 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java @@ -72,7 +72,7 @@ public class CreateBucketCmd extends BaseAsyncCreateCmd implements UserCmd { description = "Id of the Object Storage Pool where bucket is created") private long objectStoragePoolId; - @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER, required = true, description = "Bucket Quota in GiB") private Integer quota; @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable bucket encryption") diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java index 8e281b20e915..f913373c04b6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java @@ -56,7 +56,7 @@ public class UpdateBucketCmd extends BaseCmd { @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING, description = "Bucket Access Policy") private String policy; - @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER, description = "Bucket Quota in GiB") private Integer quota; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java index 6fc098295f64..aaad7f985fc3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java @@ -127,6 +127,30 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total number of snapshots available for this account") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this account", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this account", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this account", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the account can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the account", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the account", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description = "the total number of templates which can be created by this account") private String templateLimit; @@ -231,6 +255,30 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total secondary storage space (in GiB) available to be used for this account", since = "4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this account", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this account", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this account", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the account can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the account", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the account", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.STATE) @Param(description = "the state of the account") private String state; @@ -386,6 +434,36 @@ public void setSnapshotAvailable(String snapshotAvailable) { this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -537,6 +615,36 @@ public void setSecondaryStorageAvailable(String secondaryStorageAvailable) { this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setDefaultZone(String defaultZoneId) { this.defaultZoneId = defaultZoneId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java index ba44f1e024f7..d7c6f96add5b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java @@ -37,18 +37,22 @@ public class BackupScheduleResponse extends BaseResponse { @Param(description = "ID of the VM") private String vmId; - @SerializedName("schedule") + @SerializedName(ApiConstants.SCHEDULE) @Param(description = "time the backup is scheduled to be taken.") private String schedule; - @SerializedName("intervaltype") + @SerializedName(ApiConstants.INTERVAL_TYPE) @Param(description = "the interval type of the backup schedule") private DateUtil.IntervalType intervalType; - @SerializedName("timezone") + @SerializedName(ApiConstants.TIMEZONE) @Param(description = "the time zone of the backup schedule") private String timezone; + @SerializedName(ApiConstants.MAX_BACKUPS) + @Param(description = "maximum number of backups retained") + private Integer maxBakups; + public String getVmName() { return vmName; } @@ -88,4 +92,8 @@ public String getTimezone() { public void setTimezone(String timezone) { this.timezone = timezone; } + + public void setMaxBakups(Integer maxBakups) { + this.maxBakups = maxBakups; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java index f2dd365452cd..cde140839ec0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java @@ -75,7 +75,7 @@ public class BucketResponse extends BaseResponseWithTagInformation implements Co private String state; @SerializedName(ApiConstants.QUOTA) - @Param(description = "Bucket Quota in GB") + @Param(description = "Bucket Quota in GiB") private Integer quota; @SerializedName(ApiConstants.ENCRYPTION) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java index 7c6ad3a91c38..74fa2cbb1e4c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java @@ -105,6 +105,30 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou @SerializedName("snapshotavailable") @Param(description="the total number of snapshots available for this domain") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this domain", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this domain", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this domain", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the domain can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the domain", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the domain", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description="the total number of templates which can be created by this domain") private String templateLimit; @@ -177,6 +201,30 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou @SerializedName("secondarystorageavailable") @Param(description="the total secondary storage space (in GiB) available to be used for this domain", since="4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this domain", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this domain", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this domain", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the domain can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the domain", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the domain", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.RESOURCE_ICON) @Param(description = "Base64 string representation of the resource icon", since = "4.16.0.0") ResourceIconResponse icon; @@ -313,6 +361,36 @@ public void setSnapshotAvailable(String snapshotAvailable) { this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -430,6 +508,36 @@ public void setSecondaryStorageAvailable(String secondaryStorageAvailable) { this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setState(String state) { this.state = state; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java index 1c63697559b5..8bdf042add08 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java @@ -140,6 +140,30 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total secondary storage space (in GiB) available to be used for this project", since = "4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this project", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this project", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this project", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the project can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the project", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the project", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.VM_LIMIT) @Param(description = "the total number of virtual machines that can be deployed by this project", since = "4.2.0") private String vmLimit; @@ -188,6 +212,30 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total number of snapshots available for this project", since = "4.2.0") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this project", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this project", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this project", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the project can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the project", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the project", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description = "the total number of templates which can be created by this project", since = "4.2.0") private String templateLimit; @@ -320,6 +368,36 @@ public void setSnapshotAvailable(String snapshotAvailable) { this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -435,6 +513,36 @@ public void setSecondaryStorageAvailable(String secondaryStorageAvailable) { this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setOwners(List> owners) { this.owners = owners; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java index f9e6df3a0386..b86723b36c41 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java @@ -84,6 +84,30 @@ public interface ResourceLimitAndCountResponse { public void setSnapshotAvailable(String snapshotAvailable); + public void setBackupLimit(String backupLimit); + + public void setBackupTotal(Long backupTotal); + + public void setBackupAvailable(String backupAvailable); + + public void setBackupStorageLimit(String backupStorageLimit); + + public void setBackupStorageTotal(Long backupStorageTotal); + + public void setBackupStorageAvailable(String backupStorageAvailable); + + void setBucketLimit(String bucketLimit); + + void setBucketTotal(Long bucketTotal); + + void setBucketAvailable(String bucketAvailable); + + void setObjectStorageLimit(String objectStorageLimit); + + void setObjectStorageTotal(Long objectStorageTotal); + + void setObjectStorageAvailable(String objectStorageAvailable); + public void setTemplateLimit(String templateLimit); public void setTemplateTotal(Long templateTotal); diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index f21f20adb33e..dffe8a032134 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -33,6 +33,28 @@ enum Status { Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged } + public enum Type { + MANUAL, HOURLY, DAILY, WEEKLY, MONTHLY; + private int max = 8; + + public void setMax(int max) { + this.max = max; + } + + public int getMax() { + return max; + } + + @Override + public String toString() { + return this.name(); + } + + public boolean equals(String snapshotType) { + return this.toString().equalsIgnoreCase(snapshotType); + } + } + class Metric { private Long backupSize = 0L; private Long dataSize = 0L; diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 8b45bb4ee5ef..cbd4b7e05966 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -19,6 +19,7 @@ import java.util.List; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; @@ -56,6 +57,86 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer "false", "Enable volume attach/detach operations for VMs that are assigned to Backup Offerings.", true); + ConfigKey BackupHourlyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.hourly", + "8", + "Maximum recurring hourly backups to be retained for an instance. If the limit is reached, early backups from the start of the hour are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring hourly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupDailyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.daily", + "8", + "Maximum recurring daily backups to be retained for an instance. If the limit is reached, backups from the start of the day are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring daily backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupWeeklyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.weekly", + "8", + "Maximum recurring weekly backups to be retained for an instance. If the limit is reached, backups from the beginning of the week are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring weekly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupMonthlyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.monthly", + "8", + "Maximum recurring monthly backups to be retained for an instance. If the limit is reached, backups from the beginning of the month are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring monthly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountBackups = new ConfigKey("Account Defaults", Long.class, + "max.account.backups", + "20", + "The default maximum number of backups that can be created for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountBackupStorage = new ConfigKey("Account Defaults", Long.class, + "max.account.backup.storage", + "400", + "The default maximum backup storage space (in GiB) that can be used for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBackups = new ConfigKey("Project Defaults", Long.class, + "max.project.backups", + "20", + "The default maximum number of backups that can be created for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBackupStorage = new ConfigKey("Project Defaults", Long.class, + "max.project.backup.storage", + "400", + "The default maximum backup storage space (in GiB) that can be used for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBackups = new ConfigKey("Domain Defaults", Long.class, + "max.domain.backups", + "40", + "The default maximum number of backups that can be created for a domain", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBackupStorage = new ConfigKey("Domain Defaults", Long.class, + "max.domain.backup.storage", + "800", + "The default maximum backup storage space (in GiB) that can be used for a domain", + false, + ConfigKey.Scope.Global, + null); + /** * List backup provider offerings * @param zoneId zone id @@ -119,9 +200,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer /** * Creates backup of a VM * @param vmId Virtual Machine ID + * @param scheduleId Virtual Machine Backup Schedule ID * @return returns operation success */ - boolean createBackup(final Long vmId); + boolean createBackup(final Long vmId, final Long scheduleId) throws ResourceAllocationException; /** * List existing backups for a VM diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index d36dfb7360f6..e3a6c3a62bd9 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -75,7 +75,7 @@ public interface BackupProvider { * @param backup * @return */ - boolean takeBackup(VirtualMachine vm); + Pair takeBackup(VirtualMachine vm); /** * Delete an existing backup @@ -104,9 +104,16 @@ public interface BackupProvider { Map getBackupMetrics(Long zoneId, List vms); /** - * This method should reconcile and create backup entries for any backups created out-of-band - * @param vm + * This method should TODO + * @param + */ + public List listRestorePoints(VirtualMachine vm); + + /** + * This method should TODO + * @param + * @param * @param metric */ - void syncBackups(VirtualMachine vm, Backup.Metric metric); + Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java index d81dd731b1ff..4ff946be9cd4 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java @@ -30,4 +30,5 @@ public interface BackupSchedule extends InternalIdentity { String getTimezone(); Date getScheduledTimestamp(); Long getAsyncJobId(); + Integer getMaxBackups(); } diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java index 7e1361d1e71f..e27ef308d7f2 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -22,10 +22,59 @@ import com.cloud.user.Account; import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.framework.config.ConfigKey; public interface BucketApiService { + ConfigKey DefaultMaxAccountBuckets = new ConfigKey("Account Defaults", Long.class, + "max.account.buckets", + "20", + "The default maximum number of buckets that can be created for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountObjectStorage = new ConfigKey("Account Defaults", Long.class, + "max.account.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBuckets = new ConfigKey("Project Defaults", Long.class, + "max.project.buckets", + "20", + "The default maximum number of buckets that can be created for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectObjectStorage = new ConfigKey("Project Defaults", Long.class, + "max.project.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBuckets = new ConfigKey("Domain Defaults", Long.class, + "max.domain.buckets", + "20", + "The default maximum number of buckets that can be created for a domain", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainObjectStorage = new ConfigKey("Domain Defaults", Long.class, + "max.domain.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for a domain", + false, + ConfigKey.Scope.Global, + null); + /** * Creates the database object for a Bucket based on the given criteria * @@ -48,7 +97,7 @@ public interface BucketApiService { boolean deleteBucket(long bucketId, Account caller); - boolean updateBucket(UpdateBucketCmd cmd, Account caller); + boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException; void getBucketUsage(); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java index f45f28b5c2c2..2511df49807e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java @@ -27,4 +27,8 @@ public interface BucketDao extends GenericDao { List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId); List searchByIds(Long[] ids); + + Long countBucketsForAccount(long accountId); + + Long calculateObjectStorageAllocationForAccount(long accountId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java index 98bef6201a15..473879d933dc 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java @@ -16,8 +16,10 @@ // under the License. package com.cloud.storage.dao; +import com.cloud.configuration.Resource; import com.cloud.storage.BucketVO; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import org.springframework.stereotype.Component; @@ -31,6 +33,8 @@ public class BucketDaoImpl extends GenericDaoBase implements Buc private SearchBuilder searchFilteringStoreId; private SearchBuilder bucketSearch; + private GenericSearchBuilder CountBucketsByAccount; + private GenericSearchBuilder CalculateBucketsQuotaByAccount; private static final String STORE_ID = "store_id"; private static final String STATE = "state"; @@ -54,6 +58,20 @@ public boolean configure(String name, Map params) throws Configu bucketSearch.and("idIN", bucketSearch.entity().getId(), SearchCriteria.Op.IN); bucketSearch.done(); + CountBucketsByAccount = createSearchBuilder(Long.class); + CountBucketsByAccount.select(null, SearchCriteria.Func.COUNT, null); + CountBucketsByAccount.and(ACCOUNT_ID, CountBucketsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CountBucketsByAccount.and(STATE, CountBucketsByAccount.entity().getState(), SearchCriteria.Op.NIN); + CountBucketsByAccount.and("removed", CountBucketsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CountBucketsByAccount.done(); + + CalculateBucketsQuotaByAccount = createSearchBuilder(SumCount.class); + CalculateBucketsQuotaByAccount.select("sum", SearchCriteria.Func.SUM, CalculateBucketsQuotaByAccount.entity().getQuota()); + CalculateBucketsQuotaByAccount.and(ACCOUNT_ID, CalculateBucketsQuotaByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CalculateBucketsQuotaByAccount.and(STATE, CalculateBucketsQuotaByAccount.entity().getState(), SearchCriteria.Op.NIN); + CalculateBucketsQuotaByAccount.and("removed", CalculateBucketsQuotaByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CalculateBucketsQuotaByAccount.done(); + return true; } @Override @@ -79,4 +97,21 @@ public List searchByIds(Long[] ids) { sc.setParameters("idIN", ids); return search(sc, null, null, false); } + + @Override + public Long countBucketsForAccount(long accountId) { + SearchCriteria sc = CountBucketsByAccount.create(); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return customSearch(sc, null).get(0); + } + + @Override + public Long calculateObjectStorageAllocationForAccount(long accountId) { + SearchCriteria sc = CalculateBucketsQuotaByAccount.create(); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + Long totalQuota = customSearch(sc, null).get(0).sum; + return (totalQuota * Resource.ResourceType.bytesToGiB); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java index fd3c0be18d22..0258c42c52ba 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java @@ -58,15 +58,19 @@ public class BackupScheduleVO implements BackupSchedule { @Column(name = "async_job_id") Long asyncJobId; + @Column(name = "max_backups") + Integer maxBackups = 0; + public BackupScheduleVO() { } - public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp) { + public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, Integer maxBackups) { this.vmId = vmId; this.scheduleType = (short) scheduleType.ordinal(); this.schedule = schedule; this.timezone = timezone; this.scheduledTimestamp = scheduledTimestamp; + this.maxBackups = maxBackups; } @Override @@ -128,4 +132,12 @@ public Long getAsyncJobId() { public void setAsyncJobId(Long asyncJobId) { this.asyncJobId = asyncJobId; } + + public Integer getMaxBackups() { + return maxBackups; + } + + public void setMaxBackups(Integer maxBackups) { + this.maxBackups = maxBackups; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index b4cd2f7badae..9ef442baff96 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -88,6 +88,9 @@ public class BackupVO implements Backup { @Column(name = "zone_id") private long zoneId; + @Column(name = "backup_interval_type") + private short backupIntervalType; + @Column(name = "backed_volumes", length = 65535) protected String backedUpVolumes; @@ -208,6 +211,14 @@ public void setZoneId(long zoneId) { this.zoneId = zoneId; } + public short getBackupIntervalType() { + return backupIntervalType; + } + + public void setBackupIntervalType(short backupIntervalType) { + this.backupIntervalType = backupIntervalType; + } + @Override public Class getEntityType() { return Backup.class; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java index 89a13245b0a0..ffd5e5a4a66b 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java @@ -35,5 +35,10 @@ public interface BackupDao extends GenericDao { List syncBackups(Long zoneId, Long vmId, List externalBackups); BackupVO getBackupVO(Backup backup); List listByOfferingId(Long backupOfferingId); + + List listBackupsByVMandIntervalType(Long vmId, Backup.Type backupType); + BackupResponse newBackupResponse(Backup backup); + public Long countBackupsForAccount(long accountId); + public Long calculateBackupStorageForAccount(long accountId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index 5a9cd0620374..b4e1a7602825 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -24,6 +24,7 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.utils.db.GenericSearchBuilder; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; @@ -60,6 +61,9 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac BackupOfferingDao backupOfferingDao; private SearchBuilder backupSearch; + private GenericSearchBuilder CountBackupsByAccount; + private GenericSearchBuilder CalculateBackupStorageByAccount; + private SearchBuilder ListBackupsByVMandIntervalType; public BackupDaoImpl() { } @@ -72,6 +76,27 @@ protected void init() { backupSearch.and("backup_offering_id", backupSearch.entity().getBackupOfferingId(), SearchCriteria.Op.EQ); backupSearch.and("zone_id", backupSearch.entity().getZoneId(), SearchCriteria.Op.EQ); backupSearch.done(); + + CountBackupsByAccount = createSearchBuilder(Long.class); + CountBackupsByAccount.select(null, SearchCriteria.Func.COUNT, null); + CountBackupsByAccount.and("account", CountBackupsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CountBackupsByAccount.and("status", CountBackupsByAccount.entity().getStatus(), SearchCriteria.Op.NIN); + CountBackupsByAccount.and("removed", CountBackupsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CountBackupsByAccount.done(); + + CalculateBackupStorageByAccount = createSearchBuilder(SumCount.class); + CalculateBackupStorageByAccount.select("sum", SearchCriteria.Func.SUM, CalculateBackupStorageByAccount.entity().getSize()); + CalculateBackupStorageByAccount.and("account", CalculateBackupStorageByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CalculateBackupStorageByAccount.and("status", CalculateBackupStorageByAccount.entity().getStatus(), SearchCriteria.Op.NIN); + CalculateBackupStorageByAccount.and("removed", CalculateBackupStorageByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CalculateBackupStorageByAccount.done(); + + ListBackupsByVMandIntervalType = createSearchBuilder(); + ListBackupsByVMandIntervalType.and("vmId", ListBackupsByVMandIntervalType.entity().getVmId(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("intervalType", ListBackupsByVMandIntervalType.entity().getBackupIntervalType(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("status", ListBackupsByVMandIntervalType.entity().getStatus(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("removed", ListBackupsByVMandIntervalType.entity().getRemoved(), SearchCriteria.Op.NULL); + ListBackupsByVMandIntervalType.done(); } @Override @@ -142,6 +167,31 @@ public List syncBackups(Long zoneId, Long vmId, List externalBac return listByVmId(zoneId, vmId); } + @Override + public Long countBackupsForAccount(long accountId) { + SearchCriteria sc = CountBackupsByAccount.create(); + sc.setParameters("account", accountId); + sc.setParameters("status", Backup.Status.Error, Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); + return customSearch(sc, null).get(0); + } + + @Override + public Long calculateBackupStorageForAccount(long accountId) { + SearchCriteria sc = CalculateBackupStorageByAccount.create(); + sc.setParameters("account", accountId); + sc.setParameters("status", Backup.Status.Error, Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); + return customSearch(sc, null).get(0).sum; + } + + @Override + public List listBackupsByVMandIntervalType(Long vmId, Backup.Type backupType) { + SearchCriteria sc = ListBackupsByVMandIntervalType.create(); + sc.setParameters("vmId", vmId); + sc.setParameters("type", backupType.ordinal()); + sc.setParameters("status", Backup.Status.BackedUp); + return listBy(sc, null); + } + @Override public BackupResponse newBackupResponse(Backup backup) { VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java index e00ccc5abd77..aac2e3bf2320 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java @@ -97,6 +97,7 @@ public BackupScheduleResponse newBackupScheduleResponse(BackupSchedule schedule) response.setIntervalType(schedule.getScheduleType()); response.setSchedule(schedule.getSchedule()); response.setTimezone(schedule.getTimezone()); + response.setMaxBakups(schedule.getMaxBackups()); response.setObjectName("backupschedule"); return response; } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 4a5a0203a15a..b01243ad989d 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -19,6 +19,10 @@ -- Schema upgrade from 4.20.1.0 to 4.21.0.0 --; +-- Add columns max_backup and backup_interval_type to backup table +ALTER TABLE `cloud`.`backup_schedule` ADD COLUMN `max_backups` int(8) default NULL COMMENT 'maximum number of backups to maintain'; +ALTER TABLE `cloud`.`backups` ADD COLUMN `backup_interval_type` int(5) COMMENT 'type of backup, e.g. manual, recurring - hourly, daily, weekly or monthly'; + -- Add console_endpoint_creator_address column to cloud.console_session table CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_creator_address', 'VARCHAR(45)'); @@ -33,4 +37,3 @@ WHERE rp.rule = 'quotaStatement' AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaCreditsList'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host', 'last_mgmt_server_id', 'bigint unsigned DEFAULT NULL COMMENT "last management server this host is connected to" AFTER `mgmt_server_id`'); - diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql index dc64380fb57b..6092fe8e845a 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql @@ -68,6 +68,14 @@ select `primary_storage_count`.`count` AS `primaryStorageTotal`, `secondary_storage_limit`.`max` AS `secondaryStorageLimit`, `secondary_storage_count`.`count` AS `secondaryStorageTotal`, + `backup_limit`.`max` AS `backupLimit`, + `backup_count`.`count` AS `backupTotal`, + `backup_storage_limit`.`max` AS `backupStorageLimit`, + `backup_storage_count`.`count` AS `backupStorageTotal`, + `bucket_limit`.`max` AS `bucketLimit`, + `bucket_count`.`count` AS `bucketTotal`, + `object_storage_limit`.`max` AS `objectStorageLimit`, + `object_storage_count`.`count` AS `objectStorageTotal`, `async_job`.`id` AS `job_id`, `async_job`.`uuid` AS `job_uuid`, `async_job`.`job_status` AS `job_status`, @@ -160,6 +168,30 @@ from `cloud`.`resource_count` secondary_storage_count ON account.id = secondary_storage_count.account_id and secondary_storage_count.type = 'secondary_storage' left join + `cloud`.`resource_limit` backup_limit ON account.id = backup_limit.account_id + and backup_limit.type = 'backup' + left join + `cloud`.`resource_count` backup_count ON account.id = backup_count.account_id + and backup_count.type = 'backup' + left join + `cloud`.`resource_limit` backup_storage_limit ON account.id = backup_storage_limit.account_id + and backup_storage_limit.type = 'backup_storage' + left join + `cloud`.`resource_count` backup_storage_count ON account.id = backup_storage_count.account_id + and backup_storage_count.type = 'backup_storage' + left join + `cloud`.`resource_limit` bucket_limit ON account.id = bucket_limit.account_id + and bucket_limit.type = 'bucket' + left join + `cloud`.`resource_count` bucket_count ON account.id = bucket_count.account_id + and bucket_count.type = 'bucket' + left join + `cloud`.`resource_limit` object_storage_limit ON account.id = object_storage_limit.account_id + and object_storage_limit.type = 'object_storage' + left join + `cloud`.`resource_count` object_storage_count ON account.id = object_storage_count.account_id + and object_storage_count.type = 'object_storage' + left join `cloud`.`async_job` ON async_job.instance_id = account.id and async_job.instance_type = 'Account' and async_job.job_status = 0; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql index 201ece95023a..c9f7bfc51e43 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql @@ -58,7 +58,15 @@ select `primary_storage_limit`.`max` AS `primaryStorageLimit`, `primary_storage_count`.`count` AS `primaryStorageTotal`, `secondary_storage_limit`.`max` AS `secondaryStorageLimit`, - `secondary_storage_count`.`count` AS `secondaryStorageTotal` + `secondary_storage_count`.`count` AS `secondaryStorageTotal`, + `backup_limit`.`max` AS `backupLimit`, + `backup_count`.`count` AS `backupTotal`, + `backup_storage_limit`.`max` AS `backupStorageLimit`, + `backup_storage_count`.`count` AS `backupStorageTotal`, + `bucket_limit`.`max` AS `bucketLimit`, + `bucket_count`.`count` AS `bucketTotal`, + `object_storage_limit`.`max` AS `objectStorageLimit`, + `object_storage_count`.`count` AS `objectStorageTotal` from `cloud`.`domain` left join @@ -132,4 +140,28 @@ from and secondary_storage_limit.type = 'secondary_storage' left join `cloud`.`resource_count` secondary_storage_count ON domain.id = secondary_storage_count.domain_id - and secondary_storage_count.type = 'secondary_storage'; + and secondary_storage_count.type = 'secondary_storage' + left join + `cloud`.`resource_limit` backup_limit ON domain.id = backup_limit.domain_id + and backup_limit.type = 'backup' + left join + `cloud`.`resource_count` backup_count ON domain.id = backup_count.domain_id + and backup_count.type = 'backup' + left join + `cloud`.`resource_limit` backup_storage_limit ON domain.id = backup_storage_limit.domain_id + and backup_storage_limit.type = 'backup_storage' + left join + `cloud`.`resource_count` backup_storage_count ON domain.id = backup_storage_count.domain_id + and backup_storage_count.type = 'backup_storage' + left join + `cloud`.`resource_limit` bucket_limit ON domain.id = bucket_limit.domain_id + and bucket_limit.type = 'bucket' + left join + `cloud`.`resource_count` bucket_count ON domain.id = bucket_count.domain_id + and bucket_count.type = 'bucket' + left join + `cloud`.`resource_limit` object_storage_limit ON domain.id = object_storage_limit.domain_id + and object_storage_limit.type = 'object_storage' + left join + `cloud`.`resource_count` object_storage_count ON domain.id = object_storage_count.domain_id + and object_storage_count.type = 'object_storage'; diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index d4b3cff0f5c1..6935d177c72c 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -24,6 +24,7 @@ import javax.inject.Inject; +import com.cloud.configuration.Resource; import com.cloud.storage.dao.VolumeDao; import org.apache.cloudstack.backup.dao.BackupDao; @@ -99,6 +100,16 @@ public Map getBackupMetrics(Long zoneId, List listRestorePoints(VirtualMachine vm) { + return null; + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + return null; + } + @Override public boolean removeVMFromBackupOffering(VirtualMachine vm) { logger.debug(String.format("Removing VM %s from backup offering by the Dummy Backup Provider", vm)); @@ -111,7 +122,7 @@ public boolean willDeleteBackupsOnOfferingRemoval() { } @Override - public boolean takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm) { logger.debug(String.format("Starting backup for VM %s on Dummy provider", vm)); BackupVO backup = new BackupVO(); @@ -119,23 +130,20 @@ public boolean takeBackup(VirtualMachine vm) { backup.setExternalId("dummy-external-id"); backup.setType("FULL"); backup.setDate(new Date()); - backup.setSize(1024L); - backup.setProtectedSize(1024000L); + backup.setSize(1024000L); + backup.setProtectedSize(1 * Resource.ResourceType.bytesToGiB); backup.setStatus(Backup.Status.BackedUp); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - return backupDao.persist(backup) != null; + backup = backupDao.persist(backup); + return new Pair<>(true, backup); } @Override public boolean deleteBackup(Backup backup, boolean forced) { return true; } - - @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 5d3d1a919330..f148c53e614d 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -46,6 +46,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; + import javax.inject.Inject; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -141,7 +142,7 @@ protected Host getVMHypervisorHost(VirtualMachine vm) { } @Override - public boolean takeBackup(final VirtualMachine vm) { + public Pair takeBackup(final VirtualMachine vm) { final Host host = getVMHypervisorHost(vm); final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); @@ -179,12 +180,16 @@ public boolean takeBackup(final VirtualMachine vm) { backupVO.setSize(answer.getSize()); backupVO.setStatus(Backup.Status.BackedUp); backupVO.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - return backupDao.update(backupVO.getId(), backupVO); + if (backupDao.update(backupVO.getId(), backupVO)) { + return new Pair<>(true, backupVO); + } else { + throw new CloudRuntimeException("Failed to update backup"); + } } else { backupVO.setStatus(Backup.Status.Failed); backupDao.remove(backupVO.getId()); + return new Pair<>(false, null); } - return Objects.nonNull(answer) && answer.getResult(); } private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { @@ -358,6 +363,7 @@ public boolean deleteBackup(Backup backup, boolean forced) { return backupDao.remove(backup.getId()); } + logger.debug("There was an error removing the backup with id " + backup.getId()); return false; } @@ -383,6 +389,16 @@ public Map getBackupMetrics(Long zoneId, List listRestorePoints(VirtualMachine vm) { + return null; + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + return null; + } + @Override public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { return Hypervisor.HypervisorType.KVM.equals(vm.getHypervisorType()); @@ -398,11 +414,6 @@ public boolean willDeleteBackupsOnOfferingRemoval() { return false; } - @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - // TODO: check and sum/return backups metrics on per VM basis - } - @Override public List listBackupOfferings(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 393e2911ac38..822688a86a35 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -29,15 +29,11 @@ import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; import org.apache.cloudstack.backup.networker.NetworkerClient; @@ -462,7 +458,7 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU } @Override - public boolean takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm) { String networkerServer; String clusterName; @@ -514,11 +510,11 @@ public boolean takeBackup(VirtualMachine vm) { if (backup != null) { backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); backupDao.persist(backup); - return true; + return new Pair<>(true, backup); } else { LOG.error("Could not register backup for vm {} with saveset Time: {}", vm, saveTime); // We need to handle this rare situation where backup is successful but can't be registered properly. - return false; + return new Pair<>(false, null); } } @@ -532,7 +528,7 @@ public boolean deleteBackup(Backup backup, boolean forced) { LOG.debug("EMC Networker successfully deleted backup with id " + externalBackupId); return true; } else { - LOG.debug("There was an error removing the backup with id " + externalBackupId + " from EMC NEtworker"); + LOG.debug("There was an error removing the backup with id " + externalBackupId + " from EMC Networker"); } return false; } @@ -550,12 +546,12 @@ public Map getBackupMetrics(Long zoneId, List vmBackups = getClient(zoneId).getBackupsForVm(vm); for ( String vmBackup : vmBackups ) { NetworkerBackup vmNwBackup = getClient(zoneId).getNetworkerBackupInfo(vmBackup); - vmBackupProtectedSize+= vmNwBackup.getSize().getValue() / 1024L; + vmBackupSize += vmNwBackup.getSize().getValue() / 1024L; } Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); LOG.debug(String.format("Metrics for VM [%s] is [backup size: %s, data size: %s].", vm, vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize())); @@ -565,83 +561,53 @@ public Map getBackupMetrics(Long zoneId, List backupsInDb = backupDao.listByVmId(null, vm.getId()); - final ArrayList backupsInNetworker = getClient(zoneId).getBackupsForVm(vm); - final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); - for (final String networkerBackupId : backupsInNetworker ) { - Long vmBackupSize=0L; - boolean backupExists = false; - for (final Backup backupInDb : backupsInDb) { - LOG.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm)); - if ( networkerBackupId.equals(backupInDb.getExternalId()) ) { - LOG.debug(String.format("Found Backup %s in both Database and Networker", backupInDb)); - backupExists = true; - removeList.remove(backupInDb.getId()); - if (metric != null) { - LOG.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", - backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), - metric.getBackupSize(), metric.getDataSize())); - ((BackupVO) backupInDb).setSize(metric.getBackupSize()); - ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); - backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); - } - break; - } - } - if (backupExists) { - continue; - } - // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts - // with the proper parameters. So we will register any backups taken on the Networker side from - // outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will - // ensure that SLA like backups will be found and registered. - NetworkerBackup strayNetworkerBackup = getClient(vm.getDataCenterId()).getNetworkerBackupInfo(networkerBackupId); - // Since running backups are already present in Networker Server but not completed - // make sure the backup is not in progress at this time. - if ( strayNetworkerBackup.getCompletionTime() != null) { - BackupVO strayBackup = new BackupVO(); - strayBackup.setVmId(vm.getId()); - strayBackup.setExternalId(strayNetworkerBackup.getId()); - strayBackup.setType(strayNetworkerBackup.getType()); - SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - try { - strayBackup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime())); - } catch (ParseException e) { - String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime()); - LOG.error(msg, e); - throw new CloudRuntimeException(msg, e); - } - strayBackup.setStatus(Backup.Status.BackedUp); - for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { - vmBackupSize += (thisVMVol.getSize() / 1024L /1024L); - } - strayBackup.setSize(vmBackupSize); - strayBackup.setProtectedSize(strayNetworkerBackup.getSize().getValue() / 1024L ); - strayBackup.setBackupOfferingId(vm.getBackupOfferingId()); - strayBackup.setAccountId(vm.getAccountId()); - strayBackup.setDomainId(vm.getDomainId()); - strayBackup.setZoneId(vm.getDataCenterId()); - LOG.debug(String.format("Creating a new entry in backups: [id: %s, uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " - + "domain_id: %s, zone_id: %s].", strayBackup.getId(), strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(), - strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(), - strayBackup.getDomainId(), strayBackup.getZoneId())); - backupDao.persist(strayBackup); - LOG.warn("Added backup found in provider [" + strayBackup + "]"); - } else { - LOG.debug ("Backup is in progress, skipping addition for this run"); - } - } - for (final Long backupIdToRemove : removeList) { - LOG.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); - backupDao.remove(backupIdToRemove); - } + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts + // with the proper parameters. So we will register any backups taken on the Networker side from + // outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will + // ensure that SLA like backups will be found and registered. + NetworkerBackup strayNetworkerBackup = getClient(vm.getDataCenterId()).getNetworkerBackupInfo(restorePoint.getId()); + + // Since running backups are already present in Networker Server but not completed + // make sure the backup is not in progress at this time. + if (strayNetworkerBackup.getCompletionTime() != null) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(strayNetworkerBackup.getId()); + backup.setType(strayNetworkerBackup.getType()); + SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + try { + backup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime())); + } catch (ParseException e) { + String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime()); + LOG.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + backup.setStatus(Backup.Status.BackedUp); + Long vmBackupProtectedSize=0L; + for (Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { + vmBackupProtectedSize += (thisVMVol.getSize() / 1024L / 1024L); } - }); + backup.setSize(strayNetworkerBackup.getSize().getValue() / 1024L); + backup.setProtectedSize(vmBackupProtectedSize); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + backupDao.persist(backup); + return backup; + } + LOG.debug ("Backup is in progress, skipping addition for this run"); + return null; + } + + @Override + public List listRestorePoints(VirtualMachine vm) { + final Long zoneId = vm.getDataCenterId(); + final ArrayList backupIds = getClient(zoneId).getBackupsForVm(vm); + List restorePoints = + backupIds.stream().map(id -> new Backup.RestorePoint(id, null, null)).collect(Collectors.toList()); + return restorePoints; } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index c120d8bd5999..0735136d15d8 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -29,8 +29,6 @@ import javax.inject.Inject; -import org.apache.cloudstack.api.ApiCommandResourceType; -import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.veeam.VeeamClient; @@ -42,20 +40,13 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; -import com.cloud.event.ActionEventUtils; -import com.cloud.event.EventTypes; -import com.cloud.event.EventVO; import com.cloud.hypervisor.Hypervisor; import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; -import com.cloud.user.User; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -220,9 +211,10 @@ public boolean willDeleteBackupsOnOfferingRemoval() { } @Override - public boolean takeBackup(final VirtualMachine vm) { + public Pair takeBackup(final VirtualMachine vm) { final VeeamClient client = getClient(vm.getDataCenterId()); - return client.startBackupJob(vm.getBackupExternalId()); + Boolean result = client.startBackupJob(vm.getBackupExternalId()); + return new Pair<>(result, null); } @Override @@ -322,78 +314,30 @@ public Map getBackupMetrics(final Long zoneId, fi return metrics; } - private List listRestorePoints(VirtualMachine vm) { - String backupName = getGuestBackupName(vm.getInstanceName(), vm.getUuid()); - return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName()); - } - - private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(List backupsInDb, Backup.RestorePoint restorePoint, Backup.Metric metric) { - for (final Backup backup : backupsInDb) { - if (restorePoint.getId().equals(backup.getExternalId())) { - if (metric != null) { - logger.debug("Update backup with [id: {}, uuid: {}, name: {}, external id: {}] from [size: {}, protected size: {}] to [size: {}, protected size: {}].", - backup.getId(), backup.getUuid(), backup.getName(), backup.getExternalId(), backup.getSize(), backup.getProtectedSize(), metric.getBackupSize(), metric.getDataSize()); - - ((BackupVO) backup).setSize(metric.getBackupSize()); - ((BackupVO) backup).setProtectedSize(metric.getDataSize()); - backupDao.update(backup.getId(), ((BackupVO) backup)); - } - return backup; - } + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(restorePoint.getId()); + backup.setType(restorePoint.getType()); + backup.setDate(restorePoint.getCreated()); + backup.setStatus(Backup.Status.BackedUp); + if (metric != null) { + backup.setSize(metric.getBackupSize()); + backup.setProtectedSize(metric.getDataSize()); } - return null; + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + backupDao.persist(backup); + return backup; } @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - List restorePoints = listRestorePoints(vm); - if (CollectionUtils.isEmpty(restorePoints)) { - logger.debug("Can't find any restore point to VM: {}", vm); - return; - } - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - final List backupsInDb = backupDao.listByVmId(null, vm.getId()); - final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); - for (final Backup.RestorePoint restorePoint : restorePoints) { - if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) { - Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(backupsInDb, restorePoint, metric); - if (existingBackupEntry != null) { - removeList.remove(existingBackupEntry.getId()); - continue; - } - - BackupVO backup = new BackupVO(); - backup.setVmId(vm.getId()); - backup.setExternalId(restorePoint.getId()); - backup.setType(restorePoint.getType()); - backup.setDate(restorePoint.getCreated()); - backup.setStatus(Backup.Status.BackedUp); - if (metric != null) { - backup.setSize(metric.getBackupSize()); - backup.setProtectedSize(metric.getDataSize()); - } - backup.setBackupOfferingId(vm.getBackupOfferingId()); - backup.setAccountId(vm.getAccountId()); - backup.setDomainId(vm.getDomainId()); - backup.setZoneId(vm.getDataCenterId()); - - logger.debug("Creating a new entry in backups: [id: {}, uuid: {}, name: {}, vm_id: {}, external_id: {}, type: {}, date: {}, backup_offering_id: {}, account_id: {}, " - + "domain_id: {}, zone_id: {}].", backup.getId(), backup.getUuid(), backup.getName(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId()); - backupDao.persist(backup); - - ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE, - String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()), - vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); - } - } - for (final Long backupIdToRemove : removeList) { - logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); - backupDao.remove(backupIdToRemove); - } - } - }); + public List listRestorePoints(VirtualMachine vm) { + String backupName = getGuestBackupName(vm.getInstanceName(), vm.getUuid()); + return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName()); } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index d911736090cb..9accc0714de9 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -844,11 +844,11 @@ public List listRestorePointsLegacy(String backupName, Stri "if ($restore) { $restore ^| Format-List } }" ); Pair response = executePowerShellCommands(cmds); - final List restorePoints = new ArrayList<>(); if (response == null || !response.first()) { - return restorePoints; + return null; } + final List restorePoints = new ArrayList<>(); for (final String block : response.second().split("\r\n\r\n")) { if (block.isEmpty()) { continue; diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index 551d96eab9af..7eb350a06439 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -290,7 +290,7 @@ public void setBucketQuota(BucketTO bucket, long storeId, long size) { RgwAdmin rgwAdmin = getRgwAdminClient(storeId); try { - rgwAdmin.setBucketQuota(bucket.getName(), -1, size); + rgwAdmin.setIndividualBucketQuota(null, bucket.getName(), -1, size * 1024 * 1024); } catch (Exception e) { throw new CloudRuntimeException(e); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index fcc4444670cf..92892118d815 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -590,7 +590,7 @@ public ResourceLimitResponse createResourceLimitResponse(ResourceLimit limit) { } resourceLimitResponse.setResourceType(limit.getType()); - if ((limit.getType() == ResourceType.primary_storage || limit.getType() == ResourceType.secondary_storage) && limit.getMax() >= 0) { + if (ResourceType.isStorageType(limit.getType()) && limit.getMax() >= 0) { resourceLimitResponse.setMax((long)Math.ceil((double)limit.getMax() / ResourceType.bytesToGiB)); } else { resourceLimitResponse.setMax(limit.getMax()); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 7d5658f6782c..bf5c4666984f 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -453,6 +453,10 @@ private static void copyResourceLimitsIntoMap(Map r resourceLimitMap.put(Resource.ResourceType.primary_storage, domainJoinVO.getPrimaryStorageLimit()); resourceLimitMap.put(Resource.ResourceType.secondary_storage, domainJoinVO.getSecondaryStorageLimit()); resourceLimitMap.put(Resource.ResourceType.project, domainJoinVO.getProjectLimit()); + resourceLimitMap.put(Resource.ResourceType.backup, domainJoinVO.getBackupLimit()); + resourceLimitMap.put(Resource.ResourceType.backup_storage, domainJoinVO.getBackupStorageLimit()); + resourceLimitMap.put(Resource.ResourceType.bucket, domainJoinVO.getBucketLimit()); + resourceLimitMap.put(Resource.ResourceType.object_storage, domainJoinVO.getObjectStorageLimit()); } private static void copyResourceLimitsFromMap(Map resourceLimitMap, DomainJoinVO domainJoinVO){ @@ -468,6 +472,10 @@ private static void copyResourceLimitsFromMap(Map r domainJoinVO.setPrimaryStorageLimit(resourceLimitMap.get(Resource.ResourceType.primary_storage)); domainJoinVO.setSecondaryStorageLimit(resourceLimitMap.get(Resource.ResourceType.secondary_storage)); domainJoinVO.setProjectLimit(resourceLimitMap.get(Resource.ResourceType.project)); + domainJoinVO.setBackupLimit(resourceLimitMap.get(Resource.ResourceType.backup)); + domainJoinVO.setBackupStorageLimit(resourceLimitMap.get(Resource.ResourceType.backup_storage)); + domainJoinVO.setBucketLimit(resourceLimitMap.get(Resource.ResourceType.bucket)); + domainJoinVO.setObjectStorageLimit(resourceLimitMap.get(Resource.ResourceType.object_storage)); } private static void setParentResourceLimitIfNeeded(Map resourceLimitMap, DomainJoinVO domainJoinVO, List domainsCopy) { @@ -486,6 +494,10 @@ private static void setParentResourceLimitIfNeeded(Map templateSizeSearch; protected GenericSearchBuilder snapshotSizeSearch; @@ -288,6 +299,10 @@ public boolean configure(final String name, final Map params) th projectResourceLimitMap.put(Resource.ResourceType.memory.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectMemory.key()))); projectResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectPrimaryStorage.key()))); projectResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), MaxProjectSecondaryStorage.value()); + projectResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxProjectBackups.key()))); + projectResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxProjectBackupStorage.key()))); + projectResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxProjectBuckets.key()))); + projectResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxProjectObjectStorage.key()))); accountResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPublicIPs.key()))); accountResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountSnapshots.key()))); @@ -301,6 +316,10 @@ public boolean configure(final String name, final Map params) th accountResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPrimaryStorage.key()))); accountResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), MaxAccountSecondaryStorage.value()); accountResourceLimitMap.put(Resource.ResourceType.project.name(), DefaultMaxAccountProjects.value()); + accountResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxAccountBackups.key()))); + accountResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxAccountBackupStorage.key()))); + accountResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxAccountBuckets.key()))); + accountResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxAccountObjectStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPublicIPs.key()))); domainResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSnapshots.key()))); @@ -314,6 +333,10 @@ public boolean configure(final String name, final Map params) th domainResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPrimaryStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSecondaryStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.project.name(), DefaultMaxDomainProjects.value()); + domainResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxDomainBackups.key()))); + domainResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxDomainBackupStorage.key()))); + domainResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxDomainBuckets.key()))); + domainResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxDomainObjectStorage.key()))); } catch (NumberFormatException e) { logger.error("NumberFormatException during configuration", e); throw new ConfigurationException("Configuration failed due to NumberFormatException, see log for the stacktrace"); @@ -390,8 +413,8 @@ public long findCorrectResourceLimitForAccount(Account account, ResourceType typ if (value < 0) { // return unlimit if value is set to negative return max; } - // convert the value from GiB to bytes in case of primary or secondary storage. - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + // convert the value from GiB to bytes in case of storage type resource. + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -431,7 +454,7 @@ public long findCorrectResourceLimitForAccount(long accountId, Long limit, Resou if (value < 0) { // return unlimit if value is set to negative return max; } - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -478,7 +501,7 @@ public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type, if (value < 0) { // return unlimit if value is set to negative return max; } - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -512,7 +535,7 @@ protected void checkDomainResourceLimit(final Account account, final Project pro String convCurrentResourceReservation = String.valueOf(currentResourceReservation); String convNumResources = String.valueOf(numResources); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convDomainResourceLimit = toHumanReadableSize(domainResourceLimit); convCurrentDomainResourceCount = toHumanReadableSize(currentDomainResourceCount); convCurrentResourceReservation = toHumanReadableSize(currentResourceReservation); @@ -557,7 +580,7 @@ protected void checkAccountResourceLimit(final Account account, final Project pr String convertedCurrentResourceReservation = String.valueOf(currentResourceReservation); String convertedNumResources = String.valueOf(numResources); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convertedAccountResourceLimit = toHumanReadableSize(accountResourceLimit); convertedCurrentResourceCount = toHumanReadableSize(currentResourceCount); convertedCurrentResourceReservation = toHumanReadableSize(currentResourceReservation); @@ -594,7 +617,7 @@ protected List lockAccountAndOwnerDomainRows(long accountId, fi public long findDefaultResourceLimitForDomain(ResourceType resourceType) { Long resourceLimit = null; resourceLimit = domainResourceLimitMap.get(resourceType.getName()); - if (resourceLimit != null && (resourceType == ResourceType.primary_storage || resourceType == ResourceType.secondary_storage)) { + if (resourceLimit != null && ResourceType.isStorageType(resourceType)) { if (! Long.valueOf(Resource.RESOURCE_UNLIMITED).equals(resourceLimit)) { resourceLimit = resourceLimit * ResourceType.bytesToGiB; } @@ -908,12 +931,14 @@ public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Intege } //Convert max storage size from GiB to bytes - if ((resourceType == ResourceType.primary_storage || resourceType == ResourceType.secondary_storage) && max >= 0) { + if (ResourceType.isStorageType(resourceType) && max >= 0) { max *= ResourceType.bytesToGiB; } ResourceOwnerType ownerType = null; Long ownerId = null; + ApiCommandResourceType ownerResourceType = null; + Long ownerResourceId = null; if (accountId != null) { Account account = _entityMgr.findById(Account.class, accountId); @@ -942,6 +967,16 @@ public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Intege ownerType = ResourceOwnerType.Account; ownerId = accountId; + + if (account.getType() == Account.Type.PROJECT) { + ownerResourceType = ApiCommandResourceType.Project; + Project project = _projectDao.findByProjectAccountId(accountId); + ownerResourceId = project.getId(); + } else { + ownerResourceType = ApiCommandResourceType.Account; + ownerResourceId = ownerId; + } + if (StringUtils.isNotEmpty(tag)) { long untaggedLimit = findCorrectResourceLimitForAccount(account, resourceType, null); if (untaggedLimit > 0 && max > untaggedLimit) { @@ -980,6 +1015,8 @@ public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Intege } ownerType = ResourceOwnerType.Domain; ownerId = domainId; + ownerResourceType = ApiCommandResourceType.Domain; + ownerResourceId = ownerId; } if (ownerId == null) { @@ -987,6 +1024,12 @@ public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Intege } ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(ownerId, ownerType, resourceType, tag); + + ActionEventUtils.onActionEvent(caller.getId(), caller.getAccountId(), + caller.getDomainId(), EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + resourceType.toString() + ", New Value: " + max, + ownerResourceId, ownerResourceType.toString()); + if (limit != null) { // Update the existing limit _resourceLimitDao.update(limit.getId(), max); @@ -1135,7 +1178,7 @@ protected boolean updateResourceCountForAccount(final long accountId, final Reso } if (logger.isDebugEnabled()) { String convertedDelta = String.valueOf(delta); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convertedDelta = toHumanReadableSize(delta); } String typeStr = StringUtils.isNotEmpty(tag) ? String.format("%s (tag: %s)", type, tag) : type.getName(); @@ -1236,6 +1279,10 @@ protected long recalculateAccountResourceCount(final long accountId, final Resou newCount = calculateVolumeCountForAccount(accountId, tag); } else if (type == Resource.ResourceType.snapshot) { newCount = _snapshotDao.countSnapshotsForAccount(accountId); + } else if (type == Resource.ResourceType.backup) { + newCount = backupDao.countBackupsForAccount(accountId); + } else if (type == Resource.ResourceType.backup_storage) { + newCount = backupDao.calculateBackupStorageForAccount(accountId); } else if (type == Resource.ResourceType.public_ip) { newCount = calculatePublicIpForAccount(accountId); } else if (type == Resource.ResourceType.template) { @@ -1254,6 +1301,10 @@ protected long recalculateAccountResourceCount(final long accountId, final Resou newCount = calculatePrimaryStorageForAccount(accountId, tag); } else if (type == Resource.ResourceType.secondary_storage) { newCount = calculateSecondaryStorageForAccount(accountId); + } else if (type == Resource.ResourceType.bucket) { + newCount = bucketDao.countBucketsForAccount(accountId); + } else if (type == ResourceType.object_storage) { + newCount = bucketDao.calculateObjectStorageAllocationForAccount(accountId); } else { throw new InvalidParameterValueException("Unsupported resource type " + type); } @@ -1270,10 +1321,9 @@ protected long recalculateAccountResourceCount(final long accountId, final Resou _resourceCountDao.persist(new ResourceCountVO(type, newCount, accountId, ResourceOwnerType.Account, tag)); } - // No need to log message for primary and secondary storage because both are recalculating the + // No need to log message for storage type resources because both are recalculating the // resource count which will not lead to any discrepancy. - if (newCount != null && !newCount.equals(oldCount) && - type != Resource.ResourceType.primary_storage && type != Resource.ResourceType.secondary_storage) { + if (newCount != null && !newCount.equals(oldCount) && !ResourceType.isStorageType(type)) { logger.warn("Discrepancy in the resource count " + "(original count=" + oldCount + " correct count = " + newCount + ") for type " + type + " for account ID " + accountId + " is fixed during resource count recalculation."); } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index a371a0647019..b066402313ce 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -164,6 +164,7 @@ import com.cloud.serializer.GsonHelper; import com.cloud.server.ManagementService; import com.cloud.server.ResourceTag; +import com.cloud.server.StatsCollector; import com.cloud.server.TaggedResourceService; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -353,9 +354,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject private BackupDao backupDao; @Inject + private StatsCollector statsCollector; + @Inject HostPodDao podDao; - protected Gson _gson; private static final List SupportedHypervisorsForVolResize = Arrays.asList(HypervisorType.KVM, HypervisorType.XenServer, @@ -5204,6 +5206,21 @@ private VmWorkJobVO createPlaceHolderWork(long instanceId) { return workJob; } + @Override + public Long getVolumePhysicalSize(ImageFormat format, String path, String chainInfo) { + VolumeStats vs = null; + if (format == ImageFormat.VHD || format == ImageFormat.QCOW2 || format == ImageFormat.RAW) { + if (path != null) { + vs = statsCollector.getVolumeStats(path); + } + } else if (format == ImageFormat.OVA) { + if (chainInfo != null) { + vs = statsCollector.getVolumeStats(chainInfo); + } + } + return (vs == null) ? null : vs.getPhysicalSize(); + } + @Override public String getConfigComponentName() { return VolumeApiService.class.getSimpleName(); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 2e52d1ccc446..816d8fd66c48 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -29,7 +29,13 @@ import java.util.stream.Collectors; import com.amazonaws.util.CollectionUtils; +import com.cloud.alert.AlertManager; +import com.cloud.configuration.Resource; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.storage.Snapshot; import com.cloud.storage.VolumeApiService; +import com.cloud.user.DomainManager; +import com.cloud.user.ResourceLimitService; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VirtualMachineManager; import javax.inject.Inject; @@ -37,6 +43,7 @@ import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.api.command.admin.backup.DeleteBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.ListBackupProviderOfferingsCmd; @@ -115,6 +122,7 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -139,6 +147,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { @Inject private AccountManager accountManager; @Inject + private DomainManager domainManager; + @Inject private VolumeDao volumeDao; @Inject private DataCenterDao dataCenterDao; @@ -164,6 +174,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { private VolumeApiService volumeApiService; @Inject private VolumeOrchestrationService volumeOrchestrationService; + @Inject + private ResourceLimitService resourceLimitMgr; + @Inject + private AlertManager alertManager; private AsyncJobDispatcher asyncJobDispatcher; private Timer backupTimer; @@ -396,8 +410,8 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), vm.getBackupOfferingId(), null, null, Backup.class.getSimpleName(), vm.getUuid()); - final BackupSchedule backupSchedule = backupScheduleDao.findByVM(vm.getId()); - if (backupSchedule != null) { + final List backupSchedules = backupScheduleDao.listByVM(vm.getId()); + for(BackupSchedule backupSchedule: backupSchedules) { backupScheduleDao.remove(backupSchedule.getId()); } result = true; @@ -415,6 +429,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { final DateUtil.IntervalType intervalType = cmd.getIntervalType(); final String scheduleString = cmd.getSchedule(); final TimeZone timeZone = TimeZone.getTimeZone(cmd.getTimezone()); + final Integer maxBackups = cmd.getMaxBackups(); if (intervalType == null) { throw new CloudRuntimeException("Invalid interval type provided"); @@ -427,12 +442,41 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { if (vm.getBackupOfferingId() == null) { throw new CloudRuntimeException("Cannot configure backup schedule for the VM without having any backup offering"); } + if (maxBackups != null && maxBackups <= 0) { + throw new InvalidParameterValueException(String.format("maxBackups [%s] for instance %s should be greater than 0.", maxBackups, vm.getName())); + } + + Backup.Type backupType = Backup.Type.valueOf(intervalType.name()); + int intervalMaxBackups = backupType.getMax(); + if (maxBackups != null && maxBackups > intervalMaxBackups) { + throw new InvalidParameterValueException(String.format("maxBackups [%s] for instance %s exceeds limit [%s] for interval type [%s].", maxBackups, vm.getName(), + intervalMaxBackups, intervalType)); + } + + Account owner = accountManager.getAccount(vm.getAccountId()); + + long accountLimit = resourceLimitMgr.findCorrectResourceLimitForAccount(owner, Resource.ResourceType.backup, null); + long domainLimit = resourceLimitMgr.findCorrectResourceLimitForDomain(domainManager.getDomain(owner.getDomainId()), Resource.ResourceType.backup, null); + if (maxBackups != null && !accountManager.isRootAdmin(owner.getId()) && ((accountLimit != -1 && maxBackups > accountLimit) || (domainLimit != -1 && maxBackups > domainLimit))) { + String message = "domain/account"; + if (owner.getType() == Account.Type.PROJECT) { + message = "domain/project"; + } + throw new InvalidParameterValueException("Max number of backups shouldn't exceed the " + message + " level backup limit"); + } final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); if (offering == null || !offering.isUserDrivenBackupAllowed()) { throw new CloudRuntimeException("The selected backup offering does not allow user-defined backup schedule"); } + if (maxBackups == null && !"veeam".equals(offering.getProvider())) { + throw new CloudRuntimeException("Please specify the maximum number of buckets to retain."); + } + if (maxBackups != null && "veeam".equals(offering.getProvider())) { + throw new CloudRuntimeException("The maximum backups to retain cannot be configured through CloudStack for Veeam. Retention is managed directly in Veeam based on the settings specified when creating the backup job."); + } + final String timezoneId = timeZone.getID(); if (!timezoneId.equals(cmd.getTimezone())) { logger.warn("Using timezone: " + timezoneId + " for running this snapshot policy as an equivalent of " + cmd.getTimezone()); @@ -447,15 +491,16 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); if (schedule == null) { - return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime)); + return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime, maxBackups)); } schedule.setScheduleType((short) intervalType.ordinal()); schedule.setSchedule(scheduleString); schedule.setTimezone(timezoneId); schedule.setScheduledTimestamp(nextDateTime); + schedule.setMaxBackups(maxBackups); backupScheduleDao.update(schedule.getId(), schedule); - return backupScheduleDao.findByVM(vmId); + return backupScheduleDao.findById(schedule.getId()); } @Override @@ -469,7 +514,7 @@ public List listBackupSchedule(final Long vmId) { @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_DELETE, eventDescription = "deleting VM backup schedule") - public boolean deleteBackupSchedule(final Long vmId) { + public boolean deleteBackupSchedule(Long vmId) { final VMInstanceVO vm = findVmById(vmId); validateForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -481,9 +526,30 @@ public boolean deleteBackupSchedule(final Long vmId) { return backupScheduleDao.remove(schedule.getId()); } + private void postCreateScheduledBackup(Backup.Type backupType, Long vmId) { + DateUtil.IntervalType intervalType = DateUtil.IntervalType.valueOf(backupType.name()); + final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); + if (schedule == null) { + return; + } + Integer maxBackups = schedule.getMaxBackups(); + if (maxBackups == null) { + return; + } + List backups = backupDao.listBackupsByVMandIntervalType(vmId, backupType); + while (backups.size() > maxBackups) { + BackupVO oldestBackup = backups.get(0); + if (deleteBackup(oldestBackup.getId(), false)) { + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, oldestBackup.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_DELETE, + "Successfully deleted oldest backup: " + oldestBackup.getId(), oldestBackup.getId(), ApiCommandResourceType.Backup.toString(), 0); + } + backups.remove(oldestBackup); + } + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating VM backup", async = true) - public boolean createBackup(final Long vmId) { + public boolean createBackup(final Long vmId, final Long scheduleId) throws ResourceAllocationException { final VMInstanceVO vm = findVmById(vmId); validateForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -501,13 +567,65 @@ public boolean createBackup(final Long vmId) { throw new CloudRuntimeException("The assigned backup offering does not allow ad-hoc user backup"); } + Backup.Type type = getBackupType(scheduleId); + Account owner = accountManager.getAccount(vm.getAccountId()); + try { + resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup); + } catch (ResourceAllocationException e) { + if (type != Backup.Type.MANUAL) { + String msg = "Backup resource limit exceeded for account id : " + owner.getId() + ". Failed to create backup"; + logger.warn(msg); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup resource limit exceeded for account id : " + owner.getId() + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + throw e; + } + + Long backupSize = 0L; + for (final Volume volume: volumeDao.findByInstance(vmId)) { + if (Volume.State.Ready.equals(volume.getState())) { + Long volumeSize = volumeApiService.getVolumePhysicalSize(volume.getFormat(), volume.getPath(), volume.getChainInfo()); + if (volumeSize == null) { + volumeSize = volume.getSize(); + } + backupSize += volumeSize; + } + } + try { + resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup_storage, backupSize); + } catch (ResourceAllocationException e) { + if (type != Backup.Type.MANUAL) { + String msg = "Backup storage space resource limit exceeded for account id : " + owner.getId() + ". Failed to create backup"; + logger.warn(msg); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + owner.getId() + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + throw e; + } + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventTypes.EVENT_VM_BACKUP_CREATE, "creating backup for VM ID:" + vm.getUuid(), vmId, ApiCommandResourceType.VirtualMachine.toString(), true, 0); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); - if (backupProvider != null && backupProvider.takeBackup(vm)) { + if (backupProvider != null) { + Pair result = backupProvider.takeBackup(vm); + if (!result.first()) { + throw new CloudRuntimeException("Failed to create VM backup"); + } + Backup backup = result.second(); + if (backup != null) { + BackupVO vmBackup = backupDao.findById(result.second().getId()); + vmBackup.setBackupIntervalType((short) type.ordinal()); + backupDao.update(vmBackup.getId(), vmBackup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + } + if (type != Backup.Type.MANUAL) { + postCreateScheduledBackup(type, vm.getId()); + } return true; } throw new CloudRuntimeException("Failed to create VM backup"); @@ -682,6 +800,29 @@ protected void tryRestoreVM(BackupVO backup, VMInstanceVO vm, BackupOffering off } } + private Backup.Type getBackupType(Long scheduleId) { + if (scheduleId.equals(Snapshot.MANUAL_POLICY_ID)) { + return Backup.Type.MANUAL; + } else { + BackupScheduleVO scheduleVO = backupScheduleDao.findById(scheduleId); + DateUtil.IntervalType intvType = scheduleVO.getScheduleType(); + return getBackupType(intvType); + } + } + + private Backup.Type getBackupType(DateUtil.IntervalType intvType) { + if (intvType.equals(DateUtil.IntervalType.HOURLY)) { + return Backup.Type.HOURLY; + } else if (intvType.equals(DateUtil.IntervalType.DAILY)) { + return Backup.Type.DAILY; + } else if (intvType.equals(DateUtil.IntervalType.WEEKLY)) { + return Backup.Type.WEEKLY; + } else if (intvType.equals(DateUtil.IntervalType.MONTHLY)) { + return Backup.Type.MONTHLY; + } + return null; + } + /** * Tries to update the state of given VM, given specified event * @param vm The VM to update its state @@ -858,6 +999,8 @@ public boolean deleteBackup(final Long backupId, final Boolean forced) { final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); boolean result = backupProvider.deleteBackup(backup, forced); if (result) { + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); return backupDao.remove(backup.getId()); } throw new CloudRuntimeException("Failed to delete the backup"); @@ -925,6 +1068,10 @@ private boolean attachVolumeToVM(Long zoneId, String restoredVolumeLocation, Lis public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); backgroundPollManager.submitTask(new BackupSyncTask(this)); + Backup.Type.HOURLY.setMax(BackupHourlyMax.value()); + Backup.Type.DAILY.setMax(BackupDailyMax.value()); + Backup.Type.WEEKLY.setMax(BackupWeeklyMax.value()); + Backup.Type.MONTHLY.setMax(BackupMonthlyMax.value()); return true; } @@ -1004,7 +1151,17 @@ public ConfigKey[] getConfigKeys() { BackupFrameworkEnabled, BackupProviderPlugin, BackupSyncPollingInterval, - BackupEnableAttachDetachVolumes + BackupEnableAttachDetachVolumes, + BackupHourlyMax, + BackupDailyMax, + BackupWeeklyMax, + BackupMonthlyMax, + DefaultMaxAccountBackups, + DefaultMaxAccountBackupStorage, + DefaultMaxProjectBackups, + DefaultMaxProjectBackupStorage, + DefaultMaxDomainBackups, + DefaultMaxDomainBackupStorage }; } @@ -1137,6 +1294,7 @@ public void scheduleBackups() { true, 0); final Map params = new HashMap(); params.put(ApiConstants.VIRTUAL_MACHINE_ID, "" + vmId); + params.put(ApiConstants.SCHEDULE_ID, "" + backupScheduleId); params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + vm.getAccountId()); params.put("ctxStartEventId", String.valueOf(eventId)); @@ -1205,7 +1363,7 @@ private VMInstanceVO findVmById(final Long vmId) { * This background task syncs backups from providers side in CloudStack db * along with creation of usage records */ - private final class BackupSyncTask extends ManagedContextRunnable implements BackgroundPollTask { + protected final class BackupSyncTask extends ManagedContextRunnable implements BackgroundPollTask { private BackupManager backupManager; public BackupSyncTask(final BackupManager backupManager) { @@ -1253,13 +1411,81 @@ private void syncBackupMetrics(final BackupProvider backupProvider, final Map backupsInDb, VirtualMachine vm, Backup.Metric metric) { + for (final Backup backupInDb : backupsInDb) { + logger.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm)); + if (restorePoint.getId().equals(backupInDb.getExternalId())) { + logger.debug(String.format("Found Backup %s in both Database and Networker", backupInDb)); + if (metric != null) { + logger.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", + backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), metric.getBackupSize(), metric.getDataSize())); + + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); + ((BackupVO) backupInDb).setSize(metric.getBackupSize()); + ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); + + backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); + } + return backupInDb; + } + } + return null; + } + + private void syncBackups(BackupProvider backupProvider, VirtualMachine vm, Backup.Metric metric) { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final List backupsInDb = backupDao.listByVmId(null, vm.getId()); + List restorePoints = backupProvider.listRestorePoints(vm); + if (restorePoints == null) { + return; + } + + final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); + for (final Backup.RestorePoint restorePoint : restorePoints) { + if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) { + Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(restorePoint, backupsInDb, vm, metric); + if (existingBackupEntry != null) { + removeList.remove(existingBackupEntry.getId()); + continue; + } + } + + Backup backup = backupProvider.createNewBackupEntryForRestorePoint(restorePoint, vm, metric); + if (backup != null) { + logger.warn("Added backup found in provider [" + backup + "]"); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + + logger.debug(String.format("Creating a new entry in backups: [id: %s, uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + + "domain_id: %s, zone_id: %s].", backup.getId(), backup.getUuid(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), + backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId())); + + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE, + String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); + } + } + for (final Long backupIdToRemove : removeList) { + logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); + Backup backup = backupDao.findById(backupIdToRemove); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + backupDao.remove(backupIdToRemove); + } + } + }); + } + private void tryToSyncVMBackups(BackupProvider backupProvider, Map metrics, VirtualMachine vm) { try { final Backup.Metric metric = metrics.get(vm); if (metric != null) { logger.debug(String.format("Trying to sync backups of VM [%s] using backup provider [%s].", vm, backupProvider.getName())); // Sync out-of-band backups - backupProvider.syncBackups(vm, metric); + syncBackups(backupProvider, vm, metric); // Emit a usage event, update usage metric for the VM by the usage server UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java index 389ca52b03bb..ca0e6291e529 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java @@ -19,9 +19,12 @@ import com.amazonaws.services.s3.internal.BucketNameUtils; import com.amazonaws.services.s3.model.IllegalBucketNameException; import com.cloud.agent.api.to.BucketTO; +import com.cloud.configuration.Resource; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ResourceLimitManagerImpl; import com.cloud.storage.BucketVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.dao.BucketDao; @@ -60,6 +63,8 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic private BucketDao _bucketDao; @Inject private AccountManager _accountMgr; + @Inject + private ResourceLimitManagerImpl resourceLimitManager; @Inject private BucketStatisticsDao _bucketStatisticsDao; @@ -98,12 +103,18 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] { + DefaultMaxAccountBuckets, + DefaultMaxAccountObjectStorage, + DefaultMaxProjectBuckets, + DefaultMaxProjectObjectStorage, + DefaultMaxDomainBuckets, + DefaultMaxDomainObjectStorage }; } @Override @ActionEvent(eventType = EventTypes.EVENT_BUCKET_CREATE, eventDescription = "creating bucket", create = true) - public Bucket allocBucket(CreateBucketCmd cmd) { + public Bucket allocBucket(CreateBucketCmd cmd) throws ResourceAllocationException { try { BucketNameUtils.validateBucketName(cmd.getBucketName()); } catch (IllegalBucketNameException e) { @@ -125,6 +136,9 @@ public Bucket allocBucket(CreateBucketCmd cmd) { return null; } + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.bucket); + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); + BucketVO bucket = new BucketVO(ownerId, owner.getDomainId(), cmd.getObjectStoragePoolId(), cmd.getBucketName(), cmd.getQuota(), cmd.isVersioning(), cmd.isEncryption(), cmd.isObjectLocking(), cmd.getPolicy()); _bucketDao.persist(bucket); @@ -146,6 +160,7 @@ public Bucket createBucket(CreateBucketCmd cmd) { try { bucketTO = new BucketTO(objectStore.createBucket(bucket, objectLock)); bucketCreated = true; + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); if (cmd.isVersioning()) { objectStore.setBucketVersioning(bucketTO); @@ -157,6 +172,7 @@ public Bucket createBucket(CreateBucketCmd cmd) { if (cmd.getQuota() != null) { objectStore.setQuota(bucketTO, cmd.getQuota()); + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); } if (cmd.getPolicy() != null) { @@ -188,6 +204,8 @@ public boolean deleteBucket(long bucketId, Account caller) { ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); if (objectStore.deleteBucket(bucketTO)) { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (bucket.getQuota() * Resource.ResourceType.bytesToGiB)); return _bucketDao.remove(bucketId); } return false; @@ -195,7 +213,7 @@ public boolean deleteBucket(long bucketId, Account caller) { @Override @ActionEvent(eventType = EventTypes.EVENT_BUCKET_UPDATE, eventDescription = "updating bucket") - public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { + public boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException { BucketVO bucket = _bucketDao.findById(cmd.getId()); BucketTO bucketTO = new BucketTO(bucket); if (bucket == null) { @@ -204,6 +222,17 @@ public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { _accountMgr.checkAccess(caller, null, true, bucket); ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + Integer quota = cmd.getQuota(); + Integer quotaDelta = null; + + if (quota != null) { + quotaDelta = quota - bucket.getQuota(); + if (quotaDelta > 0) { + Account owner = _accountMgr.getActiveAccountById(bucket.getAccountId()); + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.object_storage, (quotaDelta * Resource.ResourceType.bytesToGiB)); + } + } + try { if (cmd.getEncryption() != null) { if (cmd.getEncryption()) { @@ -231,6 +260,11 @@ public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { if (cmd.getQuota() != null) { objectStore.setQuota(bucketTO, cmd.getQuota()); bucket.setQuota(cmd.getQuota()); + if (quotaDelta > 0) { + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (quotaDelta * Resource.ResourceType.bytesToGiB)); + } else { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, ((-quotaDelta) * Resource.ResourceType.bytesToGiB)); + } } _bucketDao.update(bucket.getId(), bucket); } catch (Exception e) { diff --git a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java index 34030626d22e..34e4632d24a0 100644 --- a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java +++ b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java @@ -23,9 +23,15 @@ import java.util.List; import java.util.Map; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; +import com.cloud.utils.db.EntityManager; + +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.commons.collections.CollectionUtils; @@ -39,6 +45,7 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -67,6 +74,7 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; +import com.cloud.user.User; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; @@ -80,6 +88,9 @@ import junit.framework.TestCase; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + @RunWith(MockitoJUnitRunner.class) public class ResourceLimitManagerImplTest extends TestCase { private Logger logger = LogManager.getLogger(ResourceLimitManagerImplTest.class); @@ -118,7 +129,10 @@ public class ResourceLimitManagerImplTest extends TestCase { VolumeDao volumeDao; @Mock UserVmDao userVmDao; + @Mock + EntityManager entityManager; + private CallContext callContext; private List hostTags = List.of("htag1", "htag2", "htag3"); private List storageTags = List.of("stag1", "stag2"); @@ -136,10 +150,15 @@ public void setUp() { } catch (IllegalAccessException | NoSuchFieldException e) { logger.error("Failed to update configurations"); } + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext.register(user, account); } @After public void tearDown() throws Exception { + CallContext.unregister(); } @Test @@ -415,6 +434,9 @@ public void testFindCorrectResourceLimitForAccount() { Mockito.when(resourceLimitDao.findByOwnerIdAndTypeAndTag(1L, Resource.ResourceOwnerType.Account, Resource.ResourceType.cpu, hostTags.get(0))).thenReturn(null); result = resourceLimitManager.findCorrectResourceLimitForAccount(account, Resource.ResourceType.cpu, hostTags.get(0)); Assert.assertEquals(defaultAccountCpuMax, result); + + result = resourceLimitManager.findCorrectResourceLimitForAccount(account, Resource.ResourceType.cpu, hostTags.get(0)); + Assert.assertEquals(defaultAccountCpuMax, result); } @Test @@ -449,24 +471,26 @@ public void testFindCorrectResourceLimitForAccountProjects() { @Test public void testFindCorrectResourceLimitForAccountId1() { -// long accountId = 1L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(true); -// long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); -// Assert.assertEquals(Resource.RESOURCE_UNLIMITED, result); -// -// accountId = 2L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); -// Long limit = 100L; -// long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, limit, Resource.ResourceType.cpu); -// Assert.assertEquals(limit.longValue(), result); -// -// long defaultAccountCpuMax = 25L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); -// Map accountResourceLimitMap = new HashMap<>(); -// accountResourceLimitMap.put(Resource.ResourceType.cpu.name(), defaultAccountCpuMax); -// resourceLimitManager.accountResourceLimitMap = accountResourceLimitMap; -// result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); -// Assert.assertEquals(defaultAccountCpuMax, result); + long accountId = 1L; + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(true); + long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); + Assert.assertEquals(Resource.RESOURCE_UNLIMITED, result); + + accountId = 2L; + AccountVO account = mock(AccountVO.class); + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); + Mockito.when(accountDao.findById(accountId)).thenReturn(account); + Long limit = 100L; + result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, limit, Resource.ResourceType.cpu); + Assert.assertEquals(limit.longValue(), result); + + long defaultAccountCpuMax = 25L; + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); + Map accountResourceLimitMap = new HashMap<>(); + accountResourceLimitMap.put(Resource.ResourceType.cpu.name(), defaultAccountCpuMax); + resourceLimitManager.accountResourceLimitMap = accountResourceLimitMap; + result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); + Assert.assertEquals(defaultAccountCpuMax, result); } @Test @@ -1223,4 +1247,65 @@ public void testDecrementVmMemoryResourceCount() { Mockito.verify(resourceLimitManager, Mockito.times(1)) .decrementResourceCountWithTag(accountId, Resource.ResourceType.memory, tag, Long.valueOf(memory)); } + + @Test + public void testUpdateResourceLimitForAccount() { + Long accountId = 1L; + Long resourceLimitId = 3L; + Integer typeId = 13; + Long maxGB = 10L; + Long maxBytes = maxGB * Resource.ResourceType.bytesToGiB; + + Account account = mock(Account.class); + when(entityManager.findById(Account.class, accountId)).thenReturn(account); + ResourceLimitVO resourceLimitVO = mock(ResourceLimitVO.class); + when(resourceLimitVO.getId()).thenReturn(resourceLimitId); + when(resourceLimitDao.findByOwnerIdAndTypeAndTag(accountId, Resource.ResourceOwnerType.Account, Resource.ResourceType.backup_storage, null)).thenReturn(resourceLimitVO); + + try (MockedStatic actionEventUtilsMockedStatic = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + resourceLimitManager.updateResourceLimit(accountId, null, typeId, maxGB, null); + + Mockito.verify(resourceLimitDao, Mockito.times(1)).update(resourceLimitId, maxBytes); + Mockito.verify(resourceLimitDao, Mockito.never()).persist(Mockito.any()); + actionEventUtilsMockedStatic.verify(() -> ActionEventUtils.onActionEvent(0L, 0L, 0L, EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + Resource.ResourceType.backup_storage.toString() + ", New Value: " + maxBytes, + accountId, ApiCommandResourceType.Account.toString())); + } + } + + @Test + public void testUpdateResourceLimitForDomain() { + Long domainId = 2L; + Long resourceLimitId = 3L; + Integer typeId = 13; + Long maxGB = 10L; + Long maxBytes = maxGB * Resource.ResourceType.bytesToGiB; + + Domain domain = mock(Domain.class); + when(domain.getParent()).thenReturn(null); + when(entityManager.findById(Domain.class, domainId)).thenReturn(domain); + ResourceLimitVO resourceLimitVO = mock(ResourceLimitVO.class); + when(resourceLimitVO.getId()).thenReturn(resourceLimitId); + when(resourceLimitDao.findByOwnerIdAndTypeAndTag(domainId, Resource.ResourceOwnerType.Domain, Resource.ResourceType.backup_storage, null)).thenReturn(resourceLimitVO); + + try (MockedStatic actionEventUtilsMockedStatic = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + resourceLimitManager.updateResourceLimit(null, domainId, typeId, maxGB, null); + + Mockito.verify(resourceLimitDao, Mockito.times(1)).update(resourceLimitId, maxBytes); + Mockito.verify(resourceLimitDao, Mockito.never()).persist(Mockito.any()); + actionEventUtilsMockedStatic.verify(() -> ActionEventUtils.onActionEvent(0L, 0L, 0L, EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + Resource.ResourceType.backup_storage.toString() + ", New Value: " + maxBytes, + domainId, ApiCommandResourceType.Domain.toString())); + } + } } diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 3bf1fb97e4d0..808511e6fdf5 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -16,22 +16,45 @@ // under the License. package org.apache.cloudstack.backup; +import com.cloud.alert.AlertManager; +import com.cloud.configuration.Resource; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.domain.Domain; import com.cloud.event.ActionEventUtils; +import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.DomainManager; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,13 +65,21 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; +import org.springframework.test.util.ReflectionTestUtils; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -72,9 +103,38 @@ public class BackupManagerTest { @Mock VolumeDao volumeDao; + @Mock + VMInstanceDao vmInstanceDao; + + @Mock + AccountManager accountManager; + + @Mock + DomainManager domainManager; + + @Mock + ResourceLimitService resourceLimitMgr; + + @Mock + BackupScheduleDao backupScheduleDao; + + @Mock + BackupDao backupDao; + + @Mock + DataCenterDao dataCenterDao; + + @Mock + AlertManager alertManager; + + private AccountVO account; + private UserVO user; + private String[] hostPossibleValues = {"127.0.0.1", "hostname"}; private String[] datastoresPossibleValues = {"e9804933-8609-4de3-bccc-6278072a496c", "datastore-name"}; private AutoCloseable closeable; + private ConfigDepotImpl configDepotImpl; + private boolean updatedConfigKeyDepot = false; @Before public void setup() throws Exception { @@ -97,11 +157,19 @@ public void setup() throws Exception { offering.setUserDrivenBackupAllowed(true); return true; }); + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext.register(user, account); } @After public void tearDown() throws Exception { closeable.close(); + if (updatedConfigKeyDepot) { + ReflectionTestUtils.setField(BackupManager.BackupFrameworkEnabled, "s_depot", configDepotImpl); + } + CallContext.unregister(); } @Test @@ -169,7 +237,7 @@ public void restoreBackedUpVolumeTestHostIpAndDatastoreUuid() { Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), Mockito.eq("127.0.0.1"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success", restoreBackedUpVolume.second()); @@ -190,7 +258,7 @@ public void restoreBackedUpVolumeTestHostIpAndDatastoreName() { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), Mockito.eq("127.0.0.1"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success2")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success2", restoreBackedUpVolume.second()); @@ -211,8 +279,8 @@ public void restoreBackedUpVolumeTestHostNameAndDatastoreUuid() { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState) )).thenReturn(new Pair(Boolean.TRUE, "Success3")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success3")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success3", restoreBackedUpVolume.second()); @@ -233,8 +301,8 @@ public void restoreBackedUpVolumeTestHostAndDatastoreName() { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success4")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Mockito.eq("hostname"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success4")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success4", restoreBackedUpVolume.second()); @@ -304,4 +372,289 @@ public void tryRestoreVMTestRestoreFails() throws NoTransitionException { } } } + + private void overrideBackupFrameworkConfigValue() { + ConfigKey configKey = BackupManager.BackupFrameworkEnabled; + this.configDepotImpl = (ConfigDepotImpl) ReflectionTestUtils.getField(configKey, "s_depot"); + ConfigDepotImpl configDepot = Mockito.mock(ConfigDepotImpl.class); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), + Mockito.eq(ConfigKey.Scope.Global), Mockito.isNull())).thenReturn("true"); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("true"); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupProviderPlugin.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("testbackupprovider"); + ReflectionTestUtils.setField(configKey, "s_depot", configDepot); + updatedConfigKeyDepot = true; + } + + @Test + public void testConfigureBackupSchedule() { + Long vmId = 1L; + Long zoneId = 2L; + Long accountId = 3L; + Long domainId = 4L; + Long backupOfferingId = 5L; + + CreateBackupScheduleCmd cmd = Mockito.mock(CreateBackupScheduleCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getTimezone()).thenReturn("GMT"); + when(cmd.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); + when(cmd.getMaxBackups()).thenReturn(8); + when(cmd.getSchedule()).thenReturn("00:00:00"); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getAccountId()).thenReturn(accountId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + + overrideBackupFrameworkConfigValue(); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + when(account.getDomainId()).thenReturn(domainId); + Domain domain = Mockito.mock(Domain.class); + when(domainManager.getDomain(domainId)).thenReturn(domain); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(account, Resource.ResourceType.backup, null)).thenReturn(8L); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domain, Resource.ResourceType.backup, null)).thenReturn(8L); + + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + when(offering.getProvider()).thenReturn("test"); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(backupScheduleDao.findByVMAndIntervalType(vmId, DateUtil.IntervalType.DAILY)).thenReturn(schedule); + + backupManager.configureBackupSchedule(cmd); + + verify(schedule, times(1)).setScheduleType((short) DateUtil.IntervalType.DAILY.ordinal()); + verify(schedule, times(1)).setSchedule("00:00:00"); + verify(schedule, times(1)).setTimezone(TimeZone.getTimeZone("GMT").getID()); + verify(schedule, times(1)).setMaxBackups(8); + } + + @Test + public void testConfigureBackupScheduleLimitReached() { + Long vmId = 1L; + Long zoneId = 2L; + Long accountId = 3L; + Long domainId = 4L; + + CreateBackupScheduleCmd cmd = Mockito.mock(CreateBackupScheduleCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getTimezone()).thenReturn("GMT"); + when(cmd.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); + when(cmd.getMaxBackups()).thenReturn(8); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + when(account.getDomainId()).thenReturn(domainId); + Domain domain = Mockito.mock(Domain.class); + when(domainManager.getDomain(domainId)).thenReturn(domain); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(account, Resource.ResourceType.backup, null)).thenReturn(10L); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domain, Resource.ResourceType.backup, null)).thenReturn(1L); + + InvalidParameterValueException exception = Assert.assertThrows(InvalidParameterValueException.class, + () -> backupManager.configureBackupSchedule(cmd)); + Assert.assertEquals(exception.getMessage(), "Max number of backups shouldn't exceed the domain/account level backup limit"); + } + + @Test + public void testCreateScheduledBackup() throws ResourceAllocationException { + Long vmId = 1L; + Long zoneId = 2L; + Long scheduleId = 3L; + Long backupOfferingId = 4L; + Long accountId = 5L; + Long backupId = 6L; + Long oldestBackupId = 7L; + Long newBackupSize = 1000000000L; + Long oldBackupSize = 400000000L; + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(vm.getId()).thenReturn(vmId); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + when(offering.getProvider()).thenReturn("test"); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(schedule.getScheduleType()).thenReturn(DateUtil.IntervalType.DAILY); + when(schedule.getMaxBackups()).thenReturn(0); + when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); + when(backupScheduleDao.findByVMAndIntervalType(vmId, DateUtil.IntervalType.DAILY)).thenReturn(schedule); + + BackupProvider backupProvider = mock(BackupProvider.class); + Backup backup = mock(Backup.class); + when(backup.getId()).thenReturn(backupId); + when(backup.getSize()).thenReturn(newBackupSize); + when(backupProvider.getName()).thenReturn("test"); + when(backupProvider.takeBackup(vm)).thenReturn(new Pair<>(true, backup)); + Map backupProvidersMap = new HashMap<>(); + backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider); + ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap); + + BackupVO backupVO = mock(BackupVO.class); + when(backupVO.getId()).thenReturn(backupId); + BackupVO oldestBackupVO = mock(BackupVO.class); + when(oldestBackupVO.getSize()).thenReturn(oldBackupSize); + when(oldestBackupVO.getId()).thenReturn(oldestBackupId); + when(oldestBackupVO.getVmId()).thenReturn(vmId); + when(oldestBackupVO.getBackupOfferingId()).thenReturn(backupOfferingId); + + when(backupDao.findById(backupId)).thenReturn(backupVO); + List backups = new ArrayList<>(List.of(oldestBackupVO)); + when(backupDao.listBackupsByVMandIntervalType(vmId, Backup.Type.DAILY)).thenReturn(backups); + when(backupDao.findByIdIncludingRemoved(oldestBackupId)).thenReturn(oldestBackupVO); + when(backupOfferingDao.findByIdIncludingRemoved(backupOfferingId)).thenReturn(offering); + when(backupProvider.deleteBackup(oldestBackupVO, false)).thenReturn(true); + when(backupDao.remove(oldestBackupVO.getId())).thenReturn(true); + + try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + Assert.assertEquals(backupManager.createBackup(vmId, scheduleId), true); + + Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); + Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); + Mockito.verify(backupDao, times(1)).update(backupVO.getId(), backupVO); + + Mockito.verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup); + Mockito.verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, oldBackupSize); + Mockito.verify(backupDao, times(1)).remove(oldestBackupId); + } + } + + @Test (expected = ResourceAllocationException.class) + public void testCreateBackupLimitReached() throws ResourceAllocationException { + Long vmId = 1L; + Long zoneId = 2L; + Long scheduleId = 3L; + Long backupOfferingId = 4L; + Long accountId = 5L; + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(schedule.getScheduleType()).thenReturn(DateUtil.IntervalType.DAILY); + when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); + + Account account = Mockito.mock(Account.class); + when(account.getId()).thenReturn(accountId); + when(accountManager.getAccount(accountId)).thenReturn(account); + Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup_storage)).when(resourceLimitMgr).checkResourceLimit(account, Resource.ResourceType.backup_storage, 0L); + + backupManager.createBackup(vmId, scheduleId); + + String msg = "Backup storage space resource limit exceeded for account id : " + accountId + ". Failed to create backup"; + Mockito.verify(alertManager, times(1)).sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + accountId + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + + @Test + public void testBackupSyncTask() { + Long dataCenterId = 1L; + Long vmId = 2L; + Long accountId = 3L; + Long backup2Id = 4L; + String restorePoint1ExternalId = "1234"; + Long backup1Size = 1 * Resource.ResourceType.bytesToGiB; + Long backup2Size = 2 * Resource.ResourceType.bytesToGiB; + Long newBackupSize = 3 * Resource.ResourceType.bytesToGiB; + Long metricSize = 4 * Resource.ResourceType.bytesToGiB; + + overrideBackupFrameworkConfigValue(); + + DataCenterVO dataCenter = mock(DataCenterVO.class); + when(dataCenter.getId()).thenReturn(dataCenterId); + when(dataCenterDao.listAllZones()).thenReturn(List.of(dataCenter)); + + BackupProvider backupProvider = mock(BackupProvider.class); + when(backupProvider.getName()).thenReturn("testbackupprovider"); + backupManager.setBackupProviders(List.of(backupProvider)); + backupManager.start(); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getAccountId()).thenReturn(accountId); + when(vmInstanceDao.listByZoneWithBackups(dataCenterId, null)).thenReturn(List.of(vm)); + Backup.Metric metric = new Backup.Metric(metricSize, null); + Map metricMap = new HashMap<>(); + metricMap.put(vm, metric); + when(backupProvider.getBackupMetrics(Mockito.anyLong(), Mockito.anyList())).thenReturn(metricMap); + + Backup.RestorePoint restorePoint1 = new Backup.RestorePoint(restorePoint1ExternalId, DateUtil.now(), "Root"); + Backup.RestorePoint restorePoint2 = new Backup.RestorePoint("12345", DateUtil.now(), "Root"); + List restorePoints = new ArrayList<>(List.of(restorePoint1, restorePoint2)); + when(backupProvider.listRestorePoints(vm)).thenReturn(restorePoints); + + BackupVO backupInDb1 = new BackupVO(); + backupInDb1.setSize(backup1Size); + backupInDb1.setExternalId(restorePoint1ExternalId); + + BackupVO backupInDb2 = new BackupVO(); + backupInDb2.setSize(backup2Size); + backupInDb2.setExternalId(null); + ReflectionTestUtils.setField(backupInDb2, "id", backup2Id); + when(backupDao.findById(backup2Id)).thenReturn(backupInDb2); + + when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(backupInDb1, backupInDb2)); + + BackupVO newBackupEntry = new BackupVO(); + newBackupEntry.setSize(newBackupSize); + when(backupProvider.createNewBackupEntryForRestorePoint(restorePoint2, vm, metric)).thenReturn(newBackupEntry); + + try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + try (MockedStatic ignored2 = Mockito.mockStatic(UsageEventUtils.class)) { + + BackupManagerImpl.BackupSyncTask backupSyncTask = backupManager.new BackupSyncTask(backupManager); + backupSyncTask.runInContext(); + + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup1Size); + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, metricSize); + Assert.assertEquals(backupInDb1.getSize(), metricSize); + + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); + + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup); + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup2Size); + } + } + } } diff --git a/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java new file mode 100644 index 000000000000..3ce855b504b4 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.agent.api.to.BucketTO; +import com.cloud.configuration.Resource; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ResourceLimitManagerImpl; +import com.cloud.storage.BucketVO; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; + +@RunWith(MockitoJUnitRunner.class) +public class BucketApiServiceImplTest { + @Spy + @InjectMocks + BucketApiServiceImpl bucketApiService; + + @Mock + AccountManager accountManager; + + @Mock + ObjectStoreDao objectStoreDao; + + @Mock + DataStoreManager dataStoreMgr; + + @Mock + private ResourceLimitManagerImpl resourceLimitManager; + + @Mock + private BucketDao bucketDao; + + @Test + public void testAllocBucket() throws ResourceAllocationException { + String bucketName = "bucket1"; + Long accountId = 1L; + Long poolId = 2L; + Long objectStoreId = 3L; + + CreateBucketCmd cmd = Mockito.mock(CreateBucketCmd.class); + Mockito.when(cmd.getBucketName()).thenReturn(bucketName); + Mockito.when(cmd.getEntityOwnerId()).thenReturn(accountId); + Mockito.when(cmd.getObjectStoragePoolId()).thenReturn(poolId); + Mockito.when(cmd.getQuota()).thenReturn(1); + + Account account = Mockito.mock(Account.class); + Mockito.when(accountManager.getActiveAccountById(accountId)).thenReturn(account); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.createUser(accountId)).thenReturn(true); + + bucketApiService.allocBucket(cmd); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).checkResourceLimit(account, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).checkResourceLimit(account, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + } + + @Test + public void testCreateBucket() { + Long objectStoreId = 1L; + Long poolId = 2L; + Long bucketId = 3L; + Long accountId = 4L; + String bucketName = "bucket1"; + + CreateBucketCmd cmd = Mockito.mock(CreateBucketCmd.class); + Mockito.when(cmd.getObjectStoragePoolId()).thenReturn(poolId); + Mockito.when(cmd.getEntityId()).thenReturn(bucketId); + Mockito.when(cmd.getQuota()).thenReturn(1); + + BucketVO bucket = new BucketVO(bucketName); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.createBucket(bucket, false)).thenReturn(bucket); + + bucketApiService.createBucket(cmd); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).incrementResourceCount(accountId, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).incrementResourceCount(accountId, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + Assert.assertEquals(bucket.getState(), Bucket.State.Created); + } + + @Test + public void testDeleteBucket() { + Long bucketId = 1L; + Long accountId = 2L; + Long objectStoreId = 3L; + String bucketName = "bucket1"; + + BucketVO bucket = new BucketVO(bucketName); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + ReflectionTestUtils.setField(bucket, "objectStoreId", objectStoreId); + ReflectionTestUtils.setField(bucket, "quota", 1); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(objectStoreId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.deleteBucket(Mockito.any(BucketTO.class))).thenReturn(true); + + bucketApiService.deleteBucket(bucketId, null); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + } + + @Test + public void testUpdateBucket() throws ResourceAllocationException { + Long bucketId = 1L; + Long objectStoreId = 2L; + Long accountId = 3L; + Integer bucketQuota = 2; + Integer cmdQuota = 1; + String bucketName = "bucket1"; + + UpdateBucketCmd cmd = Mockito.mock(UpdateBucketCmd.class); + Mockito.when(cmd.getId()).thenReturn(bucketId); + Mockito.when(cmd.getQuota()).thenReturn(cmdQuota); + + BucketVO bucket = new BucketVO(bucketName); + ReflectionTestUtils.setField(bucket, "quota", bucketQuota); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + ReflectionTestUtils.setField(bucket, "objectStoreId", objectStoreId); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + + Account account = Mockito.mock(Account.class); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(objectStoreId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + + bucketApiService.updateBucket(cmd, null); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.object_storage, (bucketQuota - cmdQuota) * Resource.ResourceType.bytesToGiB); + } +} diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index e07962d63d0a..7fdd8fab4147 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -432,6 +432,9 @@ "label.backupofferingname": "Backup offering", "label.backup.repository.add": "Add backup repository", "label.backup.repository.remove": "Remove backup repository", +"label.backuplimit": "Backup Limits", +"label.backup.storage": "Backup Storage", +"label.backupstoragelimit": "Backup Storage Limits (GiB)", "label.balance": "Balance", "label.bandwidth": "Bandwidth", "label.baremetal.dhcp.devices": "Bare metal DHCP devices", @@ -460,6 +463,7 @@ "label.brocade.vcs.address": "Vcs switch address", "label.browser": "Browser", "label.bucket": "Bucket", +"label.bucketlimit": "Bucket Limits", "label.by.account": "By Account", "label.by.domain": "By domain", "label.by.level": "By level", @@ -1394,6 +1398,10 @@ "label.max.primary.storage": "Max. primary (GiB)", "label.max.secondary.storage": "Max. secondary (GiB)", "label.max.migrations": "Max. migrations", +"label.maxbackup": "Max. Backups", +"label.maxbackupstorage": "Max. Backup Storage (GiB)", +"label.maxbackups.to.retain": "Max. Backups to retain", +"label.maxbucket": "Max. Buckets", "label.maxcpu": "Max. CPU cores", "label.maxcpunumber": "Max CPU cores", "label.maxdatavolumeslimit": "Max data volumes limit", @@ -1406,6 +1414,7 @@ "label.maxmembers": "Max members", "label.maxmemory": "Max. memory (MiB)", "label.maxnetwork": "Max. Networks", +"label.maxobjectstorage": "Max. Object Storage (GiB)", "label.maxprimarystorage": "Max. primary storage (GiB)", "label.maxproject": "Max. projects", "label.maxpublicip": "Max. public IPs", @@ -1592,6 +1601,7 @@ "label.oauth.verification": "OAuth verification", "label.ocfs2": "OCFS2", "label.object.storage" : "Object Storage", +"label.objectstoragelimit": "Object Storage Limits (GiB)", "label.object.presigned.url": "Presigned URL", "label.object.presigned.url.description" : "Presigned URL of the object in order to access it without authentication.", "label.object.url.description" : "URL of the object", @@ -2626,7 +2636,7 @@ "label.objectstorageid": "Object Storage Pool", "label.bucket.update": "Update Bucket", "label.bucket.delete": "Delete Bucket", -"label.quotagb": "Quota in GB", +"label.quotagib": "Quota in GiB", "label.encryption": "Encryption", "label.versioning": "Versioning", "label.objectlocking": "Object Lock", diff --git a/ui/src/components/view/ListResourceTable.vue b/ui/src/components/view/ListResourceTable.vue index a95927f00cf0..f8fdc5b5adad 100644 --- a/ui/src/components/view/ListResourceTable.vue +++ b/ui/src/components/view/ListResourceTable.vue @@ -51,7 +51,7 @@