Nas provider support for backup storage stats

Author: abh1sar

api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java

@@ -129,4 +129,11 @@ public interface BackupProvider {
      * @return a pair of Used size and Total size for the backup storage
      */
     Pair<Long, Long> getBackupStorageStats(Long zoneId);
+
+    /**
+     * Gets the backup storage usage (Used, Total) from the plugin and stores it in db
+     * @param zoneId the zone for which to return metrics
+     */
+    void syncBackupStorageStats(Long zoneId);
+
 }

api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java

@@ -28,7 +28,9 @@ public interface BackupRepository extends InternalIdentity, Identity {
     String getType();
     String getAddress();
     String getMountOptions();
+    void setUsedBytes(Long usedBytes);
     Long getCapacityBytes();
     Long getUsedBytes();
+    void setCapacityBytes(Long capacityBytes);
     Date getCreated();
 }

core/src/main/java/org/apache/cloudstack/backup/BackupStorageStatsAnswer.java

@@ -0,0 +1,50 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.backup;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
+
+public class BackupStorageStatsAnswer extends Answer {
+    private Long totalSize;
+    private Long usedSize;
+
+    public BackupStorageStatsAnswer(final Command command, final boolean success, final String details) {
+        super(command, success, details);
+        this.totalSize = 0L;
+        this.usedSize = 0L;
+    }
+
+    public Long getTotalSize() {
+        return totalSize;
+    }
+
+    public void setTotalSize(Long totalSize) {
+        this.totalSize = totalSize;
+    }
+
+    public Long getUsedSize() {
+        return usedSize;
+    }
+
+    public void setUsedSize(Long usedSize) {
+        this.usedSize = usedSize;
+    }
+}

core/src/main/java/org/apache/cloudstack/backup/GetBackupStorageStatsCommand.java

@@ -0,0 +1,66 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.backup;
+
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.LogLevel;
+
+public class GetBackupStorageStatsCommand extends Command {
+    private String backupRepoType;
+    private String backupRepoAddress;
+    @LogLevel(LogLevel.Log4jLevel.Off)
+    private String mountOptions;
+
+    public GetBackupStorageStatsCommand(String backupRepoType, String backupRepoAddress, String mountOptions) {
+        super();
+        this.backupRepoType = backupRepoType;
+        this.backupRepoAddress = backupRepoAddress;
+        this.mountOptions = mountOptions;
+    }
+
+    public String getBackupRepoType() {
+        return backupRepoType;
+    }
+
+    public void setBackupRepoType(String backupRepoType) {
+        this.backupRepoType = backupRepoType;
+    }
+
+    public String getBackupRepoAddress() {
+        return backupRepoAddress;
+    }
+
+    public void setBackupRepoAddress(String backupRepoAddress) {
+        this.backupRepoAddress = backupRepoAddress;
+    }
+
+    public String getMountOptions() {
+        return mountOptions == null ? "" : mountOptions;
+    }
+
+    public void setMountOptions(String mountOptions) {
+        this.mountOptions = mountOptions;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return true;
+    }
+}

engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java

@@ -144,11 +144,21 @@ public Long getUsedBytes() {
         return usedBytes;
     }
 
+    @Override
+    public void setUsedBytes(Long usedBytes) {
+        this.usedBytes = usedBytes;
+    }
+
     @Override
     public Long getCapacityBytes() {
         return capacityBytes;
     }
 
+    @Override
+    public void setCapacityBytes(Long capacityBytes) {
+        this.capacityBytes = capacityBytes;
+    }
+
     public Date getCreated() {
         return created;
     }

engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java

@@ -28,4 +28,6 @@ public interface BackupRepositoryDao extends GenericDao<BackupRepositoryVO, Long
     List<BackupRepository> listByZoneAndProvider(Long zoneId, String provider);
 
     BackupRepository findByBackupOfferingId(Long backupOfferingId);
+
+    boolean updateCapacity(BackupRepository backupRepository, Long capacityBytes, Long usedBytes);
 }

engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java

@@ -64,4 +64,12 @@ public BackupRepository findByBackupOfferingId(Long backupOfferingId) {
         }
         return findByUuid(offering.getExternalId());
     }
+
+    @Override
+    public boolean updateCapacity(BackupRepository backupRepository, Long capacityBytes, Long usedBytes) {
+            BackupRepositoryVO repository = findById(backupRepository.getId());
+            repository.setCapacityBytes(capacityBytes);
+            repository.setUsedBytes(usedBytes);
+            return update(repository.getId(), repository);
+    }
 }

plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java

@@ -183,6 +183,10 @@ public Pair<Long, Long> getBackupStorageStats(Long zoneId) {
         return new Pair<>(8L * 1024 * 1024 * 1024, 10L * 1024 * 1024 * 1024);
     }
 
+    @Override
+    public void syncBackupStorageStats(Long zoneId) {
+    }
+
     @Override
     public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) {
         return true;

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java

@@ -57,6 +57,7 @@
 import java.util.HashMap;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Random;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
@@ -69,6 +70,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co
     @Inject
     private BackupRepositoryDao backupRepositoryDao;
 
+    @Inject
+    private BackupRepositoryService backupRepositoryService;
+
     @Inject
     private HostDao hostDao;
 
@@ -90,6 +94,19 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co
     @Inject
     BackupManager backupManager;
 
+    private Host getUpHostInZone(Long zoneId) {
+        List<HostVO> hosts = hostDao.listByDataCenterIdAndHypervisorType(zoneId, Hypervisor.HypervisorType.KVM);
+        if (hosts.size() == 0) {
+            return null;
+        }
+
+        Random random = new Random();
+        int randomIndex = random.nextInt(hosts.size());
+        HostVO hostInZone = hosts.get(randomIndex);
+        LOG.debug("Found Host {} in zone {}", zoneId);
+        return hostInZone;
+    }
+
     protected Host getLastVMHypervisorHost(VirtualMachine vm) {
         Long hostId = vm.getLastHostId();
         if (hostId == null) {
@@ -110,13 +127,7 @@ protected Host getLastVMHypervisorHost(VirtualMachine vm) {
             }
         }
         // Try to find any Host in the zone
-        for (final HostVO hostInZone : hostDao.listByDataCenterIdAndHypervisorType(host.getDataCenterId(), Hypervisor.HypervisorType.KVM)) {
-            if (hostInZone.getStatus() == Status.Up) {
-                LOG.debug("Found Host {} in zone {}", hostInZone, host.getDataCenterId());
-                return hostInZone;
-            }
-        }
-        return null;
+        return getUpHostInZone(host.getDataCenterId());
     }
 
     protected Host getVMHypervisorHost(VirtualMachine vm) {
@@ -434,7 +445,36 @@ public boolean supportsInstanceFromBackup() {
 
     @Override
     public Pair<Long, Long> getBackupStorageStats(Long zoneId) {
-        return new Pair<>(0L, 0L);
+        final List<BackupRepository> repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName());
+        Long totalSize = 0L;
+        Long usedSize = 0L;
+        for (final BackupRepository repository : repositories) {
+            if (repository.getCapacityBytes() != null) {
+                totalSize += repository.getCapacityBytes();
+            }
+            if (repository.getUsedBytes() != null) {
+                usedSize += repository.getUsedBytes();
+            }
+        }
+        return new Pair<>(usedSize, totalSize);
+    }
+
+    @Override
+    public void syncBackupStorageStats(Long zoneId) {
+        final List<BackupRepository> repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName());
+        final Host host = getUpHostInZone(zoneId);
+        for (final BackupRepository repository : repositories) {
+            GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand(repository.getType(), repository.getAddress(), repository.getMountOptions());
+            BackupStorageStatsAnswer answer;
+            try {
+                answer = (BackupStorageStatsAnswer) agentManager.send(host.getId(), command);
+                backupRepositoryDao.updateCapacity(repository, answer.getTotalSize(), answer.getUsedSize());
+            } catch (AgentUnavailableException e) {
+                logger.warn("Unable to contact backend control plane to get backup stats for repository: {}", repository.getName());
+            } catch (OperationTimedoutException e) {
+                logger.warn("Operation to get backup stats timed out for the repository: " + repository.getName());
+            }
+        }
     }
 
     @Override

plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java

@@ -645,6 +645,10 @@ public Pair<Long, Long> getBackupStorageStats(Long zoneId) {
         return new Pair<>(0L, 0L);
     }
 
+    @Override
+    public void syncBackupStorageStats(Long zoneId) {
+    }
+
     @Override
     public boolean willDeleteBackupsOnOfferingRemoval() { return false; }