Skip to content

Commit aeb0710

Browse files
DaanHooglandDaanHoogland
committed
resolve merge problems in the backup framework
1 parent 24b7c66 commit aeb0710

File tree

5 files changed

+131
-2
lines changed

5 files changed

+131
-2
lines changed

‎api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,4 +116,11 @@ public interface BackupProvider {
116116
* @param metric
117117
*/
118118
Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric);
119+
120+
/**
121+
* This method should reconcile and create backup entries for any backups created out-of-band
122+
* @param vm
123+
* @param metric
124+
*/
125+
void syncBackups(VirtualMachine vm, Backup.Metric metric);
119126
}

‎plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,4 +146,8 @@ public Pair<Boolean, Backup> takeBackup(VirtualMachine vm) {
146146
public boolean deleteBackup(Backup backup, boolean forced) {
147147
return true;
148148
}
149+
150+
@Override
151+
public void syncBackups(VirtualMachine vm, Backup.Metric metric) {
152+
}
149153
}

‎plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.cloud.utils.exception.CloudRuntimeException;
3737
import com.cloud.vm.VirtualMachine;
3838
import com.cloud.vm.dao.VMInstanceDao;
39+
3940
import org.apache.cloudstack.backup.dao.BackupDao;
4041
import org.apache.cloudstack.backup.dao.BackupOfferingDao;
4142
import org.apache.cloudstack.backup.dao.BackupRepositoryDao;
@@ -414,6 +415,11 @@ public boolean willDeleteBackupsOnOfferingRemoval() {
414415
return false;
415416
}
416417

418+
@Override
419+
public void syncBackups(VirtualMachine vm, Backup.Metric metric) {
420+
// TODO: check and sum/return backups metrics on per VM basis
421+
}
422+
417423
@Override
418424
public List<BackupOffering> listBackupOfferings(Long zoneId) {
419425
final List<BackupRepository> repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName());

‎plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java‎

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.cloud.host.Status;
2222
import com.cloud.host.dao.HostDao;
2323
import com.cloud.hypervisor.Hypervisor;
24+
import com.cloud.utils.script.Script;
2425
import com.cloud.storage.StoragePoolHostVO;
2526
import com.cloud.storage.Volume;
2627
import com.cloud.storage.VolumeVO;
@@ -29,11 +30,16 @@
2930
import com.cloud.utils.Pair;
3031
import com.cloud.utils.Ternary;
3132
import com.cloud.utils.component.AdapterBase;
33+
import com.cloud.utils.db.Transaction;
34+
import com.cloud.utils.db.TransactionCallbackNoReturn;
35+
import com.cloud.utils.db.TransactionStatus;
3236
import com.cloud.utils.exception.CloudRuntimeException;
3337
import com.cloud.utils.ssh.SshHelper;
3438
import com.cloud.vm.VMInstanceVO;
3539
import com.cloud.vm.VirtualMachine;
3640
import com.cloud.vm.dao.VMInstanceDao;
41+
42+
import org.apache.cloudstack.api.InternalIdentity;
3743
import org.apache.cloudstack.backup.dao.BackupDao;
3844
import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl;
3945
import org.apache.cloudstack.backup.networker.NetworkerClient;
@@ -44,7 +50,9 @@
4450
import org.apache.logging.log4j.LogManager;
4551
import org.apache.xml.utils.URI;
4652
import org.apache.cloudstack.backup.networker.api.NetworkerBackup;
53+
4754
import javax.inject.Inject;
55+
4856
import java.net.URISyntaxException;
4957
import java.security.KeyManagementException;
5058
import java.security.NoSuchAlgorithmException;
@@ -60,7 +68,6 @@
6068
import java.util.regex.Matcher;
6169
import java.util.regex.Pattern;
6270
import java.util.stream.Collectors;
63-
import com.cloud.utils.script.Script;
6471

6572
public class NetworkerBackupProvider extends AdapterBase implements BackupProvider, Configurable {
6673

@@ -560,6 +567,86 @@ public Map<VirtualMachine, Backup.Metric> getBackupMetrics(Long zoneId, List<Vir
560567
return metrics;
561568
}
562569

570+
@Override
571+
public void syncBackups(VirtualMachine vm, Backup.Metric metric) {
572+
final Long zoneId = vm.getDataCenterId();
573+
Transaction.execute(new TransactionCallbackNoReturn() {
574+
@Override
575+
public void doInTransactionWithoutResult(TransactionStatus status) {
576+
final List<Backup> backupsInDb = backupDao.listByVmId(null, vm.getId());
577+
final ArrayList<String> backupsInNetworker = getClient(zoneId).getBackupsForVm(vm);
578+
final List<Long> removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList());
579+
for (final String networkerBackupId : backupsInNetworker ) {
580+
Long vmBackupSize=0L;
581+
boolean backupExists = false;
582+
for (final Backup backupInDb : backupsInDb) {
583+
LOG.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm));
584+
if ( networkerBackupId.equals(backupInDb.getExternalId()) ) {
585+
LOG.debug(String.format("Found Backup %s in both Database and Networker", backupInDb));
586+
backupExists = true;
587+
removeList.remove(backupInDb.getId());
588+
if (metric != null) {
589+
LOG.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].",
590+
backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(),
591+
metric.getBackupSize(), metric.getDataSize()));
592+
((BackupVO) backupInDb).setSize(metric.getBackupSize());
593+
((BackupVO) backupInDb).setProtectedSize(metric.getDataSize());
594+
backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb));
595+
}
596+
break;
597+
}
598+
}
599+
if (backupExists) {
600+
continue;
601+
}
602+
// Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts
603+
// with the proper parameters. So we will register any backups taken on the Networker side from
604+
// outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will
605+
// ensure that SLA like backups will be found and registered.
606+
NetworkerBackup strayNetworkerBackup = getClient(vm.getDataCenterId()).getNetworkerBackupInfo(networkerBackupId);
607+
// Since running backups are already present in Networker Server but not completed
608+
// make sure the backup is not in progress at this time.
609+
if ( strayNetworkerBackup.getCompletionTime() != null) {
610+
BackupVO strayBackup = new BackupVO();
611+
strayBackup.setVmId(vm.getId());
612+
strayBackup.setExternalId(strayNetworkerBackup.getId());
613+
strayBackup.setType(strayNetworkerBackup.getType());
614+
SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
615+
try {
616+
strayBackup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime()));
617+
} catch (ParseException e) {
618+
String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime());
619+
LOG.error(msg, e);
620+
throw new CloudRuntimeException(msg, e);
621+
}
622+
strayBackup.setStatus(Backup.Status.BackedUp);
623+
for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) {
624+
vmBackupSize += (thisVMVol.getSize() / 1024L /1024L);
625+
}
626+
strayBackup.setSize(vmBackupSize);
627+
strayBackup.setProtectedSize(strayNetworkerBackup.getSize().getValue() / 1024L );
628+
strayBackup.setBackupOfferingId(vm.getBackupOfferingId());
629+
strayBackup.setAccountId(vm.getAccountId());
630+
strayBackup.setDomainId(vm.getDomainId());
631+
strayBackup.setZoneId(vm.getDataCenterId());
632+
LOG.debug(String.format("Creating a new entry in backups: [id: %s, uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, "
633+
+ "domain_id: %s, zone_id: %s].", strayBackup.getId(), strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(),
634+
strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(),
635+
strayBackup.getDomainId(), strayBackup.getZoneId()));
636+
backupDao.persist(strayBackup);
637+
LOG.warn("Added backup found in provider [" + strayBackup + "]");
638+
} else {
639+
LOG.debug ("Backup is in progress, skipping addition for this run");
640+
}
641+
}
642+
for (final Long backupIdToRemove : removeList) {
643+
LOG.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove));
644+
backupDao.remove(backupIdToRemove);
645+
}
646+
}
647+
});
648+
}
649+
563650
@Override
564651
public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) {
565652
// Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts

‎plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929

3030
import javax.inject.Inject;
3131

32+
import org.apache.cloudstack.api.ApiCommandResourceType;
33+
import org.apache.cloudstack.api.InternalIdentity;
3234
import org.apache.cloudstack.backup.Backup.Metric;
3335
import org.apache.cloudstack.backup.dao.BackupDao;
3436
import org.apache.cloudstack.backup.veeam.VeeamClient;
@@ -40,15 +42,21 @@
4042

4143
import com.cloud.agent.AgentManager;
4244
import com.cloud.agent.api.Answer;
45+
import com.cloud.event.ActionEventUtils;
46+
import com.cloud.event.EventTypes;
47+
import com.cloud.event.EventVO;
4348
import com.cloud.hypervisor.Hypervisor;
4449
import com.cloud.dc.VmwareDatacenter;
45-
import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap;
4650
import com.cloud.dc.dao.VmwareDatacenterDao;
51+
import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap;
4752
import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao;
4853
import com.cloud.storage.dao.VolumeDao;
4954
import com.cloud.user.User;
5055
import com.cloud.utils.Pair;
5156
import com.cloud.utils.component.AdapterBase;
57+
import com.cloud.utils.db.Transaction;
58+
import com.cloud.utils.db.TransactionCallbackNoReturn;
59+
import com.cloud.utils.db.TransactionStatus;
5260
import com.cloud.utils.exception.CloudRuntimeException;
5361
import com.cloud.vm.VMInstanceVO;
5462
import com.cloud.vm.VirtualMachine;
@@ -344,6 +352,23 @@ public List<Backup.RestorePoint> listRestorePoints(VirtualMachine vm) {
344352
return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName());
345353
}
346354

355+
private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(List<Backup> backupsInDb, Backup.RestorePoint restorePoint, Backup.Metric metric) {
356+
for (final Backup backup : backupsInDb) {
357+
if (restorePoint.getId().equals(backup.getExternalId())) {
358+
if (metric != null) {
359+
logger.debug("Update backup with [id: {}, uuid: {}, name: {}, external id: {}] from [size: {}, protected size: {}] to [size: {}, protected size: {}].",
360+
backup.getId(), backup.getUuid(), backup.getName(), backup.getExternalId(), backup.getSize(), backup.getProtectedSize(), metric.getBackupSize(), metric.getDataSize());
361+
362+
((BackupVO) backup).setSize(metric.getBackupSize());
363+
((BackupVO) backup).setProtectedSize(metric.getDataSize());
364+
backupDao.update(backup.getId(), ((BackupVO) backup));
365+
}
366+
return backup;
367+
}
368+
}
369+
return null;
370+
}
371+
347372
@Override
348373
public void syncBackups(VirtualMachine vm, Backup.Metric metric) {
349374
List<Backup.RestorePoint> restorePoints = listRestorePoints(vm);

0 commit comments

Comments
 (0)