Skip to content

Commit 69d2c10

Browse files
sureshanapartisureshanaparti
committed
Retry moveDatastoreFile when there is any file access issue
1 parent c9073c4 commit 69d2c10

File tree

2 files changed

+36
-10
lines changed

2 files changed

+36
-10
lines changed

‎plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.util.Arrays;
2121
import java.util.List;
2222

23+
import com.vmware.vim25.ManagedObjectReference;
2324
import org.apache.cloudstack.framework.config.ConfigKey;
2425
import org.apache.cloudstack.framework.config.Configurable;
2526
import org.apache.logging.log4j.Logger;
@@ -193,21 +194,21 @@ public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmNam
193194
if (ds.fileExists(vmdkFullCloneModeLegacyPair[i])) {
194195
LOGGER.info("sync " + vmdkFullCloneModeLegacyPair[i] + "->" + vmdkFullCloneModePair[i]);
195196

196-
ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true);
197+
moveDatastoreFile(ds, vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true);
197198
}
198199
}
199200

200201
for (int i=1; i<vmdkLinkedCloneModeLegacyPair.length; i++) {
201202
if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[i])) {
202203
LOGGER.info("sync " + vmdkLinkedCloneModeLegacyPair[i] + "->" + vmdkLinkedCloneModePair[i]);
203204

204-
ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true);
205+
moveDatastoreFile(ds, vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true);
205206
}
206207
}
207208

208209
if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[0])) {
209210
LOGGER.info("sync " + vmdkLinkedCloneModeLegacyPair[0] + "->" + vmdkLinkedCloneModePair[0]);
210-
ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[0], dcMo.getMor(), true);
211+
moveDatastoreFile(ds, vmdkLinkedCloneModeLegacyPair[0], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[0], dcMo.getMor(), true);
211212
}
212213

213214
// Note: we will always return a path
@@ -242,14 +243,14 @@ public static void syncVolumeToRootFolder(DatacenterMO dcMo, DatastoreMO ds, Str
242243
String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, String.format("%s-%s",vmdkName, linkedCloneExtension));
243244

244245
LOGGER.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath);
245-
ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true);
246+
moveDatastoreFile(ds, companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true);
246247
}
247248
}
248249

249250
// move the identity VMDK file the last
250251
String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + ".vmdk");
251252
LOGGER.info("Fixup folder-synchronization. move " + fileDsFullPath + " -> " + targetPath);
252-
ds.moveDatastoreFile(fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true);
253+
moveDatastoreFile(ds, fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true);
253254

254255
try {
255256
if (folderName != null) {
@@ -287,7 +288,7 @@ public static void moveVolumeToRootFolder(DatacenterMO dcMo, List<String> detach
287288
DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, file.getFileName());
288289
if (!targetFile.getPath().equalsIgnoreCase(file.getPath())) {
289290
LOGGER.info("Move " + file.getPath() + " -> " + targetFile.getPath());
290-
dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true);
291+
moveDatastoreFile(dsMo, file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true);
291292

292293
List<String> vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*")));
293294
// add flat file format to the above list
@@ -297,7 +298,7 @@ public static void moveVolumeToRootFolder(DatacenterMO dcMo, List<String> detach
297298
String pairTargetFilePath = targetFile.getCompanionPath(String.format("%s-%s", file.getFileBaseName(), linkedCloneExtension));
298299
if (dsMo.fileExists(pairSrcFilePath)) {
299300
LOGGER.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath);
300-
dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true);
301+
moveDatastoreFile(dsMo, pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true);
301302
}
302303
}
303304
}
@@ -429,6 +430,31 @@ public static String getDatastoreVolumePath(DatastoreMO dsMo, String vmName, Str
429430
return dsMo.searchFileInSubFolders(volumePath + ".vmdk", false, null);
430431
}
431432

433+
public static boolean moveDatastoreFile(final DatastoreMO dsMo, String srcFilePath, ManagedObjectReference morSrcDc, ManagedObjectReference morDestDs,
434+
String destFilePath, ManagedObjectReference morDestDc, boolean forceOverwrite) throws Exception {
435+
final int retry = 20;
436+
int retryAttempt = 0;
437+
while (++retryAttempt <= retry) {
438+
try {
439+
LOGGER.debug(String.format("Move datastore file %s, attempt #%d", srcFilePath, retryAttempt));
440+
return dsMo.moveDatastoreFile(srcFilePath, morSrcDc, morDestDs, destFilePath, morDestDc, forceOverwrite);
441+
} catch (Exception e) {
442+
LOGGER.info(String.format("Got exception while moving datastore file %s ", srcFilePath), e);
443+
if (e.getMessage() != null && e.getMessage().contains("Unable to access file")) {
444+
LOGGER.debug(String.format("Failed to move datastore file %s. Retrying", srcFilePath));
445+
try {
446+
Thread.sleep(1000);
447+
} catch (InterruptedException ie) {
448+
LOGGER.debug(String.format("Waiting to move datastore file %s been interrupted: ", srcFilePath));
449+
}
450+
} else {
451+
throw e;
452+
}
453+
}
454+
}
455+
return false;
456+
}
457+
432458
@Override
433459
public String getConfigComponentName() {
434460
return VmwareStorageLayoutHelper.class.getSimpleName();

‎plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -682,9 +682,9 @@ public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) {
682682
String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairManagedDatastorePath(dsMo, null,
683683
managedStoragePoolRootVolumeName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, false);
684684

685-
dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true);
685+
VmwareStorageLayoutHelper.moveDatastoreFile(dsMo, vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true);
686686
for (int i=1; i<vmwareLayoutFilePair.length; i++) {
687-
dsMo.moveDatastoreFile(vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
687+
VmwareStorageLayoutHelper.moveDatastoreFile(dsMo, vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
688688
}
689689

690690
String folderToDelete = dsMo.getDatastorePath(managedStoragePoolRootVolumeName, true);
@@ -945,7 +945,7 @@ private String createVMAndFolderWithVMName(VmwareContext context, VmwareHypervis
945945
String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag);
946946

947947
for (int i = 0; i < vmwareLayoutFilePair.length; i++) {
948-
dsMo.moveDatastoreFile(vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
948+
VmwareStorageLayoutHelper.moveDatastoreFile(dsMo, vmwareLayoutFilePair[i], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[i], dcMo.getMor(), true);
949949
}
950950

951951
logger.info("detach disks from volume-wrapper VM and destroy {}", vmdkName);

0 commit comments

Comments
 (0)