Skip to content

Commit 0620b7f

Browse files
winterhazelwinterhazel
committed
Migrate public templates that have URLs on data migration across secondary storages
1 parent 2654890 commit 0620b7f

File tree

4 files changed

+107
-25
lines changed

4 files changed

+107
-25
lines changed

‎api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.cloud.storage;
1818

1919
import java.util.Date;
20+
import java.util.List;
2021

2122
import org.apache.cloudstack.api.InternalIdentity;
2223

@@ -25,6 +26,8 @@ public static enum Status {
2526
UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED
2627
}
2728

29+
List<Status> PENDING_DOWNLOAD_STATES = List.of(Status.NOT_DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS);
30+
2831
String getInstallPath();
2932

3033
long getTemplateId();

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,9 +208,7 @@ protected List<DataObject> getAllReadyTemplates(DataStore srcDataStore, Map<Data
208208
List<TemplateInfo> files = new LinkedList<>();
209209
for (TemplateDataStoreVO template : templates) {
210210
VMTemplateVO templateVO = templateDao.findById(template.getTemplateId());
211-
if (template.getState() == ObjectInDataStoreStateMachine.State.Ready && templateVO != null &&
212-
(!templateVO.isPublicTemplate() || (templateVO.isPublicTemplate() && templateVO.getUrl() == null)) &&
213-
templateVO.getHypervisorType() != Hypervisor.HypervisorType.Simulator && templateVO.getParentTemplateId() == null) {
211+
if (shouldMigrateTemplate(template, templateVO)) {
214212
files.add(templateFactory.getTemplate(template.getTemplateId(), srcDataStore));
215213
}
216214
}
@@ -231,6 +229,34 @@ protected List<DataObject> getAllReadyTemplates(DataStore srcDataStore, Map<Data
231229
return getAllReadyTemplates(srcDataStore, childTemplates, templates);
232230
}
233231

232+
/**
233+
* Returns whether a template should be migrated. A template should be migrated if:
234+
* <ol>
235+
* <li>its state is ready, and</li>
236+
* <li>its hypervisor type is not simulator, and</li>
237+
* <li>it is not a child template.</li>
238+
* </ol>
239+
*/
240+
protected boolean shouldMigrateTemplate(TemplateDataStoreVO template, VMTemplateVO templateVO) {
241+
if (template.getState() != State.Ready) {
242+
logger.debug("Template [{}] should not be migrated as it is not ready.", template);
243+
return false;
244+
}
245+
246+
if (templateVO.getHypervisorType() == Hypervisor.HypervisorType.Simulator) {
247+
logger.debug("Template [{}] should not be migrated as its hypervisor type is simulator.", template);
248+
return false;
249+
}
250+
251+
if (templateVO.getParentTemplateId() != null) {
252+
logger.debug("Template [{}] should not be migrated as it has a parent template.", template);
253+
return false;
254+
}
255+
256+
logger.debug("Template [{}] should be migrated.", template);
257+
return true;
258+
}
259+
234260
/** Returns parent snapshots and snapshots that do not have any children; snapshotChains comprises of the snapshot chain info
235261
* for each parent snapshot and the cumulative size of the chain - this is done to ensure that all the snapshots in a chain
236262
* are migrated to the same datastore

‎engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java‎

Lines changed: 75 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424

2525
import javax.inject.Inject;
2626

27+
import com.cloud.storage.VMTemplateStorageResourceAssoc;
28+
import com.cloud.utils.exception.CloudRuntimeException;
2729
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
2830
import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService;
2931
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@@ -118,26 +120,21 @@ public AsyncCallFuture<DataObjectResult> migrateData(DataObject srcDataObject, D
118120
}
119121
} else if (srcDataObject instanceof TemplateInfo && templateChain != null && templateChain.containsKey(srcDataObject)) {
120122
for (TemplateInfo templateInfo : templateChain.get(srcDataObject).first()) {
123+
if (templateIsOnDestination(templateInfo, destDatastore)) {
124+
res.setResult("Template already exists on destination.");
125+
res.setSuccess(true);
126+
logger.debug("Deleting template {} from source data store [{}].", srcDataObject.getTO().toString(),
127+
srcDataObject.getDataStore().getTO().toString());
128+
srcDataObject.getDataStore().delete(srcDataObject);
129+
future.complete(res);
130+
continue;
131+
}
121132
destDataObject = destDatastore.create(templateInfo);
122133
templateInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested);
123134
destDataObject.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested);
124135
migrateJob(future, templateInfo, destDataObject, destDatastore);
125136
}
126-
}
127-
else {
128-
// Check if template in destination store, if yes, do not proceed
129-
if (srcDataObject instanceof TemplateInfo) {
130-
logger.debug("Checking if template present at destination");
131-
TemplateDataStoreVO templateStoreVO = templateStoreDao.findByStoreTemplate(destDatastore.getId(), srcDataObject.getId());
132-
if (templateStoreVO != null) {
133-
String msg = "Template already exists in destination store";
134-
logger.debug(msg);
135-
res.setResult(msg);
136-
res.setSuccess(true);
137-
future.complete(res);
138-
return future;
139-
}
140-
}
137+
} else {
141138
destDataObject = destDatastore.create(srcDataObject);
142139
srcDataObject.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested);
143140
destDataObject.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested);
@@ -160,6 +157,69 @@ public AsyncCallFuture<DataObjectResult> migrateData(DataObject srcDataObject, D
160157
return future;
161158
}
162159

160+
/**
161+
* Returns a boolean indicating whether a template is ready on the provided data store. If the template is being downloaded,
162+
* waits until the download finishes.
163+
* @param srcDataObject the template.
164+
* @param destDatastore the data store.
165+
*/
166+
protected boolean templateIsOnDestination(DataObject srcDataObject, DataStore destDatastore) {
167+
if (!(srcDataObject instanceof TemplateInfo)) {
168+
return false;
169+
}
170+
171+
String templateAsString = srcDataObject.getTO().toString();
172+
String destDatastoreAsString = destDatastore.getTO().toString();
173+
TemplateDataStoreVO templateStoreVO;
174+
175+
long timer = getTemplateDownloadTimeoutInSeconds();
176+
int secondsToSleep = 10;
177+
int previousDownloadPercentage = -1;
178+
179+
while (true) {
180+
templateStoreVO = templateStoreDao.findByStoreTemplate(destDatastore.getId(), srcDataObject.getId());
181+
if (templateStoreVO == null) {
182+
logger.debug("{} is not present at destination [{}].", templateAsString, destDatastoreAsString);
183+
return false;
184+
}
185+
VMTemplateStorageResourceAssoc.Status downloadState = templateStoreVO.getDownloadState();
186+
if (downloadState == null || !VMTemplateStorageResourceAssoc.PENDING_DOWNLOAD_STATES.contains(downloadState)) {
187+
break;
188+
}
189+
if (previousDownloadPercentage == templateStoreVO.getDownloadPercent()) {
190+
timer -= secondsToSleep;
191+
} else {
192+
timer = getTemplateDownloadTimeoutInSeconds();
193+
}
194+
if (timer <= 0) {
195+
throw new CloudRuntimeException(String.format("Timeout while waiting for %s to be downloaded to image store [%s]. " +
196+
"The download percentage has not changed for %d seconds.", templateAsString, destDatastoreAsString, getTemplateDownloadTimeoutInSeconds()));
197+
}
198+
waitForTemplateDownload(secondsToSleep, templateAsString, destDatastoreAsString);
199+
}
200+
201+
if (templateStoreVO.getState() == ObjectInDataStoreStateMachine.State.Ready) {
202+
logger.debug("{} already exists on destination [{}].", templateAsString, destDatastoreAsString);
203+
return true;
204+
}
205+
return false;
206+
}
207+
208+
protected long getTemplateDownloadTimeoutInSeconds() {
209+
return 1800L;
210+
}
211+
212+
protected void waitForTemplateDownload(int secondsToSleep, String templateAsString, String destDatastoreAsString) {
213+
logger.debug("{} is being downloaded to destination [{}]; we will verify in {} seconds if the download has finished.",
214+
templateAsString, destDatastoreAsString, secondsToSleep);
215+
try {
216+
Thread.sleep(secondsToSleep * 1000L);
217+
} catch (InterruptedException e) {
218+
logger.warn("[ignored] interrupted while waiting for template {} download to finish before trying to migrate it to data store [{}].",
219+
templateAsString, destDatastoreAsString);
220+
}
221+
}
222+
163223
protected void migrateJob(AsyncCallFuture<DataObjectResult> future, DataObject srcDataObject, DataObject destDataObject, DataStore destDatastore) throws ExecutionException, InterruptedException {
164224
MigrateDataContext<DataObjectResult> context = new MigrateDataContext<DataObjectResult>(null, future, srcDataObject, destDataObject, destDatastore);
165225
AsyncCallbackDispatcher<SecondaryStorageServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);

‎server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,6 @@ protected TemplateJoinDaoImpl() {
151151
activeTmpltSearch.and("store_id", activeTmpltSearch.entity().getDataStoreId(), SearchCriteria.Op.EQ);
152152
activeTmpltSearch.and("type", activeTmpltSearch.entity().getTemplateType(), SearchCriteria.Op.EQ);
153153
activeTmpltSearch.and("templateState", activeTmpltSearch.entity().getTemplateState(), SearchCriteria.Op.EQ);
154-
activeTmpltSearch.and().op("public", activeTmpltSearch.entity().isPublicTemplate(), SearchCriteria.Op.EQ);
155-
activeTmpltSearch.or().op("publicNoUrl", activeTmpltSearch.entity().isPublicTemplate(), SearchCriteria.Op.EQ);
156-
activeTmpltSearch.and("url", activeTmpltSearch.entity().getUrl(), SearchCriteria.Op.NULL);
157-
activeTmpltSearch.cp();
158-
activeTmpltSearch.cp();
159154
activeTmpltSearch.done();
160155

161156
publicTmpltSearch = createSearchBuilder();
@@ -687,8 +682,6 @@ public List<TemplateJoinVO> listActiveTemplates(long storeId) {
687682
sc.setParameters("store_id", storeId);
688683
sc.setParameters("type", TemplateType.USER);
689684
sc.setParameters("templateState", VirtualMachineTemplate.State.Active);
690-
sc.setParameters("public", Boolean.FALSE);
691-
sc.setParameters("publicNoUrl",Boolean.TRUE);
692685
return searchIncludingRemoved(sc, null, null, false);
693686
}
694687

0 commit comments

Comments
 (0)