Skip to content

Commit 9a6e0e3

Browse files
DaanHooglandDaanHoogland
authored
Merge branch 'main' into nas-br-partb
2 parents f6f0ba6 + 019f2c6 commit 9a6e0e3

File tree

43 files changed

+956
-191
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+956
-191
lines changed

‎api/src/main/java/com/cloud/user/AccountService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ User createUser(String userName, String password, String firstName, String lastN
116116

117117
void checkAccess(Account account, AccessType accessType, boolean sameOwner, String apiName, ControlledEntity... entities) throws PermissionDeniedException;
118118

119+
void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource);
120+
119121
Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly);
120122

121123
/**

‎debian/control‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ Description: CloudStack server library
2424

2525
Package: cloudstack-agent
2626
Architecture: all
27-
Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, cifs-utils, lsb-release, ufw, apparmor, cpu-checker
27+
Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, cifs-utils, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd
2828
Recommends: init-system-helpers
2929
Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
3030
Description: CloudStack agent

‎engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -126,14 +126,7 @@ public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) {
126126

127127
logger.debug(String.format("Starting to remove all volume_stats rows older than [%s].", limitDate));
128128

129-
long totalRemoved = 0;
130-
long removed;
131-
132-
do {
133-
removed = expunge(sc, limitPerQuery);
134-
totalRemoved += removed;
135-
logger.trace(String.format("Removed [%s] volume_stats rows on the last update and a sum of [%s] volume_stats rows older than [%s] until now.", removed, totalRemoved, limitDate));
136-
} while (limitPerQuery > 0 && removed >= limitPerQuery);
129+
long totalRemoved = batchExpunge(sc, limitPerQuery);
137130

138131
logger.info(String.format("Removed a total of [%s] volume_stats rows older than [%s].", totalRemoved, limitDate));
139132
}

‎engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -123,14 +123,7 @@ public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) {
123123

124124
logger.debug(String.format("Starting to remove all vm_stats rows older than [%s].", limitDate));
125125

126-
long totalRemoved = 0;
127-
long removed;
128-
129-
do {
130-
removed = expunge(sc, limitPerQuery);
131-
totalRemoved += removed;
132-
logger.trace(String.format("Removed [%s] vm_stats rows on the last update and a sum of [%s] vm_stats rows older than [%s] until now.", removed, totalRemoved, limitDate));
133-
} while (limitPerQuery > 0 && removed >= limitPerQuery);
126+
long totalRemoved = batchExpunge(sc, limitPerQuery);
134127

135128
logger.info(String.format("Removed a total of [%s] vm_stats rows older than [%s].", totalRemoved, limitDate));
136129
}

‎engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java‎

Lines changed: 41 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1958,25 +1958,26 @@ protected MigrationOptions createFullCloneMigrationOptions(VolumeInfo srcVolumeI
19581958
* - Full clones (no backing file): Take snapshot of the VM prior disk creation
19591959
* Return this information
19601960
*/
1961-
protected void setVolumeMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo,
1962-
VirtualMachineTO vmTO, Host srcHost, StoragePoolVO destStoragePool) {
1963-
if (!destStoragePool.isManaged()) {
1964-
String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo);
1965-
1966-
String srcPoolUuid = srcVolumeInfo.getDataStore().getUuid();
1967-
StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId());
1968-
Storage.StoragePoolType srcPoolType = srcPool.getPoolType();
1969-
1970-
MigrationOptions migrationOptions;
1971-
if (StringUtils.isNotBlank(srcVolumeBackingFile)) {
1972-
migrationOptions = createLinkedCloneMigrationOptions(srcVolumeInfo, destVolumeInfo,
1973-
srcVolumeBackingFile, srcPoolUuid, srcPoolType);
1974-
} else {
1975-
migrationOptions = createFullCloneMigrationOptions(srcVolumeInfo, vmTO, srcHost, srcPoolUuid, srcPoolType);
1976-
}
1977-
migrationOptions.setTimeout(StorageManager.KvmStorageOnlineMigrationWait.value());
1978-
destVolumeInfo.setMigrationOptions(migrationOptions);
1961+
protected void setVolumeMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, VirtualMachineTO vmTO, Host srcHost, StoragePoolVO destStoragePool,
1962+
MigrationOptions.Type migrationType) {
1963+
if (destStoragePool.isManaged()) {
1964+
return;
1965+
}
1966+
1967+
String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo);
1968+
1969+
String srcPoolUuid = srcVolumeInfo.getDataStore().getUuid();
1970+
StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId());
1971+
Storage.StoragePoolType srcPoolType = srcPool.getPoolType();
1972+
1973+
MigrationOptions migrationOptions;
1974+
if (MigrationOptions.Type.LinkedClone.equals(migrationType)) {
1975+
migrationOptions = createLinkedCloneMigrationOptions(srcVolumeInfo, destVolumeInfo, srcVolumeBackingFile, srcPoolUuid, srcPoolType);
1976+
} else {
1977+
migrationOptions = createFullCloneMigrationOptions(srcVolumeInfo, vmTO, srcHost, srcPoolUuid, srcPoolType);
19791978
}
1979+
migrationOptions.setTimeout(StorageManager.KvmStorageOnlineMigrationWait.value());
1980+
destVolumeInfo.setMigrationOptions(migrationOptions);
19801981
}
19811982

19821983
/**
@@ -2007,6 +2008,7 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
20072008
Map<VolumeInfo, VolumeInfo> srcVolumeInfoToDestVolumeInfo = new HashMap<>();
20082009

20092010
boolean managedStorageDestination = false;
2011+
boolean migrateNonSharedInc = false;
20102012
for (Map.Entry<VolumeInfo, DataStore> entry : volumeDataStoreMap.entrySet()) {
20112013
VolumeInfo srcVolumeInfo = entry.getKey();
20122014
DataStore destDataStore = entry.getValue();
@@ -2024,15 +2026,8 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
20242026
continue;
20252027
}
20262028

2027-
VMTemplateVO vmTemplate = _vmTemplateDao.findById(vmInstance.getTemplateId());
2028-
if (srcVolumeInfo.getTemplateId() != null &&
2029-
Objects.nonNull(vmTemplate) &&
2030-
!Arrays.asList(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME, VM_IMPORT_DEFAULT_TEMPLATE_NAME).contains(vmTemplate.getName())) {
2031-
logger.debug(String.format("Copying template [%s] of volume [%s] from source storage pool [%s] to target storage pool [%s].", srcVolumeInfo.getTemplateId(), srcVolumeInfo.getId(), sourceStoragePool.getId(), destStoragePool.getId()));
2032-
copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, sourceStoragePool, destDataStore, destStoragePool, destHost);
2033-
} else {
2034-
logger.debug(String.format("Skipping copy template from source storage pool [%s] to target storage pool [%s] before migration due to volume [%s] does not have a template.", sourceStoragePool.getId(), destStoragePool.getId(), srcVolumeInfo.getId()));
2035-
}
2029+
MigrationOptions.Type migrationType = decideMigrationTypeAndCopyTemplateIfNeeded(destHost, vmInstance, srcVolumeInfo, sourceStoragePool, destStoragePool, destDataStore);
2030+
migrateNonSharedInc = migrateNonSharedInc || MigrationOptions.Type.LinkedClone.equals(migrationType);
20362031

20372032
VolumeVO destVolume = duplicateVolumeOnAnotherStorage(srcVolume, destStoragePool);
20382033
VolumeInfo destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore);
@@ -2044,7 +2039,7 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
20442039
// move the volume from Ready to Migrating
20452040
destVolumeInfo.processEvent(Event.MigrationRequested);
20462041

2047-
setVolumeMigrationOptions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost, destStoragePool);
2042+
setVolumeMigrationOptions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost, destStoragePool, migrationType);
20482043

20492044
// create a volume on the destination storage
20502045
destDataStore.getDriver().createAsync(destDataStore, destVolumeInfo, null);
@@ -2059,7 +2054,7 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
20592054

20602055
_volumeDao.update(destVolume.getId(), destVolume);
20612056

2062-
postVolumeCreationActions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost);
2057+
postVolumeCreationActions(srcVolumeInfo, destVolumeInfo);
20632058

20642059
destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore);
20652060

@@ -2110,8 +2105,6 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
21102105
VMInstanceVO vm = _vmDao.findById(vmTO.getId());
21112106
boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
21122107

2113-
boolean migrateNonSharedInc = isSourceAndDestinationPoolTypeOfNfs(volumeDataStoreMap);
2114-
21152108
MigrateCommand migrateCommand = new MigrateCommand(vmTO.getName(), destHost.getPrivateIpAddress(), isWindows, vmTO, true);
21162109
migrateCommand.setWait(StorageManager.KvmStorageOnlineMigrationWait.value());
21172110
migrateCommand.setMigrateStorage(migrateStorage);
@@ -2161,6 +2154,22 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
21612154
}
21622155
}
21632156

2157+
private MigrationOptions.Type decideMigrationTypeAndCopyTemplateIfNeeded(Host destHost, VMInstanceVO vmInstance, VolumeInfo srcVolumeInfo, StoragePoolVO sourceStoragePool, StoragePoolVO destStoragePool, DataStore destDataStore) {
2158+
VMTemplateVO vmTemplate = _vmTemplateDao.findById(vmInstance.getTemplateId());
2159+
String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo);
2160+
if (StringUtils.isNotBlank(srcVolumeBackingFile) && supportStoragePoolType(destStoragePool.getPoolType(), StoragePoolType.Filesystem) &&
2161+
srcVolumeInfo.getTemplateId() != null &&
2162+
Objects.nonNull(vmTemplate) &&
2163+
!Arrays.asList(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME, VM_IMPORT_DEFAULT_TEMPLATE_NAME).contains(vmTemplate.getName())) {
2164+
logger.debug(String.format("Copying template [%s] of volume [%s] from source storage pool [%s] to target storage pool [%s].", srcVolumeInfo.getTemplateId(), srcVolumeInfo.getId(), sourceStoragePool.getId(), destStoragePool.getId()));
2165+
copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, sourceStoragePool, destDataStore, destStoragePool, destHost);
2166+
return MigrationOptions.Type.LinkedClone;
2167+
}
2168+
logger.debug(String.format("Skipping copy template from source storage pool [%s] to target storage pool [%s] before migration due to volume [%s] does not have a " +
2169+
"template or we are doing full clone migration.", sourceStoragePool.getId(), destStoragePool.getId(), srcVolumeInfo.getId()));
2170+
return MigrationOptions.Type.FullClone;
2171+
}
2172+
21642173
protected String formatMigrationElementsAsJsonToDisplayOnLog(String objectName, Object object, Object from, Object to){
21652174
return String.format("{%s: \"%s\", from: \"%s\", to:\"%s\"}", objectName, object, from, to);
21662175
}
@@ -2422,7 +2431,7 @@ protected void updateCopiedTemplateReference(VolumeInfo srcVolumeInfo, VolumeInf
24222431
/**
24232432
* Handle post destination volume creation actions depending on the migrating volume type: full clone or linked clone
24242433
*/
2425-
protected void postVolumeCreationActions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, VirtualMachineTO vmTO, Host srcHost) {
2434+
protected void postVolumeCreationActions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) {
24262435
MigrationOptions migrationOptions = destVolumeInfo.getMigrationOptions();
24272436
if (migrationOptions != null) {
24282437
if (migrationOptions.getType() == MigrationOptions.Type.LinkedClone && migrationOptions.isCopySrcTemplate()) {

‎framework/db/src/main/java/com/cloud/utils/db/GenericDao.java‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -247,14 +247,6 @@ public interface GenericDao<T, ID extends Serializable> {
247247

248248
int expungeList(List<ID> ids);
249249

250-
/**
251-
* Delete the entity beans specified by the search criteria with a given limit
252-
* @param sc Search criteria
253-
* @param limit Maximum number of rows that will be affected
254-
* @return Number of rows deleted
255-
*/
256-
int expunge(SearchCriteria<T> sc, long limit);
257-
258250
/**
259251
* expunge the removed rows.
260252
*/

‎framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java‎

Lines changed: 29 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1244,13 +1244,6 @@ public boolean expunge(final ID id) {
12441244
}
12451245
}
12461246

1247-
// FIXME: Does not work for joins.
1248-
@Override
1249-
public int expunge(final SearchCriteria<T> sc, long limit) {
1250-
Filter filter = new Filter(limit);
1251-
return expunge(sc, filter);
1252-
}
1253-
12541247
@Override
12551248
public int expunge(final SearchCriteria<T> sc, final Filter filter) {
12561249
if (sc == null) {
@@ -1400,22 +1393,39 @@ protected List<Attribute> addJoins(StringBuilder str, Collection<JoinBuilder<Sea
14001393
onClause.append("?");
14011394
joinAttrList.add(join.getFirstAttributes()[i]);
14021395
} else {
1403-
onClause.append(joinedTableNames.getOrDefault(join.getFirstAttributes()[i].table, join.getFirstAttributes()[i].table))
1404-
.append(".")
1405-
.append(join.getFirstAttributes()[i].columnName);
1396+
if ((join.getFirstAttributes()[i].table == null && join.getFirstAttributes()[i].value == null) ||
1397+
(join.getSecondAttribute()[i].table == null && join.getSecondAttribute()[i].value == null)) {
1398+
onClause.append(joinedTableNames.getOrDefault(join.getSecondAttribute()[i].table, join.getFirstAttributes()[i].table))
1399+
.append(".");
1400+
if (join.getFirstAttributes()[i].table == null && join.getFirstAttributes()[i].value == null) {
1401+
onClause.append(join.getSecondAttribute()[i].columnName);
1402+
} else {
1403+
onClause.append(join.getFirstAttributes()[i].columnName);
1404+
}
1405+
1406+
} else {
1407+
onClause.append(joinedTableNames.getOrDefault(join.getFirstAttributes()[i].table, join.getFirstAttributes()[i].table))
1408+
.append(".")
1409+
.append(join.getFirstAttributes()[i].columnName);
1410+
}
14061411
}
1407-
onClause.append("=");
1408-
if (join.getSecondAttribute()[i].getValue() != null) {
1409-
onClause.append("?");
1410-
joinAttrList.add(join.getSecondAttribute()[i]);
1412+
if ((join.getFirstAttributes()[i].table == null && join.getFirstAttributes()[i].value == null) ||
1413+
(join.getSecondAttribute()[i].table == null && join.getSecondAttribute()[i].value == null)) {
1414+
onClause.append(" IS NULL");
14111415
} else {
1412-
if(!joinTableAlias.equals(joinTableName)) {
1413-
onClause.append(joinTableAlias);
1416+
onClause.append("=");
1417+
if (join.getSecondAttribute()[i].getValue() != null) {
1418+
onClause.append("?");
1419+
joinAttrList.add(join.getSecondAttribute()[i]);
14141420
} else {
1415-
onClause.append(joinTableName);
1421+
if (!joinTableAlias.equals(joinTableName)) {
1422+
onClause.append(joinTableAlias);
1423+
} else {
1424+
onClause.append(joinTableName);
1425+
}
1426+
onClause.append(".")
1427+
.append(join.getSecondAttribute()[i].columnName);
14161428
}
1417-
onClause.append(".")
1418-
.append(join.getSecondAttribute()[i].columnName);
14191429
}
14201430
}
14211431
onClause.append(" ");

‎packaging/el8/cloud.spec‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ Requires: java-17-openjdk
102102
Requires: tzdata-java
103103
Requires: %{name}-common = %{_ver}
104104
Requires: libvirt
105+
Requires: libvirt-daemon-driver-storage-rbd
105106
Requires: ebtables
106107
Requires: iptables
107108
Requires: ethtool

‎plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121

2222
import javax.inject.Inject;
2323

24+
import com.cloud.user.Account;
25+
26+
import org.apache.cloudstack.api.ACL;
2427
import org.apache.cloudstack.api.APICommand;
2528
import org.apache.cloudstack.api.ApiConstants;
2629
import org.apache.cloudstack.api.BaseCmd;
@@ -40,6 +43,7 @@ public class QuotaBalanceCmd extends BaseCmd {
4043
@Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which statement needs to be generated")
4144
private String accountName;
4245

46+
@ACL
4347
@Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.")
4448
private Long domainId;
4549

@@ -51,6 +55,7 @@ public class QuotaBalanceCmd extends BaseCmd {
5155
ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS)
5256
private Date startDate;
5357

58+
@ACL
5459
@Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified account")
5560
private Long accountId;
5661

@@ -104,7 +109,14 @@ public void setStartDate(Date startDate) {
104109

105110
@Override
106111
public long getEntityOwnerId() {
107-
return _accountService.getActiveAccountByName(accountName, domainId).getAccountId();
112+
if (accountId != null) {
113+
return accountId;
114+
}
115+
Account account = _accountService.getActiveAccountByName(accountName, domainId);
116+
if (account != null) {
117+
return account.getAccountId();
118+
}
119+
return Account.ACCOUNT_ID_SYSTEM;
108120
}
109121

110122
@Override

‎plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaCreditsCmd.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import com.cloud.user.Account;
2020

21+
import org.apache.cloudstack.api.ACL;
2122
import org.apache.cloudstack.api.APICommand;
2223
import org.apache.cloudstack.api.ApiConstants;
2324
import org.apache.cloudstack.api.ApiErrorCode;
@@ -46,6 +47,7 @@ public class QuotaCreditsCmd extends BaseCmd {
4647
@Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which quota credits need to be added")
4748
private String accountName;
4849

50+
@ACL
4951
@Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Domain for which quota credits need to be added")
5052
private Long domainId;
5153

@@ -130,6 +132,10 @@ public void execute() {
130132

131133
@Override
132134
public long getEntityOwnerId() {
135+
Account account = _accountService.getActiveAccountByName(accountName, domainId);
136+
if (account != null) {
137+
return account.getAccountId();
138+
}
133139
return Account.ACCOUNT_ID_SYSTEM;
134140
}
135141

0 commit comments

Comments
 (0)