Events listing and deletion optimizations

winterhazel · winterhazel · commit cea607d5993b · 2025-08-13T08:42:16.000-03:00
diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/event/DeleteEventsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/event/DeleteEventsCmd.java
@@ -31,7 +31,6 @@
 import org.apache.cloudstack.context.CallContext;
 
 import com.cloud.event.Event;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.user.Account;
 
 @APICommand(name = "deleteEvents", description = "Delete one or more events.", responseObject = SuccessResponse.class, entityType = {Event.class},
@@ -97,17 +96,12 @@ public long getEntityOwnerId() {
 
     @Override
     public void execute() {
-        if (ids == null && type == null && endDate == null) {
-            throw new InvalidParameterValueException("either ids, type or enddate must be specified");
-        } else if (startDate != null && endDate == null) {
-            throw new InvalidParameterValueException("enddate must be specified with startdate parameter");
-        }
         boolean result = _mgr.deleteEvents(this);
         if (result) {
             SuccessResponse response = new SuccessResponse(getCommandName());
             setResponseObject(response);
         } else {
-            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to delete Events, one or more parameters has invalid values");
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Unable to delete any events. One or more parameters have invalid values.");
         }
     }
 }
diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/event/ListEventsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/event/ListEventsCmd.java
@@ -127,7 +127,6 @@ public boolean getArchived() {
 
     @Override
     public void execute() {
-
         ListResponse<EventResponse> response = _queryService.searchForEvents(this);
         response.setResponseName(getCommandName());
         setResponseObject(response);
diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDao.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDao.java
@@ -42,5 +42,7 @@ public interface DomainDao extends GenericDao<DomainVO, Long> {
 
     List<Long> getDomainChildrenIds(String path);
 
+    List<Long> getDomainAndChildrenIds(long domainId);
+
     boolean domainIdListContainsAccessibleDomain(String domainIdList, Account caller, Long domainId);
 }
diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java
@@ -19,6 +19,7 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -238,6 +239,15 @@ public List<Long> getDomainChildrenIds(String path) {
         return customSearch(sc, null);
     }
 
+    @Override
+    public List<Long> getDomainAndChildrenIds(long domainId) {
+        DomainVO domain = findById(domainId);
+        if (domain != null) {
+            return getDomainChildrenIds(domain.getPath());
+        }
+        return new ArrayList<>();
+    }
+
     @Override
     public boolean isChildDomain(Long parentId, Long childId) {
         if ((parentId == null) || (childId == null)) {
diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java
@@ -27,12 +27,12 @@
 public interface EventDao extends GenericDao<EventVO, Long> {
     public List<EventVO> searchAllEvents(SearchCriteria<EventVO> sc, Filter filter);
 
-    public List<EventVO> listOlderEvents(Date oldTime);
-
     EventVO findCompletedEvent(long startId);
 
     public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Date startDate, Date endDate, List<Long> accountIds);
 
     public void archiveEvents(List<EventVO> events);
 
+    long purgeAll(List<Long> ids, Date startDate, Date endDate, Date limitDate, String type, Long accountId, List<Long> domainIds,
+                  long limitPerQuery);
 }
diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java
@@ -20,6 +20,7 @@
 import java.util.List;
 
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Component;
 
 import com.cloud.event.Event.State;
@@ -46,9 +47,12 @@ public EventDaoImpl() {
         ToArchiveOrDeleteEventSearch = createSearchBuilder();
         ToArchiveOrDeleteEventSearch.and("id", ToArchiveOrDeleteEventSearch.entity().getId(), Op.IN);
         ToArchiveOrDeleteEventSearch.and("type", ToArchiveOrDeleteEventSearch.entity().getType(), Op.EQ);
+        ToArchiveOrDeleteEventSearch.and("accountId", ToArchiveOrDeleteEventSearch.entity().getAccountId(), Op.EQ);
         ToArchiveOrDeleteEventSearch.and("accountIds", ToArchiveOrDeleteEventSearch.entity().getAccountId(), Op.IN);
+        ToArchiveOrDeleteEventSearch.and("domainIds", ToArchiveOrDeleteEventSearch.entity().getDomainId(), Op.IN);
         ToArchiveOrDeleteEventSearch.and("createdDateB", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.BETWEEN);
         ToArchiveOrDeleteEventSearch.and("createdDateL", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LTEQ);
+        ToArchiveOrDeleteEventSearch.and("createdDateLT", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LT);
         ToArchiveOrDeleteEventSearch.and("archived", ToArchiveOrDeleteEventSearch.entity().getArchived(), Op.EQ);
         ToArchiveOrDeleteEventSearch.done();
     }
@@ -58,16 +62,6 @@ public List<EventVO> searchAllEvents(SearchCriteria<EventVO> sc, Filter filter)
         return listIncludingRemovedBy(sc, filter);
     }
 
-    @Override
-    public List<EventVO> listOlderEvents(Date oldTime) {
-        if (oldTime == null)
-            return null;
-        SearchCriteria<EventVO> sc = createSearchCriteria();
-        sc.addAnd("createDate", SearchCriteria.Op.LT, oldTime);
-        sc.addAnd("archived", SearchCriteria.Op.EQ, false);
-        return listIncludingRemovedBy(sc, null);
-    }
-
     @Override
     public EventVO findCompletedEvent(long startId) {
         SearchCriteria<EventVO> sc = CompletedEventSearch.create();
@@ -112,4 +106,28 @@ public void archiveEvents(List<EventVO> events) {
             txn.close();
         }
     }
+
+    @Override
+    public long purgeAll(List<Long> ids, Date startDate, Date endDate, Date limitDate, String type, Long accountId,
+                         List<Long> domainIds, long limitPerQuery) {
+        SearchCriteria<EventVO> sc = ToArchiveOrDeleteEventSearch.create();
+
+        if (CollectionUtils.isNotEmpty(ids)) {
+            sc.setParameters("id", ids.toArray(new Object[0]));
+        }
+        if (startDate != null && endDate != null) {
+            sc.setParameters("createdDateB", startDate, endDate);
+        } else if (endDate != null) {
+            sc.setParameters("createdDateL", endDate);
+        }
+        sc.setParametersIfNotNull("createdDateLT", limitDate);
+        sc.setParametersIfNotNull("type", type);
+        sc.setParametersIfNotNull("accountId", accountId);
+        if (CollectionUtils.isNotEmpty(domainIds)) {
+            sc.setParameters("domainIds", domainIds.toArray(new Object[0]));
+        }
+        sc.setParameters("archived", false);
+
+        return batchExpunge(sc, limitPerQuery);
+    }
 }
diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java
@@ -25,7 +25,10 @@ public class DbUpgradeUtils {
 
     public static void addIndexIfNeeded(Connection conn, String tableName, String... columnNames) {
         String indexName = dao.generateIndexName(tableName, columnNames);
+        addIndexWithNameIfNeeded(conn, tableName, indexName, columnNames);
+    }
 
+    public static void addIndexWithNameIfNeeded(Connection conn, String tableName, String indexName, String... columnNames) {
         if (!dao.indexExists(conn, tableName, indexName)) {
             dao.createIndex(conn, tableName, indexName, columnNames);
         }
diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42020.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42020.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+public class Upgrade42010to42020 implements DbUpgrade {
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[]{"4.20.1.0", "4.20.2.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.20.2.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-42010to42020.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[]{script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+        addIndexes(conn);
+    }
+
+    private void addIndexes(Connection conn) {
+        DbUpgradeUtils.addIndexWithNameIfNeeded(conn, "event", "i_event__multiple_columns_for_generic_search",
+                "account_id", "domain_id", "archived",  "display", "resource_type", "resource_id", "start_id", "type", "level",
+                "created", "id");
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        final String scriptFile = "META-INF/db/schema-42010to42020-cleanup.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[]{script};
+    }
+}
diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42020-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42020-cleanup.sql
@@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade cleanup from 4.20.1.0 to 4.20.2.0
+--;
diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42020.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42020.sql
@@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.20.1.0 to 4.20.2.0
+--;
diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
@@ -865,7 +865,7 @@ private Pair<List<Long>, Integer> searchForEventIdsAndCount(ListEventsCmd cmd) {
         searchFilter.addOrderBy(EventVO.class, "id", false);
 
         SearchBuilder<EventVO> eventSearchBuilder = eventDao.createSearchBuilder();
-        eventSearchBuilder.select(null, Func.DISTINCT, eventSearchBuilder.entity().getId());
+        eventSearchBuilder.select(null, Func.NATIVE, eventSearchBuilder.entity().getId());
         accountMgr.buildACLSearchBuilder(eventSearchBuilder, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
 
         eventSearchBuilder.and("id", eventSearchBuilder.entity().getId(), SearchCriteria.Op.EQ);
diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
@@ -527,7 +527,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
     public static final ConfigKey<Long> DELETE_QUERY_BATCH_SIZE = new ConfigKey<>("Advanced", Long.class, "delete.query.batch.size", "0",
             "Indicates the limit applied while deleting entries in bulk. With this, the delete query will apply the limit as many times as necessary," +
                     " to delete all the entries. This is advised when retaining several days of records, which can lead to slowness. <= 0 means that no limit will " +
-                    "be applied. Default value is 0. For now, this is used for deletion of vm & volume stats only.", true);
+                    "be applied. Default value is 0. For now, this is used for deletion of VM stats, volume stats and events.", true);
 
     private static final String IOPS_READ_RATE = "IOPS Read";
     private static final String IOPS_WRITE_RATE = "IOPS Write";
diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java
@@ -640,6 +640,7 @@
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import com.cloud.agent.AgentManager;
@@ -1224,31 +1225,34 @@ public boolean archiveEvents(final ArchiveEventsCmd cmd) {
     }
 
     @Override
-    public boolean deleteEvents(final DeleteEventsCmd cmd) {
-        final Account caller = getCaller();
-        final List<Long> ids = cmd.getIds();
-        boolean result = true;
-        List<Long> permittedAccountIds = new ArrayList<>();
+    public boolean deleteEvents(DeleteEventsCmd cmd) {
+        List<Long> ids = cmd.getIds();
+        String type = cmd.getType();
+        Date startDate = cmd.getStartDate();
+        Date endDate = cmd.getEndDate();
 
-        if (_accountMgr.isNormalUser(caller.getId()) || caller.getType() == Account.Type.PROJECT) {
-            permittedAccountIds.add(caller.getId());
-        } else {
-            final DomainVO domain = _domainDao.findById(caller.getDomainId());
-            final List<Long> permittedDomainIds = _domainDao.getDomainChildrenIds(domain.getPath());
-            permittedAccountIds = _accountDao.getAccountIdsForDomains(permittedDomainIds);
+        if (CollectionUtils.isEmpty(ids) && ObjectUtils.allNull(type, endDate)) {
+            throw new InvalidParameterValueException("Either 'ids', 'type' or 'enddate' must be specified.");
+        }
+        if (startDate != null && endDate == null) {
+            throw new InvalidParameterValueException("'startdate' must be specified with 'enddate' parameter.");
         }
 
-        final List<EventVO> events = _eventDao.listToArchiveOrDeleteEvents(ids, cmd.getType(), cmd.getStartDate(), cmd.getEndDate(), permittedAccountIds);
-        final ControlledEntity[] sameOwnerEvents = events.toArray(new ControlledEntity[events.size()]);
-        _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, false, sameOwnerEvents);
+        Long accountId = null;
+        List<Long> domainIds = null;
+        Account caller = getCaller();
+        Account.Type callerType = caller.getType();
 
-        if (ids != null && events.size() < ids.size()) {
-            return false;
-        }
-        for (final EventVO event : events) {
-            _eventDao.remove(event.getId());
+        if (callerType == Account.Type.NORMAL || callerType == Account.Type.PROJECT) {
+            accountId = caller.getId();
+        } else {
+            domainIds = _domainDao.getDomainAndChildrenIds(caller.getDomainId());
         }
-        return result;
+
+        long totalRemoved = _eventDao.purgeAll(ids, startDate, endDate, null, type, accountId, domainIds,
+                ConfigurationManagerImpl.DELETE_QUERY_BATCH_SIZE.value());
+
+        return totalRemoved > 0;
     }
 
     @Override
@@ -4086,12 +4090,10 @@ protected void runInContext() {
                     final Calendar purgeCal = Calendar.getInstance();
                     purgeCal.add(Calendar.DAY_OF_YEAR, -_purgeDelay);
                     final Date purgeTime = purgeCal.getTime();
-                    logger.debug("Deleting events older than: " + purgeTime.toString());
-                    final List<EventVO> oldEvents = _eventDao.listOlderEvents(purgeTime);
-                    logger.debug("Found " + oldEvents.size() + " events to be purged");
-                    for (final EventVO event : oldEvents) {
-                        _eventDao.expunge(event.getId());
-                    }
+                    logger.debug("Starting to purge all event rows older than [{}].", purgeTime);
+                    long totalRemoved = _eventDao.purgeAll(null, null, null, purgeTime, null, null, null,
+                            ConfigurationManagerImpl.DELETE_QUERY_BATCH_SIZE.value());
+                    logger.info("Purged a total of [{}] event rows older than [{}].", totalRemoved, purgeTime);
                 } catch (final Exception e) {
                     logger.error("Exception ", e);
                 } finally {

Original file line number	Diff line number	Diff line change
`@@ -42,5 +42,7 @@ public interface DomainDao extends GenericDao<DomainVO, Long> {`
`42`	`42`
`43`	`43`	`List<Long> getDomainChildrenIds(String path);`
`44`	`44`
	`45`	`+ List<Long> getDomainAndChildrenIds(long domainId);`
	`46`	`+`
`45`	`47`	`boolean domainIdListContainsAccessibleDomain(String domainIdList, Account caller, Long domainId);`
`46`	`48`	`}`
Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,10 @@ public class DbUpgradeUtils {`
`25`	`25`
`26`	`26`	`public static void addIndexIfNeeded(Connection conn, String tableName, String... columnNames) {`
`27`	`27`	`String indexName = dao.generateIndexName(tableName, columnNames);`
	`28`	`+ addIndexWithNameIfNeeded(conn, tableName, indexName, columnNames);`
	`29`	`+ }`
`28`	`30`
	`31`	`+ public static void addIndexWithNameIfNeeded(Connection conn, String tableName, String indexName, String... columnNames) {`
`29`	`32`	`if (!dao.indexExists(conn, tableName, indexName)) {`
`30`	`33`	`dao.createIndex(conn, tableName, indexName, columnNames);`
`31`	`34`	`}`