delete expunge (hapifhir#2131)

Added delete _expunge=true

Author: fil512

hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java

@@ -916,6 +916,95 @@ public enum Pointcut {
 		"org.hl7.fhir.instance.model.api.IBaseResource"
 	),
 
+	/**
+	 * <b>Storage Hook:</b>
+	 * Invoked when a set of resources are about to be deleted and expunged via url like http://localhost/Patient?active=false&_expunge=true
+	 * <p>
+	 * Hooks may accept the following parameters:
+	 * </p>
+	 * <ul>
+	 * <li>
+	 * ca.uhn.fhir.rest.api.server.RequestDetails - A bean containing details about the request that is about to be processed, including details such as the
+	 * resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been
+	 * pulled out of the servlet request. Note that the bean
+	 * properties are not all guaranteed to be populated, depending on how early during processing the
+	 * exception occurred. <b>Note that this parameter may be null in contexts where the request is not
+	 * known, such as while processing searches</b>
+	 * </li>
+	 * <li>
+	 * ca.uhn.fhir.rest.server.servlet.ServletRequestDetails - A bean containing details about the request that is about to be processed, including details such as the
+	 * resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been
+	 * pulled out of the servlet request. This parameter is identical to the RequestDetails parameter above but will
+	 * only be populated when operating in a RestfulServer implementation. It is provided as a convenience.
+	 * </li>
+	 * <li>
+	 * java.lang.String - Contains the url used to delete and expunge the resources
+	 * </li>
+	 * </ul>
+	 * <p>
+	 * Hooks should return <code>void</code>. They may choose to throw an exception however, in
+	 * which case the delete expunge will not occur.
+	 * </p>
+	 */
+
+	STORAGE_PRE_DELETE_EXPUNGE(
+		void.class,
+		"ca.uhn.fhir.rest.api.server.RequestDetails",
+		"ca.uhn.fhir.rest.server.servlet.ServletRequestDetails",
+		"java.lang.String"
+	),
+
+	/**
+	 * <b>Storage Hook:</b>
+	 * Invoked when a batch of resource pids are about to be deleted and expunged via url like http://localhost/Patient?active=false&_expunge=true
+	 * <p>
+	 * Hooks may accept the following parameters:
+	 * </p>
+	 * <ul>
+	 * <li>
+	 * java.lang.String - the name of the resource type being deleted
+	 * </li>
+	 * <li>
+	 * java.util.List - the list of Long pids of the resources about to be deleted
+	 * </li>
+	 * <li>
+	 * java.util.concurrent.atomic.AtomicLong - holds a running tally of all entities deleted so far.
+	 * If the pointcut callback deletes any entities, then this parameter should be incremented by the total number
+	 * of additional entities deleted.
+	 * </li>
+	 * <li>
+	 * ca.uhn.fhir.rest.api.server.RequestDetails - A bean containing details about the request that is about to be processed, including details such as the
+	 * resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been
+	 * pulled out of the servlet request. Note that the bean
+	 * properties are not all guaranteed to be populated, depending on how early during processing the
+	 * exception occurred. <b>Note that this parameter may be null in contexts where the request is not
+	 * known, such as while processing searches</b>
+	 * </li>
+	 * <li>
+	 * ca.uhn.fhir.rest.server.servlet.ServletRequestDetails - A bean containing details about the request that is about to be processed, including details such as the
+	 * resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been
+	 * pulled out of the servlet request. This parameter is identical to the RequestDetails parameter above but will
+	 * only be populated when operating in a RestfulServer implementation. It is provided as a convenience.
+	 * </li>
+	 * <li>
+	 * java.lang.String - Contains the url used to delete and expunge the resources
+	 * </li>
+	 * </ul>
+	 * <p>
+	 * Hooks should return <code>void</code>. They may choose to throw an exception however, in
+	 * which case the delete expunge will not occur.
+	 * </p>
+	 */
+
+	STORAGE_PRE_DELETE_EXPUNGE_PID_LIST(
+		void.class,
+		"java.lang.String",
+		"java.util.List",
+		"java.util.concurrent.atomic.AtomicLong",
+		"ca.uhn.fhir.rest.api.server.RequestDetails",
+		"ca.uhn.fhir.rest.server.servlet.ServletRequestDetails"
+	),
+
 	/**
 	 * <b>Storage Hook:</b>
 	 * Invoked when one or more resources may be returned to the user, whether as a part of a READ,

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/5_2_0/2131-delete-expunge.yaml

@@ -0,0 +1,10 @@
+---
+type: add
+issue: 2131
+title: "A new _expunge parameter has been added to the DELETE operation when deleting multiple resources via a URL.  For
+example DELETE http://www.example.com:8000/Observation?_expunge=true or
+DELETE http://www.example.com:8000/Observation?status=cancelled&_expunge=true.  When the _expunge parameter is provided to DELETE
+then the matched resources and all of their history will be both deleted and expunged from the database.  This will
+perform considerably faster than doing the delete and expunge separately.  Note that Expunge must be enabled on the
+server for this to work."
+

hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/config/DaoConfig.java

@@ -144,6 +144,7 @@ public class DaoConfig {
 	private IdStrategyEnum myResourceServerIdStrategy = IdStrategyEnum.SEQUENTIAL_NUMERIC;
 	private boolean myMarkResourcesForReindexingUponSearchParameterChange;
 	private boolean myExpungeEnabled;
+	private boolean myDeleteExpungeEnabled;
 	private int myExpungeBatchSize = DEFAULT_EXPUNGE_BATCH_SIZE;
 	private int myReindexThreadCount;
 	private int myExpungeThreadCount;
@@ -1308,6 +1309,42 @@ public boolean isExpungeEnabled() {
 		return myExpungeEnabled;
 	}
 
+	/**
+	 * If set to <code>true</code> (default is <code>false</code>), the _expunge parameter on the DELETE
+	 * operation will be enabled on this server. DELETE _expunge removes all data associated with a resource in a highly performant
+	 * way, skipping most of the the checks that are enforced with usual DELETE operations.  The only check
+	 * that is performed before deleting the resources and their indexes is that no other resources reference the resources about to
+	 * be deleted.  This operation is potentially dangerous since it allows
+	 * a client to physically delete data in a way that can not be recovered (without resorting
+	 * to backups).
+	 * <p>
+	 * It is recommended to not enable this setting without appropriate security
+	 * in place on your server to prevent non-administrators from using this
+	 * operation.
+	 * </p>
+	 */
+	public void setDeleteExpungeEnabled(boolean theDeleteExpungeEnabled) {
+		myDeleteExpungeEnabled = theDeleteExpungeEnabled;
+	}
+
+	/**
+	 * If set to <code>true</code> (default is <code>false</code>), the _expunge parameter on the DELETE
+	 * operation will be enabled on this server. DELETE _expunge removes all data associated with a resource in a highly performant
+	 * way, skipping most of the the checks that are enforced with usual DELETE operations.  The only check
+	 * that is performed before deleting the data is that no other resources reference the resources about to
+	 * be deleted.  This operation is potentially dangerous since it allows
+	 * a client to physically delete data in a way that can not be recovered (without resorting
+	 * to backups).
+	 * <p>
+	 * It is recommended to not enable this setting without appropriate security
+	 * in place on your server to prevent non-administrators from using this
+	 * operation.
+	 * </p>
+	 */
+	public boolean isDeleteExpungeEnabled() {
+		return myDeleteExpungeEnabled;
+	}
+
 	/**
 	 * If set to <code>true</code> (default is <code>false</code>), the $expunge operation
 	 * will be enabled on this server. This operation is potentially dangerous since it allows

hapi-fhir-jpaserver-api/src/main/java/ca/uhn/fhir/jpa/api/model/DeleteMethodOutcome.java

@@ -32,6 +32,8 @@
 public class DeleteMethodOutcome extends MethodOutcome {
 
 	private List<ResourceTable> myDeletedEntities;
+	private long myExpungedResourcesCount;
+	private long myExpungedEntitiesCount;
 
 	public List<ResourceTable> getDeletedEntities() {
 		return myDeletedEntities;
@@ -42,4 +44,21 @@ public DeleteMethodOutcome setDeletedEntities(List<ResourceTable> theDeletedEnti
 		return this;
 	}
 
+	public long getExpungedResourcesCount() {
+		return myExpungedResourcesCount;
+	}
+
+	public DeleteMethodOutcome setExpungedResourcesCount(long theExpungedResourcesCount) {
+		myExpungedResourcesCount = theExpungedResourcesCount;
+		return this;
+	}
+
+	public long getExpungedEntitiesCount() {
+		return myExpungedEntitiesCount;
+	}
+
+	public DeleteMethodOutcome setExpungedEntitiesCount(long theExpungedEntitiesCount) {
+		myExpungedEntitiesCount = theExpungedEntitiesCount;
+		return this;
+	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java

@@ -33,6 +33,7 @@
 import ca.uhn.fhir.jpa.api.model.DeleteMethodOutcome;
 import ca.uhn.fhir.jpa.api.model.ExpungeOptions;
 import ca.uhn.fhir.jpa.api.model.ExpungeOutcome;
+import ca.uhn.fhir.jpa.dao.expunge.DeleteExpungeService;
 import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
 import ca.uhn.fhir.jpa.delete.DeleteConflictService;
 import ca.uhn.fhir.jpa.model.entity.BaseHasResource;
@@ -51,6 +52,7 @@
 import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
 import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider;
 import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
+import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
 import ca.uhn.fhir.model.api.IQueryParameterType;
@@ -102,6 +104,7 @@
 import org.jetbrains.annotations.Nullable;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Required;
+import org.springframework.data.domain.SliceImpl;
 import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.TransactionDefinition;
 import org.springframework.transaction.annotation.Propagation;
@@ -151,6 +154,10 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
 	private HapiTransactionService myTransactionService;
 	@Autowired(required = false)
 	protected IFulltextSearchSvc mySearchDao;
+	@Autowired
+	private MatchUrlService myMatchUrlService;
+	@Autowired
+	private DeleteExpungeService myDeleteExpungeService;
 
 	private IInstanceValidatorModule myInstanceValidator;
 	private String myResourceName;
@@ -456,6 +463,7 @@ public DaoMethodOutcome delete(IIdType theId, DeleteConflictList theDeleteConfli
 			.add(RequestDetails.class, theRequestDetails)
 			.addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
 			.add(TransactionDetails.class, theTransactionDetails);
+
 		if (theTransactionDetails.isAcceptingDeferredInterceptorBroadcasts()) {
 			theTransactionDetails.addDeferredInterceptorBroadcast(Pointcut.STORAGE_PRECOMMIT_RESOURCE_DELETED, hookParams);
 		} else {
@@ -499,14 +507,31 @@ public DeleteMethodOutcome deleteByUrl(String theUrl, DeleteConflictList deleteC
 
 	@Nonnull
 	private DeleteMethodOutcome doDeleteByUrl(String theUrl, DeleteConflictList deleteConflicts, RequestDetails theRequest) {
-		Set<ResourcePersistentId> resourceIds = myMatchResourceUrlService.processMatchUrl(theUrl, myResourceType, theRequest);
+		RuntimeResourceDefinition resourceDef = getContext().getResourceDefinition(myResourceType);
+		SearchParameterMap paramMap = myMatchUrlService.translateMatchUrl(theUrl, resourceDef);
+		paramMap.setLoadSynchronous(true);
+
+		Set<ResourcePersistentId> resourceIds = myMatchResourceUrlService.search(paramMap, myResourceType, theRequest);
+
 		if (resourceIds.size() > 1) {
 			if (!myDaoConfig.isAllowMultipleDelete()) {
 				throw new PreconditionFailedException(getContext().getLocalizer().getMessageSanitized(BaseHapiFhirDao.class, "transactionOperationWithMultipleMatchFailure", "DELETE", theUrl, resourceIds.size()));
 			}
 		}
 
-		return deletePidList(theUrl, resourceIds, deleteConflicts, theRequest);
+		if (paramMap.isDeleteExpunge()) {
+			return deleteExpunge(theUrl, theRequest, resourceIds);
+		} else {
+			return deletePidList(theUrl, resourceIds, deleteConflicts, theRequest);
+		}
+	}
+
+	private DeleteMethodOutcome deleteExpunge(String theUrl, RequestDetails theTheRequest, Set<ResourcePersistentId> theResourceIds) {
+		if (!myDaoConfig.isExpungeEnabled() || !myDaoConfig.isDeleteExpungeEnabled()) {
+			throw new MethodNotAllowedException("_expunge is not enabled on this server");
+		}
+
+		return myDeleteExpungeService.expungeByResourcePids(theUrl, myResourceName, new SliceImpl<>(ResourcePersistentId.toLongList(theResourceIds)), theTheRequest);
 	}
 
 	@NotNull

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/MatchResourceUrlService.java

@@ -27,12 +27,12 @@
 import ca.uhn.fhir.interceptor.api.Pointcut;
 import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
 import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
-import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
 import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
 import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.jpa.util.JpaInterceptorBroadcaster;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
+import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
@@ -55,23 +55,24 @@ public class MatchResourceUrlService {
 	private IInterceptorBroadcaster myInterceptorBroadcaster;
 
 	public <R extends IBaseResource> Set<ResourcePersistentId> processMatchUrl(String theMatchUrl, Class<R> theResourceType, RequestDetails theRequest) {
-		StopWatch sw = new StopWatch();
-
 		RuntimeResourceDefinition resourceDef = myContext.getResourceDefinition(theResourceType);
-
 		SearchParameterMap paramMap = myMatchUrlService.translateMatchUrl(theMatchUrl, resourceDef);
-		paramMap.setLoadSynchronous(true);
-
 		if (paramMap.isEmpty() && paramMap.getLastUpdated() == null) {
 			throw new InvalidRequestException("Invalid match URL[" + theMatchUrl + "] - URL has no search parameters");
 		}
+		paramMap.setLoadSynchronous(true);
 
+		return search(paramMap, theResourceType, theRequest);
+	}
+
+	public <R extends IBaseResource> Set<ResourcePersistentId> search(SearchParameterMap theParamMap, Class<R> theResourceType, RequestDetails theRequest) {
+		StopWatch sw = new StopWatch();
 		IFhirResourceDao<R> dao = myDaoRegistry.getResourceDao(theResourceType);
 		if (dao == null) {
 			throw new InternalErrorException("No DAO for resource type: " + theResourceType.getName());
 		}
 
-		Set<ResourcePersistentId> retVal = dao.searchForIds(paramMap, theRequest);
+		Set<ResourcePersistentId> retVal = dao.searchForIds(theParamMap, theRequest);
 
 		// Interceptor broadcast: JPA_PERFTRACE_INFO
 		if (JpaInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_INFO, myInterceptorBroadcaster, theRequest)) {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IEmpiLinkDao.java

@@ -20,6 +20,7 @@
  * #L%
  */
 
+import ca.uhn.fhir.empi.api.EmpiMatchResultEnum;
 import ca.uhn.fhir.jpa.entity.EmpiLink;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
@@ -32,4 +33,8 @@ public interface IEmpiLinkDao extends JpaRepository<EmpiLink, Long> {
 	@Modifying
 	@Query("DELETE FROM EmpiLink f WHERE myPersonPid = :pid OR myTargetPid = :pid")
 	int deleteWithAnyReferenceToPid(@Param("pid") Long thePid);
+
+	@Modifying
+	@Query("DELETE FROM EmpiLink f WHERE (myPersonPid = :pid OR myTargetPid = :pid) AND myMatchResult <> :matchResult")
+	int deleteWithAnyReferenceToPidAndMatchResultNot(@Param("pid") Long thePid, @Param("matchResult")EmpiMatchResultEnum theMatchResult);
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IResourceLinkDao.java

@@ -35,5 +35,8 @@ public interface IResourceLinkDao extends JpaRepository<ResourceLink, Long> {
 	void deleteByResourceId(@Param("resId") Long theResourcePid);
 
 	@Query("SELECT t FROM ResourceLink t WHERE t.mySourceResourcePid = :resId")
-	List<ResourceLink> findAllForResourceId(@Param("resId") Long thePatientId);
+	List<ResourceLink> findAllForSourceResourceId(@Param("resId") Long thePatientId);
+
+	@Query("SELECT t FROM ResourceLink t WHERE t.myTargetResourcePid in :resIds")
+	List<ResourceLink> findWithTargetPidIn(@Param("resIds") List<Long> thePids);
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/empi/EmpiLinkDeleteSvc.java

@@ -20,6 +20,7 @@
  * #L%
  */
 
+import ca.uhn.fhir.empi.api.EmpiMatchResultEnum;
 import ca.uhn.fhir.jpa.dao.data.IEmpiLinkDao;
 import ca.uhn.fhir.jpa.dao.index.IdHelperService;
 import org.hl7.fhir.instance.model.api.IBaseResource;
@@ -50,4 +51,13 @@ public int deleteWithAnyReferenceTo(IBaseResource theResource) {
 		}
 		return removed;
 	}
+
+	public int deleteNonRedirectWithWithAnyReferenceTo(IBaseResource theResource) {
+		Long pid = myIdHelperService.getPidOrThrowException(theResource.getIdElement(), null);
+		int removed =  myEmpiLinkDao.deleteWithAnyReferenceToPidAndMatchResultNot(pid, EmpiMatchResultEnum.REDIRECT);
+		if (removed > 0) {
+			ourLog.info("Removed {} non-redirect EMPI links with references to {}", removed, theResource.getIdElement().toVersionless());
+		}
+		return removed;
+	}
 }