7002 supporting reindex of deleted resources (#7143)

* - adding initial failing test
- adding support for 'includeDeletedResources' to reindexProvider and ReindexJobParameters.

* - adding initial failing test
- adding support for 'includeDeletedResources' to reindexProvider and ReindexJobParameters.

* wip

* initial commit - implementation, small refactoring, lots of tests, docs

* spotless

* fix 2 failing tests

* fix 1 more failing test

* spotless

* remove todo, fixme's

* changelog

* changelog

* changelog

* apply suggestion on Batch2DaoSvcImpl

Co-authored-by: volodymyr-korzh <[email protected]>

* apply suggestion on Batch2DaoSvcImpl

Co-authored-by: volodymyr-korzh <[email protected]>

* code review suggestions

* code review suggestions part 1

* review suggestion: change include deleted enums / name

* review suggestion: update a couple error messages

* refactor getOrCreateFirstPredicateBuilder and getOrCreateResourceTablePredicateBuilder

* add null check

* add tests for general search with include deleted

* spotless

* remove public from test

* update error message in test

* fix test: use default value instead of null for getbean

* spotless

* fix the mocks

* fix failing test due to change in behaviour from review

* move static methods to bottom

* add nonnull for review

---------

Co-authored-by: peartree <[email protected]>
Co-authored-by: jdar <[email protected]>
Co-authored-by: volodymyr-korzh <[email protected]>

Author: 4 people

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/Constants.java

@@ -220,7 +220,7 @@ public class Constants {
 	public static final String PARAM_CONTENT_URL = "http://hl7.org/fhir/SearchParameter/Resource-content";
 	public static final String PARAM_COUNT = "_count";
 	public static final String PARAM_OFFSET = "_offset";
-	public static final String PARAM_DELETE = "_delete";
+	public static final String PARAM_INCLUDE_DELETED = "_includeDeleted";
 	public static final String PARAM_ELEMENTS = "_elements";
 	public static final String PARAM_ELEMENTS_EXCLUDE_MODIFIER = ":exclude";
 	public static final String PARAM_FORMAT = "_format";

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/SearchIncludeDeletedEnum.java

@@ -0,0 +1,75 @@
+/*
+ * #%L
+ * HAPI FHIR - Core Library
+ * %%
+ * Copyright (C) 2014 - 2025 Smile CDR, Inc.
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+package ca.uhn.fhir.rest.api;
+
+import ca.uhn.fhir.i18n.Msg;
+import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
+import ca.uhn.fhir.util.UrlUtil;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public enum SearchIncludeDeletedEnum {
+
+	/**
+	 * default, search on the non-deleted resources
+	 */
+	NEVER("never"),
+
+	/**
+	 * search on the deleted resources only
+	 */
+	EXCLUSIVE("exclusive"),
+
+	/**
+	 * Search on the non-deleted resources and deleted resources.
+	 */
+	BOTH("both");
+
+	private static Map<String, SearchIncludeDeletedEnum> ourCodeToEnum;
+	private final String myCode;
+
+	SearchIncludeDeletedEnum(String theCode) {
+		myCode = theCode;
+	}
+
+	public String getCode() {
+		return myCode;
+	}
+
+	public static SearchIncludeDeletedEnum fromCode(String theCode) {
+		Map<String, SearchIncludeDeletedEnum> codeToEnum = ourCodeToEnum;
+		if (codeToEnum == null) {
+			codeToEnum = new HashMap<>();
+			for (SearchIncludeDeletedEnum next : values()) {
+				codeToEnum.put(next.getCode(), next);
+			}
+			ourCodeToEnum = codeToEnum;
+		}
+
+		SearchIncludeDeletedEnum retVal = codeToEnum.get(theCode);
+		if (retVal == null) {
+			throw new InvalidRequestException(Msg.code(2741) + "Invalid \"" + Constants.PARAM_INCLUDE_DELETED
+					+ "\" mode: " + UrlUtil.sanitizeUrlPart(theCode));
+		}
+
+		return retVal;
+	}
+}

hapi-fhir-base/src/main/resources/ca/uhn/fhir/i18n/hapi-messages.properties

@@ -225,3 +225,5 @@ ca.uhn.fhir.jpa.interceptor.validation.RuleRequireProfileDeclaration.noMatchingP
 ca.uhn.fhir.jpa.interceptor.validation.RuleRequireProfileDeclaration.illegalProfile=Resource of type "{0}" must not declare conformance to profile: {1}
 
 ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl.invalidUseOfSearchIdentifier=Unsupported search modifier(s): "{0}" for resource type "{1}". Valid search modifiers are: {2}
+
+ca.uhn.fhir.jpa.searchparam.MatchUrlService.noResourceType = Conditional URL does not include a resource type, but includes parameters which require a resource type

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/8_4_0/7002-support-reindex-of-deleted-resources.yaml

@@ -0,0 +1,7 @@
+---
+type: add
+issue: 7002
+jira: SMILE-9203
+title: "Added support for reindexing deleted resources. This can be done by adding the new `_includeDeleted` parameter
+to the Parameters resource upon submitting the `$reindex` job. 
+See <a href=\"https://smilecdr.com/docs/fhir_repository/search_parameter_reindexing.html#reindex-batch-job\">the reindex documentation</a> for more information."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/Batch2DaoSvcImpl.java

@@ -31,11 +31,14 @@
 import ca.uhn.fhir.jpa.api.pid.TypedResourcePid;
 import ca.uhn.fhir.jpa.api.pid.TypedResourceStream;
 import ca.uhn.fhir.jpa.api.svc.IBatch2DaoSvc;
+import ca.uhn.fhir.jpa.dao.ISearchBuilder;
+import ca.uhn.fhir.jpa.dao.SearchBuilderFactory;
 import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
 import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService;
 import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.model.dao.JpaPid;
+import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
 import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.model.primitive.IdDt;
@@ -45,16 +48,20 @@
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.util.DateRangeUtil;
 import ca.uhn.fhir.util.Logs;
+import ca.uhn.fhir.util.UrlUtil;
 import jakarta.annotation.Nonnull;
 import jakarta.annotation.Nullable;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 import org.hl7.fhir.instance.model.api.IIdType;
 
 import java.util.Date;
+import java.util.UUID;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 
+import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
+
 public class Batch2DaoSvcImpl implements IBatch2DaoSvc {
 	private static final org.slf4j.Logger ourLog = Logs.getBatchTroubleshootingLog();
 
@@ -72,6 +79,8 @@ public class Batch2DaoSvcImpl implements IBatch2DaoSvc {
 
 	private final PartitionSettings myPartitionSettings;
 
+	private final SearchBuilderFactory<JpaPid> mySearchBuilderFactory;
+
 	@Override
 	public boolean isAllResourceTypeSupported() {
 		return true;
@@ -84,20 +93,24 @@ public Batch2DaoSvcImpl(
 			DaoRegistry theDaoRegistry,
 			FhirContext theFhirContext,
 			IHapiTransactionService theTransactionService,
-			PartitionSettings thePartitionSettings) {
+			PartitionSettings thePartitionSettings,
+			SearchBuilderFactory<JpaPid> theSearchBuilderFactory) {
 		myResourceTableDao = theResourceTableDao;
 		myResourceLinkDao = theResourceLinkDao;
 		myMatchUrlService = theMatchUrlService;
 		myDaoRegistry = theDaoRegistry;
 		myFhirContext = theFhirContext;
 		myTransactionService = theTransactionService;
 		myPartitionSettings = thePartitionSettings;
+		mySearchBuilderFactory = theSearchBuilderFactory;
 	}
 
 	@Override
 	public IResourcePidStream fetchResourceIdStream(
 			Date theStart, Date theEnd, RequestPartitionId theRequestPartitionId, String theUrl) {
+
 		if (StringUtils.isBlank(theUrl)) {
+			// first scenario
 			return makeStreamResult(
 					theRequestPartitionId, () -> streamResourceIdsNoUrl(theStart, theEnd, theRequestPartitionId));
 		} else {
@@ -116,21 +129,55 @@ private Stream<TypedResourcePid> streamResourceIdsWithUrl(
 			Date theStart, Date theEnd, String theUrl, RequestPartitionId theRequestPartitionId) {
 		validateUrl(theUrl);
 
-		SearchParameterMap searchParamMap = parseQuery(theUrl);
+		String resourceType = UrlUtil.determineResourceTypeInResourceUrl(myFhirContext, theUrl);
+
+		// Search in all partitions if no partition is provided
+		ourLog.debug("No partition id detected in request - searching all partitions");
+		RequestPartitionId thePartitionId = defaultIfNull(theRequestPartitionId, RequestPartitionId.allPartitions());
+
+		SearchParameterMap searchParamMap;
+		SystemRequestDetails request = new SystemRequestDetails();
+		request.setRequestPartitionId(thePartitionId);
+
+		if (isNoResourceTypeProvidedInUrl(theUrl, resourceType)) {
+			searchParamMap = parseQuery(theUrl, null);
+		} else {
+			searchParamMap = parseQuery(theUrl);
+		}
+
 		searchParamMap.setLastUpdated(DateRangeUtil.narrowDateRange(searchParamMap.getLastUpdated(), theStart, theEnd));
 
-		String resourceType = theUrl.substring(0, theUrl.indexOf('?'));
-		IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(resourceType);
+		if (isNoResourceTypeProvidedInUrl(theUrl, resourceType)) {
+			return searchForResourceIdsAndType(thePartitionId, request, searchParamMap);
+		}
 
-		SystemRequestDetails request = new SystemRequestDetails().setRequestPartitionId(theRequestPartitionId);
+		IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(resourceType);
 
 		return dao.searchForIdStream(searchParamMap, request, null).map(pid -> new TypedResourcePid(resourceType, pid));
 	}
 
-	private static TypedResourcePid typedPidFromQueryArray(Object[] thePidTypeDateArray) {
-		JpaPid pid = (JpaPid) thePidTypeDateArray[0];
-		String resourceType = (String) thePidTypeDateArray[1];
-		return new TypedResourcePid(resourceType, pid);
+	/**
+	 * Since the resource type is not specified, query the DB for resources matching the params and return resource ID and type
+	 *
+	 * @param theRequestPartitionId the partition to search on
+	 * @param theRequestDetails the theRequestDetails details
+	 * @param theSearchParams the search params
+	 * @return Stream of typed resource pids
+	 */
+	private Stream<TypedResourcePid> searchForResourceIdsAndType(
+			RequestPartitionId theRequestPartitionId,
+			SystemRequestDetails theRequestDetails,
+			SearchParameterMap theSearchParams) {
+		ISearchBuilder<JpaPid> builder = mySearchBuilderFactory.newSearchBuilder(null, null);
+		return myTransactionService
+				.withRequest(theRequestDetails)
+				.search(() -> builder.createQueryStream(
+						theSearchParams,
+						new SearchRuntimeDetails(
+								theRequestDetails, UUID.randomUUID().toString()),
+						theRequestDetails,
+						theRequestPartitionId))
+				.map(pid -> new TypedResourcePid(pid.getResourceType(), pid));
 	}
 
 	@Nonnull
@@ -155,6 +202,7 @@ private Stream<TypedResourcePid> streamResourceIdsNoUrl(
 			Date theStart, Date theEnd, RequestPartitionId theRequestPartitionId) {
 		Integer defaultPartitionId = myPartitionSettings.getDefaultPartitionId();
 		Stream<Object[]> rowStream;
+
 		if (theRequestPartitionId == null || theRequestPartitionId.isAllPartitions()) {
 			ourLog.debug("Search for resources - all partitions");
 			rowStream = myResourceTableDao.streamIdsTypesAndUpdateTimesOfResourcesWithinUpdatedRangeOrderedFromOldest(
@@ -184,22 +232,38 @@ public IResourcePidList fetchResourceIdsPage(
 		return null;
 	}
 
-	private static void validateUrl(@Nonnull String theUrl) {
-		if (!theUrl.contains("?")) {
-			throw new InternalErrorException(Msg.code(2422) + "this should never happen: URL is missing a '?'");
-		}
-	}
-
 	@Nonnull
 	private SearchParameterMap parseQuery(String theUrl) {
 		String resourceType = theUrl.substring(0, theUrl.indexOf('?'));
 		RuntimeResourceDefinition def = myFhirContext.getResourceDefinition(resourceType);
 
-		SearchParameterMap searchParamMap = myMatchUrlService.translateMatchUrl(theUrl, def);
+		return parseQuery(theUrl, def);
+	}
+
+	@Nonnull
+	private SearchParameterMap parseQuery(
+			String theUrl, @Nullable RuntimeResourceDefinition theRuntimeResourceDefinition) {
+		SearchParameterMap searchParamMap = myMatchUrlService.translateMatchUrl(theUrl, theRuntimeResourceDefinition);
 		// this matches idx_res_type_del_updated
 		searchParamMap.setSort(new SortSpec(Constants.PARAM_LASTUPDATED).setChain(new SortSpec(Constants.PARAM_PID)));
 		// TODO this limits us to 2G resources.
 		searchParamMap.setLoadSynchronousUpTo(Integer.MAX_VALUE);
 		return searchParamMap;
 	}
+
+	private static TypedResourcePid typedPidFromQueryArray(Object[] thePidTypeDateArray) {
+		JpaPid pid = (JpaPid) thePidTypeDateArray[0];
+		String resourceType = (String) thePidTypeDateArray[1];
+		return new TypedResourcePid(resourceType, pid);
+	}
+
+	private static void validateUrl(@Nonnull String theUrl) {
+		if (!theUrl.contains("?")) {
+			throw new InternalErrorException(Msg.code(2422) + "this should never happen: URL is missing a '?'");
+		}
+	}
+
+	private static boolean isNoResourceTypeProvidedInUrl(String theUrl, String resourceType) {
+		return (resourceType == null || resourceType.isBlank()) && theUrl.indexOf('?') == 0;
+	}
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/Batch2SupportConfig.java

@@ -27,6 +27,7 @@
 import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
 import ca.uhn.fhir.jpa.batch2.Batch2DaoSvcImpl;
 import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
+import ca.uhn.fhir.jpa.dao.SearchBuilderFactory;
 import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao;
 import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
 import ca.uhn.fhir.jpa.dao.expunge.ResourceTableFKProvider;
@@ -49,15 +50,17 @@ public IBatch2DaoSvc batch2DaoSvc(
 			DaoRegistry theDaoRegistry,
 			FhirContext theFhirContext,
 			IHapiTransactionService theTransactionService,
-			PartitionSettings thePartitionSettings) {
+			PartitionSettings thePartitionSettings,
+			SearchBuilderFactory theSearchBuilderFactory) {
 		return new Batch2DaoSvcImpl(
 				theResourceTableDao,
 				theResourceLinkDao,
 				theMatchUrlService,
 				theDaoRegistry,
 				theFhirContext,
 				theTransactionService,
-				thePartitionSettings);
+				thePartitionSettings,
+				theSearchBuilderFactory);
 	}
 
 	@Bean

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaConfig.java

@@ -190,6 +190,7 @@
 import ca.uhn.fhir.replacereferences.ReplaceReferencesPatchBundleSvc;
 import ca.uhn.fhir.replacereferences.ReplaceReferencesProvenanceSvc;
 import ca.uhn.fhir.replacereferences.UndoReplaceReferencesSvc;
+import ca.uhn.fhir.rest.api.SearchIncludeDeletedEnum;
 import ca.uhn.fhir.rest.api.server.RequestDetails;
 import ca.uhn.fhir.rest.api.server.storage.IDeleteExpungeJobSubmitter;
 import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
@@ -596,8 +597,8 @@ public PersistedJpaBundleProviderFactory persistedJpaBundleProviderFactory() {
 	}
 
 	@Bean
-	public SearchBuilderFactory searchBuilderFactory() {
-		return new SearchBuilderFactory();
+	public SearchBuilderFactory<JpaPid> searchBuilderFactory() {
+		return new SearchBuilderFactory<>();
 	}
 
 	@Bean
@@ -725,8 +726,9 @@ public ResourceLinkPredicateBuilder newResourceLinkPredicateBuilder(
 
 	@Bean
 	@Scope("prototype")
-	public ResourceTablePredicateBuilder newResourceTablePredicateBuilder(SearchQueryBuilder theSearchBuilder) {
-		return new ResourceTablePredicateBuilder(theSearchBuilder);
+	public ResourceTablePredicateBuilder newResourceTablePredicateBuilder(
+			SearchQueryBuilder theSearchBuilder, SearchIncludeDeletedEnum theSearchIncludeDeleted) {
+		return new ResourceTablePredicateBuilder(theSearchBuilder, theSearchIncludeDeleted);
 	}
 
 	@Bean