diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java index 0ed2de94855a..c2da71730f74 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java @@ -22,7 +22,6 @@ import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.config.BaseBatch2Config; import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; -import ca.uhn.fhir.jpa.bulk.export.job.BulkExportJobConfig; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkMetadataViewRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; @@ -30,10 +29,8 @@ import jakarta.persistence.EntityManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; @Configuration -@Import({BulkExportJobConfig.class}) public class JpaBatch2Config extends BaseBatch2Config { @Bean diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/job/BulkExportJobConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/job/BulkExportJobConfig.java deleted file mode 100644 index 2c6286bc78f5..000000000000 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/job/BulkExportJobConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/*- - * #%L - * HAPI FHIR JPA Server - * %% - * Copyright (C) 2014 - 2025 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package ca.uhn.fhir.jpa.bulk.export.job; - -import ca.uhn.fhir.mdm.svc.MdmExpansionCacheSvc; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * Spring batch Job configuration file. Contains all necessary plumbing to run a - * Bulk Export job. - */ -@Configuration -public class BulkExportJobConfig { - - @Bean - public MdmExpansionCacheSvc mdmExpansionCacheSvc() { - return new MdmExpansionCacheSvc(); - } -} diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessor.java index 79485f5d3d90..e37c52bb6e49 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessor.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessor.java @@ -37,7 +37,7 @@ import ca.uhn.fhir.jpa.model.search.SearchBuilderLoadIncludesParameters; import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.mdm.svc.MdmExpandersHolder; +import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.model.primitive.IdDt; import ca.uhn.fhir.rest.api.server.SystemRequestDetails; @@ -86,7 +86,7 @@ public class JpaBulkExportProcessor implements IBulkExportProcessor { private EntityManager myEntityManager; private IHapiTransactionService myHapiTransactionService; private ISearchParamRegistry mySearchParamRegistry; - private MdmExpandersHolder myMdmExpandersHolder; + private Optional myMdmLinkExpandSvc; @Autowired public JpaBulkExportProcessor( @@ -99,7 +99,7 @@ public JpaBulkExportProcessor( EntityManager theEntityManager, IHapiTransactionService theHapiTransactionService, ISearchParamRegistry theSearchParamRegistry, - MdmExpandersHolder theMdmExpandersHolder) { + Optional theMdmLinkExpandSvc) { myContext = theContext; myBulkExportHelperSvc = theBulkExportHelperSvc; myStorageSettings = theStorageSettings; @@ -109,7 +109,7 @@ public JpaBulkExportProcessor( myEntityManager = theEntityManager; myHapiTransactionService = theHapiTransactionService; mySearchParamRegistry = theSearchParamRegistry; - myMdmExpandersHolder = theMdmExpandersHolder; + myMdmLinkExpandSvc = theMdmLinkExpandSvc; } @Override @@ -345,15 +345,6 @@ protected RuntimeSearchParam getPatientSearchParamForCurrentResourceType(String return searchParam; } - @Override - public void expandMdmResources(List theResources) { - for (IBaseResource resource : theResources) { - if (!PATIENT_BULK_EXPORT_FORWARD_REFERENCE_RESOURCE_TYPES.contains(resource.fhirType())) { - myMdmExpandersHolder.getBulkExportMDMResourceExpanderInstance().annotateResource(resource); - } - } - } - /** * For Patient **/ @@ -389,7 +380,7 @@ private void validateSearchParametersForGroup(SearchParameterMap expandedSpMap, /** * Given the local myGroupId, perform an expansion to retrieve all resource IDs of member patients. * if myMdmEnabled is set to true, we also attempt to also expand it into matched - * patients. + * patients, assuming MDM is configured. * * @return a Set of Strings representing the resource IDs of all members of a group. */ @@ -404,10 +395,14 @@ private LinkedHashSet getExpandedPatientList( LinkedHashSet patientPidsToExport = new LinkedHashSet<>(members); if (theParameters.isExpandMdm()) { - RequestPartitionId partitionId = theParameters.getPartitionIdOrAllPartitions(); - patientPidsToExport.addAll(myMdmExpandersHolder - .getBulkExportMDMResourceExpanderInstance() - .expandGroup(theParameters.getGroupId(), partitionId)); + if (myMdmLinkExpandSvc.isPresent()) { + RequestPartitionId partitionId = theParameters.getPartitionIdOrAllPartitions(); + IMdmLinkExpandSvc iMdmLinkExpandSvc = myMdmLinkExpandSvc.get(); + patientPidsToExport.addAll(iMdmLinkExpandSvc.expandGroup(theParameters.getGroupId(), partitionId)); + } else { + ourLog.warn( + "Attempted to perform MDM expansion during a group-level export operation, but no IMdmLinkExpandSvc was configured. Is MDM Configured correctly?"); + } } return patientPidsToExport; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaBulkExportConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaBulkExportConfig.java index f08fffc689b6..39d5c85e573b 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaBulkExportConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/JpaBulkExportConfig.java @@ -29,12 +29,14 @@ import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.jpa.model.dao.JpaPid; -import ca.uhn.fhir.mdm.svc.MdmExpandersHolder; +import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; import jakarta.persistence.EntityManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.Optional; + @Configuration public class JpaBulkExportConfig { @Bean @@ -48,7 +50,7 @@ public IBulkExportProcessor jpaBulkExportProcessor( EntityManager theEntityManager, IHapiTransactionService theHapiTransactionService, ISearchParamRegistry theSearchParamRegistry, - MdmExpandersHolder theMdmExpandersHolder) { + Optional theMdmLinkExpandSvc) { return new JpaBulkExportProcessor( theFhirContext, theBulkExportHelperService, @@ -59,7 +61,7 @@ public IBulkExportProcessor jpaBulkExportProcessor( theEntityManager, theHapiTransactionService, theSearchParamRegistry, - theMdmExpandersHolder); + theMdmLinkExpandSvc); } @Bean diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/MdmJpaConfig.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/MdmJpaConfig.java index acf2b94bff19..a7d3aac9931f 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/MdmJpaConfig.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/MdmJpaConfig.java @@ -30,47 +30,23 @@ import ca.uhn.fhir.jpa.entity.MdmLink; import ca.uhn.fhir.jpa.model.dao.JpaPid; import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; +import ca.uhn.fhir.mdm.api.IMdmSettings; import ca.uhn.fhir.mdm.dao.IMdmLinkDao; import ca.uhn.fhir.mdm.dao.IMdmLinkImplFactory; -import ca.uhn.fhir.mdm.svc.BulkExportMdmEidMatchOnlyResourceExpander; -import ca.uhn.fhir.mdm.svc.BulkExportMdmResourceExpander; +import ca.uhn.fhir.mdm.svc.DisabledMdmLinkExpandSvc; import ca.uhn.fhir.mdm.svc.MdmEidMatchOnlyExpandSvc; -import ca.uhn.fhir.mdm.svc.MdmExpandersHolder; -import ca.uhn.fhir.mdm.svc.MdmExpansionCacheSvc; import ca.uhn.fhir.mdm.svc.MdmLinkExpandSvc; import ca.uhn.fhir.mdm.svc.MdmSearchExpansionSvc; +import ca.uhn.fhir.mdm.util.EIDHelper; +import jakarta.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; @Configuration public class MdmJpaConfig { - - @Bean - public MdmExpandersHolder mdmLinkExpandSvcHolder( - FhirContext theFhirContext, - IMdmLinkExpandSvc theMdmLinkExpandSvc, - MdmEidMatchOnlyExpandSvc theMdmEidMatchOnlyLinkExpandSvc, - BulkExportMdmEidMatchOnlyResourceExpander theBulkExportMdmEidMatchOnlyResourceExpander, - BulkExportMdmResourceExpander theBulkExportMdmResourceExpander) { - return new MdmExpandersHolder( - theFhirContext, - theMdmLinkExpandSvc, - theMdmEidMatchOnlyLinkExpandSvc, - theBulkExportMdmResourceExpander, - theBulkExportMdmEidMatchOnlyResourceExpander); - } - - @Bean - public MdmEidMatchOnlyExpandSvc mdmEidMatchOnlyLinkExpandSvc(DaoRegistry theDaoRegistry) { - return new MdmEidMatchOnlyExpandSvc(theDaoRegistry); - } - - @Bean - @Primary - public IMdmLinkExpandSvc mdmLinkExpandSvc() { - return new MdmLinkExpandSvc(); - } + private static final Logger ourLog = LoggerFactory.getLogger(MdmJpaConfig.class); @Bean public MdmSearchExpansionSvc mdmSearchExpansionSvc() { @@ -83,33 +59,41 @@ public IMdmLinkDao mdmLinkDao() { } @Bean - public BulkExportMdmResourceExpander bulkExportMDMResourceExpander( - MdmExpansionCacheSvc theMdmExpansionCacheSvc, - IMdmLinkDao theMdmLinkDao, - IIdHelperService theIdHelperService, - DaoRegistry theDaoRegistry, - FhirContext theFhirContext) { - return new BulkExportMdmResourceExpander( - theMdmExpansionCacheSvc, theMdmLinkDao, theIdHelperService, theDaoRegistry, theFhirContext); + public IMdmLinkImplFactory mdmLinkImplFactory() { + return new JpaMdmLinkImplFactory(); } + /** + * Based on the rules laid out in the {@link IMdmSettings} file, construct an {@link IMdmLinkExpandSvc} that is suitable + */ @Bean - public BulkExportMdmEidMatchOnlyResourceExpander bulkExportMDMEidMatchOnlyResourceExpander( + public IMdmLinkExpandSvc mdmLinkExpandSvc( + EIDHelper theEidHelper, + @Nullable IMdmSettings theMdmSettings, DaoRegistry theDaoRegistry, - MdmEidMatchOnlyExpandSvc theMdmEidMatchOnlyLinkExpandSvc, FhirContext theFhirContext, - IIdHelperService theIdHelperService) { - return new BulkExportMdmEidMatchOnlyResourceExpander( - theDaoRegistry, theMdmEidMatchOnlyLinkExpandSvc, theFhirContext, theIdHelperService); + IIdHelperService theIdHelperService) { + if (theMdmSettings == null) { + return new DisabledMdmLinkExpandSvc(); + } + if (theMdmSettings.supportsLinkBasedExpansion()) { + return new MdmLinkExpandSvc(); + } else if (theMdmSettings.supportsEidBasedExpansion()) { + return new MdmEidMatchOnlyExpandSvc(theDaoRegistry, theFhirContext, theIdHelperService, theEidHelper); + } + return new DisabledMdmLinkExpandSvc(); } @Bean - public IMdmLinkImplFactory mdmLinkImplFactory() { - return new JpaMdmLinkImplFactory(); + EIDHelper eidHelper(FhirContext theFhirContext, @Nullable IMdmSettings theMdmSettings) { + if (theMdmSettings == null) { + ourLog.warn("Loading up an EID helper without an IMdmSetting bean defined! MDM will _not_ work!"); + } + return new EIDHelper(theFhirContext, theMdmSettings); } @Bean - public IMdmClearHelperSvc helperSvc(IDeleteExpungeSvc theDeleteExpungeSvc) { + public IMdmClearHelperSvc mdmClearHelperSvc(IDeleteExpungeSvc theDeleteExpungeSvc) { return new MdmClearHelperSvcImpl(theDeleteExpungeSvc); } } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessorTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessorTest.java index 3a1adefeffc1..afaf39e946ff 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessorTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/bulk/export/svc/JpaBulkExportProcessorTest.java @@ -8,7 +8,8 @@ import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.api.svc.IIdHelperService; -import ca.uhn.fhir.mdm.svc.IBulkExportMdmResourceExpander; +import ca.uhn.fhir.mdm.api.IMdmLink; +import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; import ca.uhn.fhir.jpa.bulk.export.model.ExportPIDIteratorParameters; import ca.uhn.fhir.jpa.dao.IResultIterator; import ca.uhn.fhir.jpa.dao.ISearchBuilder; @@ -19,7 +20,6 @@ import ca.uhn.fhir.jpa.model.search.SearchBuilderLoadIncludesParameters; import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.mdm.svc.MdmExpandersHolder; import ca.uhn.fhir.model.primitive.IdDt; import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; import ca.uhn.fhir.rest.api.server.RequestDetails; @@ -52,6 +52,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; @@ -131,10 +132,10 @@ public JpaPid next() { private IIdHelperService myIdHelperService; @Mock - private MdmExpandersHolder myMdmExpandersHolder; + private Optional myOptionalMdmLinkExpanderService; @Mock - private IBulkExportMdmResourceExpander myBulkExportMDMResourceExpander; + private IMdmLinkExpandSvc myMdmLinkExpanderService; @Mock private ISearchParamRegistry mySearchParamRegistry; @@ -299,9 +300,10 @@ public void getResourcePidIterator_groupExportStyleWithPatientResource_returnsIt final JpaPid mdmExpandedPatientId = JpaPid.fromId(4567L); if (theMdm) { - when(myMdmExpandersHolder.getBulkExportMDMResourceExpanderInstance()).thenReturn(myBulkExportMDMResourceExpander); // mock the call to expandGroup method of the expander - when(myBulkExportMDMResourceExpander.expandGroup(parameters.getGroupId(), getPartitionIdFromParams(thePartitioned))) + when(myOptionalMdmLinkExpanderService.isPresent()).thenReturn(true); + when(myOptionalMdmLinkExpanderService.get()).thenReturn(myMdmLinkExpanderService); + when(myMdmLinkExpanderService.expandGroup(parameters.getGroupId(), getPartitionIdFromParams(thePartitioned))) .thenReturn(Set.of(mdmExpandedPatientId)); } @@ -395,9 +397,10 @@ public void getResourcePidIterator_groupExportStyleWithNonPatientResource_return .thenReturn(observationResultsIterator); if (theMdm) { - when(myMdmExpandersHolder.getBulkExportMDMResourceExpanderInstance()).thenReturn(myBulkExportMDMResourceExpander); // mock the call to expandGroup method of the expander - when(myBulkExportMDMResourceExpander.expandGroup(parameters.getGroupId(), getPartitionIdFromParams(thePartitioned))) + when(myOptionalMdmLinkExpanderService.isPresent()).thenReturn(true); + when(myOptionalMdmLinkExpanderService.get()).thenReturn(myMdmLinkExpanderService); + when(myMdmLinkExpanderService.expandGroup(parameters.getGroupId(), getPartitionIdFromParams(thePartitioned))) .thenReturn(Collections.emptySet()); } diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/BaseMdmR4Test.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/BaseMdmR4Test.java index d88775506b71..fd7f9214c8f3 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/BaseMdmR4Test.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/BaseMdmR4Test.java @@ -28,6 +28,7 @@ import ca.uhn.fhir.mdm.api.MdmLinkSourceEnum; import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; import ca.uhn.fhir.mdm.dao.IMdmLinkDao; +import ca.uhn.fhir.mdm.interceptor.MdmStorageInterceptor; import ca.uhn.fhir.mdm.model.MdmTransactionContext; import ca.uhn.fhir.mdm.rules.config.MdmSettings; import ca.uhn.fhir.mdm.rules.svc.MdmResourceMatcherSvc; @@ -60,6 +61,7 @@ import java.util.Optional; import java.util.function.Function; +import static java.util.stream.Collectors.joining; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.slf4j.LoggerFactory.getLogger; @@ -91,6 +93,8 @@ abstract public class BaseMdmR4Test extends BaseJpaR4Test { .setValue("555-555-5555"); private static final String NAME_GIVEN_FRANK = "Frank"; + @Autowired(required = false) + private MdmStorageInterceptor myMdmStorageInterceptor; @Autowired protected IFhirResourceDaoPatient myPatientDao; @Autowired @@ -155,6 +159,37 @@ protected void saveLink(MdmLink theMdmLink) { protected GoldenResourceMatchingAssert mdmAssertThat(IAnyResource theResource) { return GoldenResourceMatchingAssert.assertThat(theResource, myIdHelperService, myMdmLinkDaoSvc); } + + + @Override + public void afterPurgeDatabase() { + boolean registeredStorageInterceptor = false; + if (myMdmStorageInterceptor != null && !myInterceptorService.getAllRegisteredInterceptors().contains(myMdmStorageInterceptor)) { + myInterceptorService.registerInterceptor(myMdmStorageInterceptor); + registeredStorageInterceptor = true; + } + runInTransaction(() -> { + myMdmLinkDao.deleteAll(); + }); + super.afterPurgeDatabase(); + + if (registeredStorageInterceptor) { + myInterceptorService.unregisterInterceptor(myMdmStorageInterceptor); + } + + } + + protected int logAllMdmLinks() { + return runInTransaction(()->{ + List links = myMdmLinkDao.findAll(); + if (links.isEmpty()) { + ourLog.info("MDM Links: NONE"); + } else { + ourLog.info("MDM Links:\n * {}", links.stream().map(t -> t.toString()).collect(joining("\n * "))); + } + return links.size(); + }); + } @Nonnull protected Patient createGoldenPatient() { return createPatient(new Patient(), true, false); diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/config/MdmConfigTest.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/config/MdmConfigTest.java deleted file mode 100644 index 437f0d4106b1..000000000000 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/config/MdmConfigTest.java +++ /dev/null @@ -1,2 +0,0 @@ -package ca.uhn.fhir.jpa.mdm.config;public class MdmConfigTest { -} diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmReadVirtualizationInterceptorTest.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmReadVirtualizationInterceptorTest.java index 4e1fac36e2d9..bf3cbd80e5d4 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmReadVirtualizationInterceptorTest.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/jpa/mdm/interceptor/MdmReadVirtualizationInterceptorTest.java @@ -9,9 +9,7 @@ import ca.uhn.fhir.jpa.mdm.helper.testmodels.MDMState; import ca.uhn.fhir.jpa.model.dao.JpaPid; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.mdm.api.IMdmSettings; import ca.uhn.fhir.mdm.interceptor.MdmReadVirtualizationInterceptor; -import ca.uhn.fhir.mdm.svc.MdmExpandersHolder; import ca.uhn.fhir.rest.api.RestOperationTypeEnum; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.SystemRequestDetails; diff --git a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/mdm/batch2/clear/MdmLinkSlowDeletionSandboxIT.java b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/mdm/batch2/clear/MdmLinkSlowDeletionSandboxIT.java index 5cba19589010..0aebcda73488 100644 --- a/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/mdm/batch2/clear/MdmLinkSlowDeletionSandboxIT.java +++ b/hapi-fhir-jpaserver-mdm/src/test/java/ca/uhn/fhir/mdm/batch2/clear/MdmLinkSlowDeletionSandboxIT.java @@ -1,6 +1,7 @@ package ca.uhn.fhir.mdm.batch2.clear; import ca.uhn.fhir.jpa.entity.MdmLink; +import ca.uhn.fhir.jpa.mdm.BaseMdmR4Test; import ca.uhn.fhir.jpa.model.dao.JpaPid; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.jpa.test.config.TestR4Config; @@ -32,17 +33,17 @@ @Disabled("Keeping as a sandbox to be used whenever we need a lot of MdmLinks in DB for performance testing") @ContextConfiguration(classes = {MdmLinkSlowDeletionSandboxIT.TestDataSource.class}) -public class MdmLinkSlowDeletionSandboxIT extends BaseJpaR4Test { +public class MdmLinkSlowDeletionSandboxIT extends BaseMdmR4Test { private static final Logger ourLog = LoggerFactory.getLogger(MdmLinkSlowDeletionSandboxIT.class); private final int ourMdmLinksToCreate = 1_000_000; private final int ourLogMdmLinksEach = 1_000; + /** + * Overridden so we don't purge the DB in between tests. + */ @Override - public void afterPurgeDatabase() { - // keep the generated data! -// super.afterPurgeDatabase(); - } + public void afterPurgeDatabase() {} @Disabled @Test @@ -54,7 +55,6 @@ void createMdmLinks() { assertTrue(totalLinks > 0); } - private void generatePatientsAndMdmLinks(int theLinkCount) { StopWatch sw = new StopWatch(); int totalMdmLinksCreated = 0; diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java index 06785e6071c4..e1b44d6669a9 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseTest.java @@ -12,6 +12,7 @@ import ca.uhn.fhir.jpa.api.model.BulkExportJobResults; import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome; import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; +import ca.uhn.fhir.jpa.bulk.config.MdmRulesWithEidMatchOnlyConfig; import ca.uhn.fhir.jpa.bulk.export.model.BulkExportResponseJson; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; @@ -20,11 +21,7 @@ import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; -import ca.uhn.fhir.mdm.api.MdmModeEnum; import ca.uhn.fhir.mdm.rules.config.MdmRuleValidator; -import ca.uhn.fhir.mdm.rules.config.MdmSettings; -import ca.uhn.fhir.mdm.rules.json.MdmRulesJson; -import ca.uhn.fhir.mdm.svc.MdmExpandersHolder; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.MethodOutcome; @@ -75,6 +72,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.ContextConfiguration; import java.io.IOException; import java.util.ArrayList; @@ -102,14 +101,13 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; - +@Import(MdmRulesWithEidMatchOnlyConfig.class) class BulkExportUseCaseTest extends BaseResourceProviderR4Test { private static final Logger ourLog = LoggerFactory.getLogger(BulkExportUseCaseTest.class); private static final String TEST_PATIENT_EID_SYS = "http://patient-eid-sys"; @Autowired private IJobCoordinator myJobCoordinator; - @Autowired private IJobPersistence myJobPersistence; @Autowired @@ -122,15 +120,12 @@ class BulkExportUseCaseTest extends BaseResourceProviderR4Test { private IInterceptorService myInterceptorService; @Autowired private MdmRuleValidator myMdmRulesValidator; - @Autowired - private MdmExpandersHolder myMdmExpandersHolder; @BeforeEach void beforeEach() { myStorageSettings.setJobFastTrackingEnabled(false); } - @Nested class SpecConformanceTests { @@ -359,27 +354,6 @@ void export_shouldNotExportBinaryResource_whenTypeParameterOmitted() throws IOEx } - private String submitBulkExportForTypes(String... theTypes) throws IOException { - return submitBulkExportForTypesWithExportId(null, theTypes); - } - - private String submitBulkExportForTypesWithExportId(String theExportId, String... theTypes) throws IOException { - String typeString = String.join(",", theTypes); - String uri = myClient.getServerBase() + "/$export?_type=" + typeString; - if (!StringUtils.isBlank(theExportId)) { - uri += "&_exportId=" + theExportId; - } - - HttpGet httpGet = new HttpGet(uri); - httpGet.addHeader(Constants.HEADER_PREFER, Constants.HEADER_PREFER_RESPOND_ASYNC); - String pollingLocation; - try (CloseableHttpResponse status = ourHttpClient.execute(httpGet)) { - Header[] headers = status.getHeaders("Content-Location"); - pollingLocation = headers[0].getValue(); - } - return pollingLocation; - } - @Nested class SystemBulkExportTests { @@ -686,11 +660,6 @@ void testExportEmptyResult() { @Nested class GroupBulkExportTests { - @AfterEach - void tearDown() { - restoreMdmSettingsToDefault(); - } - @Test void testGroupExportSuccessfulyExportsPatientForwardReferences() { BundleBuilder bb = new BundleBuilder(myFhirContext); @@ -800,6 +769,58 @@ void testGroupBulkExportMembershipShouldNotExpandIntoOtherGroups() { assertThat(firstMap.get("Group")).hasSize(1); } + @Test + void testGroupExportWithMdmEnabled_EidMatchOnly() { + BundleBuilder bb = new BundleBuilder(myFhirContext); + + //In this test, we create two patients with the same Eid value for the eid system specified in mdm rules + //and 2 observations referencing one of each of these patients + //Create a group that contains one of the patients. + //When we export the group, we should get both patients and the 2 observations + //in the export as the other patient should be mdm expanded + //based on having the same eid value + Patient pat1 = new Patient(); + pat1.setId("pat-1"); + pat1.addIdentifier(new Identifier().setSystem(TEST_PATIENT_EID_SYS).setValue("the-patient-eid-value")); + bb.addTransactionUpdateEntry(pat1); + + Observation obs1 = new Observation(); + obs1.setId("obs-1"); + obs1.setSubject(new Reference("Patient/pat-1")); + bb.addTransactionUpdateEntry(obs1); + + Patient pat2 = new Patient(); + pat2.setId("pat-2"); + pat2.addIdentifier(new Identifier().setSystem(TEST_PATIENT_EID_SYS).setValue("the-patient-eid-value")); + bb.addTransactionUpdateEntry(pat2); + + Observation obs2 = new Observation(); + obs2.setId("obs-2"); + obs2.setSubject(new Reference("Patient/pat-2")); + bb.addTransactionUpdateEntry(obs2); + + Group group = new Group(); + group.setId("Group/mdm-group"); + group.setActive(true); + group.addMember().getEntity().setReference("Patient/pat-1"); + bb.addTransactionUpdateEntry(group); + + myClient.transaction().withBundle(bb.getBundle()).execute(); + + BulkExportJobResults bulkExportJobResults = startGroupBulkExportJobAndAwaitCompletionForMdmExpand(new HashSet<>(), new HashSet<>(), "mdm-group", true); + Map> exportedResourcesMap = convertJobResultsToResources(bulkExportJobResults); + + assertThat(exportedResourcesMap.keySet()).hasSize(3); + List exportedGroups = exportedResourcesMap.get("Group"); + assertResourcesIds(exportedGroups, "Group/mdm-group"); + + List exportedPatients = exportedResourcesMap.get("Patient"); + assertResourcesIds(exportedPatients, "Patient/pat-1", "Patient/pat-2"); + + List exportedObservations = exportedResourcesMap.get("Observation"); + assertResourcesIds(exportedObservations, "Observation/obs-1", "Observation/obs-2"); + } + @Test void testDifferentTypesDoNotUseCachedResults() { Patient patient = new Patient(); @@ -1772,6 +1793,26 @@ void testSystemBulkExportWithHistory_WithClientAssignedIds() { } + private String submitBulkExportForTypes(String... theTypes) throws IOException { + return submitBulkExportForTypesWithExportId(null, theTypes); + } + + private String submitBulkExportForTypesWithExportId(String theExportId, String... theTypes) throws IOException { + String typeString = String.join(",", theTypes); + String uri = myClient.getServerBase() + "/$export?_type=" + typeString; + if (!StringUtils.isBlank(theExportId)) { + uri += "&_exportId=" + theExportId; + } + + HttpGet httpGet = new HttpGet(uri); + httpGet.addHeader(Constants.HEADER_PREFER, Constants.HEADER_PREFER_RESPOND_ASYNC); + String pollingLocation; + try (CloseableHttpResponse status = ourHttpClient.execute(httpGet)) { + Header[] headers = status.getHeaders("Content-Location"); + pollingLocation = headers[0].getValue(); + } + return pollingLocation; + } private Map>> convertJobResultsToResourceVersionMap(BulkExportJobResults theBulkExportJobResults) { Map> exportedResourcesByType = convertJobResultsToResources(theBulkExportJobResults); @@ -1962,77 +2003,10 @@ private IIdType createPatient(int i) { return myPatientDao.create(p, mySrd).getId(); } - @Test - void testGroupExportWithMdmEnabled_EidMatchOnly() { - - createAndSetMdmSettingsForEidMatchOnly(); - BundleBuilder bb = new BundleBuilder(myFhirContext); - - //In this test, we create two patients with the same Eid value for the eid system specified in mdm rules - //and 2 observations referencing one of each of these patients - //Create a group that contains one of the patients. - //When we export the group, we should get both patients and the 2 observations - //in the export as the other patient should be mdm expanded - //based on having the same eid value - Patient pat1 = new Patient(); - pat1.setId("pat-1"); - pat1.addIdentifier(new Identifier().setSystem(TEST_PATIENT_EID_SYS).setValue("the-patient-eid-value")); - bb.addTransactionUpdateEntry(pat1); - - Observation obs1 = new Observation(); - obs1.setId("obs-1"); - obs1.setSubject(new Reference("Patient/pat-1")); - bb.addTransactionUpdateEntry(obs1); - - Patient pat2 = new Patient(); - pat2.setId("pat-2"); - pat2.addIdentifier(new Identifier().setSystem(TEST_PATIENT_EID_SYS).setValue("the-patient-eid-value")); - bb.addTransactionUpdateEntry(pat2); - - Observation obs2 = new Observation(); - obs2.setId("obs-2"); - obs2.setSubject(new Reference("Patient/pat-2")); - bb.addTransactionUpdateEntry(obs2); - Group group = new Group(); - group.setId("Group/mdm-group"); - group.setActive(true); - group.addMember().getEntity().setReference("Patient/pat-1"); - bb.addTransactionUpdateEntry(group); - - myClient.transaction().withBundle(bb.getBundle()).execute(); - BulkExportJobResults bulkExportJobResults = startGroupBulkExportJobAndAwaitCompletionForMdmExpand(new HashSet<>(), new HashSet<>(), "mdm-group", true); - Map> exportedResourcesMap = convertJobResultsToResources(bulkExportJobResults); - assertThat(exportedResourcesMap.keySet()).hasSize(3); - List exportedGroups = exportedResourcesMap.get("Group"); - assertResourcesIds(exportedGroups, "Group/mdm-group"); - List exportedPatients = exportedResourcesMap.get("Patient"); - assertResourcesIds(exportedPatients, "Patient/pat-1", "Patient/pat-2"); - - List exportedObservations = exportedResourcesMap.get("Observation"); - assertResourcesIds(exportedObservations, "Observation/obs-1", "Observation/obs-2"); - - } - - - private void createAndSetMdmSettingsForEidMatchOnly() { - MdmSettings mdmSettings = new MdmSettings(myMdmRulesValidator); - mdmSettings.setEnabled(true); - mdmSettings.setMdmMode(MdmModeEnum.MATCH_ONLY); - MdmRulesJson rules = new MdmRulesJson(); - rules.setMdmTypes(List.of("Patient")); - rules.addEnterpriseEIDSystem("Patient", TEST_PATIENT_EID_SYS); - mdmSettings.setMdmRules(rules); - - myMdmExpandersHolder.setMdmSettings(mdmSettings); - } - - private void restoreMdmSettingsToDefault() { - myMdmExpandersHolder.setMdmSettings(new MdmSettings(myMdmRulesValidator)); - } private static void assertResourcesIds(List theResources, String... theExpectedResourceIds) { assertThat(theResources).hasSize(theExpectedResourceIds.length); @@ -2279,4 +2253,6 @@ private void verifyBulkExportResults(@SuppressWarnings("SameParameterValue") Str private BulkExportJobResults startPatientBulkExportJobAndAwaitResults(HashSet theTypes, HashSet theFilters, @SuppressWarnings("SameParameterValue") String thePatientId) { return startBulkExportJobAndAwaitCompletion(BulkExportJobParameters.ExportStyle.PATIENT, theTypes, theFilters, thePatientId, false); } + + } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/config/MdmRulesWithEidMatchOnlyConfig.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/config/MdmRulesWithEidMatchOnlyConfig.java new file mode 100644 index 000000000000..66bd1dfef5eb --- /dev/null +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/config/MdmRulesWithEidMatchOnlyConfig.java @@ -0,0 +1,81 @@ +package ca.uhn.fhir.jpa.bulk.config; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.api.svc.IDeleteExpungeSvc; +import ca.uhn.fhir.jpa.api.svc.IIdHelperService; +import ca.uhn.fhir.jpa.api.svc.IMdmClearHelperSvc; +import ca.uhn.fhir.jpa.bulk.mdm.MdmClearHelperSvcImpl; +import ca.uhn.fhir.jpa.dao.mdm.JpaMdmLinkImplFactory; +import ca.uhn.fhir.jpa.dao.mdm.MdmLinkDaoJpaImpl; +import ca.uhn.fhir.jpa.entity.MdmLink; +import ca.uhn.fhir.jpa.model.dao.JpaPid; +import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; +import ca.uhn.fhir.mdm.api.IMdmRuleValidator; +import ca.uhn.fhir.mdm.api.IMdmSettings; +import ca.uhn.fhir.mdm.api.MdmModeEnum; +import ca.uhn.fhir.mdm.dao.IMdmLinkDao; +import ca.uhn.fhir.mdm.dao.IMdmLinkImplFactory; +import ca.uhn.fhir.mdm.rules.config.MdmSettings; +import ca.uhn.fhir.mdm.rules.json.MdmRulesJson; +import ca.uhn.fhir.mdm.svc.MdmEidMatchOnlyExpandSvc; +import ca.uhn.fhir.mdm.svc.MdmSearchExpansionSvc; +import ca.uhn.fhir.mdm.util.EIDHelper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import java.util.List; + +@Configuration +public class MdmRulesWithEidMatchOnlyConfig { + public static final String TEST_PATIENT_EID_SYS = "http://patient-eid-sys"; + + @Bean + @Primary //Override any random test mdm settings that are setup above. + public IMdmSettings mdmSettings(IMdmRuleValidator theMdmRuleValidator) { + MdmSettings mdmSettings = new MdmSettings(theMdmRuleValidator); + mdmSettings.setEnabled(true); + mdmSettings.setMdmMode(MdmModeEnum.MATCH_ONLY); + MdmRulesJson rules = new MdmRulesJson(); + rules.setMdmTypes(List.of("Patient")); + rules.addEnterpriseEIDSystem("Patient", TEST_PATIENT_EID_SYS); + mdmSettings.setMdmRules(rules); + return mdmSettings; + } + + @Bean + public MdmSearchExpansionSvc mdmSearchExpansionSvc() { + return new MdmSearchExpansionSvc(); + } + + @Bean + public IMdmLinkDao mdmLinkDao() { + return new MdmLinkDaoJpaImpl(); + } + + @Bean + public IMdmLinkImplFactory mdmLinkImplFactory() { + return new JpaMdmLinkImplFactory(); + } + + @Bean + public IMdmLinkExpandSvc mdmLinkExpandSvc( + EIDHelper theEidHelper, + IMdmSettings theMdmSettings, + DaoRegistry theDaoRegistry, + FhirContext theFhirContext, + IIdHelperService theIdHelperService) { + return new MdmEidMatchOnlyExpandSvc(theDaoRegistry, theFhirContext, theIdHelperService, theEidHelper); + } + + @Bean + public EIDHelper eidHelper(FhirContext theFhirContext, IMdmSettings theMdmSettings) { + return new EIDHelper(theFhirContext, theMdmSettings); + } + + @Bean + public IMdmClearHelperSvc mdmClearHelperSvc(IDeleteExpungeSvc theDeleteExpungeSvc) { + return new MdmClearHelperSvcImpl(theDeleteExpungeSvc); + } +} diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java index 9215ebebeaa4..181acd2d311f 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java @@ -107,7 +107,6 @@ import ca.uhn.fhir.jpa.util.MemoryCacheService; import ca.uhn.fhir.jpa.util.ResourceCountCache; import ca.uhn.fhir.jpa.validation.ValidationSettings; -import ca.uhn.fhir.mdm.interceptor.MdmStorageInterceptor; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.parser.StrictErrorHandler; import ca.uhn.fhir.rest.api.Constants; @@ -235,7 +234,7 @@ @ContextConfiguration(classes = { TestR4Config.class, ReplaceReferencesAppCtx.class, // Batch job - MergeAppCtx.class // Batch job + MergeAppCtx.class, // Batch job }) public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuilder { public static final String MY_VALUE_SET = "my-value-set"; @@ -550,8 +549,6 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil @Autowired protected ValidationSettings myValidationSettings; @Autowired - protected IMdmLinkJpaRepository myMdmLinkRepository; - @Autowired protected IMdmLinkJpaRepository myMdmLinkHistoryDao; @Autowired private IValidationSupport myJpaValidationSupportChainR4; @@ -561,9 +558,7 @@ public abstract class BaseJpaR4Test extends BaseJpaTest implements ITestDataBuil @Autowired protected IResourceSearchUrlDao myResourceSearchUrlDao; @Autowired - private IInterceptorService myInterceptorService; - @Autowired(required = false) - private MdmStorageInterceptor myMdmStorageInterceptor; + protected IInterceptorService myInterceptorService; @Autowired protected TestDaoSearch myTestDaoSearch; @Autowired @@ -651,41 +646,29 @@ public void afterPurgeDatabase() { ourLog.info("Pausing Schedulers"); mySchedulerService.pause(); - myTerminologyDeferredStorageSvc.logQueueForUnitTest(); if (!myTermDeferredStorageSvc.isStorageQueueEmpty(true)) { - ourLog.warn("There is deferred terminology storage stuff still in the queue. Please verify your tests clean up ok."); + ourLog.warn("There is deferred terminology storage stuff still in the queue. Please verify your tests clean up ok. Please call myTerminologyDeferredStorageSvc.logQueueForUnitTest() to find out what is in there in your test."); if (myTermDeferredStorageSvc instanceof TermDeferredStorageSvcImpl t) { t.clearDeferred(); } } - boolean registeredStorageInterceptor = false; - if (myMdmStorageInterceptor != null && !myInterceptorService.getAllRegisteredInterceptors().contains(myMdmStorageInterceptor)) { - myInterceptorService.registerInterceptor(myMdmStorageInterceptor); - registeredStorageInterceptor = true; - } - try { - runInTransaction(() -> { - myMdmLinkHistoryDao.deleteAll(); - myMdmLinkDao.deleteAll(); - }); - purgeDatabase(myStorageSettings, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataScheduleHelper); - - myBatch2JobHelper.cancelAllJobsAndAwaitCancellation(); - runInTransaction(() -> { - myWorkChunkRepository.deleteAll(); - myJobInstanceRepository.deleteAll(); - }); - } finally { - if (registeredStorageInterceptor) { - myInterceptorService.unregisterInterceptor(myMdmStorageInterceptor); - } - } + + runInTransaction(() -> { + myMdmLinkHistoryDao.deleteAll(); + }); + purgeDatabase(myStorageSettings, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataScheduleHelper); + + myBatch2JobHelper.cancelAllJobsAndAwaitCancellation(); + runInTransaction(() -> { + myWorkChunkRepository.deleteAll(); + myJobInstanceRepository.deleteAll(); + }); // restart the jobs ourLog.info("Restarting the schedulers"); mySchedulerService.unpause(); - ourLog.info("5 - " + getClass().getSimpleName() + ".afterPurgeDatabases"); + ourLog.info("Finished executing afterPurgeDatabases() for class " + getClass().getSimpleName()); } @BeforeEach diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java index 148c4b4d3174..9250b142ac86 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaTest.java @@ -65,7 +65,6 @@ import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao; import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao; import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao; -import ca.uhn.fhir.jpa.entity.MdmLink; import ca.uhn.fhir.jpa.entity.TermConcept; import ca.uhn.fhir.jpa.entity.TermConceptDesignation; import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink; @@ -105,7 +104,6 @@ import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; import ca.uhn.fhir.jpa.util.MemoryCacheService; -import ca.uhn.fhir.mdm.dao.IMdmLinkDao; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.RequestDetails; @@ -230,8 +228,6 @@ public abstract class BaseJpaTest extends BaseTest { protected ServletRequestDetails mySrd; protected InterceptorService mySrdInterceptorService; @Autowired - protected IMdmLinkDao myMdmLinkDao; - @Autowired protected FhirContext myFhirContext; @Autowired protected JpaStorageSettings myStorageSettings; @@ -592,22 +588,6 @@ protected void logAllResourcesOfType(String type) { } } - protected int countAllMdmLinks() { - return runInTransaction(()-> myMdmLinkDao.findAll().size()); - } - - protected int logAllMdmLinks() { - return runInTransaction(()->{ - List links = myMdmLinkDao.findAll(); - if (links.isEmpty()) { - ourLog.info("MDM Links: NONE"); - } else { - ourLog.info("MDM Links:\n * {}", links.stream().map(t -> t.toString()).collect(joining("\n * "))); - } - return links.size(); - }); - } - public void logAllResourceLinks() { runInTransaction(() -> { ourLog.info("Resource Links:\n * {}", myResourceLinkDao.findAll().stream().map(ResourceLink::toString).collect(Collectors.joining("\n * "))); diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestJPAConfig.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestJPAConfig.java index 58bf58380037..30ee6ceda32b 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestJPAConfig.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestJPAConfig.java @@ -143,4 +143,16 @@ public IEmailSender emailSender(){ public IMdmRuleValidator mdmRuleValidator(FhirContext theFhirContext, ISearchParamRegistry theSearchParamRetriever) { return new MdmRuleValidator(theFhirContext, theSearchParamRetriever); } + + /** + * N.B GGG: This bean only exists to support our existing busted test infrastructure that smooshes persistence contexts + * together with MDM contexts. In the future, this will be ripped out into mdm-specific JPA config for test. For now, + * if you need to override this MDM behaviour, add @Primary to your IMdmSettings bean definition in your implementing test class. + */ + @Bean + public IMdmSettings mdmSettings(IMdmRuleValidator theMdmRuleValidator) { + MdmSettings mdmSettings = new MdmSettings(theMdmRuleValidator); + mdmSettings.setEnabled(true); + return mdmSettings; + } } diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java index a948773e6cd4..eb8580209590 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java @@ -71,10 +71,10 @@ @Configuration @Import({ + TestJPAConfig.class, JpaR4Config.class, PackageLoaderConfig.class, TestHapiJpaConfig.class, - TestJPAConfig.class, SubscriptionTopicConfig.class, TestHSearchAddInConfig.DefaultLuceneHeap.class, JpaBatch2Config.class, diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkExpandSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkExpandSvc.java index bd52509dc8f3..7453f187ee6f 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkExpandSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmLinkExpandSvc.java @@ -20,6 +20,7 @@ package ca.uhn.fhir.mdm.api; import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.model.dao.JpaPid; import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; @@ -41,4 +42,15 @@ Set expandMdmByGoldenResourcePid( RequestPartitionId theRequestPartitionId, IResourcePersistentId theGoldenResourcePid); Set expandMdmByGoldenResourceId(RequestPartitionId theRequestPartitionId, IIdType theId); + + /** + * For the Group resource with the given id, returns all the persistent id ofs + * the members of the group + the mdm matched resources to a member in the group + */ + Set expandGroup(String groupResourceId, RequestPartitionId requestPartitionId); + + /** + * annotates the given resource to be exported with the implementation specific extra information if applicable + */ + void annotateResource(IBaseResource resource); } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSettings.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSettings.java index 4c64fe9e8a68..f2d815024456 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSettings.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/api/IMdmSettings.java @@ -53,6 +53,21 @@ default String getSupportedMdmTypes() { return getMdmRules().getMdmTypes().stream().collect(Collectors.joining(", ")); } + /** + * Link based expansion (using the {@link IMdmLink}) table is enabled if MDM is running in MATCH_AND_LINK mode. + */ + default boolean supportsLinkBasedExpansion() { + return getMode().equals(MdmModeEnum.MATCH_AND_LINK); + } + + /** + * EID-Based expansion is supported if MDM is running in MATCH_ONLY mode, and at least one EID system is defined. + */ + default boolean supportsEidBasedExpansion() { + return getMode().equals(MdmModeEnum.MATCH_ONLY) + && !getMdmRules().getEnterpriseEIDSystems().isEmpty(); + } + int getCandidateSearchLimit(); String getGoldenResourcePartitionName(); diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/BulkExportMdmEidMatchOnlyResourceExpander.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/BulkExportMdmEidMatchOnlyResourceExpander.java deleted file mode 100644 index 2239db8e6412..000000000000 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/BulkExportMdmEidMatchOnlyResourceExpander.java +++ /dev/null @@ -1,123 +0,0 @@ -/*- - * #%L - * HAPI FHIR - Master Data Management - * %% - * Copyright (C) 2014 - 2025 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package ca.uhn.fhir.mdm.svc; - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; -import ca.uhn.fhir.jpa.api.svc.IIdHelperService; -import ca.uhn.fhir.jpa.api.svc.ResolveIdentityMode; -import ca.uhn.fhir.jpa.model.dao.JpaPid; -import ca.uhn.fhir.rest.api.server.SystemRequestDetails; -import ca.uhn.fhir.util.FhirTerser; -import org.hl7.fhir.instance.model.api.IBaseReference; -import org.hl7.fhir.instance.model.api.IBaseResource; -import org.hl7.fhir.instance.model.api.IIdType; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Implementation of {@link IBulkExportMdmResourceExpander} that handles bulk export resource expansion - * when MDM mode is Match-Only and Eid Systems defined in mdm rules. - * - *

This expander is used during bulk export operations to expand Group resources by resolving - * MDM matching resources for the members in the group. Resources are - * matched based on just eids rather than the full MDM golden resource relationships.

- */ -public class BulkExportMdmEidMatchOnlyResourceExpander implements IBulkExportMdmResourceExpander { - - private final DaoRegistry myDaoRegistry; - private final MdmEidMatchOnlyExpandSvc myMdmEidMatchOnlyLinkExpandSvc; - private final FhirContext myFhirContext; - private final IIdHelperService myIdHelperService; - - /** - * Constructor - */ - public BulkExportMdmEidMatchOnlyResourceExpander( - DaoRegistry theDaoRegistry, - MdmEidMatchOnlyExpandSvc theMdmEidMatchOnlyLinkExpandSvc, - FhirContext theFhirContext, - IIdHelperService theIdHelperService) { - myDaoRegistry = theDaoRegistry; - myMdmEidMatchOnlyLinkExpandSvc = theMdmEidMatchOnlyLinkExpandSvc; - myFhirContext = theFhirContext; - myIdHelperService = theIdHelperService; - } - - /** - * Expands a Group resource and returns the Group members' resource persistent ids. - * The returned ids consists of group members + all MDM matched resources based on EID only. - * - *

This method:

- *
    - *
  1. Reads the specified Group resource
    2. - *
  2. Extracts all member entity references from the Group
    3. - *
  3. For each member, uses EID matching to find all resources that have the same EID as the member, using eid system specified in mdm rules
    4. - *
  4. Converts the expanded resource IDs to persistent IDs (PIDs)
    5. - *
- * - * @param groupResourceId The ID of the Group resource to expand - * @param requestPartitionId The request partition ID - * @return A set of {@link JpaPid} objects representing all expanded resources - */ - @Override - public Set expandGroup(String groupResourceId, RequestPartitionId requestPartitionId) { - // Read the Group resource - SystemRequestDetails srd = SystemRequestDetails.forRequestPartitionId(requestPartitionId); - IIdType groupId = myFhirContext.getVersion().newIdType(groupResourceId); - IFhirResourceDao groupDao = myDaoRegistry.getResourceDao("Group"); - IBaseResource groupResource = groupDao.read(groupId, srd); - - Set allResourceIds = new HashSet<>(); - FhirTerser terser = myFhirContext.newTerser(); - // Extract all member.entity references from the Group resource - List memberEntities = - terser.getValues(groupResource, "Group.member.entity", IBaseReference.class); - // mdm expand each member based on eid - for (IBaseReference entityRef : memberEntities) { - if (!entityRef.getReferenceElement().isEmpty()) { - IIdType memberId = entityRef.getReferenceElement(); - Set expanded = - myMdmEidMatchOnlyLinkExpandSvc.expandMdmBySourceResourceId(requestPartitionId, memberId); - allResourceIds.addAll(expanded); - } - } - // Convert all resourceIds to IIdType and resolve in batch - List idTypes = allResourceIds.stream() - .map(id -> myFhirContext.getVersion().newIdType(id)) - .collect(Collectors.toList()); - List pidList = myIdHelperService.resolveResourcePids( - requestPartitionId, - idTypes, - ResolveIdentityMode.excludeDeleted().cacheOk()); - return new HashSet<>(pidList); - } - - @Override - public void annotateResource(IBaseResource resource) { - // This function is normally used to add golden resource id to the exported resources, - // but in the Eid-based match only mode, there isn't any golden resource, so nothing to do here - } -} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/BulkExportMdmResourceExpander.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/BulkExportMdmResourceExpander.java deleted file mode 100644 index 06b12dac3a71..000000000000 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/BulkExportMdmResourceExpander.java +++ /dev/null @@ -1,228 +0,0 @@ -/*- - * #%L - * HAPI FHIR - Master Data Management - * %% - * Copyright (C) 2014 - 2025 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package ca.uhn.fhir.mdm.svc; - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.context.RuntimeSearchParam; -import ca.uhn.fhir.fhirpath.IFhirPath; -import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.api.dao.DaoRegistry; -import ca.uhn.fhir.jpa.api.model.PersistentIdToForcedIdMap; -import ca.uhn.fhir.jpa.api.svc.IIdHelperService; -import ca.uhn.fhir.jpa.model.dao.JpaPid; -import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; -import ca.uhn.fhir.mdm.dao.IMdmLinkDao; -import ca.uhn.fhir.mdm.model.MdmPidTuple; -import ca.uhn.fhir.model.primitive.IdDt; -import ca.uhn.fhir.rest.api.server.SystemRequestDetails; -import ca.uhn.fhir.util.ExtensionUtil; -import ca.uhn.fhir.util.HapiExtensions; -import ca.uhn.fhir.util.SearchParameterUtil; -import org.apache.commons.lang3.StringUtils; -import org.hl7.fhir.instance.model.api.IBaseExtension; -import org.hl7.fhir.instance.model.api.IBaseReference; -import org.hl7.fhir.instance.model.api.IBaseResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -/** - * Implementation of MDM resource expansion for bulk export operations. - * Expands group memberships via MDM links and annotates exported resources with golden resource references. - */ -public class BulkExportMdmResourceExpander implements IBulkExportMdmResourceExpander { - private static final Logger ourLog = LoggerFactory.getLogger(BulkExportMdmResourceExpander.class); - - private final MdmExpansionCacheSvc myMdmExpansionCacheSvc; - private final IMdmLinkDao myMdmLinkDao; - private final IIdHelperService myIdHelperService; - private final DaoRegistry myDaoRegistry; - private final FhirContext myContext; - private IFhirPath myFhirPath; - - public BulkExportMdmResourceExpander( - MdmExpansionCacheSvc theMdmExpansionCacheSvc, - IMdmLinkDao theMdmLinkDao, - IIdHelperService theIdHelperService, - DaoRegistry theDaoRegistry, - FhirContext theFhirContext) { - myMdmExpansionCacheSvc = theMdmExpansionCacheSvc; - myMdmLinkDao = theMdmLinkDao; - myIdHelperService = theIdHelperService; - myDaoRegistry = theDaoRegistry; - myContext = theFhirContext; - } - - @Override - public Set expandGroup(String theGroupResourceId, RequestPartitionId theRequestPartitionId) { - IdDt groupId = new IdDt(theGroupResourceId); - SystemRequestDetails requestDetails = new SystemRequestDetails(); - requestDetails.setRequestPartitionId(theRequestPartitionId); - IBaseResource group = myDaoRegistry.getResourceDao("Group").read(groupId, requestDetails); - JpaPid pidOrNull = myIdHelperService.getPidOrNull(theRequestPartitionId, group); - // Attempt to perform MDM Expansion of membership - return performMembershipExpansionViaMdmTable(pidOrNull); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private Set performMembershipExpansionViaMdmTable(JpaPid pidOrNull) { - List> goldenPidTargetPidTuples = - myMdmLinkDao.expandPidsFromGroupPidGivenMatchResult(pidOrNull, MdmMatchResultEnum.MATCH); - - Set uniquePids = new HashSet<>(); - goldenPidTargetPidTuples.forEach(tuple -> { - uniquePids.add(tuple.getGoldenPid()); - uniquePids.add(tuple.getSourcePid()); - }); - populateMdmResourceCache(goldenPidTargetPidTuples); - return uniquePids; - } - - /** - * @param thePidTuples - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - private void populateMdmResourceCache(List> thePidTuples) { - if (myMdmExpansionCacheSvc.hasBeenPopulated()) { - return; - } - // First, convert this zipped set of tuples to a map of - // { - // patient/gold-1 -> [patient/1, patient/2] - // patient/gold-2 -> [patient/3, patient/4] - // } - Map> goldenResourceToSourcePidMap = new HashMap<>(); - extract(thePidTuples, goldenResourceToSourcePidMap); - - // Next, lets convert it to an inverted index for fast lookup - // { - // patient/1 -> patient/gold-1 - // patient/2 -> patient/gold-1 - // patient/3 -> patient/gold-2 - // patient/4 -> patient/gold-2 - // } - Map sourceResourceIdToGoldenResourceIdMap = new HashMap<>(); - goldenResourceToSourcePidMap.forEach((key, value) -> { - String goldenResourceId = - myIdHelperService.translatePidIdToForcedIdWithCache(key).orElse(key.toString()); - PersistentIdToForcedIdMap pidsToForcedIds = myIdHelperService.translatePidsToForcedIds(value); - - Set sourceResourceIds = pidsToForcedIds.getResolvedResourceIds(); - - sourceResourceIds.forEach( - sourceResourceId -> sourceResourceIdToGoldenResourceIdMap.put(sourceResourceId, goldenResourceId)); - }); - - // Now that we have built our cached expansion, store it. - myMdmExpansionCacheSvc.setCacheContents(sourceResourceIdToGoldenResourceIdMap); - } - - private void extract( - List> theGoldenPidTargetPidTuples, - Map> theGoldenResourceToSourcePidMap) { - for (MdmPidTuple goldenPidTargetPidTuple : theGoldenPidTargetPidTuples) { - JpaPid goldenPid = goldenPidTargetPidTuple.getGoldenPid(); - JpaPid sourcePid = goldenPidTargetPidTuple.getSourcePid(); - theGoldenResourceToSourcePidMap - .computeIfAbsent(goldenPid, key -> new HashSet<>()) - .add(sourcePid); - } - } - - private RuntimeSearchParam getRuntimeSearchParam(IBaseResource theResource) { - Optional oPatientSearchParam = - SearchParameterUtil.getOnlyPatientSearchParamForResourceType(myContext, theResource.fhirType()); - if (!oPatientSearchParam.isPresent()) { - String errorMessage = String.format( - "[%s] has no search parameters that are for patients, so it is invalid for Group Bulk Export!", - theResource.fhirType()); - throw new IllegalArgumentException(Msg.code(2242) + errorMessage); - } else { - return oPatientSearchParam.get(); - } - } - - @Override - public void annotateResource(IBaseResource iBaseResource) { - Optional patientReference = getPatientReference(iBaseResource); - if (patientReference.isPresent()) { - addGoldenResourceExtension(iBaseResource, patientReference.get()); - } else { - ourLog.error( - "Failed to find the patient reference information for resource {}. This is a bug, " - + "as all resources which can be exported via Group Bulk Export must reference a patient.", - iBaseResource); - } - } - - private Optional getPatientReference(IBaseResource iBaseResource) { - String fhirPath; - - RuntimeSearchParam runtimeSearchParam = getRuntimeSearchParam(iBaseResource); - fhirPath = getPatientFhirPath(runtimeSearchParam); - - if (iBaseResource.fhirType().equalsIgnoreCase("Patient")) { - return Optional.of(iBaseResource.getIdElement().getIdPart()); - } else { - Optional optionalReference = - getFhirParser().evaluateFirst(iBaseResource, fhirPath, IBaseReference.class); - if (optionalReference.isPresent()) { - return optionalReference.map(theIBaseReference -> - theIBaseReference.getReferenceElement().getIdPart()); - } else { - return Optional.empty(); - } - } - } - - private void addGoldenResourceExtension(IBaseResource iBaseResource, String sourceResourceId) { - String goldenResourceId = myMdmExpansionCacheSvc.getGoldenResourceId(sourceResourceId); - IBaseExtension extension = ExtensionUtil.getOrCreateExtension( - iBaseResource, HapiExtensions.ASSOCIATED_GOLDEN_RESOURCE_EXTENSION_URL); - if (!StringUtils.isBlank(goldenResourceId)) { - ExtensionUtil.setExtension(myContext, extension, "reference", prefixPatient(goldenResourceId)); - } - } - - private String prefixPatient(String theResourceId) { - return "Patient/" + theResourceId; - } - - private IFhirPath getFhirParser() { - if (myFhirPath == null) { - myFhirPath = myContext.newFhirPath(); - } - return myFhirPath; - } - - private String getPatientFhirPath(RuntimeSearchParam theRuntimeParam) { - String path = theRuntimeParam.getPath(); - // GGG: Yes this is a stupid hack, but by default this runtime search param will return stuff like - // Observation.subject.where(resolve() is Patient) which unfortunately our FHIRpath evaluator doesn't play - // nicely with - // our FHIRPath evaluator. - if (path.contains(".where")) { - path = path.substring(0, path.indexOf(".where")); - } - return path; - } -} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/DisabledMdmLinkExpandSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/DisabledMdmLinkExpandSvc.java new file mode 100644 index 000000000000..331e605db561 --- /dev/null +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/DisabledMdmLinkExpandSvc.java @@ -0,0 +1,72 @@ +/*- + * #%L + * HAPI FHIR - Master Data Management + * %% + * Copyright (C) 2014 - 2025 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package ca.uhn.fhir.mdm.svc; + +import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.model.dao.JpaPid; +import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; +import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; + +import java.util.Set; + +public class DisabledMdmLinkExpandSvc implements IMdmLinkExpandSvc { + @Override + public Set expandMdmBySourceResource(RequestPartitionId theRequestPartitionId, IBaseResource theResource) { + return Set.of(); + } + + @Override + public Set expandMdmBySourceResourceId(RequestPartitionId theRequestPartitionId, IIdType theId) { + return Set.of(); + } + + @Override + public Set expandMdmBySourceResourcePid( + RequestPartitionId theRequestPartitionId, IResourcePersistentId theSourceResourcePid) { + return Set.of(); + } + + @Override + public Set expandMdmByGoldenResourceId( + RequestPartitionId theRequestPartitionId, IResourcePersistentId theGoldenResourcePid) { + return Set.of(); + } + + @Override + public Set expandMdmByGoldenResourcePid( + RequestPartitionId theRequestPartitionId, IResourcePersistentId theGoldenResourcePid) { + return Set.of(); + } + + @Override + public Set expandMdmByGoldenResourceId(RequestPartitionId theRequestPartitionId, IIdType theId) { + return Set.of(); + } + + @Override + public Set expandGroup(String groupResourceId, RequestPartitionId requestPartitionId) { + return Set.of(); + } + + @Override + public void annotateResource(IBaseResource resource) {} +} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/IBulkExportMdmResourceExpander.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/IBulkExportMdmResourceExpander.java deleted file mode 100644 index 22b83a6cb4b2..000000000000 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/IBulkExportMdmResourceExpander.java +++ /dev/null @@ -1,43 +0,0 @@ -/*- - * #%L - * HAPI FHIR - Master Data Management - * %% - * Copyright (C) 2014 - 2025 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package ca.uhn.fhir.mdm.svc; - -import ca.uhn.fhir.interceptor.model.RequestPartitionId; -import ca.uhn.fhir.jpa.model.dao.JpaPid; -import org.hl7.fhir.instance.model.api.IBaseResource; - -import java.util.Set; - -/** - * Interface for mdm expanding Group resources on group bulk export - */ -public interface IBulkExportMdmResourceExpander { - - /** - * For the Group resource with the given id, returns all the persistent id ofs - * the members of the group + the mdm matched resources to a member in the group - */ - Set expandGroup(String groupResourceId, RequestPartitionId requestPartitionId); - - /** - * annotates the given resource to be exported with the implementation specific extra information if applicable - */ - void annotateResource(IBaseResource resource); -} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmEidMatchOnlyExpandSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmEidMatchOnlyExpandSvc.java index 3422455e9fde..c4fe62e0d4bb 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmEidMatchOnlyExpandSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmEidMatchOnlyExpandSvc.java @@ -19,10 +19,14 @@ */ package ca.uhn.fhir.mdm.svc; +import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; +import ca.uhn.fhir.jpa.api.svc.IIdHelperService; +import ca.uhn.fhir.jpa.api.svc.ResolveIdentityMode; +import ca.uhn.fhir.jpa.model.dao.JpaPid; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; import ca.uhn.fhir.mdm.model.CanonicalEID; @@ -31,9 +35,12 @@ import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; import ca.uhn.fhir.rest.param.TokenOrListParam; import ca.uhn.fhir.rest.param.TokenParam; +import ca.uhn.fhir.util.FhirTerser; +import org.hl7.fhir.instance.model.api.IBaseReference; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; +import java.util.stream.Collectors; import java.util.*; /** @@ -46,11 +53,18 @@ public class MdmEidMatchOnlyExpandSvc implements IMdmLinkExpandSvc { private EIDHelper myEidHelper; - public MdmEidMatchOnlyExpandSvc(DaoRegistry theDaoRegistry) { - myDaoRegistry = theDaoRegistry; - } + private IIdHelperService myIdHelperService; - public void setMyEidHelper(EIDHelper theEidHelper) { + private FhirContext myFhirContext; + + public MdmEidMatchOnlyExpandSvc( + DaoRegistry theDaoRegistry, + FhirContext theFhirContext, + IIdHelperService theIdHelperService, + EIDHelper theEidHelper) { + myDaoRegistry = theDaoRegistry; + myFhirContext = theFhirContext; + myIdHelperService = theIdHelperService; myEidHelper = theEidHelper; } @@ -118,4 +132,57 @@ public Set expandMdmByGoldenResourceId(RequestPartitionId theRequestPart // return an emtpy set to rather than an exception to not affect existing code return Collections.emptySet(); } + /** + * Expands a Group resource and returns the Group members' resource persistent ids. + * The returned ids consists of group members + all MDM matched resources based on EID only. + * + *

This method:

+ *
    + *
  1. Reads the specified Group resource
    2. + *
  2. Extracts all member entity references from the Group
    3. + *
  3. For each member, uses EID matching to find all resources that have the same EID as the member, using eid system specified in mdm rules
    4. + *
  4. Converts the expanded resource IDs to persistent IDs (PIDs)
    5. + *
+ * + * @param groupResourceId The ID of the Group resource to expand + * @param requestPartitionId The request partition ID + * @return A set of {@link JpaPid} objects representing all expanded resources + */ + @Override + public Set expandGroup(String groupResourceId, RequestPartitionId requestPartitionId) { + // Read the Group resource + SystemRequestDetails srd = SystemRequestDetails.forRequestPartitionId(requestPartitionId); + IIdType groupId = myFhirContext.getVersion().newIdType(groupResourceId); + IFhirResourceDao groupDao = myDaoRegistry.getResourceDao("Group"); + IBaseResource groupResource = groupDao.read(groupId, srd); + + Set allResourceIds = new HashSet<>(); + FhirTerser terser = myFhirContext.newTerser(); + // Extract all member.entity references from the Group resource + List memberEntities = + terser.getValues(groupResource, "Group.member.entity", IBaseReference.class); + // mdm expand each member based on eid + for (IBaseReference entityRef : memberEntities) { + if (!entityRef.getReferenceElement().isEmpty()) { + IIdType memberId = entityRef.getReferenceElement(); + Set expanded = this.expandMdmBySourceResourceId(requestPartitionId, memberId); + allResourceIds.addAll(expanded); + } + } + // Convert all resourceIds to IIdType and resolve in batch + List idTypes = allResourceIds.stream() + .map(id -> myFhirContext.getVersion().newIdType(id)) + .collect(Collectors.toList()); + List pidList = myIdHelperService.resolveResourcePids( + requestPartitionId, + idTypes, + ResolveIdentityMode.excludeDeleted().cacheOk()); + return new HashSet<>(pidList); + } + + @Override + public void annotateResource(IBaseResource resource) { + // This function is normally used to add golden resource id to the exported resources, + // but in the Eid-based match only mode, there isn't any golden resource, so nothing to do here + } } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmExpandersHolder.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmExpandersHolder.java deleted file mode 100644 index f3c423d09abf..000000000000 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmExpandersHolder.java +++ /dev/null @@ -1,181 +0,0 @@ -/*- - * #%L - * HAPI FHIR - Master Data Management - * %% - * Copyright (C) 2014 - 2025 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package ca.uhn.fhir.mdm.svc; - -import ca.uhn.fhir.context.FhirContext; -import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; -import ca.uhn.fhir.mdm.api.IMdmSettings; -import ca.uhn.fhir.mdm.api.MdmModeEnum; -import ca.uhn.fhir.mdm.util.EIDHelper; - -/** - * Holder class that manages two different MDM expansion implementation approaches based on MdmSettings. - *

- * This class addresses the dependency injection challenge where MdmSettings may not be available when these - * service objects are created. It holds references to both implementation approaches and determines which ones - * to use based on the MdmSettings. setMdmSettings method should be called when MdmSettings is constructed or updated - * for proper functioning. - *

- * The two implementation approaches are: - *

- * 1. Eid Match-Only Mode: A simplified approach where resources are matched based solely on - * Enterprise ID (EID) values without creating golden resources or MDM links. This mode is activated when - * MdmSettings specify MATCH_ONLY mode and EID systems are defined. - *

- * 2. Full MDM Mode: The complete MDM solution that creates golden resources and manages - * MDM links between resources. - *

- * Each approach has two service implementations, one for mdm expansion for searches and the other for mdm expansion for group bulk export: - *

- * Eid Match-Only Mode implementations: - * - MdmEidMatchOnlyLinkExpandSvc - * - BulkExportMdmEidMatchOnlyResourceExpander - *

- * Full MDM Mode implementations: - * - MdmLinkExpandSvc - * - BulkExportMdmResourceExpander - *

- * - * This class holds references to these service objects rather than creating them by itself, because some of these service objects have Spring annotations - * like @Transactional, for those annotations to work the objects need to be created by Spring itself. - */ -public class MdmExpandersHolder { - - /** MDM configuration settings used to determine which implementation to use */ - private IMdmSettings myMdmSettings; - - /** Cached instance of the selected link expand service */ - private IMdmLinkExpandSvc myLinkExpandSvcInstanceToUse; - - /** Cached instance of the selected bulk export resource expander */ - private IBulkExportMdmResourceExpander myBulkExportMDMResourceExpanderInstanceToUse; - - /** Full MDM link expand service implementation - * We have to use the interface as the type here instead of concrete implementing class MdmLinkExpandSvc - * because the class has Spring annotations like @Transactional which rely on a Proxy interface implementation and doesn't work if concrete classes are used as beans. */ - private final IMdmLinkExpandSvc myMdmLinkExpandSvc; - - /** EID-only match expand service implementation */ - private final MdmEidMatchOnlyExpandSvc myMdmEidMatchOnlyExpandSvc; - - /** Full MDM bulk export resource expander implementation */ - private final BulkExportMdmResourceExpander myBulkExportMDMResourceExpander; - - /** EID-only match bulk export resource expander implementation */ - private final BulkExportMdmEidMatchOnlyResourceExpander myBulkExportMDMEidMatchOnlyResourceExpander; - - private final FhirContext myFhirContext; - - public MdmExpandersHolder( - FhirContext theFhirContext, - IMdmLinkExpandSvc theMdmLinkExpandSvc, - MdmEidMatchOnlyExpandSvc theMdmEidMatchOnlyLinkExpandSvc, - BulkExportMdmResourceExpander theBulkExportMDMResourceExpander, - BulkExportMdmEidMatchOnlyResourceExpander theBulkExportMDMEidMatchOnlyResourceExpander) { - - myFhirContext = theFhirContext; - myMdmLinkExpandSvc = theMdmLinkExpandSvc; - myMdmEidMatchOnlyExpandSvc = theMdmEidMatchOnlyLinkExpandSvc; - myBulkExportMDMResourceExpander = theBulkExportMDMResourceExpander; - myBulkExportMDMEidMatchOnlyResourceExpander = theBulkExportMDMEidMatchOnlyResourceExpander; - } - - /** - * Returns the appropriate expand service instance appropriate for the mdm settings - */ - public IMdmLinkExpandSvc getLinkExpandSvcInstance() { - if (myLinkExpandSvcInstanceToUse != null) { - // we already determined instance to use, just return it - return myLinkExpandSvcInstanceToUse; - } - - myLinkExpandSvcInstanceToUse = determineExpandSvsInstanceToUse(); - - return myLinkExpandSvcInstanceToUse; - } - - /** - * Returns the appropriate bulk export resource expander instance appropriate for the mdm settings - */ - public IBulkExportMdmResourceExpander getBulkExportMDMResourceExpanderInstance() { - if (myBulkExportMDMResourceExpanderInstanceToUse != null) { - // we already determined instance to use, just return it - return myBulkExportMDMResourceExpanderInstanceToUse; - } - - myBulkExportMDMResourceExpanderInstanceToUse = determineBulkExportMDMResourceExpanderInstanceToUse(); - - return myBulkExportMDMResourceExpanderInstanceToUse; - } - - /** - * Determines which bulk export resource expander to use based on MDM mode and EID configuration. - */ - public IBulkExportMdmResourceExpander determineBulkExportMDMResourceExpanderInstanceToUse() { - if (isMatchOnlyWithEidSystems()) { - return myBulkExportMDMEidMatchOnlyResourceExpander; - } else { - return myBulkExportMDMResourceExpander; - } - } - - /** - * Determines if MDM is configured in MATCH_ONLY mode and EID systems are defined in the MDM rules. - */ - private boolean isMatchOnlyWithEidSystems() { - - if (myMdmSettings == null) { - // if mdmSettings is not set yet, assume we are using the full mdm mode - // to not break existing code, because previously we were just using the - // full mdm implementation without checking the mdm settings. - // This would be called again when mdmSettings setter is called - return false; - } - boolean isMatchOnly = myMdmSettings.getMode() == MdmModeEnum.MATCH_ONLY; - boolean hasEidSystems = false; - if (myMdmSettings.getMdmRules() != null) { - hasEidSystems = myMdmSettings.getMdmRules().getEnterpriseEIDSystems() != null - && !myMdmSettings.getMdmRules().getEnterpriseEIDSystems().isEmpty(); - } - return isMatchOnly && hasEidSystems; - } - - /** - * Determines which expand service to use and configures it if necessary. - */ - private IMdmLinkExpandSvc determineExpandSvsInstanceToUse() { - if (isMatchOnlyWithEidSystems()) { - myMdmEidMatchOnlyExpandSvc.setMyEidHelper(new EIDHelper(myFhirContext, myMdmSettings)); - return myMdmEidMatchOnlyExpandSvc; - } else { - return myMdmLinkExpandSvc; - } - } - - /** - * Sets the MDM settings and immediately determines which service implementations to use. - * This method is called after MDM settings become available during application startup. - */ - public void setMdmSettings(IMdmSettings theMdmSettings) { - myMdmSettings = theMdmSettings; - myLinkExpandSvcInstanceToUse = determineExpandSvsInstanceToUse(); - myBulkExportMDMResourceExpanderInstanceToUse = determineBulkExportMDMResourceExpanderInstanceToUse(); - } -} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmExpansionCacheSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmExpansionCacheSvc.java deleted file mode 100644 index 1bc409ba3123..000000000000 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmExpansionCacheSvc.java +++ /dev/null @@ -1,106 +0,0 @@ -/*- - * #%L - * HAPI FHIR - Master Data Management - * %% - * Copyright (C) 2014 - 2025 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ -package ca.uhn.fhir.mdm.svc; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static org.slf4j.LoggerFactory.getLogger; - -/** - * The purpose of this class is to share context between steps of a given GroupBulkExport job. - * - * This cache allows you to port state between reader/processor/writer. In this case, we are maintaining - * a cache of Source Resource ID -> Golden Resource ID, so that we can annotate outgoing resources with their golden owner - * if applicable. - * - */ -public class MdmExpansionCacheSvc { - private static final Logger ourLog = getLogger(MdmExpansionCacheSvc.class); - - private final ConcurrentHashMap mySourceToGoldenIdCache = new ConcurrentHashMap<>(); - - /** - * Lookup a given resource's golden resource ID in the cache. Note that if you pass this function the resource ID of a - * golden resource, it will just return itself. - * - * @param theSourceId the resource ID of the source resource ,e.g. PAT123 - * @return the resource ID of the associated golden resource. - */ - public String getGoldenResourceId(String theSourceId) { - ourLog.debug(buildLogMessage("About to lookup cached resource ID " + theSourceId)); - String goldenResourceId = mySourceToGoldenIdCache.get(theSourceId); - - // A golden resources' golden resource ID is itself. - if (StringUtils.isBlank(goldenResourceId)) { - if (mySourceToGoldenIdCache.containsValue(theSourceId)) { - goldenResourceId = theSourceId; - } - } - return goldenResourceId; - } - - private String buildLogMessage(String theMessage) { - return buildLogMessage(theMessage, false); - } - - /** - * Builds a log message, potentially enriched with the cache content. - * - * @param message The log message - * @param theAddCacheContentContent If true, will annotate the log message with the current cache contents. - * @return a built log message, which may include the cache content. - */ - public String buildLogMessage(String message, boolean theAddCacheContentContent) { - StringBuilder builder = new StringBuilder(); - builder.append(message); - if (ourLog.isDebugEnabled() || theAddCacheContentContent) { - builder.append("\n").append("Current cache content is:").append("\n"); - mySourceToGoldenIdCache.entrySet().stream().forEach(entry -> builder.append(entry.getKey()) - .append(" -> ") - .append(entry.getValue()) - .append("\n")); - return builder.toString(); - } - return builder.toString(); - } - - /** - * Populate the cache - * - * @param theSourceResourceIdToGoldenResourceIdMap the source ID -> golden ID map to populate the cache with. - */ - public void setCacheContents(Map theSourceResourceIdToGoldenResourceIdMap) { - if (mySourceToGoldenIdCache.isEmpty()) { - this.mySourceToGoldenIdCache.putAll(theSourceResourceIdToGoldenResourceIdMap); - } - } - - /** - * Since this cache is used at @JobScope, we can skip a whole whack of expansions happening by simply checking - * if one of our child steps has populated the cache yet. . - */ - public boolean hasBeenPopulated() { - return !mySourceToGoldenIdCache.isEmpty(); - } -} diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmLinkExpandSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmLinkExpandSvc.java index 82fb144b9282..a080672d4a13 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmLinkExpandSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmLinkExpandSvc.java @@ -19,41 +19,70 @@ */ package ca.uhn.fhir.mdm.svc; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.context.RuntimeSearchParam; +import ca.uhn.fhir.fhirpath.IFhirPath; +import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.interceptor.model.RequestPartitionId; +import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.svc.IIdHelperService; +import ca.uhn.fhir.jpa.model.dao.JpaPid; import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; import ca.uhn.fhir.mdm.api.MdmMatchResultEnum; import ca.uhn.fhir.mdm.dao.IMdmLinkDao; import ca.uhn.fhir.mdm.log.Logs; import ca.uhn.fhir.mdm.model.MdmPidTuple; +import ca.uhn.fhir.model.primitive.IdDt; +import ca.uhn.fhir.rest.api.server.SystemRequestDetails; import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; +import ca.uhn.fhir.util.ExtensionUtil; +import ca.uhn.fhir.util.HapiExtensions; +import ca.uhn.fhir.util.SearchParameterUtil; import jakarta.annotation.Nonnull; +import org.apache.commons.lang3.StringUtils; +import org.hl7.fhir.instance.model.api.IBaseExtension; +import org.hl7.fhir.instance.model.api.IBaseReference; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -@Service @Transactional public class MdmLinkExpandSvc implements IMdmLinkExpandSvc { private static final Logger ourLog = Logs.getMdmTroubleshootingLog(); + @Autowired + private FhirContext myContext; + @Autowired IMdmLinkDao myMdmLinkDao; @Autowired IIdHelperService myIdHelperService; + @Autowired + private DaoRegistry myDaoRegistry; + + private IFhirPath myFhirPath; + public MdmLinkExpandSvc() {} + private IFhirPath getFhirPath() { + if (myFhirPath == null) { + myFhirPath = myContext.newFhirPath(); + } + return myFhirPath; + } + /** * Given a source resource, perform MDM expansion and return all the resource IDs of all resources that are * MDM-Matched to this resource. @@ -172,4 +201,99 @@ static Set flattenTuple(RequestPartitionId theRequestPart return Collections.emptySet(); } + + @Override + public Set expandGroup(String theGroupResourceId, RequestPartitionId theRequestPartitionId) { + IdDt groupId = new IdDt(theGroupResourceId); + SystemRequestDetails requestDetails = new SystemRequestDetails(); + requestDetails.setRequestPartitionId(theRequestPartitionId); + IBaseResource group = myDaoRegistry.getResourceDao("Group").read(groupId, requestDetails); + JpaPid pidOrNull = (JpaPid) myIdHelperService.getPidOrNull(theRequestPartitionId, group); + // Attempt to perform MDM Expansion of membership + return performMembershipExpansionViaMdmTable(pidOrNull); + } + + @Override + public void annotateResource(IBaseResource iBaseResource) { + Optional patientReference = getPatientReference(iBaseResource); + if (patientReference.isPresent()) { + addGoldenResourceExtension(iBaseResource, patientReference.get()); + } else { + ourLog.error( + "Failed to find the patient reference information for resource {}. This is a bug, " + + "as all resources which can be exported via Group Bulk Export must reference a patient.", + iBaseResource); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private Set performMembershipExpansionViaMdmTable(JpaPid pidOrNull) { + List> goldenPidTargetPidTuples = + myMdmLinkDao.expandPidsFromGroupPidGivenMatchResult(pidOrNull, MdmMatchResultEnum.MATCH); + + Set uniquePids = new HashSet<>(); + goldenPidTargetPidTuples.forEach(tuple -> { + uniquePids.add(tuple.getGoldenPid()); + uniquePids.add(tuple.getSourcePid()); + }); + return uniquePids; + } + + private Optional getPatientReference(IBaseResource iBaseResource) { + String fhirPath; + + RuntimeSearchParam runtimeSearchParam = getRuntimeSearchParam(iBaseResource); + fhirPath = getPatientFhirPath(runtimeSearchParam); + + if (iBaseResource.fhirType().equalsIgnoreCase("Patient")) { + return Optional.of(iBaseResource.getIdElement().getIdPart()); + } else { + Optional optionalReference = + getFhirPath().evaluateFirst(iBaseResource, fhirPath, IBaseReference.class); + if (optionalReference.isPresent()) { + return optionalReference.map(theIBaseReference -> + theIBaseReference.getReferenceElement().getIdPart()); + } else { + return Optional.empty(); + } + } + } + + private void addGoldenResourceExtension(IBaseResource iBaseResource, String sourceResourceId) { + // TODO, reimplement this, it is currently completely broken given the distributed nature of the job. + String goldenResourceId = ""; // TODO we must be able to fetch this, for now, will be no-op + if (!StringUtils.isBlank(goldenResourceId)) { + IBaseExtension extension = ExtensionUtil.getOrCreateExtension( + iBaseResource, HapiExtensions.ASSOCIATED_GOLDEN_RESOURCE_EXTENSION_URL); + ExtensionUtil.setExtension(myContext, extension, "reference", prefixPatient(goldenResourceId)); + } + } + + private String prefixPatient(String theResourceId) { + return "Patient/" + theResourceId; + } + + private String getPatientFhirPath(RuntimeSearchParam theRuntimeParam) { + String path = theRuntimeParam.getPath(); + // GGG: Yes this is a stupid hack, but by default this runtime search param will return stuff like + // Observation.subject.where(resolve() is Patient) which unfortunately our FHIRpath evaluator doesn't play + // nicely with + if (path.contains(".where")) { + path = path.substring(0, path.indexOf(".where")); + } + return path; + } + + private RuntimeSearchParam getRuntimeSearchParam(IBaseResource theResource) { + Optional oPatientSearchParam = + SearchParameterUtil.getOnlyPatientSearchParamForResourceType(myContext, theResource.fhirType()); + if (!oPatientSearchParam.isPresent()) { + String errorMessage = String.format( + "[%s] has no search parameters that are for patients, so it is invalid for Group Bulk Export!", + theResource.fhirType()); + throw new IllegalArgumentException(Msg.code(2242) + errorMessage); + } else { + return oPatientSearchParam.get(); + } + } } diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmSearchExpansionSvc.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmSearchExpansionSvc.java index f762d2b756b4..da357aa99770 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmSearchExpansionSvc.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/svc/MdmSearchExpansionSvc.java @@ -55,10 +55,10 @@ public class MdmSearchExpansionSvc { private FhirContext myFhirContext; @Autowired - private IRequestPartitionHelperSvc myRequestPartitionHelperSvc; + private IMdmLinkExpandSvc myMdmLinkExpandSvc; @Autowired - private MdmExpandersHolder myMdmExpandersHolder; + private IRequestPartitionHelperSvc myRequestPartitionHelperSvc; /** * This method looks through all the reference parameters within a {@link SearchParameterMap} @@ -141,8 +141,6 @@ private void expandAnyReferenceParameters( IParamTester theParamTester, MdmSearchExpansionResults theResultsToPopulate) { - IMdmLinkExpandSvc mdmLinkExpandSvc = myMdmExpandersHolder.getLinkExpandSvcInstance(); - List toRemove = new ArrayList<>(); List toAdd = new ArrayList<>(); for (IQueryParameterType iQueryParameterType : orList) { @@ -152,12 +150,12 @@ private void expandAnyReferenceParameters( // First, attempt to expand as a source resource. IIdType sourceId = newId(refParam.getValue()); Set expandedResourceIds = - mdmLinkExpandSvc.expandMdmBySourceResourceId(theRequestPartitionId, sourceId); + myMdmLinkExpandSvc.expandMdmBySourceResourceId(theRequestPartitionId, sourceId); // If we failed, attempt to expand as a golden resource if (expandedResourceIds.isEmpty()) { expandedResourceIds = - mdmLinkExpandSvc.expandMdmByGoldenResourceId(theRequestPartitionId, sourceId); + myMdmLinkExpandSvc.expandMdmByGoldenResourceId(theRequestPartitionId, sourceId); } // Rebuild the search param list. @@ -238,11 +236,10 @@ private void expandIdParameter( } else if (mdmExpand) { ourLog.debug("_id parameter must be expanded out from: {}", id.getValue()); - IMdmLinkExpandSvc mdmLinkExpandSvc = myMdmExpandersHolder.getLinkExpandSvcInstance(); - Set expandedResourceIds = mdmLinkExpandSvc.expandMdmBySourceResourceId(theRequestPartitionId, id); + Set expandedResourceIds = myMdmLinkExpandSvc.expandMdmBySourceResourceId(theRequestPartitionId, id); if (expandedResourceIds.isEmpty()) { - expandedResourceIds = mdmLinkExpandSvc.expandMdmByGoldenResourceId(theRequestPartitionId, id); + expandedResourceIds = myMdmLinkExpandSvc.expandMdmByGoldenResourceId(theRequestPartitionId, id); } // Rebuild diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/EIDHelper.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/EIDHelper.java index 9930cc7690e2..d0134847bcb7 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/EIDHelper.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/EIDHelper.java @@ -26,14 +26,12 @@ import org.hl7.fhir.instance.model.api.IAnyResource; import org.hl7.fhir.instance.model.api.IBaseResource; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; -@Service public class EIDHelper { private final FhirContext myFhirContext; diff --git a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/GoldenResourceHelper.java b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/GoldenResourceHelper.java index 4b1b061375bb..f2a3e80aadf4 100644 --- a/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/GoldenResourceHelper.java +++ b/hapi-fhir-server-mdm/src/main/java/ca/uhn/fhir/mdm/util/GoldenResourceHelper.java @@ -40,7 +40,6 @@ import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.List; @@ -52,7 +51,6 @@ import static ca.uhn.fhir.context.FhirVersionEnum.R4; import static ca.uhn.fhir.context.FhirVersionEnum.R5; -@Service public class GoldenResourceHelper { private static final Logger ourLog = Logs.getMdmTroubleshootingLog(); diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourceAndWriteBinaryStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourceAndWriteBinaryStep.java index d9e8a82c33a7..64b9f7bea724 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourceAndWriteBinaryStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourceAndWriteBinaryStep.java @@ -48,6 +48,7 @@ import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryMatchResult; import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher; import ca.uhn.fhir.jpa.util.RandomTextUtils; +import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.IBundleProvider; @@ -91,6 +92,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; +import static ca.uhn.fhir.batch2.jobs.export.BulkDataExportUtil.PATIENT_BULK_EXPORT_FORWARD_REFERENCE_RESOURCE_TYPES; import static ca.uhn.fhir.batch2.jobs.imprt.BulkImportAppCtx.PARAM_MAXIMUM_BATCH_SIZE_DEFAULT; import static ca.uhn.fhir.rest.api.Constants.PARAM_ID; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -133,6 +135,9 @@ public class ExpandResourceAndWriteBinaryStep @Autowired private IBulkDataExportHistoryHelper myExportHelper; + @Autowired(required = false) + private Optional myMdmLinkExpandSvc; + private volatile ResponseTerminologyTranslationSvc myResponseTerminologyTranslationSvc; /** @@ -523,7 +528,16 @@ public void accept(List theResources) throws JobExecutionFailedEx // if necessary, expand resources if (parameters.isExpandMdm()) { - myBulkExportProcessor.expandMdmResources(theResources); + if (myMdmLinkExpandSvc.isPresent()) { + for (IBaseResource resource : theResources) { + if (!PATIENT_BULK_EXPORT_FORWARD_REFERENCE_RESOURCE_TYPES.contains(resource.fhirType())) { + myMdmLinkExpandSvc.get().annotateResource(resource); + } + } + } else { + ourLog.warn( + "Attempted to annotate a resource with an extension identifying it's associated golden resource, but no IMdmLinkExpandSvc was configured. Is MDM configured correctly?"); + } } // Normalize terminology diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java index dbf60025b7a1..df503f49f2de 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/ExpandResourcesStep.java @@ -42,6 +42,7 @@ import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryMatchResult; import ca.uhn.fhir.jpa.searchparam.matcher.InMemoryResourceMatcher; +import ca.uhn.fhir.mdm.api.IMdmLinkExpandSvc; import ca.uhn.fhir.parser.IParser; import ca.uhn.fhir.rest.api.server.IBundleProvider; import ca.uhn.fhir.rest.api.server.SystemRequestDetails; @@ -66,6 +67,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static ca.uhn.fhir.batch2.jobs.export.BulkDataExportUtil.PATIENT_BULK_EXPORT_FORWARD_REFERENCE_RESOURCE_TYPES; import static ca.uhn.fhir.rest.api.Constants.PARAM_ID; import static org.slf4j.LoggerFactory.getLogger; @@ -100,6 +102,9 @@ public class ExpandResourcesStep @Autowired private InterceptorService myInterceptorService; + @Autowired(required = false) + private Optional myMdmLinkExpandSvc; + private volatile ResponseTerminologyTranslationSvc myResponseTerminologyTranslationSvc; @Nonnull @@ -139,7 +144,16 @@ public RunOutcome run( // if necessary, expand resources if (parameters.isExpandMdm()) { - myBulkExportProcessor.expandMdmResources(allResources); + if (myMdmLinkExpandSvc.isPresent()) { + for (IBaseResource resource : allResources) { + if (!PATIENT_BULK_EXPORT_FORWARD_REFERENCE_RESOURCE_TYPES.contains(resource.fhirType())) { + myMdmLinkExpandSvc.get().annotateResource(resource); + } + } + } else { + ourLog.warn( + "Attempted to annotate a resource with an extension identifying it's associated golden resource, but no IMdmLinkExpandSvc was configured. Is MDM configured correctly?"); + } } // Normalize terminology diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/api/IBulkExportProcessor.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/api/IBulkExportProcessor.java index c1b297ee4b4e..a70ab7a3e29e 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/api/IBulkExportProcessor.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/api/IBulkExportProcessor.java @@ -21,10 +21,8 @@ import ca.uhn.fhir.jpa.bulk.export.model.ExportPIDIteratorParameters; import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; -import org.hl7.fhir.instance.model.api.IBaseResource; import java.util.Iterator; -import java.util.List; public interface IBulkExportProcessor { @@ -34,10 +32,4 @@ public interface IBulkExportProcessor { * @return */ Iterator getResourcePidIterator(ExportPIDIteratorParameters theParams); - - /** - * Does the MDM expansion of resources if necessary - * @param theResources - the list of resources to expand - */ - void expandMdmResources(List theResources); }