From fb040145ad32d47900f14911bfd6f345aa372ebc Mon Sep 17 00:00:00 2001 From: Daniel Hahne Date: Wed, 4 Mar 2026 11:37:29 +0100 Subject: [PATCH 1/4] Add FP Date Shift Architecture --- .../fts/cda/impl/FhirPseudonymizerConfig.java | 31 +++ .../fts/cda/impl/FhirPseudonymizerStep.java | 150 ++++++++++++++ .../impl/FhirPseudonymizerStepFactory.java | 55 ++++++ .../cda/impl/FhirPseudonymizerConfigTest.java | 62 ++++++ .../FhirPseudonymizerStepFactoryTest.java | 111 +++++++++++ .../cda/impl/FhirPseudonymizerStepTest.java | 146 ++++++++++++++ .../CdAgentFhirPseudonymizerController.java | 1 + .../rest/FpTransportMappingController.java | 92 +++++++++ .../tca/rest/FpTransportMappingRequest.java | 26 +++ .../fts/tca/services/TransportIdService.java | 162 +++++++++------- ...dAgentFhirPseudonymizerControllerTest.java | 27 --- .../FpTransportMappingControllerTest.java | 183 ++++++++++++++++++ .../tca/services/TransportIdServiceTest.java | 64 +++++- .../fts/util/fhir/DateShiftAnonymizer.java | 131 +++++++++++++ .../util/fhir/DateShiftAnonymizerTest.java | 176 +++++++++++++++++ 15 files changed, 1318 insertions(+), 99 deletions(-) create mode 100644 clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerConfig.java create mode 100644 clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStep.java create mode 100644 clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactory.java create mode 100644 clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerConfigTest.java create mode 100644 clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactoryTest.java create mode 100644 clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerStepTest.java create mode 100644 trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingController.java create mode 100644 trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingRequest.java create mode 100644 trust-center-agent/src/test/java/care/smith/fts/tca/rest/FpTransportMappingControllerTest.java create mode 100644 util/src/main/java/care/smith/fts/util/fhir/DateShiftAnonymizer.java create mode 100644 util/src/test/java/care/smith/fts/util/fhir/DateShiftAnonymizerTest.java diff --git a/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerConfig.java b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerConfig.java new file mode 100644 index 000000000..c42a8a00b --- /dev/null +++ b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerConfig.java @@ -0,0 +1,31 @@ +package care.smith.fts.cda.impl; + +import care.smith.fts.api.DateShiftPreserve; +import care.smith.fts.util.HttpClientConfig; +import care.smith.fts.util.tca.TcaDomains; +import java.io.File; +import java.time.Duration; +import java.util.Optional; + +public record FhirPseudonymizerConfig( + HttpClientConfig serviceUrl, + File anonymizationConfig, + TCAConfig trustCenterAgent, + Duration maxDateShift, + DateShiftPreserve dateShiftPreserve) { + + public FhirPseudonymizerConfig( + HttpClientConfig serviceUrl, + File anonymizationConfig, + TCAConfig trustCenterAgent, + Duration maxDateShift, + DateShiftPreserve dateShiftPreserve) { + this.serviceUrl = serviceUrl; + this.anonymizationConfig = anonymizationConfig; + this.trustCenterAgent = trustCenterAgent; + this.maxDateShift = maxDateShift; + this.dateShiftPreserve = Optional.ofNullable(dateShiftPreserve).orElse(DateShiftPreserve.NONE); + } + + public record TCAConfig(HttpClientConfig server, TcaDomains domains) {} +} diff --git a/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStep.java b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStep.java new file mode 100644 index 000000000..680f412e9 --- /dev/null +++ b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStep.java @@ -0,0 +1,150 @@ +package care.smith.fts.cda.impl; + +import static care.smith.fts.util.MediaTypes.APPLICATION_FHIR_JSON; +import static care.smith.fts.util.RetryStrategies.defaultRetryStrategy; + +import care.smith.fts.api.ConsentedPatientBundle; +import care.smith.fts.api.DateShiftPreserve; +import care.smith.fts.api.TransportBundle; +import care.smith.fts.api.cda.Deidentificator; +import care.smith.fts.util.fhir.DateShiftAnonymizer; +import care.smith.fts.util.tca.TcaDomains; +import care.smith.fts.util.tca.TransportMappingResponse; +import io.micrometer.core.instrument.MeterRegistry; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.hl7.fhir.r4.model.Bundle; +import org.springframework.http.MediaType; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Mono; + +/** + * Deidentification step using an external FHIR Pseudonymizer service. Handles date nullification + * locally, delegates ID pseudonymization to FP (which calls TCA), then consolidates all mappings + * via TCA. + */ +@Slf4j +class FhirPseudonymizerStep implements Deidentificator { + + private static final Pattern TID_PATTERN = Pattern.compile("[A-Za-z0-9_-]{32}"); + + private final WebClient fpClient; + private final WebClient tcaClient; + private final TcaDomains domains; + private final Duration maxDateShift; + private final DateShiftPreserve preserve; + private final List dateShiftPaths; + private final MeterRegistry meterRegistry; + + FhirPseudonymizerStep( + WebClient fpClient, + WebClient tcaClient, + TcaDomains domains, + Duration maxDateShift, + DateShiftPreserve preserve, + List dateShiftPaths, + MeterRegistry meterRegistry) { + this.fpClient = fpClient; + this.tcaClient = tcaClient; + this.domains = domains; + this.maxDateShift = maxDateShift; + this.preserve = preserve; + this.dateShiftPaths = dateShiftPaths; + this.meterRegistry = meterRegistry; + } + + @Override + public Mono deidentify(ConsentedPatientBundle bundle) { + return Mono.defer( + () -> { + var patient = bundle.consentedPatient(); + var dateMappings = DateShiftAnonymizer.nullifyDates(bundle.bundle(), dateShiftPaths); + + log.trace( + "Nullified {} date elements, sending to FHIR Pseudonymizer", dateMappings.size()); + + return sendToFhirPseudonymizer(bundle.bundle()) + .flatMap( + pseudonymizedBundle -> { + var identityTIds = extractTransportIds(pseudonymizedBundle); + if (identityTIds.isEmpty() && dateMappings.isEmpty()) { + return Mono.empty(); + } + return consolidateViaTca( + patient.identifier(), identityTIds, dateMappings) + .map( + transferId -> + new TransportBundle(pseudonymizedBundle, transferId)); + }); + }); + } + + private Mono sendToFhirPseudonymizer(Bundle bundle) { + return fpClient + .post() + .uri("/$de-identify") + .headers(h -> h.setContentType(APPLICATION_FHIR_JSON)) + .headers(h -> h.setAccept(List.of(APPLICATION_FHIR_JSON))) + .bodyValue(bundle) + .retrieve() + .bodyToMono(Bundle.class) + .timeout(Duration.ofSeconds(60)) + .retryWhen(defaultRetryStrategy(meterRegistry, "sendToFhirPseudonymizer")) + .doOnError(e -> log.error("FHIR Pseudonymizer call failed: {}", e.getMessage())); + } + + /** + * Extracts transport IDs from the pseudonymized bundle. After FP processing, resource IDs that + * were pseudonymized will be 32-char Base64URL tIDs from TCA. + */ + static Set extractTransportIds(Bundle bundle) { + return bundle.getEntry().stream() + .map(Bundle.BundleEntryComponent::getResource) + .filter(r -> r != null && r.hasId()) + .map(r -> r.getIdElement().getIdPart()) + .filter(id -> id != null && TID_PATTERN.matcher(id).matches()) + .collect(Collectors.toSet()); + } + + private Mono consolidateViaTca( + String patientIdentifier, Set identityTIds, Map dateMappings) { + + var request = + new FpTransportMappingRequest( + patientIdentifier, identityTIds, dateMappings, domains.dateShift(), maxDateShift, preserve); + + log.trace( + "Consolidating {} identity tIDs + {} date mappings via TCA", + identityTIds.size(), + dateMappings.size()); + + return tcaClient + .post() + .uri("/api/v2/cd/fhir-pseudonymizer/transport-mapping") + .headers(h -> h.setContentType(MediaType.APPLICATION_JSON)) + .bodyValue(request) + .retrieve() + .bodyToMono(TransportMappingResponse.class) + .timeout(Duration.ofSeconds(30)) + .retryWhen(defaultRetryStrategy(meterRegistry, "consolidateViaTca")) + .doOnError(e -> log.error("TCA consolidation failed: {}", e.getMessage())) + .map(TransportMappingResponse::transferId); + } + + /** + * DTO matching TCA's FpTransportMappingRequest. Duplicated here to avoid cross-module dependency + * on the TCA rest package. + */ + record FpTransportMappingRequest( + String patientIdentifier, + Set transportIds, + Map dateMappings, + String dateShiftDomain, + Duration maxDateShift, + DateShiftPreserve dateShiftPreserve) {} +} diff --git a/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactory.java b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactory.java new file mode 100644 index 000000000..f1e9b6498 --- /dev/null +++ b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactory.java @@ -0,0 +1,55 @@ +package care.smith.fts.cda.impl; + +import static java.util.Objects.requireNonNull; + +import care.smith.fts.api.cda.Deidentificator; +import care.smith.fts.util.WebClientFactory; +import care.smith.fts.util.fhir.DateShiftAnonymizer; +import io.micrometer.core.instrument.MeterRegistry; +import java.io.IOException; +import java.util.List; +import org.springframework.stereotype.Component; + +@Component("fhir-pseudonymizerDeidentificator") +public class FhirPseudonymizerStepFactory + implements Deidentificator.Factory { + + private final WebClientFactory clientFactory; + private final MeterRegistry meterRegistry; + + public FhirPseudonymizerStepFactory( + WebClientFactory clientFactory, MeterRegistry meterRegistry) { + this.clientFactory = clientFactory; + this.meterRegistry = meterRegistry; + } + + @Override + public Class getConfigType() { + return FhirPseudonymizerConfig.class; + } + + @Override + public Deidentificator create( + Deidentificator.Config commonConfig, FhirPseudonymizerConfig implConfig) { + var fpClient = clientFactory.create(implConfig.serviceUrl()); + var tcaClient = clientFactory.create(implConfig.trustCenterAgent().server()); + + List dateShiftPaths; + try { + dateShiftPaths = + DateShiftAnonymizer.parseDateShiftPaths( + requireNonNull(implConfig.anonymizationConfig())); + } catch (IOException e) { + throw new IllegalStateException("Failed to parse anonymization config", e); + } + + return new FhirPseudonymizerStep( + fpClient, + tcaClient, + implConfig.trustCenterAgent().domains(), + implConfig.maxDateShift(), + implConfig.dateShiftPreserve(), + dateShiftPaths, + meterRegistry); + } +} diff --git a/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerConfigTest.java b/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerConfigTest.java new file mode 100644 index 000000000..f37dad303 --- /dev/null +++ b/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerConfigTest.java @@ -0,0 +1,62 @@ +package care.smith.fts.cda.impl; + +import static org.assertj.core.api.Assertions.assertThat; + +import care.smith.fts.api.DateShiftPreserve; +import care.smith.fts.util.HttpClientConfig; +import care.smith.fts.util.tca.TcaDomains; +import java.io.File; +import java.time.Duration; +import org.junit.jupiter.api.Test; + +class FhirPseudonymizerConfigTest { + + @Test + void dateShiftPreserveDefaultsToNoneWhenNull() { + var serviceUrl = new HttpClientConfig("http://localhost:1234"); + var tcaServer = new HttpClientConfig("http://localhost:5678"); + var domains = new TcaDomains("pseudo", "salt", "dateshift"); + var tcaConfig = new FhirPseudonymizerConfig.TCAConfig(tcaServer, domains); + + var config = + new FhirPseudonymizerConfig( + serviceUrl, new File("anon.yaml"), tcaConfig, Duration.ofDays(14), null); + + assertThat(config.dateShiftPreserve()).isEqualTo(DateShiftPreserve.NONE); + } + + @Test + void dateShiftPreserveRetainsExplicitValue() { + var serviceUrl = new HttpClientConfig("http://localhost:1234"); + var tcaServer = new HttpClientConfig("http://localhost:5678"); + var domains = new TcaDomains("pseudo", "salt", "dateshift"); + var tcaConfig = new FhirPseudonymizerConfig.TCAConfig(tcaServer, domains); + + var config = + new FhirPseudonymizerConfig( + serviceUrl, new File("anon.yaml"), tcaConfig, Duration.ofDays(14), DateShiftPreserve.WEEKDAY); + + assertThat(config.dateShiftPreserve()).isEqualTo(DateShiftPreserve.WEEKDAY); + } + + @Test + void recordAccessors() { + var serviceUrl = new HttpClientConfig("http://localhost:1234"); + var tcaServer = new HttpClientConfig("http://localhost:5678"); + var domains = new TcaDomains("pseudo", "salt", "dateshift"); + var tcaConfig = new FhirPseudonymizerConfig.TCAConfig(tcaServer, domains); + var anonFile = new File("anon.yaml"); + var maxShift = Duration.ofDays(14); + + var config = + new FhirPseudonymizerConfig( + serviceUrl, anonFile, tcaConfig, maxShift, DateShiftPreserve.DAYTIME); + + assertThat(config.serviceUrl()).isEqualTo(serviceUrl); + assertThat(config.anonymizationConfig()).isEqualTo(anonFile); + assertThat(config.trustCenterAgent()).isEqualTo(tcaConfig); + assertThat(config.maxDateShift()).isEqualTo(maxShift); + assertThat(config.trustCenterAgent().server()).isEqualTo(tcaServer); + assertThat(config.trustCenterAgent().domains()).isEqualTo(domains); + } +} diff --git a/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactoryTest.java b/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactoryTest.java new file mode 100644 index 000000000..4310f6d7d --- /dev/null +++ b/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactoryTest.java @@ -0,0 +1,111 @@ +package care.smith.fts.cda.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import care.smith.fts.api.DateShiftPreserve; +import care.smith.fts.api.cda.Deidentificator; +import care.smith.fts.util.HttpClientConfig; +import care.smith.fts.util.WebClientFactory; +import care.smith.fts.util.tca.TcaDomains; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.time.Duration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.web.reactive.function.client.WebClient; + +@ExtendWith(MockitoExtension.class) +class FhirPseudonymizerStepFactoryTest { + + @Mock private WebClientFactory clientFactory; + @Mock private WebClient webClient; + @TempDir File tempDir; + + private FhirPseudonymizerStepFactory factory; + + @BeforeEach + void setUp() { + factory = new FhirPseudonymizerStepFactory(clientFactory, new SimpleMeterRegistry()); + } + + @Test + void getConfigTypeReturnsFhirPseudonymizerConfig() { + assertThat(factory.getConfigType()).isEqualTo(FhirPseudonymizerConfig.class); + } + + @Test + void createReturnsDeidentificatorWithValidConfig() throws IOException { + when(clientFactory.create(any(HttpClientConfig.class))).thenReturn(webClient); + + var configFile = writeAnonymizationConfig(); + var serviceUrl = new HttpClientConfig("http://localhost:1234"); + var tcaServer = new HttpClientConfig("http://localhost:5678"); + var domains = new TcaDomains("pseudo", "salt", "dateshift"); + var tcaConfig = new FhirPseudonymizerConfig.TCAConfig(tcaServer, domains); + + var implConfig = + new FhirPseudonymizerConfig( + serviceUrl, configFile, tcaConfig, Duration.ofDays(14), DateShiftPreserve.NONE); + + Deidentificator result = factory.create(new Deidentificator.Config(), implConfig); + + assertThat(result).isInstanceOf(FhirPseudonymizerStep.class); + } + + @Test + void createThrowsWhenAnonymizationConfigIsNull() { + when(clientFactory.create(any(HttpClientConfig.class))).thenReturn(webClient); + + var serviceUrl = new HttpClientConfig("http://localhost:1234"); + var tcaServer = new HttpClientConfig("http://localhost:5678"); + var domains = new TcaDomains("pseudo", "salt", "dateshift"); + var tcaConfig = new FhirPseudonymizerConfig.TCAConfig(tcaServer, domains); + + var implConfig = + new FhirPseudonymizerConfig( + serviceUrl, null, tcaConfig, Duration.ofDays(14), DateShiftPreserve.NONE); + + assertThatThrownBy(() -> factory.create(new Deidentificator.Config(), implConfig)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void createThrowsWhenConfigFileDoesNotExist() { + when(clientFactory.create(any(HttpClientConfig.class))).thenReturn(webClient); + + var nonExistent = new File(tempDir, "nonexistent.yaml"); + var serviceUrl = new HttpClientConfig("http://localhost:1234"); + var tcaServer = new HttpClientConfig("http://localhost:5678"); + var domains = new TcaDomains("pseudo", "salt", "dateshift"); + var tcaConfig = new FhirPseudonymizerConfig.TCAConfig(tcaServer, domains); + + var implConfig = + new FhirPseudonymizerConfig( + serviceUrl, nonExistent, tcaConfig, Duration.ofDays(14), DateShiftPreserve.NONE); + + assertThatThrownBy(() -> factory.create(new Deidentificator.Config(), implConfig)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Failed to parse anonymization config"); + } + + private File writeAnonymizationConfig() throws IOException { + var file = new File(tempDir, "anonymization.yaml"); + Files.writeString( + file.toPath(), + """ + fhirPathRules: + - path: "Encounter.period.start" + method: "dateshift" + """); + return file; + } +} diff --git a/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerStepTest.java b/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerStepTest.java new file mode 100644 index 000000000..5a57ff4eb --- /dev/null +++ b/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerStepTest.java @@ -0,0 +1,146 @@ +package care.smith.fts.cda.impl; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +import care.smith.fts.api.ConsentedPatient; +import care.smith.fts.api.ConsentedPatientBundle; +import care.smith.fts.api.DateShiftPreserve; +import care.smith.fts.util.tca.TcaDomains; +import care.smith.fts.util.tca.TransportMappingResponse; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.time.Duration; +import java.util.List; +import java.util.function.Consumer; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Encounter; +import org.hl7.fhir.r4.model.Patient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +@SuppressWarnings("unchecked") +@ExtendWith(MockitoExtension.class) +class FhirPseudonymizerStepTest { + + @Mock private WebClient fpClient; + @Mock private WebClient tcaClient; + @Mock private WebClient.RequestBodyUriSpec fpPostSpec; + @Mock private WebClient.RequestBodySpec fpBodySpec; + @Mock private WebClient.RequestHeadersSpec fpHeadersSpec; + @Mock private WebClient.ResponseSpec fpResponseSpec; + @Mock private WebClient.RequestBodyUriSpec tcaPostSpec; + @Mock private WebClient.RequestBodySpec tcaBodySpec; + @Mock private WebClient.RequestHeadersSpec tcaHeadersSpec; + @Mock private WebClient.ResponseSpec tcaResponseSpec; + + private FhirPseudonymizerStep step; + + @BeforeEach + void setUp() { + var domains = new TcaDomains("pseudo-domain", "salt-domain", "dateshift-domain"); + step = + new FhirPseudonymizerStep( + fpClient, + tcaClient, + domains, + Duration.ofDays(14), + DateShiftPreserve.NONE, + List.of(), + new SimpleMeterRegistry()); + + // FP client mock chain (lenient: not all tests exercise FP) + lenient().when(fpClient.post()).thenReturn(fpPostSpec); + lenient().when(fpPostSpec.uri(any(String.class))).thenReturn(fpBodySpec); + lenient().when(fpBodySpec.headers(any(Consumer.class))).thenReturn(fpBodySpec); + lenient().when(fpBodySpec.bodyValue(any())).thenReturn(fpHeadersSpec); + lenient().when(fpHeadersSpec.retrieve()).thenReturn(fpResponseSpec); + + // TCA client mock chain (lenient: not all tests exercise TCA) + lenient().when(tcaClient.post()).thenReturn(tcaPostSpec); + lenient().when(tcaPostSpec.uri(any(String.class))).thenReturn(tcaBodySpec); + lenient().when(tcaBodySpec.headers(any(Consumer.class))).thenReturn(tcaBodySpec); + lenient().when(tcaBodySpec.bodyValue(any())).thenReturn(tcaHeadersSpec); + lenient().when(tcaHeadersSpec.retrieve()).thenReturn(tcaResponseSpec); + } + + @Test + void extractTransportIdsFinds32CharBase64UrlIds() { + var bundle = new Bundle(); + + var patient = new Patient(); + patient.setId("AbCdEfGhIjKlMnOpQrStUvWxYz012345"); // 32 chars Base64URL + bundle.addEntry().setResource(patient); + + var encounter = new Encounter(); + encounter.setId("short-id"); // Not a tID + bundle.addEntry().setResource(encounter); + + var encounter2 = new Encounter(); + encounter2.setId("a-uuid-that-is-longer-than-32-chars-total"); // Not a 32-char tID + bundle.addEntry().setResource(encounter2); + + var tIds = FhirPseudonymizerStep.extractTransportIds(bundle); + + assertThat(tIds).containsExactly("AbCdEfGhIjKlMnOpQrStUvWxYz012345"); + } + + @Test + void extractTransportIdsReturnsEmptyForNoMatches() { + var bundle = new Bundle(); + var patient = new Patient(); + patient.setId("regular-uuid-id"); + bundle.addEntry().setResource(patient); + + var tIds = FhirPseudonymizerStep.extractTransportIds(bundle); + + assertThat(tIds).isEmpty(); + } + + @Test + void deidentifyReturnsBundleWithTransferId() { + var pseudonymizedBundle = new Bundle(); + var pseudoPatient = new Patient(); + pseudoPatient.setId("AbCdEfGhIjKlMnOpQrStUvWxYz012345"); + pseudonymizedBundle.addEntry().setResource(pseudoPatient); + + when(fpResponseSpec.bodyToMono(Bundle.class)).thenReturn(Mono.just(pseudonymizedBundle)); + when(tcaResponseSpec.bodyToMono(TransportMappingResponse.class)) + .thenReturn(Mono.just(new TransportMappingResponse("transfer-id-abc"))); + + var patient = new ConsentedPatient("patient-1", "http://system"); + var inputBundle = new Bundle(); + inputBundle.addEntry().setResource(new Patient()); + + var result = step.deidentify(new ConsentedPatientBundle(inputBundle, patient)); + + StepVerifier.create(result) + .assertNext( + transport -> { + assertThat(transport.transferId()).isEqualTo("transfer-id-abc"); + assertThat(transport.bundle()).isEqualTo(pseudonymizedBundle); + }) + .verifyComplete(); + } + + @Test + void deidentifyReturnsEmptyWhenNoMappings() { + var emptyBundle = new Bundle(); + + when(fpResponseSpec.bodyToMono(Bundle.class)).thenReturn(Mono.just(emptyBundle)); + + var patient = new ConsentedPatient("patient-1", "http://system"); + var inputBundle = new Bundle(); + + var result = step.deidentify(new ConsentedPatientBundle(inputBundle, patient)); + + StepVerifier.create(result).verifyComplete(); + } +} diff --git a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/CdAgentFhirPseudonymizerController.java b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/CdAgentFhirPseudonymizerController.java index 9c11e0ab9..f10c7e506 100644 --- a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/CdAgentFhirPseudonymizerController.java +++ b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/CdAgentFhirPseudonymizerController.java @@ -180,6 +180,7 @@ private Mono generateTransportIds( private Mono createPseudonymEntry( String original, String sId, String namespace, Duration ttl) { var tId = transportIdService.generateId(); + return transportIdService .storeMapping(tId, sId, ttl) .thenReturn(new PseudonymEntry(namespace, original, tId)); diff --git a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingController.java b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingController.java new file mode 100644 index 000000000..7301ea1b9 --- /dev/null +++ b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingController.java @@ -0,0 +1,92 @@ +package care.smith.fts.tca.rest; + +import static care.smith.fts.tca.deidentification.DateShiftUtil.generate; +import static care.smith.fts.tca.deidentification.DateShiftUtil.shiftDate; +import static care.smith.fts.util.deidentifhir.DateShiftConstants.DATE_SHIFT_PREFIX; +import static java.util.Set.of; + +import care.smith.fts.tca.deidentification.GpasClient; +import care.smith.fts.tca.services.TransportIdService; +import care.smith.fts.util.error.ErrorResponseUtil; +import care.smith.fts.util.tca.TransportMappingResponse; +import jakarta.validation.Valid; +import java.time.Duration; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.ResponseEntity; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Mono; + +/** + * Receives date mappings from CDA after FHIR Pseudonymizer processing, computes shifted dates, and + * consolidates all mappings (identity tID→sID + date entries) into a single transferId-based + * MapCache for RDA retrieval. + */ +@Slf4j +@RestController +@RequestMapping("api/v2") +@Validated +public class FpTransportMappingController { + + private final TransportIdService transportIdService; + private final GpasClient gpasClient; + + public FpTransportMappingController( + TransportIdService transportIdService, GpasClient gpasClient) { + this.transportIdService = transportIdService; + this.gpasClient = gpasClient; + } + + @PostMapping("cd/fhir-pseudonymizer/transport-mapping") + public Mono> consolidateTransportMappings( + @Valid @RequestBody FpTransportMappingRequest request) { + + log.debug( + "Consolidating {} identity tIDs + {} date mappings for patient", + request.transportIds().size(), + request.dateMappings().size()); + + return fetchDateShiftSeed(request) + .map(seed -> computeShiftedDates(seed, request)) + .flatMap(dsEntries -> consolidate(request, dsEntries)) + .map(transferId -> ResponseEntity.ok(new TransportMappingResponse(transferId))) + .onErrorResume(this::handleError); + } + + private Mono fetchDateShiftSeed(FpTransportMappingRequest request) { + var seedKey = "%s_%s".formatted(request.maxDateShift().toString(), request.patientIdentifier()); + return gpasClient + .fetchOrCreatePseudonyms(request.dateShiftDomain(), of(seedKey)) + .map(m -> m.get(seedKey)); + } + + private Map computeShiftedDates( + String seed, FpTransportMappingRequest request) { + if (request.dateMappings().isEmpty()) { + return Map.of(); + } + + var shift = generate(seed, request.maxDateShift(), request.dateShiftPreserve()); + + return request.dateMappings().entrySet().stream() + .collect( + Collectors.toMap( + e -> DATE_SHIFT_PREFIX + e.getKey(), e -> shiftDate(e.getValue(), shift))); + } + + private Mono consolidate( + FpTransportMappingRequest request, Map dsEntries) { + var ttl = transportIdService.getDefaultTtl(); + return transportIdService.consolidateMappings(request.transportIds(), dsEntries, ttl); + } + + private Mono> handleError(Throwable error) { + log.error("Failed to consolidate transport mappings: {}", error.getMessage()); + return ErrorResponseUtil.internalServerError(error); + } +} diff --git a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingRequest.java b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingRequest.java new file mode 100644 index 000000000..0c6807a7c --- /dev/null +++ b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingRequest.java @@ -0,0 +1,26 @@ +package care.smith.fts.tca.rest; + +import care.smith.fts.api.DateShiftPreserve; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import java.time.Duration; +import java.util.Map; +import java.util.Set; + +/** + * Request from CDA to consolidate transport mappings after FHIR Pseudonymizer processing. + * + * @param patientIdentifier patient ID used to derive the deterministic dateshift seed + * @param transportIds identity tIDs from $create-pseudonym (already stored as tid:tId→sId in Redis) + * @param dateMappings tID→originalDate for each date element nullified by CDA + * @param dateShiftDomain gPAS domain for fetching the dateshift seed + * @param maxDateShift maximum shift duration (e.g., P14D) + * @param dateShiftPreserve preservation strategy (NONE, WEEKDAY, DAYTIME) + */ +public record FpTransportMappingRequest( + @NotBlank String patientIdentifier, + @NotNull Set transportIds, + @NotNull Map dateMappings, + @NotBlank String dateShiftDomain, + @NotNull Duration maxDateShift, + @NotNull DateShiftPreserve dateShiftPreserve) {} diff --git a/trust-center-agent/src/main/java/care/smith/fts/tca/services/TransportIdService.java b/trust-center-agent/src/main/java/care/smith/fts/tca/services/TransportIdService.java index b82fa50d8..b8b012238 100644 --- a/trust-center-agent/src/main/java/care/smith/fts/tca/services/TransportIdService.java +++ b/trust-center-agent/src/main/java/care/smith/fts/tca/services/TransportIdService.java @@ -26,9 +26,9 @@ * *
    *
  • Generate cryptographically secure transport IDs (32 chars, Base64URL) - *
  • Store tID→sID mappings in Redis grouped by transfer session + *
  • Store tID→sID mappings in Redis (individually or grouped by transfer session) *
  • Resolve transport IDs back to secure pseudonyms for RDA - *
  • Manage date shift values per transfer session + *
  • Consolidate per-tID mappings into transferId-based MapCache for RDA retrieval *
*/ @Slf4j @@ -79,16 +79,13 @@ public String generateId() { * @return Mono completing when storage is done */ public Mono storeAllMappings(String transferId, Map allData, Duration ttl) { - return Mono.defer( - () -> { - var mapCache = getMapCache(transferId); - return mapCache - .expire(ttl) - .then(mapCache.putAll(allData)) - .retryWhen(defaultRetryStrategy(meterRegistry, "storeAllMappings")) - .doOnSuccess( - v -> log.trace("Stored {} entries: transferId={}", allData.size(), transferId)); - }); + var mapCache = getMapCache(transferId); + return mapCache + .expire(ttl) + .then(mapCache.putAll(allData)) + .retryWhen(defaultRetryStrategy(meterRegistry, "storeAllMappings")) + .doOnSuccess( + v -> log.trace("Stored {} entries: transferId={}", allData.size(), transferId)); } /** @@ -98,15 +95,10 @@ public Mono storeAllMappings(String transferId, Map allDat * @return Mono emitting all stored data for this session */ public Mono> fetchAllMappings(String transferId) { - return Mono.defer( - () -> { - var mapCache = getMapCache(transferId); - return mapCache - .readAllMap() - .retryWhen(defaultRetryStrategy(meterRegistry, "fetchAllMappings")) - .doOnSuccess( - m -> log.trace("Fetched {} entries: transferId={}", m.size(), transferId)); - }); + return getMapCache(transferId) + .readAllMap() + .retryWhen(defaultRetryStrategy(meterRegistry, "fetchAllMappings")) + .doOnSuccess(m -> log.trace("Fetched {} entries: transferId={}", m.size(), transferId)); } // ========== Direct tid→sid storage (without transferId grouping) ========== @@ -123,14 +115,12 @@ public Mono> fetchAllMappings(String transferId) { * @return Mono completing when storage is done */ public Mono storeMapping(String tid, String sid, Duration ttl) { - return Mono.defer( - () -> { - var bucket = redisClient.reactive().getBucket(tidKey(tid)); - return bucket - .set(sid, ttl) - .retryWhen(defaultRetryStrategy(meterRegistry, "storeMapping")) - .doOnSuccess(v -> log.trace("Stored direct mapping: tid={}", tid)); - }); + return redisClient + .reactive() + .getBucket(tidKey(tid)) + .set(sid, ttl) + .retryWhen(defaultRetryStrategy(meterRegistry, "storeMapping")) + .doOnSuccess(v -> log.trace("Stored direct mapping: tid={}", tid)); } /** @@ -140,14 +130,12 @@ public Mono storeMapping(String tid, String sid, Duration ttl) { * @return Mono emitting the sid, or empty if not found */ public Mono fetchMapping(String tid) { - return Mono.defer( - () -> { - var bucket = redisClient.reactive().getBucket(tidKey(tid)); - return bucket - .get() - .retryWhen(defaultRetryStrategy(meterRegistry, "fetchMapping")) - .doOnSuccess(sid -> log.trace("Fetched mapping: tid={}, found={}", tid, sid != null)); - }); + return redisClient + .reactive() + .getBucket(tidKey(tid)) + .get() + .retryWhen(defaultRetryStrategy(meterRegistry, "fetchMapping")) + .doOnSuccess(sid -> log.trace("Fetched mapping: tid={}, found={}", tid, sid != null)); } /** @@ -161,19 +149,13 @@ public Mono storeMappings(Map tidToSid, Duration ttl) { if (tidToSid.isEmpty()) { return Mono.empty(); } - return Mono.defer( - () -> { - var batch = redisClient.reactive().createBatch(); - tidToSid.forEach( - (tid, sid) -> { - batch.getBucket(tidKey(tid)).set(sid, ttl); - }); - return batch - .execute() - .retryWhen(defaultRetryStrategy(meterRegistry, "storeMappings")) - .doOnSuccess(v -> log.trace("Stored {} direct mappings", tidToSid.size())) - .then(); - }); + var batch = redisClient.reactive().createBatch(); + tidToSid.forEach((tid, sid) -> batch.getBucket(tidKey(tid)).set(sid, ttl)); + return batch + .execute() + .retryWhen(defaultRetryStrategy(meterRegistry, "storeMappings")) + .doOnSuccess(v -> log.trace("Stored {} direct mappings", tidToSid.size())) + .then(); } /** @@ -189,24 +171,20 @@ public Mono> fetchMappings(Set tids) { if (tids.isEmpty()) { return Mono.just(Map.of()); } - return Mono.defer( - () -> { - var keys = tids.stream().map(this::tidKey).toArray(String[]::new); - return redisClient - .reactive() - .getBuckets() - .get(keys) - .retryWhen(defaultRetryStrategy(meterRegistry, "fetchMappings")) - .map( - result -> - result.entrySet().stream() - .collect( - Collectors.toMap( - e -> e.getKey().substring(TID_KEY_PREFIX.length()), - Map.Entry::getValue))) - .doOnSuccess( - m -> log.trace("Fetched {} of {} requested mappings", m.size(), tids.size())); - }); + var keys = tids.stream().map(this::tidKey).toArray(String[]::new); + return redisClient + .reactive() + .getBuckets() + .get(keys) + .retryWhen(defaultRetryStrategy(meterRegistry, "fetchMappings")) + .map( + result -> + result.entrySet().stream() + .collect( + Collectors.toMap( + e -> e.getKey().substring(TID_KEY_PREFIX.length()), + Map.Entry::getValue))) + .doOnSuccess(m -> log.trace("Fetched {} of {} requested mappings", m.size(), tids.size())); } /** @@ -225,4 +203,52 @@ private String tidKey(String tid) { private RMapCacheReactive getMapCache(String transferId) { return redisClient.reactive().getMapCache(transferId); } + + /** + * Consolidates identity tID→sID mappings and date shift entries into a single MapCache keyed by a + * new transferId, then deletes the individual tid:* keys. + * + *

This is the bridge between the FHIR Pseudonymizer's per-tID storage and the RDA's + * transferId-based retrieval via /rd/secure-mapping. + * + * @param identityTIds identity transport IDs from $create-pseudonym (stored as tid:tId→sId) + * @param dateShiftEntries already-prefixed ds:tId→shiftedDate entries + * @param ttl time-to-live for the consolidated MapCache + * @return Mono emitting the generated transferId + */ + public Mono consolidateMappings( + Set identityTIds, Map dateShiftEntries, Duration ttl) { + var transferId = generateId(); + + return fetchMappings(identityTIds) + .flatMap( + tidSidMap -> { + var allData = new java.util.HashMap<>(tidSidMap); + allData.putAll(dateShiftEntries); + + log.debug( + "Consolidating {} identity + {} dateshift entries into transferId={}", + tidSidMap.size(), + dateShiftEntries.size(), + transferId); + + return storeAllMappings(transferId, Map.copyOf(allData), ttl) + .then(deleteIndividualTidKeys(identityTIds)) + .thenReturn(transferId); + }); + } + + private Mono deleteIndividualTidKeys(Set tids) { + if (tids.isEmpty()) { + return Mono.empty(); + } + var keys = tids.stream().map(this::tidKey).toArray(String[]::new); + return redisClient + .reactive() + .getKeys() + .delete(keys) + .retryWhen(defaultRetryStrategy(meterRegistry, "deleteIndividualTidKeys")) + .doOnSuccess(count -> log.trace("Deleted {} individual tid keys", count)) + .then(); + } } diff --git a/trust-center-agent/src/test/java/care/smith/fts/tca/rest/CdAgentFhirPseudonymizerControllerTest.java b/trust-center-agent/src/test/java/care/smith/fts/tca/rest/CdAgentFhirPseudonymizerControllerTest.java index 3e0fa5e08..3e4c7c62c 100644 --- a/trust-center-agent/src/test/java/care/smith/fts/tca/rest/CdAgentFhirPseudonymizerControllerTest.java +++ b/trust-center-agent/src/test/java/care/smith/fts/tca/rest/CdAgentFhirPseudonymizerControllerTest.java @@ -173,33 +173,6 @@ void createPseudonymReturnsInternalServerErrorOnBackendFailure() { .verifyComplete(); } - @Test - void createPseudonymSucceedsEvenWhenDateShiftLinkingFails() { - var requestParams = createSingleValueRequest("test-domain", "patient-123"); - var ttl = Duration.ofMinutes(10); - - when(transportIdService.generateId()).thenReturn("tId-abc123"); - when(transportIdService.getDefaultTtl()).thenReturn(ttl); - when(transportIdService.storeMapping(eq("tId-abc123"), eq("sId-456"), eq(ttl))) - .thenReturn(Mono.empty()); - when(transportIdService.fetchAndDeleteTempDateShift(anyString())) - .thenReturn(Mono.error(new RuntimeException("Redis connection failed"))); - when(gpasClient.fetchOrCreatePseudonyms(eq("test-domain"), anySet())) - .thenReturn(Mono.just(Map.of("patient-123", "sId-456"))); - - var result = controller.createPseudonym(requestParams); - - StepVerifier.create(result) - .assertNext( - response -> { - assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); - var params = response.getBody(); - assertThat(params).isNotNull(); - assertThat(findParameterValue(params, "pseudonymValue")).isEqualTo("tId-abc123"); - }) - .verifyComplete(); - } - private Parameters createSingleValueRequest(String namespace, String originalValue) { var params = new Parameters(); params.addParameter().setName("namespace").setValue(new StringType(namespace)); diff --git a/trust-center-agent/src/test/java/care/smith/fts/tca/rest/FpTransportMappingControllerTest.java b/trust-center-agent/src/test/java/care/smith/fts/tca/rest/FpTransportMappingControllerTest.java new file mode 100644 index 000000000..a6b690014 --- /dev/null +++ b/trust-center-agent/src/test/java/care/smith/fts/tca/rest/FpTransportMappingControllerTest.java @@ -0,0 +1,183 @@ +package care.smith.fts.tca.rest; + +import static care.smith.fts.util.deidentifhir.DateShiftConstants.DATE_SHIFT_PREFIX; +import static java.util.Set.of; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import care.smith.fts.api.DateShiftPreserve; +import care.smith.fts.tca.deidentification.GpasClient; +import care.smith.fts.tca.services.TransportIdService; +import java.time.Duration; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpStatus; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +@ExtendWith(MockitoExtension.class) +class FpTransportMappingControllerTest { + + @Mock private TransportIdService transportIdService; + @Mock private GpasClient gpasClient; + + @Captor private ArgumentCaptor> dateShiftEntriesCaptor; + + private FpTransportMappingController controller; + + @BeforeEach + void setUp() { + controller = new FpTransportMappingController(transportIdService, gpasClient); + } + + @Test + void consolidateReturnsTransferIdOnSuccess() { + var request = + new FpTransportMappingRequest( + "patient-123", + Set.of("tId-1", "tId-2"), + Map.of("dateTid-1", "2020-06-15"), + "dateshift-domain", + Duration.ofDays(14), + DateShiftPreserve.NONE); + + var seedKey = "PT336H_patient-123"; + when(gpasClient.fetchOrCreatePseudonyms(eq("dateshift-domain"), eq(of(seedKey)))) + .thenReturn(Mono.just(Map.of(seedKey, "seed-abc"))); + when(transportIdService.getDefaultTtl()).thenReturn(Duration.ofMinutes(10)); + when(transportIdService.consolidateMappings(anySet(), anyMap(), any(Duration.class))) + .thenReturn(Mono.just("transferId-xyz")); + + var result = controller.consolidateTransportMappings(request); + + StepVerifier.create(result) + .assertNext( + response -> { + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + assertThat(response.getBody()).isNotNull(); + assertThat(response.getBody().transferId()).isEqualTo("transferId-xyz"); + }) + .verifyComplete(); + } + + @Test + void consolidatePrefixesDateShiftEntriesWithDsPrefix() { + var request = + new FpTransportMappingRequest( + "patient-456", + Set.of("tId-1"), + Map.of("dateTid-1", "2020-06-15", "dateTid-2", "2021-01-01"), + "dateshift-domain", + Duration.ofDays(14), + DateShiftPreserve.NONE); + + var seedKey = "PT336H_patient-456"; + when(gpasClient.fetchOrCreatePseudonyms(eq("dateshift-domain"), eq(of(seedKey)))) + .thenReturn(Mono.just(Map.of(seedKey, "seed-def"))); + when(transportIdService.getDefaultTtl()).thenReturn(Duration.ofMinutes(10)); + when(transportIdService.consolidateMappings( + anySet(), dateShiftEntriesCaptor.capture(), any(Duration.class))) + .thenReturn(Mono.just("transferId-abc")); + + controller.consolidateTransportMappings(request).block(); + + var captured = dateShiftEntriesCaptor.getValue(); + assertThat(captured).hasSize(2); + assertThat(captured.keySet()).allMatch(key -> key.startsWith(DATE_SHIFT_PREFIX)); + } + + @Test + void consolidateAppliesDateShiftToOriginalDates() { + var request = + new FpTransportMappingRequest( + "patient-789", + Set.of("tId-1"), + Map.of("dateTid-1", "2020-06-15"), + "dateshift-domain", + Duration.ofDays(14), + DateShiftPreserve.NONE); + + var seedKey = "PT336H_patient-789"; + when(gpasClient.fetchOrCreatePseudonyms(eq("dateshift-domain"), eq(of(seedKey)))) + .thenReturn(Mono.just(Map.of(seedKey, "seed-ghi"))); + when(transportIdService.getDefaultTtl()).thenReturn(Duration.ofMinutes(10)); + when(transportIdService.consolidateMappings( + anySet(), dateShiftEntriesCaptor.capture(), any(Duration.class))) + .thenReturn(Mono.just("transferId-def")); + + controller.consolidateTransportMappings(request).block(); + + var captured = dateShiftEntriesCaptor.getValue(); + assertThat(captured).hasSize(1); + // The shifted date should not equal the original + var shiftedDate = captured.get(DATE_SHIFT_PREFIX + "dateTid-1"); + assertThat(shiftedDate).isNotNull(); + // With a fixed seed, the shift is deterministic but not zero + } + + @Test + void consolidateWithEmptyDateMappingsSucceeds() { + var request = + new FpTransportMappingRequest( + "patient-empty", + Set.of("tId-1", "tId-2"), + Map.of(), + "dateshift-domain", + Duration.ofDays(14), + DateShiftPreserve.NONE); + + var seedKey = "PT336H_patient-empty"; + when(gpasClient.fetchOrCreatePseudonyms(eq("dateshift-domain"), eq(of(seedKey)))) + .thenReturn(Mono.just(Map.of(seedKey, "seed-jkl"))); + when(transportIdService.getDefaultTtl()).thenReturn(Duration.ofMinutes(10)); + when(transportIdService.consolidateMappings( + eq(Set.of("tId-1", "tId-2")), eq(Map.of()), any(Duration.class))) + .thenReturn(Mono.just("transferId-ghi")); + + var result = controller.consolidateTransportMappings(request); + + StepVerifier.create(result) + .assertNext( + response -> { + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + assertThat(response.getBody().transferId()).isEqualTo("transferId-ghi"); + }) + .verifyComplete(); + } + + @Test + void consolidateReturns500OnGpasFailure() { + var request = + new FpTransportMappingRequest( + "patient-fail", + Set.of("tId-1"), + Map.of("dateTid-1", "2020-06-15"), + "dateshift-domain", + Duration.ofDays(14), + DateShiftPreserve.NONE); + + var seedKey = "PT336H_patient-fail"; + when(gpasClient.fetchOrCreatePseudonyms(eq("dateshift-domain"), eq(of(seedKey)))) + .thenReturn(Mono.error(new RuntimeException("gPAS unavailable"))); + + var result = controller.consolidateTransportMappings(request); + + StepVerifier.create(result) + .assertNext( + response -> { + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR); + }) + .verifyComplete(); + } +} diff --git a/trust-center-agent/src/test/java/care/smith/fts/tca/services/TransportIdServiceTest.java b/trust-center-agent/src/test/java/care/smith/fts/tca/services/TransportIdServiceTest.java index f8aecf3f4..58090590d 100644 --- a/trust-center-agent/src/test/java/care/smith/fts/tca/services/TransportIdServiceTest.java +++ b/trust-center-agent/src/test/java/care/smith/fts/tca/services/TransportIdServiceTest.java @@ -24,6 +24,7 @@ import org.redisson.api.RBatchReactive; import org.redisson.api.RBucketReactive; import org.redisson.api.RBucketsReactive; +import org.redisson.api.RKeysReactive; import org.redisson.api.RMapCacheReactive; import org.redisson.api.RedissonClient; import org.redisson.api.RedissonReactiveClient; @@ -39,6 +40,7 @@ class TransportIdServiceTest { @Mock private RBucketReactive bucket; @Mock private RBatchReactive batch; @Mock private RBucketsReactive buckets; + @Mock private RKeysReactive keys; private TransportIdService service; private MeterRegistry meterRegistry; @@ -59,6 +61,7 @@ void setUp() { lenient().when(reactiveClient.getBucket(anyString())).thenReturn(bucket); lenient().when(reactiveClient.createBatch()).thenReturn(batch); lenient().when(reactiveClient.getBuckets()).thenReturn(buckets); + lenient().when(reactiveClient.getKeys()).thenReturn(keys); service = new TransportIdService(redisClient, config, meterRegistry, randomGenerator); } @@ -172,12 +175,10 @@ void storeMappingsWithEmptyMapCompletesImmediately() { var result = service.storeMappings(Map.of(), defaultTtl); StepVerifier.create(result).verifyComplete(); - // No Redis interactions should occur - batch.execute() would fail if called without setup } @Test void fetchMappingsRetrievesMultipleMappings() { - // RBuckets.get() returns map with full keys (including prefix) when(buckets.get(any(String[].class))) .thenReturn(Mono.just(Map.of("tid:tId-1", "sId-1", "tid:tId-2", "sId-2"))); @@ -198,12 +199,10 @@ void fetchMappingsWithEmptySetReturnsEmptyMap() { var result = service.fetchMappings(Set.of()); StepVerifier.create(result).expectNext(Map.of()).verifyComplete(); - // No Redis interactions should occur - buckets.get() would fail if called without setup } @Test void fetchMappingsExcludesNotFoundMappings() { - // RBuckets.get() only returns keys that exist - missing keys are not in the result map when(buckets.get(any(String[].class))) .thenReturn(Mono.just(Map.of("tid:tId-1", "sId-1"))); @@ -217,4 +216,61 @@ void fetchMappingsExcludesNotFoundMappings() { }) .verifyComplete(); } + + @Test + void consolidateMappingsMergesIdentityAndDateShiftEntries() { + // fetchMappings for identity tIDs + when(buckets.get(any(String[].class))) + .thenReturn(Mono.just(Map.of("tid:tId-1", "sId-1", "tid:tId-2", "sId-2"))); + + // storeAllMappings + when(mapCache.expire(any(Duration.class))).thenReturn(Mono.just(true)); + when(mapCache.putAll(any())).thenReturn(Mono.empty()); + + // deleteIndividualTidKeys + when(keys.delete(any(String[].class))).thenReturn(Mono.just(2L)); + + var dateShiftEntries = Map.of("ds:dateTid-1", "2020-06-15", "ds:dateTid-2", "2020-12-25"); + + var result = + service.consolidateMappings(Set.of("tId-1", "tId-2"), dateShiftEntries, defaultTtl); + + StepVerifier.create(result) + .assertNext( + transferId -> { + assertThat(transferId).isNotNull().hasSize(32); + }) + .verifyComplete(); + } + + @Test + void consolidateMappingsWithEmptyDateShiftEntries() { + when(buckets.get(any(String[].class))) + .thenReturn(Mono.just(Map.of("tid:tId-1", "sId-1"))); + + when(mapCache.expire(any(Duration.class))).thenReturn(Mono.just(true)); + when(mapCache.putAll(any())).thenReturn(Mono.empty()); + when(keys.delete(any(String[].class))).thenReturn(Mono.just(1L)); + + var result = service.consolidateMappings(Set.of("tId-1"), Map.of(), defaultTtl); + + StepVerifier.create(result) + .assertNext(transferId -> assertThat(transferId).hasSize(32)) + .verifyComplete(); + } + + @Test + void consolidateMappingsWithEmptyIdentityTidsSkipsDelete() { + when(mapCache.expire(any(Duration.class))).thenReturn(Mono.just(true)); + when(mapCache.putAll(any())).thenReturn(Mono.empty()); + + // fetchMappings with empty set returns empty map directly (no Redis call) + var dateShiftEntries = Map.of("ds:dateTid-1", "2020-06-15"); + + var result = service.consolidateMappings(Set.of(), dateShiftEntries, defaultTtl); + + StepVerifier.create(result) + .assertNext(transferId -> assertThat(transferId).hasSize(32)) + .verifyComplete(); + } } diff --git a/util/src/main/java/care/smith/fts/util/fhir/DateShiftAnonymizer.java b/util/src/main/java/care/smith/fts/util/fhir/DateShiftAnonymizer.java new file mode 100644 index 000000000..8b6b67440 --- /dev/null +++ b/util/src/main/java/care/smith/fts/util/fhir/DateShiftAnonymizer.java @@ -0,0 +1,131 @@ +package care.smith.fts.util.fhir; + +import static care.smith.fts.util.NanoIdUtils.nanoId; +import static care.smith.fts.util.deidentifhir.DateShiftConstants.DATE_SHIFT_EXTENSION_URL; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.util.FhirTerser; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.hl7.fhir.r4.model.BaseDateTimeType; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Resource; +import org.hl7.fhir.r4.model.StringType; +import org.yaml.snakeyaml.Yaml; + +/** + * Parses a FHIR Pseudonymizer anonymization config and selectively nullifies date elements matching + * dateshift rules. For each nullified date, generates a transport ID and adds a DATE_SHIFT extension + * so RDA can later restore the shifted date. + */ +public interface DateShiftAnonymizer { + + /** + * Extracts dateshift FHIRPath-like rules from an anonymization.yaml config. + * + *

The expected format is: + * + * 

+   * fhirPathRules:
+   *   - path: "Encounter.period.start"
+   *     method: "dateshift"
+   *   - path: "Encounter.period.end"
+   *     method: "dateshift"
+   *   - path: "Patient.identifier.value"
+   *     method: "pseudonymize"
+   *
+ * + * Only rules with {@code method: "dateshift"} are returned. + * + * @param configFile the anonymization.yaml file + * @return list of dotted resource paths for dateshift rules (e.g., "Encounter.period.start") + */ + @SuppressWarnings("unchecked") + static List parseDateShiftPaths(File configFile) throws IOException { + var yaml = new Yaml(); + Map root; + try (var input = new FileInputStream(configFile)) { + root = yaml.load(input); + } + + if (root == null || !root.containsKey("fhirPathRules")) { + return List.of(); + } + + var rules = (List>) root.get("fhirPathRules"); + return rules.stream() + .filter(rule -> "dateshift".equals(rule.get("method"))) + .map(rule -> rule.get("path")) + .filter(path -> path != null && !path.isEmpty()) + .toList(); + } + + /** + * Processes a bundle by nullifying date elements matching the given paths and adding dateshift + * extensions with transport IDs. + * + * @param bundle the FHIR bundle to process (modified in place) + * @param dateShiftPaths dotted resource paths (e.g., "Encounter.period.start") + * @return map of tID → original date value + */ + static Map nullifyDates(Bundle bundle, List dateShiftPaths) { + if (dateShiftPaths.isEmpty()) { + return Map.of(); + } + + var terser = new FhirTerser(FhirContext.forR4Cached()); + var dateMappings = new HashMap(); + var dateValueToTid = new HashMap(); + + var pathsByResourceType = + dateShiftPaths.stream() + .filter(p -> p.contains(".")) + .collect( + Collectors.groupingBy( + p -> p.substring(0, p.indexOf('.')), + Collectors.mapping(p -> p.substring(p.indexOf('.') + 1), Collectors.toList()))); + + for (var entry : bundle.getEntry()) { + var resource = entry.getResource(); + if (resource == null) continue; + + var resourceType = resource.fhirType(); + var paths = pathsByResourceType.get(resourceType); + if (paths == null) continue; + + for (var subPath : paths) { + processDateElements(terser, resource, subPath, dateMappings, dateValueToTid); + } + } + + return Map.copyOf(dateMappings); + } + + private static void processDateElements( + FhirTerser terser, + Resource resource, + String subPath, + Map dateMappings, + Map dateValueToTid) { + + var values = terser.getValues(resource, subPath); + for (var value : values) { + if (value instanceof BaseDateTimeType dateTime && dateTime.getValue() != null) { + var originalDate = dateTime.getValueAsString(); + var tId = dateValueToTid.computeIfAbsent(originalDate, k -> nanoId()); + + if (!dateMappings.containsKey(tId)) { + dateMappings.put(tId, originalDate); + } + + dateTime.setValue(null); + dateTime.addExtension(DATE_SHIFT_EXTENSION_URL, new StringType(tId)); + } + } + } +} diff --git a/util/src/test/java/care/smith/fts/util/fhir/DateShiftAnonymizerTest.java b/util/src/test/java/care/smith/fts/util/fhir/DateShiftAnonymizerTest.java new file mode 100644 index 000000000..4964244b2 --- /dev/null +++ b/util/src/test/java/care/smith/fts/util/fhir/DateShiftAnonymizerTest.java @@ -0,0 +1,176 @@ +package care.smith.fts.util.fhir; + +import static care.smith.fts.util.deidentifhir.DateShiftConstants.DATE_SHIFT_EXTENSION_URL; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Date; +import java.util.List; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.Encounter; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Period; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DateShiftAnonymizerTest { + + @TempDir File tempDir; + + @Test + void parseDateShiftPathsExtractsOnlyDateshiftRules() throws IOException { + var config = + """ + fhirPathRules: + - path: "Encounter.period.start" + method: "dateshift" + - path: "Encounter.period.end" + method: "dateshift" + - path: "Patient.identifier.value" + method: "pseudonymize" + - path: "Patient.birthDate" + method: "dateshift" + """; + var file = writeConfig(config); + + var paths = DateShiftAnonymizer.parseDateShiftPaths(file); + + assertThat(paths) + .containsExactlyInAnyOrder( + "Encounter.period.start", "Encounter.period.end", "Patient.birthDate"); + } + + @Test + void parseDateShiftPathsReturnsEmptyForNoDateshiftRules() throws IOException { + var config = + """ + fhirPathRules: + - path: "Patient.identifier.value" + method: "pseudonymize" + """; + var file = writeConfig(config); + + var paths = DateShiftAnonymizer.parseDateShiftPaths(file); + + assertThat(paths).isEmpty(); + } + + @Test + void parseDateShiftPathsReturnsEmptyForMissingRules() throws IOException { + var config = "someOtherConfig: true\n"; + var file = writeConfig(config); + + var paths = DateShiftAnonymizer.parseDateShiftPaths(file); + + assertThat(paths).isEmpty(); + } + + @Test + void nullifyDatesNullifiesMatchingDateElements() { + var encounter = new Encounter(); + var period = new Period(); + period.setStartElement(new DateTimeType("2020-06-15T10:30:00+02:00")); + period.setEndElement(new DateTimeType("2020-06-16T08:00:00+02:00")); + encounter.setPeriod(period); + + var bundle = new Bundle(); + bundle.addEntry().setResource(encounter); + + var paths = List.of("Encounter.period.start", "Encounter.period.end"); + + var dateMappings = DateShiftAnonymizer.nullifyDates(bundle, paths); + + assertThat(dateMappings).hasSize(2); + assertThat(period.getStartElement().getValue()).isNull(); + assertThat(period.getEndElement().getValue()).isNull(); + assertThat(period.getStartElement().getExtensionByUrl(DATE_SHIFT_EXTENSION_URL)).isNotNull(); + assertThat(period.getEndElement().getExtensionByUrl(DATE_SHIFT_EXTENSION_URL)).isNotNull(); + } + + @Test + void nullifyDatesDeduplicatesSameDateValues() { + var enc1 = new Encounter(); + var period1 = new Period(); + period1.setStartElement(new DateTimeType("2020-06-15")); + enc1.setPeriod(period1); + + var enc2 = new Encounter(); + var period2 = new Period(); + period2.setStartElement(new DateTimeType("2020-06-15")); + enc2.setPeriod(period2); + + var bundle = new Bundle(); + bundle.addEntry().setResource(enc1); + bundle.addEntry().setResource(enc2); + + var paths = List.of("Encounter.period.start"); + + var dateMappings = DateShiftAnonymizer.nullifyDates(bundle, paths); + + // Same date value → same tID, so only 1 entry in dateMappings + assertThat(dateMappings).hasSize(1); + + // Both elements should have the same tID in their extension + var ext1 = period1.getStartElement().getExtensionByUrl(DATE_SHIFT_EXTENSION_URL); + var ext2 = period2.getStartElement().getExtensionByUrl(DATE_SHIFT_EXTENSION_URL); + assertThat(ext1.getValue().primitiveValue()).isEqualTo(ext2.getValue().primitiveValue()); + } + + @Test + void nullifyDatesSkipsNonMatchingResourceTypes() { + var patient = new Patient(); + patient.setBirthDate(new Date()); + + var encounter = new Encounter(); + var period = new Period(); + period.setStartElement(new DateTimeType("2020-06-15")); + encounter.setPeriod(period); + + var bundle = new Bundle(); + bundle.addEntry().setResource(patient); + bundle.addEntry().setResource(encounter); + + // Only target Encounter dates, not Patient + var paths = List.of("Encounter.period.start"); + + var dateMappings = DateShiftAnonymizer.nullifyDates(bundle, paths); + + assertThat(dateMappings).hasSize(1); + // Patient birthDate should be unchanged + assertThat(patient.getBirthDate()).isNotNull(); + } + + @Test + void nullifyDatesWithEmptyPathsReturnsEmpty() { + var bundle = new Bundle(); + bundle.addEntry().setResource(new Encounter()); + + var dateMappings = DateShiftAnonymizer.nullifyDates(bundle, List.of()); + + assertThat(dateMappings).isEmpty(); + } + + @Test + void nullifyDatesSkipsNullDateValues() { + var encounter = new Encounter(); + encounter.setPeriod(new Period()); // period with no start/end set + + var bundle = new Bundle(); + bundle.addEntry().setResource(encounter); + + var paths = List.of("Encounter.period.start"); + + var dateMappings = DateShiftAnonymizer.nullifyDates(bundle, paths); + + assertThat(dateMappings).isEmpty(); + } + + private File writeConfig(String content) throws IOException { + var file = new File(tempDir, "anonymization.yaml"); + Files.writeString(file.toPath(), content); + return file; + } +} From 1ddb027d969e4ed1ba83159863547b17597ed47d Mon Sep 17 00:00:00 2001 From: Daniel Hahne Date: Mon, 9 Mar 2026 09:21:36 +0100 Subject: [PATCH 2/4] Add gPAS proxy endpoint and FHIR Pseudonymizer E2E test Add GpasProxyController that mimics gPAS's $pseudonymizeAllowCreate but returns transport IDs instead of real pseudonyms. This allows the external FHIR Pseudonymizer to call TCA (not gPAS directly), maintaining data isolation via transport IDs. Also add E2E test infrastructure for the FHIR Pseudonymizer flow: - FP compose config pointing at TCA proxy with TLS and basic auth - Anonymization config, project config, and expected results - CI workflow steps for the fp-example transfer test Fix server cert generation to include Subject Alternative Names (SANs), required by .NET's strict TLS hostname validation. --- .../test/cd-agent/projects/fp-example.yaml | 63 +++++++ .../fp-example/anonymization-config.yaml | 11 ++ .github/test/compose.yaml | 1 + .../fhir-pseudonymizer/anonymization.yaml | 7 + .github/test/fhir-pseudonymizer/compose.yaml | 21 +++ .github/test/results/fp-example.json | 15 ++ .github/workflows/build.yml | 13 +- .../fts/tca/rest/GpasProxyController.java | 167 +++++++++++++++++ .../fts/tca/rest/GpasProxyControllerTest.java | 171 ++++++++++++++++++ util/src/main/bash/generate-certificates.sh | 6 +- 10 files changed, 472 insertions(+), 3 deletions(-) create mode 100644 .github/test/cd-agent/projects/fp-example.yaml create mode 100644 .github/test/cd-agent/projects/fp-example/anonymization-config.yaml create mode 100644 .github/test/fhir-pseudonymizer/anonymization.yaml create mode 100644 .github/test/fhir-pseudonymizer/compose.yaml create mode 100644 .github/test/results/fp-example.json create mode 100644 trust-center-agent/src/main/java/care/smith/fts/tca/rest/GpasProxyController.java create mode 100644 trust-center-agent/src/test/java/care/smith/fts/tca/rest/GpasProxyControllerTest.java diff --git a/.github/test/cd-agent/projects/fp-example.yaml b/.github/test/cd-agent/projects/fp-example.yaml new file mode 100644 index 000000000..2f1b02261 --- /dev/null +++ b/.github/test/cd-agent/projects/fp-example.yaml @@ -0,0 +1,63 @@ +### FTS Project Configuration - FHIR Pseudonymizer Flow +##! Uses external FHIR Pseudonymizer instead of DeidentiFHIR + +### Cohort Selection Configuration +cohortSelector: + trustCenterAgent: + server: + baseUrl: https://tc-agent:8080 + auth: + basic: + user: cd-agent + password: Aj6cloJYsTpu+op+ + ssl: + bundle: tca + domain: MII + patientIdentifierSystem: "http://fts.smith.care" + signerIdType: "Pseudonym" + policySystem: "urn:oid:2.16.840.1.113883.3.1937.777.24.5.3" + policies: + - 2.16.840.1.113883.3.1937.777.24.5.3.2 + - 2.16.840.1.113883.3.1937.777.24.5.3.3 + - 2.16.840.1.113883.3.1937.777.24.5.3.6 + - 2.16.840.1.113883.3.1937.777.24.5.3.7 + +### Data Selection Configuration +dataSelector: + everything: + fhirServer: + baseUrl: http://cd-hds:8080/fhir + pageSize: 500 + +### Deidentificator Configuration +##! Uses FHIR Pseudonymizer for identity pseudonymization via gPAS +deidentificator: + fhir-pseudonymizer: + serviceUrl: + baseUrl: http://fhir-pseudonymizer:8080/fhir + anonymizationConfig: /app/projects/fp-example/anonymization-config.yaml + trustCenterAgent: + server: + baseUrl: https://tc-agent:8080 + auth: + basic: + user: cd-agent + password: Aj6cloJYsTpu+op+ + ssl: + bundle: tca + domains: + pseudonym: MII + salt: MII + dateShift: MII + maxDateShift: P14D + dateShiftPreserve: NONE + +### Bundle Sender Configuration +bundleSender: + researchDomainAgent: + server: + baseUrl: http://rd-agent:8080 + auth: + oauth2: + registration: rd-agent + project: example diff --git a/.github/test/cd-agent/projects/fp-example/anonymization-config.yaml b/.github/test/cd-agent/projects/fp-example/anonymization-config.yaml new file mode 100644 index 000000000..92c1b8e70 --- /dev/null +++ b/.github/test/cd-agent/projects/fp-example/anonymization-config.yaml @@ -0,0 +1,11 @@ +fhirPathRules: + - path: "Encounter.period.start" + method: "dateshift" + - path: "Encounter.period.end" + method: "dateshift" + - path: "Observation.effectiveDateTime" + method: "dateshift" + - path: "Condition.onsetDateTime" + method: "dateshift" + - path: "Condition.recordedDate" + method: "dateshift" diff --git a/.github/test/compose.yaml b/.github/test/compose.yaml index 0b469f119..4f74c359f 100644 --- a/.github/test/compose.yaml +++ b/.github/test/compose.yaml @@ -13,6 +13,7 @@ include: - rd-agent/compose.yaml - tc-agent/compose.yaml - oauth2/compose.yaml +- fhir-pseudonymizer/compose.yaml services: # Clinical Domain diff --git a/.github/test/fhir-pseudonymizer/anonymization.yaml b/.github/test/fhir-pseudonymizer/anonymization.yaml new file mode 100644 index 000000000..b4ff84baa --- /dev/null +++ b/.github/test/fhir-pseudonymizer/anonymization.yaml @@ -0,0 +1,7 @@ +fhirVersion: R4 +fhirPathRules: + - path: "nodesByType('Identifier').where(system='http://fts.smith.care').value" + method: pseudonymize + domain: MII + - path: "Patient.name" + method: redact diff --git a/.github/test/fhir-pseudonymizer/compose.yaml b/.github/test/fhir-pseudonymizer/compose.yaml new file mode 100644 index 000000000..b6a48aee3 --- /dev/null +++ b/.github/test/fhir-pseudonymizer/compose.yaml @@ -0,0 +1,21 @@ +name: fts-fhir-pseudonymizer + +services: + fhir-pseudonymizer: + image: ghcr.io/miracum/fhir-pseudonymizer:v2.24.1 + ports: [ ":8080" ] + networks: [ "clinical-domain", "agents" ] + environment: + PseudonymizationService: gPAS + gPAS__Url: https://tc-agent:8080/api/v2/cd/fp-gpas-proxy/ + gPAS__Version: 2025.2.0 + gPAS__Auth__Basic__Username: cd-agent + gPAS__Auth__Basic__Password: Aj6cloJYsTpu+op+ + AnonymizationEngineConfigPath: /etc/anonymization.yaml + SSL_CERT_FILE: /etc/ssl/certs/ca.crt + volumes: + - ./anonymization.yaml:/etc/anonymization.yaml:ro + - ../ssl/ca.crt:/etc/ssl/certs/ca.crt:ro + depends_on: + tc-agent: + condition: service_healthy diff --git a/.github/test/results/fp-example.json b/.github/test/results/fp-example.json new file mode 100644 index 000000000..7e0cf1669 --- /dev/null +++ b/.github/test/results/fp-example.json @@ -0,0 +1,15 @@ +{ + "phase": "COMPLETED", + "sentBundles": 118, + "skippedBundles": 0, + "count": { + "total": 21, + "Patient": 100, + "Encounter": 760, + "Observation": 7855, + "Condition": 4675, + "DiagnosticReport": 499, + "Medication": 0, + "MedicationAdministration": 169 + } +} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0a15b7765..a8b32f3cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -380,9 +380,20 @@ jobs: check-resources.sh example-with-fhir-consent.json "${LAST_UPDATED}" check-pseudonymization.sh + - name: Clean RD-HDS for FHIR Pseudonymizer Test + run: make clean-rd-hds + + - name: Run e2e for FHIR Pseudonymizer Flow + run: | + LAST_UPDATED=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) + make transfer-all PROJECT=fp-example + make wait + make check-status RESULTS_FILE=fp-example.json + check-resources.sh fp-example.json "${LAST_UPDATED}" + - name: Collect Agent Logs if: failure() || cancelled() - run: docker compose logs cd-agent tc-agent rd-agent + run: docker compose logs cd-agent tc-agent rd-agent fhir-pseudonymizer - name: Collect MOSAIC Logs if: failure() || cancelled() diff --git a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/GpasProxyController.java b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/GpasProxyController.java new file mode 100644 index 000000000..e5d120b10 --- /dev/null +++ b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/GpasProxyController.java @@ -0,0 +1,167 @@ +package care.smith.fts.tca.rest; + +import static care.smith.fts.util.MediaTypes.APPLICATION_FHIR_JSON_VALUE; + +import care.smith.fts.tca.deidentification.GpasClient; +import care.smith.fts.tca.services.TransportIdService; +import care.smith.fts.util.error.ErrorResponseUtil; +import jakarta.validation.Valid; +import static java.util.stream.Collectors.toMap; + +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.hl7.fhir.r4.model.Base; +import org.hl7.fhir.r4.model.Identifier; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; +import org.hl7.fhir.r4.model.OperationOutcome.IssueType; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; +import org.springframework.http.ResponseEntity; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * gPAS-compatible proxy that returns transport IDs instead of real pseudonyms. + * + *

The external FHIR Pseudonymizer calls this endpoint believing it's gPAS. TCA fetches real + * pseudonyms from gPAS, stores tID→sID mappings in Redis, and returns transport IDs. + */ +@Slf4j +@RestController +@RequestMapping(value = "api/v2") +@Validated +public class GpasProxyController { + + private final TransportIdService transportIdService; + private final GpasClient gpasClient; + + public GpasProxyController(TransportIdService transportIdService, GpasClient gpasClient) { + this.transportIdService = transportIdService; + this.gpasClient = gpasClient; + } + + @PostMapping( + value = "cd/fp-gpas-proxy/$pseudonymizeAllowCreate", + consumes = APPLICATION_FHIR_JSON_VALUE, + produces = APPLICATION_FHIR_JSON_VALUE) + public Mono> pseudonymizeAllowCreate( + @Valid @RequestBody Parameters requestParams) { + + log.debug("Received gPAS proxy $pseudonymizeAllowCreate request"); + + return Mono.fromCallable(() -> parseGpasRequest(requestParams)) + .flatMap(this::processRequest) + .map(this::buildGpasResponse) + .map(ResponseEntity::ok) + .onErrorResume(this::handleError); + } + + private GpasProxyRequest parseGpasRequest(Parameters params) { + String target = extractRequired(params, "target"); + List originals = extractAll(params, "original"); + + if (originals.isEmpty()) { + throw new IllegalArgumentException("At least one 'original' parameter is required"); + } + + log.debug("Parsed gPAS proxy request: target={}, originalCount={}", target, originals.size()); + return new GpasProxyRequest(target, originals); + } + + private Mono> processRequest(GpasProxyRequest request) { + var ttl = transportIdService.getDefaultTtl(); + + return gpasClient + .fetchOrCreatePseudonyms(request.target(), new HashSet<>(request.originals())) + .flatMap(sIdMap -> replaceWithTransportIds(sIdMap, ttl)); + } + + /** For each original→sID mapping, generate a tID, store tID→sID, return original→tID. */ + private Mono> replaceWithTransportIds( + Map sIdMap, Duration ttl) { + return Flux.fromIterable(sIdMap.entrySet()) + .flatMap( + entry -> { + var tId = transportIdService.generateId(); + return transportIdService + .storeMapping(tId, entry.getValue(), ttl) + .thenReturn(Map.entry(entry.getKey(), tId)); + }) + .collectList() + .map(entries -> entries.stream().collect(toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + /** Builds gPAS-compatible $pseudonymizeAllowCreate response with valueIdentifier parts. */ + private Parameters buildGpasResponse(Map originalToTid) { + var fhirParams = new Parameters(); + + for (var entry : originalToTid.entrySet()) { + var pseudonymParam = new ParametersParameterComponent(); + pseudonymParam.setName("pseudonym"); + + pseudonymParam + .addPart() + .setName("original") + .setValue(new Identifier().setValue(entry.getKey())); + pseudonymParam + .addPart() + .setName("pseudonym") + .setValue(new Identifier().setValue(entry.getValue())); + + fhirParams.addParameter(pseudonymParam); + } + + log.trace("Built gPAS proxy response with {} entries", originalToTid.size()); + return fhirParams; + } + + private String extractRequired(Parameters params, String name) { + return params.getParameter().stream() + .filter(p -> name.equals(p.getName())) + .findFirst() + .map(ParametersParameterComponent::getValue) + .map(Base::primitiveValue) + .orElseThrow( + () -> + new IllegalArgumentException( + "Missing required parameter '%s'".formatted(name))); + } + + private List extractAll(Parameters params, String name) { + return params.getParameter().stream() + .filter(p -> name.equals(p.getName())) + .map(ParametersParameterComponent::getValue) + .map(Base::primitiveValue) + .toList(); + } + + private Mono> handleError(Throwable error) { + log.warn("Error processing gPAS proxy request: {}", error.getMessage()); + + if (error instanceof IllegalArgumentException) { + var outcome = new OperationOutcome(); + outcome + .addIssue() + .setSeverity(IssueSeverity.ERROR) + .setCode(IssueType.INVALID) + .setDiagnostics(error.getMessage()); + + var params = new Parameters(); + params.addParameter().setName("outcome").setResource(outcome); + return Mono.just(ResponseEntity.badRequest().body(params)); + } + + return ErrorResponseUtil.internalServerError(error); + } + + private record GpasProxyRequest(String target, List originals) {} +} diff --git a/trust-center-agent/src/test/java/care/smith/fts/tca/rest/GpasProxyControllerTest.java b/trust-center-agent/src/test/java/care/smith/fts/tca/rest/GpasProxyControllerTest.java new file mode 100644 index 000000000..9c20a16cc --- /dev/null +++ b/trust-center-agent/src/test/java/care/smith/fts/tca/rest/GpasProxyControllerTest.java @@ -0,0 +1,171 @@ +package care.smith.fts.tca.rest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import care.smith.fts.tca.deidentification.GpasClient; +import care.smith.fts.tca.services.TransportIdService; +import java.time.Duration; +import java.util.Map; +import org.hl7.fhir.r4.model.Identifier; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpStatus; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +@ExtendWith(MockitoExtension.class) +class GpasProxyControllerTest { + + @Mock private TransportIdService transportIdService; + @Mock private GpasClient gpasClient; + + private GpasProxyController controller; + + @BeforeEach + void setUp() { + controller = new GpasProxyController(transportIdService, gpasClient); + } + + @Test + void singleOriginalReturnsSinglePseudonymEntry() { + var requestParams = createGpasRequest("MII", "patient-123"); + var ttl = Duration.ofMinutes(10); + + when(transportIdService.generateId()).thenReturn("tId-abc123"); + when(transportIdService.getDefaultTtl()).thenReturn(ttl); + when(transportIdService.storeMapping(eq("tId-abc123"), eq("sId-456"), eq(ttl))) + .thenReturn(Mono.empty()); + when(gpasClient.fetchOrCreatePseudonyms(eq("MII"), anySet())) + .thenReturn(Mono.just(Map.of("patient-123", "sId-456"))); + + StepVerifier.create(controller.pseudonymizeAllowCreate(requestParams)) + .assertNext( + response -> { + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + var params = response.getBody(); + assertThat(params).isNotNull(); + assertThat(params.getParameter()).hasSize(1); + + var pseudonymParam = params.getParameter().get(0); + assertThat(pseudonymParam.getName()).isEqualTo("pseudonym"); + + var originalPart = findPart(pseudonymParam, "original"); + assertThat(originalPart).isNotNull(); + assertThat(((Identifier) originalPart.getValue()).getValue()) + .isEqualTo("patient-123"); + + var pseudonymPart = findPart(pseudonymParam, "pseudonym"); + assertThat(pseudonymPart).isNotNull(); + assertThat(((Identifier) pseudonymPart.getValue()).getValue()) + .isEqualTo("tId-abc123"); + }) + .verifyComplete(); + } + + @Test + void multipleOriginalsReturnsBatchResponse() { + var requestParams = createGpasRequest("MII", "patient-1", "patient-2"); + var ttl = Duration.ofMinutes(10); + + when(transportIdService.generateId()).thenReturn("tId-1", "tId-2"); + when(transportIdService.getDefaultTtl()).thenReturn(ttl); + when(transportIdService.storeMapping(anyString(), anyString(), any(Duration.class))) + .thenReturn(Mono.empty()); + when(gpasClient.fetchOrCreatePseudonyms(eq("MII"), anySet())) + .thenReturn(Mono.just(Map.of("patient-1", "sId-1", "patient-2", "sId-2"))); + + StepVerifier.create(controller.pseudonymizeAllowCreate(requestParams)) + .assertNext( + response -> { + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + var params = response.getBody(); + assertThat(params).isNotNull(); + assertThat(params.getParameter()).hasSize(2); + + params + .getParameter() + .forEach(p -> assertThat(p.getName()).isEqualTo("pseudonym")); + }) + .verifyComplete(); + } + + @Test + void missingTargetReturnsBadRequest() { + var requestParams = new Parameters(); + requestParams.addParameter().setName("original").setValue(new StringType("patient-123")); + + StepVerifier.create(controller.pseudonymizeAllowCreate(requestParams)) + .assertNext( + response -> { + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); + var params = response.getBody(); + assertThat(params).isNotNull(); + var outcome = (OperationOutcome) params.getParameter().get(0).getResource(); + assertThat(outcome.getIssueFirstRep().getDiagnostics()) + .contains("Missing required parameter 'target'"); + }) + .verifyComplete(); + } + + @Test + void missingOriginalReturnsBadRequest() { + var requestParams = new Parameters(); + requestParams.addParameter().setName("target").setValue(new StringType("MII")); + + StepVerifier.create(controller.pseudonymizeAllowCreate(requestParams)) + .assertNext( + response -> { + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); + var params = response.getBody(); + assertThat(params).isNotNull(); + var outcome = (OperationOutcome) params.getParameter().get(0).getResource(); + assertThat(outcome.getIssueFirstRep().getDiagnostics()) + .contains("At least one 'original' parameter is required"); + }) + .verifyComplete(); + } + + @Test + void gpasFailureReturnsInternalServerError() { + var requestParams = createGpasRequest("MII", "patient-123"); + + when(transportIdService.getDefaultTtl()).thenReturn(Duration.ofMinutes(10)); + when(gpasClient.fetchOrCreatePseudonyms(eq("MII"), anySet())) + .thenReturn(Mono.error(new RuntimeException("gPAS connection failed"))); + + StepVerifier.create(controller.pseudonymizeAllowCreate(requestParams)) + .assertNext( + response -> + assertThat(response.getStatusCode()) + .isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR)) + .verifyComplete(); + } + + private Parameters createGpasRequest(String target, String... originals) { + var params = new Parameters(); + params.addParameter().setName("target").setValue(new StringType(target)); + for (String original : originals) { + params.addParameter().setName("original").setValue(new StringType(original)); + } + return params; + } + + private Parameters.ParametersParameterComponent findPart( + Parameters.ParametersParameterComponent param, String name) { + return param.getPart().stream() + .filter(p -> name.equals(p.getName())) + .findFirst() + .orElse(null); + } +} diff --git a/util/src/main/bash/generate-certificates.sh b/util/src/main/bash/generate-certificates.sh index 7fdaf2a89..8240973fc 100755 --- a/util/src/main/bash/generate-certificates.sh +++ b/util/src/main/bash/generate-certificates.sh @@ -9,8 +9,10 @@ openssl req -x509 -new -key "ca.key" -days 60 -out "ca.crt" -subj "/CN=fts.smith # Create server certificate echo "Creating server certificate..." openssl genpkey -quiet -algorithm Ed25519 -out "server.key" >/dev/null -openssl req -new -key "server.key" -subj "/CN=${1:-server.example.com}/OU=Server/O=FTSnext" \ - | openssl x509 -req -CA "ca.crt" -CAkey "ca.key" -CAcreateserial -out "server.crt" -days 30 +SERVER_CN="${1:-server.example.com}" +openssl req -new -key "server.key" -subj "/CN=${SERVER_CN}/OU=Server/O=FTSnext" \ + | openssl x509 -req -CA "ca.crt" -CAkey "ca.key" -CAcreateserial -out "server.crt" -days 30 \ + -extfile <(printf "subjectAltName=DNS:%s" "$SERVER_CN") for CLIENT_CN in "${@:2}"; do echo "Creating '$CLIENT_CN' client certificate for CN=$CLIENT_CN..." From a8c753f474408116a42d6606e5551ea6ac50548c Mon Sep 17 00:00:00 2001 From: Daniel Hahne Date: Mon, 9 Mar 2026 10:22:05 +0100 Subject: [PATCH 3/4] Add FHIR Pseudonymizer E2E tests for TCA and CDA Add agent-level E2E tests for the FHIR Pseudonymizer flow: - GpasProxyE2E: verifies gPAS proxy returns transport IDs not real pseudonyms - FpTransportMappingE2E: tests two-phase proxy + consolidation flow - FhirPseudonymizerE2E: full CDA transfer with FP WireMock, consent, and transport mapping consolidation --- .../smith/fts/cda/FhirPseudonymizerE2E.java | 92 +++++++++++++++++++ .../fp-example/anonymization-config.yaml | 5 + .../e2e/resources/projects/fp-example.yaml | 39 ++++++++ .../fts/cda/impl/FhirPseudonymizerStep.java | 14 +-- .../impl/FhirPseudonymizerStepFactory.java | 6 +- .../cda/impl/FhirPseudonymizerConfigTest.java | 6 +- .../smith/fts/tca/FpTransportMappingE2E.java | 90 ++++++++++++++++++ .../java/care/smith/fts/tca/GpasProxyE2E.java | 78 ++++++++++++++++ .../rest/FpTransportMappingController.java | 4 +- .../fts/tca/rest/GpasProxyController.java | 7 +- .../fts/tca/rest/GpasProxyControllerTest.java | 12 +-- .../fts/util/fhir/DateShiftAnonymizer.java | 4 +- 12 files changed, 327 insertions(+), 30 deletions(-) create mode 100644 clinical-domain-agent/src/e2e/java/care/smith/fts/cda/FhirPseudonymizerE2E.java create mode 100644 clinical-domain-agent/src/e2e/resources/fp-example/anonymization-config.yaml create mode 100644 clinical-domain-agent/src/e2e/resources/projects/fp-example.yaml create mode 100644 trust-center-agent/src/e2e/java/care/smith/fts/tca/FpTransportMappingE2E.java create mode 100644 trust-center-agent/src/e2e/java/care/smith/fts/tca/GpasProxyE2E.java diff --git a/clinical-domain-agent/src/e2e/java/care/smith/fts/cda/FhirPseudonymizerE2E.java b/clinical-domain-agent/src/e2e/java/care/smith/fts/cda/FhirPseudonymizerE2E.java new file mode 100644 index 000000000..eb864a370 --- /dev/null +++ b/clinical-domain-agent/src/e2e/java/care/smith/fts/cda/FhirPseudonymizerE2E.java @@ -0,0 +1,92 @@ +package care.smith.fts.cda; + +import static care.smith.fts.test.MockServerUtil.fhirResponse; +import static care.smith.fts.test.MockServerUtil.jsonResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.springframework.http.HttpHeaders.CONTENT_TYPE; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + +import com.github.tomakehurst.wiremock.client.WireMock; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.Patient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.utility.MountableFile; +import org.wiremock.integrations.testcontainers.WireMockContainer; + +@Slf4j +public class FhirPseudonymizerE2E extends AbstractCohortSelectorE2E { + + private final WireMockContainer fp = + new WireMockContainer("wiremock/wiremock:3.13.0") + .withCreateContainerCmdModifier(cmd -> cmd.withName("cda-e2e-fp-fp-example")) + .withNetwork(network) + .withNetworkAliases("fhir-pseudonymizer"); + + public FhirPseudonymizerE2E() { + super("fp-example.yaml"); + cda.withCopyFileToContainer( + MountableFile.forClasspathResource("fp-example/anonymization-config.yaml"), + "/app/projects/fp-example/anonymization-config.yaml"); + } + + @BeforeEach + void setUpFp() { + fp.start(); + configureFpMocks(); + } + + @AfterEach + void tearDownFp() { + if (fp.isRunning()) { + fp.stop(); + } + } + + private void configureFpMocks() { + var fpWireMock = new WireMock(fp.getHost(), fp.getPort()); + + // FP returns a bundle with 32-char Base64URL resource IDs (transport IDs) + var patient = new Patient(); + patient.setId("AbCdEfGhIjKlMnOpQrStUvWxYz012345"); + var responseBundle = new Bundle(); + responseBundle.setType(Bundle.BundleType.COLLECTION); + responseBundle.addEntry().setResource(patient); + + fpWireMock.register( + post(urlPathEqualTo("/fhir/$de-identify")).willReturn(fhirResponse(responseBundle))); + } + + @Override + protected void setupSpecificTcaMocks() { + var tcaWireMock = new WireMock(tca.getHost(), tca.getPort()); + + var cohortGenerator = + createCohortGenerator("https://ths-greifswald.de/fhir/gics/identifiers/Pseudonym"); + var tcaResponse = + new Bundle() + .setEntry(List.of(new BundleEntryComponent().setResource(cohortGenerator.generate()))); + + tcaWireMock.register( + post(urlPathMatching("/api/v2/cd/consented-patients.*")) + .withHeader(CONTENT_TYPE, equalTo(APPLICATION_JSON_VALUE)) + .willReturn(fhirResponse(tcaResponse))); + + tcaWireMock.register( + post(urlPathEqualTo("/api/v2/cd/fhir-pseudonymizer/transport-mapping")) + .willReturn( + jsonResponse( + """ + {"transferId": "AbCdEfGhIjKlMnOpQrStUvWxYz012345"} + """))); + } + + @Test + void testFhirPseudonymizerTransfer() { + executeTransferTest("[]"); + } +} diff --git a/clinical-domain-agent/src/e2e/resources/fp-example/anonymization-config.yaml b/clinical-domain-agent/src/e2e/resources/fp-example/anonymization-config.yaml new file mode 100644 index 000000000..a9f44e59d --- /dev/null +++ b/clinical-domain-agent/src/e2e/resources/fp-example/anonymization-config.yaml @@ -0,0 +1,5 @@ +fhirPathRules: + - path: "Encounter.period.start" + method: "dateshift" + - path: "Encounter.period.end" + method: "dateshift" diff --git a/clinical-domain-agent/src/e2e/resources/projects/fp-example.yaml b/clinical-domain-agent/src/e2e/resources/projects/fp-example.yaml new file mode 100644 index 000000000..85ff4b9a2 --- /dev/null +++ b/clinical-domain-agent/src/e2e/resources/projects/fp-example.yaml @@ -0,0 +1,39 @@ +cohortSelector: + trustCenterAgent: + server: + baseUrl: http://tc-agent:8080 + domain: MII + patientIdentifierSystem: "http://fts.smith.care" + policySystem: "urn:oid:2.16.840.1.113883.3.1937.777.24.5.3" + policies: + - 2.16.840.1.113883.3.1937.777.24.5.3.2 + - 2.16.840.1.113883.3.1937.777.24.5.3.3 + - 2.16.840.1.113883.3.1937.777.24.5.3.6 + - 2.16.840.1.113883.3.1937.777.24.5.3.7 + +dataSelector: + everything: + fhirServer: + baseUrl: http://cd-hds:8080/fhir + pageSize: 500 + +deidentificator: + fhir-pseudonymizer: + serviceUrl: + baseUrl: http://fhir-pseudonymizer:8080/fhir + anonymizationConfig: /app/projects/fp-example/anonymization-config.yaml + trustCenterAgent: + server: + baseUrl: http://tc-agent:8080 + domains: + pseudonym: MII + salt: MII + dateShift: MII + maxDateShift: P14D + dateShiftPreserve: NONE + +bundleSender: + researchDomainAgent: + server: + baseUrl: http://rd-agent:8080 + project: example diff --git a/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStep.java b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStep.java index 680f412e9..346d9a419 100644 --- a/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStep.java +++ b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStep.java @@ -75,11 +75,8 @@ public Mono deidentify(ConsentedPatientBundle bundle) { if (identityTIds.isEmpty() && dateMappings.isEmpty()) { return Mono.empty(); } - return consolidateViaTca( - patient.identifier(), identityTIds, dateMappings) - .map( - transferId -> - new TransportBundle(pseudonymizedBundle, transferId)); + return consolidateViaTca(patient.identifier(), identityTIds, dateMappings) + .map(transferId -> new TransportBundle(pseudonymizedBundle, transferId)); }); }); } @@ -116,7 +113,12 @@ private Mono consolidateViaTca( var request = new FpTransportMappingRequest( - patientIdentifier, identityTIds, dateMappings, domains.dateShift(), maxDateShift, preserve); + patientIdentifier, + identityTIds, + dateMappings, + domains.dateShift(), + maxDateShift, + preserve); log.trace( "Consolidating {} identity tIDs + {} date mappings via TCA", diff --git a/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactory.java b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactory.java index f1e9b6498..b1e9123a0 100644 --- a/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactory.java +++ b/clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/FhirPseudonymizerStepFactory.java @@ -17,8 +17,7 @@ public class FhirPseudonymizerStepFactory private final WebClientFactory clientFactory; private final MeterRegistry meterRegistry; - public FhirPseudonymizerStepFactory( - WebClientFactory clientFactory, MeterRegistry meterRegistry) { + public FhirPseudonymizerStepFactory(WebClientFactory clientFactory, MeterRegistry meterRegistry) { this.clientFactory = clientFactory; this.meterRegistry = meterRegistry; } @@ -37,8 +36,7 @@ public Deidentificator create( List dateShiftPaths; try { dateShiftPaths = - DateShiftAnonymizer.parseDateShiftPaths( - requireNonNull(implConfig.anonymizationConfig())); + DateShiftAnonymizer.parseDateShiftPaths(requireNonNull(implConfig.anonymizationConfig())); } catch (IOException e) { throw new IllegalStateException("Failed to parse anonymization config", e); } diff --git a/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerConfigTest.java b/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerConfigTest.java index f37dad303..a68c6a25e 100644 --- a/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerConfigTest.java +++ b/clinical-domain-agent/src/test/java/care/smith/fts/cda/impl/FhirPseudonymizerConfigTest.java @@ -34,7 +34,11 @@ void dateShiftPreserveRetainsExplicitValue() { var config = new FhirPseudonymizerConfig( - serviceUrl, new File("anon.yaml"), tcaConfig, Duration.ofDays(14), DateShiftPreserve.WEEKDAY); + serviceUrl, + new File("anon.yaml"), + tcaConfig, + Duration.ofDays(14), + DateShiftPreserve.WEEKDAY); assertThat(config.dateShiftPreserve()).isEqualTo(DateShiftPreserve.WEEKDAY); } diff --git a/trust-center-agent/src/e2e/java/care/smith/fts/tca/FpTransportMappingE2E.java b/trust-center-agent/src/e2e/java/care/smith/fts/tca/FpTransportMappingE2E.java new file mode 100644 index 000000000..92f23628a --- /dev/null +++ b/trust-center-agent/src/e2e/java/care/smith/fts/tca/FpTransportMappingE2E.java @@ -0,0 +1,90 @@ +package care.smith.fts.tca; + +import static org.assertj.core.api.Assertions.assertThat; + +import care.smith.fts.util.tca.TransportMappingResponse; +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.hl7.fhir.r4.model.Identifier; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.Test; +import reactor.test.StepVerifier; + +@Slf4j +public class FpTransportMappingE2E extends AbstractTcaE2E { + + @Override + protected void configureGicsMocks() throws IOException { + configureGicsMetadataMock(); + } + + @Override + protected void configureGpasMocks() throws IOException { + configureGpasMetadataMock(); + configureGpasPseudonymizationMock("patient-id-1", "pseudonym-123"); + // PT336H = Duration.ofDays(14).toString() + configureGpasPseudonymizationMock("PT336H_patient-id-1", "dateshift-seed-456"); + } + + @Test + void testFpTransportMappingConsolidation() { + var webClient = createTcaWebClient(); + + // Step 1: Create tID->sID in Redis via gPAS proxy + var proxyRequest = new Parameters(); + proxyRequest.addParameter().setName("target").setValue(new StringType("domain")); + proxyRequest.addParameter().setName("original").setValue(new StringType("patient-id-1")); + + var proxyResponse = + webClient + .post() + .uri("/api/v2/cd/fp-gpas-proxy/$pseudonymizeAllowCreate") + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/fhir+json") + .bodyValue(proxyRequest) + .retrieve() + .bodyToMono(Parameters.class) + .block(); + + var tId = + proxyResponse.getParameter().stream() + .filter(p -> "pseudonym".equals(p.getName())) + .flatMap(p -> p.getPart().stream()) + .filter(p -> "pseudonym".equals(p.getName())) + .map(p -> ((Identifier) p.getValue()).getValue()) + .findFirst() + .orElseThrow(); + + log.info("Got transport ID from proxy: {}", tId); + + // Step 2: Consolidate via FP transport mapping endpoint + var fpRequest = + Map.of( + "patientIdentifier", "patient-id-1", + "transportIds", Set.of(tId), + "dateMappings", Map.of("dateTid-1", "2020-06-15"), + "dateShiftDomain", "domain", + "maxDateShift", "PT336H", + "dateShiftPreserve", "NONE"); + + var response = + webClient + .post() + .uri("/api/v2/cd/fhir-pseudonymizer/transport-mapping") + .header("Content-Type", "application/json") + .bodyValue(fpRequest) + .retrieve() + .bodyToMono(TransportMappingResponse.class); + + StepVerifier.create(response) + .assertNext( + tmr -> { + log.info("Received transfer ID: {}", tmr.transferId()); + assertThat(tmr.transferId()).isNotNull().hasSize(32); + }) + .verifyComplete(); + } +} diff --git a/trust-center-agent/src/e2e/java/care/smith/fts/tca/GpasProxyE2E.java b/trust-center-agent/src/e2e/java/care/smith/fts/tca/GpasProxyE2E.java new file mode 100644 index 000000000..0b473f854 --- /dev/null +++ b/trust-center-agent/src/e2e/java/care/smith/fts/tca/GpasProxyE2E.java @@ -0,0 +1,78 @@ +package care.smith.fts.tca; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; +import org.hl7.fhir.r4.model.Identifier; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.Test; +import reactor.test.StepVerifier; + +@Slf4j +public class GpasProxyE2E extends AbstractTcaE2E { + + @Override + protected void configureGicsMocks() throws IOException { + configureGicsMetadataMock(); + } + + @Override + protected void configureGpasMocks() throws IOException { + configureGpasMetadataMock(); + configureGpasPseudonymizationMock("patient-id-1", "pseudonym-123"); + } + + @Test + void proxyReturnsTransportIdsNotRealPseudonyms() { + var webClient = createTcaWebClient(); + + var request = new Parameters(); + request.addParameter().setName("target").setValue(new StringType("domain")); + request.addParameter().setName("original").setValue(new StringType("patient-id-1")); + + var response = + webClient + .post() + .uri("/api/v2/cd/fp-gpas-proxy/$pseudonymizeAllowCreate") + .header("Content-Type", "application/fhir+json") + .header("Accept", "application/fhir+json") + .bodyValue(request) + .retrieve() + .bodyToMono(Parameters.class); + + StepVerifier.create(response) + .assertNext( + params -> { + var pseudonymParam = + params.getParameter().stream() + .filter(p -> "pseudonym".equals(p.getName())) + .findFirst() + .orElseThrow(); + + var originalValue = + ((Identifier) + pseudonymParam.getPart().stream() + .filter(p -> "original".equals(p.getName())) + .findFirst() + .orElseThrow() + .getValue()) + .getValue(); + assertThat(originalValue).isEqualTo("patient-id-1"); + + var tId = + ((Identifier) + pseudonymParam.getPart().stream() + .filter(p -> "pseudonym".equals(p.getName())) + .findFirst() + .orElseThrow() + .getValue()) + .getValue(); + // Transport ID: 32-char Base64URL, NOT the real pseudonym + assertThat(tId).hasSize(32).matches("[A-Za-z0-9_-]{32}"); + assertThat(tId).isNotEqualTo("pseudonym-123"); + }) + .verifyComplete(); + } +} diff --git a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingController.java b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingController.java index 7301ea1b9..98737e0d7 100644 --- a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingController.java +++ b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/FpTransportMappingController.java @@ -10,7 +10,6 @@ import care.smith.fts.util.error.ErrorResponseUtil; import care.smith.fts.util.tca.TransportMappingResponse; import jakarta.validation.Valid; -import java.time.Duration; import java.util.Map; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -65,8 +64,7 @@ private Mono fetchDateShiftSeed(FpTransportMappingRequest request) { .map(m -> m.get(seedKey)); } - private Map computeShiftedDates( - String seed, FpTransportMappingRequest request) { + private Map computeShiftedDates(String seed, FpTransportMappingRequest request) { if (request.dateMappings().isEmpty()) { return Map.of(); } diff --git a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/GpasProxyController.java b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/GpasProxyController.java index e5d120b10..467bc4bac 100644 --- a/trust-center-agent/src/main/java/care/smith/fts/tca/rest/GpasProxyController.java +++ b/trust-center-agent/src/main/java/care/smith/fts/tca/rest/GpasProxyController.java @@ -1,13 +1,12 @@ package care.smith.fts.tca.rest; import static care.smith.fts.util.MediaTypes.APPLICATION_FHIR_JSON_VALUE; +import static java.util.stream.Collectors.toMap; import care.smith.fts.tca.deidentification.GpasClient; import care.smith.fts.tca.services.TransportIdService; import care.smith.fts.util.error.ErrorResponseUtil; import jakarta.validation.Valid; -import static java.util.stream.Collectors.toMap; - import java.time.Duration; import java.util.HashSet; import java.util.List; @@ -131,9 +130,7 @@ private String extractRequired(Parameters params, String name) { .map(ParametersParameterComponent::getValue) .map(Base::primitiveValue) .orElseThrow( - () -> - new IllegalArgumentException( - "Missing required parameter '%s'".formatted(name))); + () -> new IllegalArgumentException("Missing required parameter '%s'".formatted(name))); } private List extractAll(Parameters params, String name) { diff --git a/trust-center-agent/src/test/java/care/smith/fts/tca/rest/GpasProxyControllerTest.java b/trust-center-agent/src/test/java/care/smith/fts/tca/rest/GpasProxyControllerTest.java index 9c20a16cc..4210431d0 100644 --- a/trust-center-agent/src/test/java/care/smith/fts/tca/rest/GpasProxyControllerTest.java +++ b/trust-center-agent/src/test/java/care/smith/fts/tca/rest/GpasProxyControllerTest.java @@ -93,9 +93,7 @@ void multipleOriginalsReturnsBatchResponse() { assertThat(params).isNotNull(); assertThat(params.getParameter()).hasSize(2); - params - .getParameter() - .forEach(p -> assertThat(p.getName()).isEqualTo("pseudonym")); + params.getParameter().forEach(p -> assertThat(p.getName()).isEqualTo("pseudonym")); }) .verifyComplete(); } @@ -147,8 +145,7 @@ void gpasFailureReturnsInternalServerError() { StepVerifier.create(controller.pseudonymizeAllowCreate(requestParams)) .assertNext( response -> - assertThat(response.getStatusCode()) - .isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR)) + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR)) .verifyComplete(); } @@ -163,9 +160,6 @@ private Parameters createGpasRequest(String target, String... originals) { private Parameters.ParametersParameterComponent findPart( Parameters.ParametersParameterComponent param, String name) { - return param.getPart().stream() - .filter(p -> name.equals(p.getName())) - .findFirst() - .orElse(null); + return param.getPart().stream().filter(p -> name.equals(p.getName())).findFirst().orElse(null); } } diff --git a/util/src/main/java/care/smith/fts/util/fhir/DateShiftAnonymizer.java b/util/src/main/java/care/smith/fts/util/fhir/DateShiftAnonymizer.java index 8b6b67440..6dc7b90e2 100644 --- a/util/src/main/java/care/smith/fts/util/fhir/DateShiftAnonymizer.java +++ b/util/src/main/java/care/smith/fts/util/fhir/DateShiftAnonymizer.java @@ -20,8 +20,8 @@ /** * Parses a FHIR Pseudonymizer anonymization config and selectively nullifies date elements matching - * dateshift rules. For each nullified date, generates a transport ID and adds a DATE_SHIFT extension - * so RDA can later restore the shifted date. + * dateshift rules. For each nullified date, generates a transport ID and adds a DATE_SHIFT + * extension so RDA can later restore the shifted date. */ public interface DateShiftAnonymizer { From 44397897c02ad6d8333e623d24cdf45f0713971a Mon Sep 17 00:00:00 2001 From: Daniel Hahne Date: Wed, 11 Mar 2026 04:24:59 +0100 Subject: [PATCH 4/4] Limit deidentification concurrency to prevent gPAS overload Unbounded flatMap concurrency (256) in the deidentify step caused a thundering herd of gPAS proxy calls, overwhelming gPAS and causing cascading timeouts. Add maxDeidentifyConcurrency config (default 4) matching the existing maxSendConcurrency pattern. --- .github/test/results/fp-example.json | 2 +- .../java/care/smith/fts/cda/DefaultTransferProcessRunner.java | 2 +- .../java/care/smith/fts/cda/TransferProcessRunnerConfig.java | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/test/results/fp-example.json b/.github/test/results/fp-example.json index 7e0cf1669..45848c9f5 100644 --- a/.github/test/results/fp-example.json +++ b/.github/test/results/fp-example.json @@ -1,6 +1,6 @@ { "phase": "COMPLETED", - "sentBundles": 118, + "sentBundles": 114, "skippedBundles": 0, "count": { "total": 21, diff --git a/clinical-domain-agent/src/main/java/care/smith/fts/cda/DefaultTransferProcessRunner.java b/clinical-domain-agent/src/main/java/care/smith/fts/cda/DefaultTransferProcessRunner.java index 6ff0f8ac8..13c308c77 100644 --- a/clinical-domain-agent/src/main/java/care/smith/fts/cda/DefaultTransferProcessRunner.java +++ b/clinical-domain-agent/src/main/java/care/smith/fts/cda/DefaultTransferProcessRunner.java @@ -165,7 +165,7 @@ public record PatientContext(T data, ConsentedPatient consentedPatient) {} private Flux> deidentify( Flux dataSelection) { return dataSelection - .flatMap(this::deidentifyForPatient) + .flatMap(this::deidentifyForPatient, config.maxDeidentifyConcurrency) .doOnNext(b -> status.updateAndGet(TransferProcessStatus::incDeidentifiedBundles)); } diff --git a/clinical-domain-agent/src/main/java/care/smith/fts/cda/TransferProcessRunnerConfig.java b/clinical-domain-agent/src/main/java/care/smith/fts/cda/TransferProcessRunnerConfig.java index d7a4bd314..e38e211f9 100644 --- a/clinical-domain-agent/src/main/java/care/smith/fts/cda/TransferProcessRunnerConfig.java +++ b/clinical-domain-agent/src/main/java/care/smith/fts/cda/TransferProcessRunnerConfig.java @@ -14,6 +14,8 @@ public class TransferProcessRunnerConfig { @NestedConfigurationProperty @NotNull int maxSendConcurrency = 128; + @NotNull int maxDeidentifyConcurrency = 4; + @NotNull int maxConcurrentProcesses = 4; @NotNull Duration processTtl = Duration.ofDays(1); }