Handle Non-Patient Compartment Resource IDs

api/src/main/java/care/smith/fts/api/ConsentedPatientBundle.java

@@ -2,4 +2,14 @@
 
 import org.hl7.fhir.r4.model.Bundle;
 
-public record ConsentedPatientBundle(Bundle bundle, ConsentedPatient consentedPatient) {}
+/**
+ * A bundle of FHIR resources for a consented patient.
+ *
+ * @param bundle the FHIR bundle containing patient resources
+ * @param consentedPatient the consented patient
+ * @param patientResourceId the FHIR resource ID of the Patient (e.g., "DGXCRR3SDVNIEB2R"), which
+ *     may differ from the patient identifier. This is needed for compartment membership checking
+ *     because resource references use the resource ID, not the patient identifier.
+ */
+public record ConsentedPatientBundle(
+    Bundle bundle, ConsentedPatient consentedPatient, String patientResourceId) {}

api/src/test/java/care/smith/fts/api/ConsentedPatientBundleTest.java

@@ -7,6 +7,6 @@
 class ConsentedPatientBundleTest {
   @Test
   void nullsAllowed() {
-    assertThatNoException().isThrownBy(() -> new ConsentedPatientBundle(null, null));
+    assertThatNoException().isThrownBy(() -> new ConsentedPatientBundle(null, null, null));
   }
 }

clinical-domain-agent/src/main/java/care/smith/fts/cda/DefaultTransferProcessRunner.java

@@ -157,7 +157,13 @@ private Flux<Result> sendBundles(Flux<TransportBundle> deidentification) {
       return deidentification
           .flatMap(b -> process.bundleSender().send(b), config.maxSendConcurrency)
           .doOnNext(b -> status.updateAndGet(TransferProcessStatus::incSentBundles))
-          .onErrorContinue((e, r) -> status.updateAndGet(TransferProcessStatus::incSkippedBundles));
+          .onErrorContinue(
+              (e, r) -> {
+                log.warn(
+                    "[Process {}] Bundle skipped due to error: {}", processId(), e.getMessage());
+                log.debug("[Process {}] Bundle skip stack trace", processId(), e);
+                status.updateAndGet(TransferProcessStatus::incSkippedBundles);
+              });
     }
 
     private void onComplete() {

clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/DeidentifhirStep.java

@@ -10,11 +10,12 @@
 import care.smith.fts.api.cda.Deidentificator;
 import care.smith.fts.cda.services.deidentifhir.DeidentifhirUtils;
 import care.smith.fts.cda.services.deidentifhir.IdatScraper;
+import care.smith.fts.cda.services.deidentifhir.IdatScraper.GatheredIds;
+import care.smith.fts.cda.services.deidentifhir.PatientCompartmentService;
 import care.smith.fts.util.error.TransferProcessException;
 import care.smith.fts.util.tca.*;
 import io.micrometer.core.instrument.MeterRegistry;
 import java.time.Duration;
-import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
@@ -32,6 +33,8 @@ class DeidentifhirStep implements Deidentificator {
   private final com.typesafe.config.Config deidentifhirConfig;
   private final com.typesafe.config.Config scraperConfig;
   private final MeterRegistry meterRegistry;
+  private final PatientCompartmentService patientCompartmentService;
+  private final boolean enableCompartmentNamespacing;
 
   public DeidentifhirStep(
       WebClient tcaClient,
@@ -40,20 +43,30 @@ public DeidentifhirStep(
       DateShiftPreserve preserve,
       com.typesafe.config.Config deidentifhirConfig,
       com.typesafe.config.Config scraperConfig,
-      MeterRegistry meterRegistry) {
+      MeterRegistry meterRegistry,
+      PatientCompartmentService patientCompartmentService,
+      boolean enableCompartmentNamespacing) {
     this.tcaClient = tcaClient;
     this.domains = domains;
     this.maxDateShift = maxDateShift;
     this.preserve = preserve;
     this.deidentifhirConfig = deidentifhirConfig;
     this.scraperConfig = scraperConfig;
     this.meterRegistry = meterRegistry;
+    this.patientCompartmentService = patientCompartmentService;
+    this.enableCompartmentNamespacing = enableCompartmentNamespacing;
   }
 
   @Override
   public Mono<TransportBundle> deidentify(ConsentedPatientBundle bundle) {
     var patient = bundle.consentedPatient();
-    var idatScraper = new IdatScraper(scraperConfig, patient);
+    var idatScraper =
+        new IdatScraper(
+            scraperConfig,
+            patient,
+            patientCompartmentService,
+            bundle.patientResourceId(),
+            enableCompartmentNamespacing);
     var ids = idatScraper.gatherIDs(bundle.bundle());
 
     return !ids.isEmpty()
@@ -76,12 +89,21 @@ public Mono<TransportBundle> deidentify(ConsentedPatientBundle bundle) {
   }
 
   private Mono<TransportMappingResponse> fetchTransportMapping(
-      ConsentedPatient patient, Set<String> ids) {
+      ConsentedPatient patient, GatheredIds ids) {
     var request =
         new TransportMappingRequest(
-            patient.id(), patient.patientIdentifierSystem(), ids, domains, maxDateShift, preserve);
+            patient.id(),
+            patient.patientIdentifierSystem(),
+            ids.compartment(),
+            ids.nonCompartment(),
+            domains,
+            maxDateShift,
+            preserve);
 
-    log.trace("Fetch transport mapping for {} IDs", ids.size());
+    log.trace(
+        "Fetch transport mapping for {} compartment + {} non-compartment IDs",
+        ids.compartment().size(),
+        ids.nonCompartment().size());
     return tcaClient
         .post()
         .uri("/api/v2/cd/transport-mapping")

clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/DeidentifhirStepConfig.java

@@ -12,19 +12,23 @@ public record DeidentifhirStepConfig(
     Duration maxDateShift,
     File deidentifhirConfig,
     File scraperConfig,
-    DateShiftPreserve dateShiftPreserve) {
+    DateShiftPreserve dateShiftPreserve,
+    Boolean enableCompartmentNamespacing) {
 
   public DeidentifhirStepConfig(
       TCAConfig trustCenterAgent,
       Duration maxDateShift,
       File deidentifhirConfig,
       File scraperConfig,
-      DateShiftPreserve dateShiftPreserve) {
+      DateShiftPreserve dateShiftPreserve,
+      Boolean enableCompartmentNamespacing) {
     this.trustCenterAgent = trustCenterAgent;
     this.maxDateShift = maxDateShift;
     this.deidentifhirConfig = deidentifhirConfig;
     this.scraperConfig = scraperConfig;
     this.dateShiftPreserve = Optional.ofNullable(dateShiftPreserve).orElse(DateShiftPreserve.NONE);
+    this.enableCompartmentNamespacing =
+        Optional.ofNullable(enableCompartmentNamespacing).orElse(false);
   }
 
   public record TCAConfig(HttpClientConfig server, TcaDomains domains) {}

clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/DeidentifhirStepFactory.java

@@ -4,6 +4,7 @@
 import static java.util.Objects.requireNonNull;
 
 import care.smith.fts.api.cda.Deidentificator;
+import care.smith.fts.cda.services.deidentifhir.PatientCompartmentService;
 import care.smith.fts.util.WebClientFactory;
 import io.micrometer.core.instrument.MeterRegistry;
 import org.springframework.stereotype.Component;
@@ -13,10 +14,15 @@ public class DeidentifhirStepFactory implements Deidentificator.Factory<Deidenti
 
   private final WebClientFactory clientFactory;
   private final MeterRegistry meterRegistry;
+  private final PatientCompartmentService patientCompartmentService;
 
-  public DeidentifhirStepFactory(WebClientFactory clientFactory, MeterRegistry meterRegistry) {
+  public DeidentifhirStepFactory(
+      WebClientFactory clientFactory,
+      MeterRegistry meterRegistry,
+      PatientCompartmentService patientCompartmentService) {
     this.clientFactory = clientFactory;
     this.meterRegistry = meterRegistry;
+    this.patientCompartmentService = patientCompartmentService;
   }
 
   @Override
@@ -36,6 +42,8 @@ public Deidentificator create(
         implConfig.dateShiftPreserve(),
         parseFile(requireNonNull(implConfig.deidentifhirConfig())),
         parseFile(requireNonNull(implConfig.scraperConfig())),
-        meterRegistry);
+        meterRegistry,
+        patientCompartmentService,
+        implConfig.enableCompartmentNamespacing());
   }
 }

clinical-domain-agent/src/main/java/care/smith/fts/cda/impl/EverythingDataSelector.java

@@ -50,8 +50,10 @@ public EverythingDataSelector(
   public Flux<ConsentedPatientBundle> select(ConsentedPatient patient) {
     return pidResolver
         .resolve(patient)
-        .flatMapMany(fhirId -> fetchEverything(patient, fhirId))
-        .map(b -> new ConsentedPatientBundle(b, patient));
+        .flatMapMany(
+            fhirId ->
+                fetchEverything(patient, fhirId)
+                    .map(b -> new ConsentedPatientBundle(b, patient, fhirId.getIdPart())));
   }
 
   private Flux<Bundle> fetchEverything(ConsentedPatient patient, IIdType fhirId) {

clinical-domain-agent/src/main/java/care/smith/fts/cda/services/deidentifhir/IdatScraper.java

@@ -7,14 +7,40 @@
 import de.ume.deidentifhir.Registry;
 import de.ume.deidentifhir.util.Handlers;
 import de.ume.deidentifhir.util.JavaCompat;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
+import lombok.extern.slf4j.Slf4j;
+import org.hl7.fhir.r4.model.Bundle;
 import org.hl7.fhir.r4.model.Resource;
 
+@Slf4j
 public class IdatScraper {
+
+  public record GatheredIds(Set<String> compartment, Set<String> nonCompartment) {
+    public boolean isEmpty() {
+      return compartment.isEmpty() && nonCompartment.isEmpty();
+    }
+  }
+
   private final Deidentifhir deidentiFHIR;
   private final ScrapingStorage scrapingStorage;
+  private final PatientCompartmentService patientCompartmentService;
+  private final String patientIdentifier;
+  private final String patientResourceId;
+  private final boolean enableCompartmentNamespacing;
+
+  public IdatScraper(
+      Config config,
+      ConsentedPatient patient,
+      PatientCompartmentService patientCompartmentService,
+      String patientResourceId,
+      boolean enableCompartmentNamespacing) {
+    this.patientCompartmentService = patientCompartmentService;
+    this.patientIdentifier = patient.id();
+    this.patientResourceId = patientResourceId;
+    this.enableCompartmentNamespacing = enableCompartmentNamespacing;
 
-  public IdatScraper(Config config, ConsentedPatient patient) {
     var keyCreator = NamespacingReplacementProvider.withNamespacing(patient.id());
     scrapingStorage = new ScrapingStorage(keyCreator);
 
@@ -38,12 +64,45 @@ public IdatScraper(Config config, ConsentedPatient patient) {
   }
 
   /**
-   * Gather all IDs contained in the provided bundle and return them as a Set.
+   * Gather all IDs contained in the provided bundle and return them separated by compartment.
+   *
+   * <p>Resources in the patient compartment will have IDs prefixed with the patient ID. Resources
+   * not in the compartment will have IDs without the patient prefix.
    *
-   * @return a Set of all IDs gathered in the Resource
+   * @return a GatheredIds record containing compartment and non-compartment IDs
    */
-  public Set<String> gatherIDs(Resource resource) {
-    deidentiFHIR.deidentify(resource);
-    return scrapingStorage.getGatheredIdats();
+  public GatheredIds gatherIDs(Bundle bundle) {
+    // Pre-compute compartment membership for all resources
+    Map<String, Boolean> membership = precomputeCompartmentMembership(bundle);
+    scrapingStorage.setCompartmentMembership(membership);
+
+    deidentiFHIR.deidentify(bundle);
+    return new GatheredIds(
+        scrapingStorage.getCompartmentIds(), scrapingStorage.getNonCompartmentIds());
+  }
+
+  private Map<String, Boolean> precomputeCompartmentMembership(Bundle bundle) {
+    if (!enableCompartmentNamespacing) {
+      // When disabled, return empty map - ScrapingStorage defaults to all-in-compartment
+      log.trace("Compartment namespacing disabled, treating all resources as in-compartment");
+      return Map.of();
+    }
+
+    Map<String, Boolean> membership = new HashMap<>();
+
+    log.trace(
+        "Checking compartment membership with patientResourceId: {} for patient identifier: {}",
+        patientResourceId,
+        patientIdentifier);
+
+    for (var entry : bundle.getEntry()) {
+      Resource r = entry.getResource();
+      String key = r.fhirType() + ":" + r.getIdPart();
+      boolean inCompartment =
+          patientCompartmentService.isInPatientCompartment(r, patientResourceId);
+      membership.put(key, inCompartment);
+    }
+
+    return membership;
   }
 }