fix: Fix enrollment and event tables [DHIS2-12600]

Author: enricocolasante

dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/ProgramObjectBundleHook.java

@@ -35,6 +35,7 @@
 import org.hisp.dhis.dxf2.metadata.objectbundle.ObjectBundle;
 import org.hisp.dhis.feedback.ErrorCode;
 import org.hisp.dhis.feedback.ErrorReport;
+import org.hisp.dhis.organisationunit.OrganisationUnitService;
 import org.hisp.dhis.preheat.PreheatIdentifier;
 import org.hisp.dhis.program.Enrollment;
 import org.hisp.dhis.program.EnrollmentStatus;
@@ -58,6 +59,8 @@ public class ProgramObjectBundleHook extends AbstractObjectBundleHook<Program> {
 
   private final ProgramStageService programStageService;
 
+  private final OrganisationUnitService organisationUnitService;
+
   private final AclService aclService;
 
   private final IdentifiableObjectManager identifiableObjectManager;
@@ -133,6 +136,8 @@ private void addProgramInstance(Program program) {
       enrollment.setProgram(program);
       enrollment.setStatus(EnrollmentStatus.ACTIVE);
       enrollment.setStoredBy("system-process");
+      enrollment.setOrganisationUnit(
+          organisationUnitService.getRootOrganisationUnits().iterator().next());
 
       identifiableObjectManager.save(enrollment);
     }

dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/ProgramObjectBundleHookTest.java

@@ -45,6 +45,8 @@
 import org.hisp.dhis.common.IdentifiableObjectManager;
 import org.hisp.dhis.feedback.ErrorCode;
 import org.hisp.dhis.feedback.ErrorReport;
+import org.hisp.dhis.organisationunit.OrganisationUnit;
+import org.hisp.dhis.organisationunit.OrganisationUnitService;
 import org.hisp.dhis.program.Enrollment;
 import org.hisp.dhis.program.EnrollmentStatus;
 import org.hisp.dhis.program.EventProgramEnrollmentService;
@@ -71,6 +73,8 @@ class ProgramObjectBundleHookTest {
 
   @Mock private ProgramStageService programStageService;
 
+  @Mock private OrganisationUnitService organisationUnitService;
+
   @Mock private AclService aclService;
 
   @Mock private IdentifiableObjectManager identifiableObjectManager;
@@ -83,6 +87,7 @@ public void setUp() {
         new ProgramObjectBundleHook(
             eventProgramEnrollmentService,
             programStageService,
+            organisationUnitService,
             aclService,
             identifiableObjectManager);
 
@@ -106,6 +111,8 @@ void verifyMissingBundleIsIgnored() {
 
   @Test
   void verifyProgramInstanceIsSavedForEventProgram() {
+    when(organisationUnitService.getRootOrganisationUnits())
+        .thenReturn(List.of(new OrganisationUnit()));
     ArgumentCaptor<Enrollment> argument = ArgumentCaptor.forClass(Enrollment.class);
 
     programA.setProgramType(ProgramType.WITHOUT_REGISTRATION);

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/HibernateEnrollmentStore.java

@@ -57,6 +57,7 @@
 import org.hisp.dhis.organisationunit.OrganisationUnit;
 import org.hisp.dhis.program.Enrollment;
 import org.hisp.dhis.program.EnrollmentStatus;
+import org.hisp.dhis.program.ProgramType;
 import org.hisp.dhis.security.acl.AclService;
 import org.hisp.dhis.tracker.export.Order;
 import org.hisp.dhis.tracker.export.Page;
@@ -192,6 +193,9 @@ private QueryWithOrderBy buildEnrollmentHql(EnrollmentQueryParams params) {
       hql += hlp.whereAnd() + "en.program.uid = '" + params.getProgram().getUid() + "'";
     }
 
+    // TODO(DHIS2-17961) This will be removed when placeholder enrollments will not exist anymore
+    hql += hlp.whereAnd() + "en.program.programType = '" + ProgramType.WITH_REGISTRATION + "'";
+
     if (params.hasEnrollmentStatus()) {
       hql += hlp.whereAnd() + "en." + STATUS + " = '" + params.getEnrollmentStatus() + "'";
     }

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/validator/event/SecurityOwnershipValidator.java

@@ -96,11 +96,7 @@ public void validate(
     // If event is newly created, or going to be deleted, capture scope
     // has to be checked
     if (program.isWithoutRegistration() || strategy.isCreate() || strategy.isDelete()) {
-      if (organisationUnit == null) {
-        log.warn(ORG_UNIT_NO_USER_ASSIGNED, event.getUid());
-      } else {
-        checkOrgUnitInCaptureScope(reporter, event, organisationUnit, bundle.getUser());
-      }
+      checkOrgUnitInCaptureScope(reporter, event, organisationUnit, bundle.getUser());
     }
 
     UID teUid = getTeUidFromEvent(bundle, event, program);
@@ -304,9 +300,7 @@ private void checkEventOrgUnitWriteAccess(
       OrganisationUnit eventOrgUnit,
       boolean isCreatableInSearchScope,
       UserDetails user) {
-    if (eventOrgUnit == null) {
-      log.warn(ORG_UNIT_NO_USER_ASSIGNED, event.getUid());
-    } else if (isCreatableInSearchScope
+    if (isCreatableInSearchScope
         ? !user.isInUserEffectiveSearchOrgUnitHierarchy(eventOrgUnit.getPath())
         : !user.isInUserHierarchy(eventOrgUnit.getPath())) {
       reporter.addError(event, ValidationCode.E1000, user, eventOrgUnit);

dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.42/V2_42_28__Make_ou_event_and_enrollment_not_null.sql

@@ -0,0 +1,61 @@
+-- Delete invalid events linked to invalid enrollments.
+delete from event e where e.enrollmentid in (
+        select en.enrollmentid
+        from enrollment en join program p on en.programid = p.programid
+        where p.type = 'WITH_REGISTRATION'
+        and en.trackedentityid is null
+    );
+
+-- Delete invalid enrollments that are part of a tracker program and
+-- have null tracked entity.
+delete from enrollment en where en.programid in (
+        select p.programid
+        from program p
+        where p.type = 'WITH_REGISTRATION'
+)
+and en.trackedentityid is null;
+
+-- Update null organisation unit of enrollments to organisation unit of one of its events
+update enrollment en set organisationunitid =
+         (select distinct ev.organisationunitid
+                  from event ev
+                  where en.enrollmentid = ev.enrollmentid
+                    and ev.organisationunitid is not null
+                  limit 1)
+where en.organisationunitid is null;
+
+-- If organisationunitid column is still null for any placeholder enrollment,
+-- update organisation unit to root organisation unit.
+-- Placeholder enrollments do not have a tracked entity, so we need to use one well-know
+-- organisation unit.
+-- Organisation unit for this enrollments is never used anyway but we still need to fill in a value.
+update enrollment en set organisationunitid = (
+        select distinct organisationunitid
+        from organisationunit
+        where parentid is null
+        limit 1
+    )
+where en.organisationunitid is null
+and (select type from program p where en.programid = p.programid) = 'WITHOUT_REGISTRATION';
+
+-- If organisationunitid column is still null for any enrollment that is not a placeholder,
+-- use the tracked entity organisation unit.
+-- Tracked entity organisation unit is guaranteed to be not null.
+update enrollment en set organisationunitid =
+         (select te.organisationunitid
+          from trackedentity te
+          where en.trackedentityid = te.trackedentityid)
+where en.organisationunitid is null
+and (select type from program p where en.programid = p.programid) = 'WITH_REGISTRATION';
+
+alter table enrollment alter column organisationunitid set not null;
+
+-- Update null organisation unit of event to organisation unit the enrollment
+-- that at this point is guaranteed to be not null.
+update event ev set organisationunitid = (
+        select organisationunitid
+        from enrollment en
+        where en.enrollmentid = ev.enrollmentid)
+where ev.organisationunitid is null;
+
+alter table event alter column organisationunitid set not null;

dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/maintenance/MaintenanceServiceTest.java

@@ -335,6 +335,7 @@ void testDeleteSoftDeletedEnrollmentLinkedToATrackedEntityDataValueAudit()
     eventA.setScheduledDate(enrollmentDate);
     eventA.setUid("UID-A");
     eventA.setAttributeOptionCombo(coA);
+    eventA.setOrganisationUnit(organisationUnit);
     manager.save(eventA);
     TrackedEntityDataValueChangeLog trackedEntityDataValueChangeLog =
         new TrackedEntityDataValueChangeLog(
@@ -364,6 +365,7 @@ void testDeleteSoftDeletedEventLinkedToARelationshipItem() {
     eventA.setScheduledDate(enrollmentDate);
     eventA.setUid(UID.generate().getValue());
     eventA.setAttributeOptionCombo(coA);
+    eventA.setOrganisationUnit(organisationUnit);
     manager.save(eventA);
     long idA = eventA.getId();
     Relationship r = new Relationship();

dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/program/ProgramMessageStoreTest.java

@@ -111,6 +111,7 @@ void setUp() {
 
     enrollmentA = new Enrollment(new Date(), new Date(), trackedEntityB, programA);
     enrollmentA.setUid(CodeGenerator.generateUid());
+    enrollmentA.setOrganisationUnit(orgUnitA);
 
     eventA = createEvent(stageA, enrollmentA, orgUnitA);
     eventA.setScheduledDate(new Date());

dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/program/notification/ProgramNotificationServiceTest.java

@@ -171,26 +171,28 @@ void setUp() {
     Date enrollmentDate = testDate2.toDate();
     enrollmentA = new Enrollment(enrollmentDate, incidenDate, trackedEntityA, programA);
     enrollmentA.setUid("UID-PIA");
+    enrollmentA.setOrganisationUnit(organisationUnitA);
     manager.save(enrollmentA);
     enrollmentB = new Enrollment(enrollmentDate, incidenDate, trackedEntityB, programB);
+    enrollmentB.setOrganisationUnit(organisationUnitB);
     manager.save(enrollmentB);
-    Event eventA = new Event(enrollmentA, stageA);
+    Event eventA = new Event(enrollmentA, stageA, organisationUnitA);
     eventA.setScheduledDate(enrollmentDate);
     eventA.setUid("UID-A");
     eventA.setAttributeOptionCombo(coA);
-    Event eventB = new Event(enrollmentA, stageB);
+    Event eventB = new Event(enrollmentA, stageB, organisationUnitA);
     eventB.setScheduledDate(enrollmentDate);
     eventB.setUid("UID-B");
     eventB.setAttributeOptionCombo(coA);
-    Event eventC = new Event(enrollmentB, stageC);
+    Event eventC = new Event(enrollmentB, stageC, organisationUnitA);
     eventC.setScheduledDate(enrollmentDate);
     eventC.setUid("UID-C");
     eventC.setAttributeOptionCombo(coA);
-    Event eventD1 = new Event(enrollmentB, stageD);
+    Event eventD1 = new Event(enrollmentB, stageD, organisationUnitA);
     eventD1.setScheduledDate(enrollmentDate);
     eventD1.setUid("UID-D1");
     eventD1.setAttributeOptionCombo(coA);
-    Event eventD2 = new Event(enrollmentB, stageD);
+    Event eventD2 = new Event(enrollmentB, stageD, organisationUnitA);
     eventD2.setScheduledDate(enrollmentDate);
     eventD2.setUid("UID-D2");
     eventD2.setAttributeOptionCombo(coA);
@@ -281,15 +283,15 @@ void testGetEventsWithScheduledNotifications() {
     cal.add(Calendar.DATE, -2);
     Date yesterday = cal.getTime();
     // Events
-    Event eventA = new Event(enrollmentA, stageA);
+    Event eventA = new Event(enrollmentA, stageA, organisationUnitA);
     eventA.setScheduledDate(tomorrow);
     eventA.setAttributeOptionCombo(coA);
     manager.save(eventA);
-    Event eventB = new Event(enrollmentB, stageB);
+    Event eventB = new Event(enrollmentB, stageB, organisationUnitA);
     eventB.setScheduledDate(today);
     eventB.setAttributeOptionCombo(coA);
     manager.save(eventB);
-    Event eventC = new Event(enrollmentB, stageC);
+    Event eventC = new Event(enrollmentB, stageC, organisationUnitA);
     eventC.setScheduledDate(yesterday);
     eventC.setAttributeOptionCombo(coA);
     manager.save(eventC);
@@ -415,8 +417,10 @@ void testGetEnrollmentsWithScheduledNotifications() {
     Date aWeekAgo = cal.getTime();
     // Enrollments
     Enrollment enrollmentC = new Enrollment(today, tomorrow, trackedEntityX, programA);
+    enrollmentC.setOrganisationUnit(organisationUnitA);
     manager.save(enrollmentC);
     Enrollment enrollmentD = new Enrollment(aWeekAgo, yesterday, trackedEntityY, programA);
+    enrollmentD.setOrganisationUnit(organisationUnitA);
     manager.save(enrollmentD);
     // Queries
     List<Enrollment> results;