Migrate legacy data from django reversion

- Migrate Per Overview version datetime data to date to avoid django
serialization error
- Add helper function for future

Author: thenav56

main/utils.py

@@ -1,8 +1,17 @@
 import requests
+import typing
+import json
+import datetime
 
 from tempfile import NamedTemporaryFile, _TemporaryFileWrapper
 from collections import defaultdict
 
+from reversion.models import Version
+from reversion.revisions import _get_options
+from django.utils.dateparse import parse_datetime, parse_date
+from django.db import models, router
+from django.contrib.contenttypes.models import ContentType
+
 
 def is_tableau(request):
     """ Checking the request for the 'tableau' parameter
@@ -60,3 +69,94 @@ def __enter__(self) -> _TemporaryFileWrapper:
     def __exit__(self, *_):
         if self.downloaded_file:
             self.downloaded_file.close()
+
+
+class DjangoReversionDataFixHelper:
+    @staticmethod
+    def _get_content_type(
+        content_type_model: typing.Type[ContentType],
+        model: typing.Type[models.Model],
+        using
+    ):
+        version_options = _get_options(model)
+        return content_type_model.objects.db_manager(using).get_for_model(
+            model,
+            for_concrete_model=version_options.for_concrete_model,
+        )
+
+    @classmethod
+    def get_for_model(
+        cls,
+        content_type_model: typing.Type[ContentType],
+        version_model: typing.Type[Version],
+        model: typing.Type[models.Model],
+    ):
+        model_db = router.db_for_write(model)
+        content_type = cls._get_content_type(content_type_model, model, version_model.objects.db)
+        return version_model.objects.filter(
+            content_type=content_type,
+            db=model_db,
+        )
+
+    @classmethod
+    def _date_field_adjust(
+        cls,
+        content_type_model: typing.Type[ContentType],
+        version_model: typing.Type[Version],
+        model: typing.Type[models.Model],
+        fields: typing.List[str],
+        parser: typing.Callable,
+        renderer: typing.Callable,
+    ):
+        updated_versions = []
+        for version in cls.get_for_model(content_type_model, version_model, model):
+            updated_serialized_data = json.loads(version.serialized_data)
+            has_changed = False
+            for field in fields:
+                if field not in updated_serialized_data[0]['fields']:
+                    continue
+                updated_value = parser(updated_serialized_data[0]['fields'][field])
+                if updated_value is None:
+                    # For other format, parser should return None
+                    continue
+                updated_serialized_data[0]['fields'][field] = renderer(updated_value)
+                has_changed = True
+            if has_changed:
+                version.serialized_data = json.dumps(updated_serialized_data)
+                updated_versions.append(version)
+
+        version_model.objects.bulk_update(updated_versions, fields=('serialized_data',))
+
+    @classmethod
+    def date_fields_to_datetime(
+        cls,
+        content_type_model: typing.Type[ContentType],
+        version_model: typing.Type[Version],
+        model: typing.Type[models.Model],
+        fields: typing.List[str],
+    ):
+        return cls._date_field_adjust(
+            content_type_model,
+            version_model,
+            model,
+            fields,
+            parse_date,
+            lambda x: datetime.datetime.combine(x, datetime.datetime.min.time()).isoformat(),
+        )
+
+    @classmethod
+    def datetime_fields_to_date(
+        cls,
+        content_type_model: typing.Type[ContentType],
+        version_model: typing.Type[Version],
+        model: typing.Type[models.Model],
+        fields: typing.List[str],
+    ):
+        return cls._date_field_adjust(
+            content_type_model,
+            version_model,
+            model,
+            fields,
+            parse_datetime,
+            lambda x: x.date().isoformat(),
+        )

per/factories.py

@@ -1,5 +1,6 @@
 import factory
 import datetime
+from factory import fuzzy
 
 from per.models import (
     Overview,
@@ -22,7 +23,7 @@ class Meta:
 
 
 class OverviewFactory(factory.django.DjangoModelFactory):
-    date_of_assessment = factory.fuzzy.FuzzyNaiveDateTime(datetime.datetime(2023, 1, 1))
+    date_of_assessment = fuzzy.FuzzyNaiveDateTime(datetime.datetime(2023, 1, 1))
     type_of_assessment = factory.SubFactory(AssessmentTypeFactory)
 
     class Meta:

per/migrations/0098_fix_reversion_data_20240208_0502.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.23 on 2024-02-08 05:02
+
+from django.db import migrations
+from main.utils import DjangoReversionDataFixHelper
+
+
+def modify_per_datetime_data_to_date(self, schema_editor):
+    ContentType = self.get_model('contenttypes', 'ContentType')
+    Version = self.get_model('reversion', 'Version')
+    Overview = self.get_model('per', 'Overview')
+    DjangoReversionDataFixHelper.datetime_fields_to_date(ContentType, Version, Overview, ['date_of_assessment'])
+
+
+def modify_per_date_data_to_datetime(self, schema_editor):
+    ContentType = self.get_model('contenttypes', 'ContentType')
+    Version = self.get_model('reversion', 'Version')
+    Overview = self.get_model('per', 'Overview')
+    DjangoReversionDataFixHelper.date_fields_to_datetime(ContentType, Version, Overview, ['date_of_assessment'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('reversion', '0001_squashed_0004_auto_20160611_1202'),
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('per', '0097_alter_opslearning_appeal_code'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            modify_per_datetime_data_to_date,  # Time data is lost here
+            reverse_code=modify_per_date_data_to_datetime
+        ),
+    ]

utils/test_utils.py

@@ -1,9 +1,20 @@
 import erp
+import json
+from collections import Counter
 from django.test import TestCase
+
+import reversion
+from reversion.errors import RevertError
+from reversion.models import Version
+from django.contrib.contenttypes.models import ContentType
+
 from api.models import Event, FieldReport, Region, Country, DisasterType, ERPGUID
 from main.mock import erp_request_side_effect_mock
+from main.utils import DjangoReversionDataFixHelper
 from unittest.mock import patch
 
+from per.models import Overview as PerOverview
+from per.factories import OverviewFactory as PerOverviewFactory
 from api.factories import disaster_type as dtFactory
 from api.factories import country as countryFactory
 from api.factories import event as eventFactory
@@ -44,3 +55,85 @@ def test_successful(self, erp_request_side_effect_mock):
         self.assertEqual(ERP.api_guid, 'FindThisGUID')
         self.assertEqual(ERP.field_report_id, report.id)
         self.assertEqual(erp_request_side_effect_mock.called, True)
+
+
+class DjangoReversionDataFixHelperTest(TestCase):
+    def get_version_qs(self):
+        return Version.objects.get_for_model(PerOverview)
+
+    def update_serialized_data(self, raw_data, new_value):
+        new_data = json.loads(raw_data)
+        new_data[0]['fields'][self.field_name] = new_value
+        return json.dumps(new_data)
+
+    def get_version_data_snapshot(self, field_name):
+        # Version data snapshot excluding self.field_name
+        version_data_snapshot = []
+        for _id, data_raw in self.get_version_qs().values_list('id', 'serialized_data').order_by('id'):
+            data = json.loads(data_raw)
+            data[0]['fields'].pop(field_name)
+            version_data_snapshot.append({
+                'id': _id,
+                'data': data,
+            })
+        return version_data_snapshot
+
+    def assert_values(self, values: dict):
+        # Make sure other values are not changed
+        assert self.version_data_snapshot == self.get_version_data_snapshot(self.field_name)
+        # Check count
+        assert self.get_version_qs().count() == sum(values.values())
+        # Check count by value
+        assert dict(Counter([
+            json.loads(data)[0]['fields'][self.field_name]
+            for data in self.get_version_qs().values_list('serialized_data', flat=True)
+        ])) == values
+
+    def confirm_version_data_serialization(self):
+        for version in self.get_version_qs().all():
+            version._local_field_dict
+
+    def setUp(self):
+        super().setUp()
+        self.field_name = 'date_of_assessment'
+        for _ in range(95):
+            reversion.create_revision()(PerOverviewFactory.create)()
+
+        versions = self.get_version_qs().all()
+        # Create dataset with different formats
+        for index, version in enumerate(versions):
+            new_value = '2022-01-01'
+            if (index % 2) == 0:
+                new_value = '2022-01-01T00:00:00'
+            version.serialized_data = self.update_serialized_data(
+                version.serialized_data,
+                new_value,
+            )
+            version.save()
+        Version.objects.bulk_update(versions, fields=('serialized_data',))
+        # Version data snapshot excluding self.field_name
+        self.version_data_snapshot = self.get_version_data_snapshot(self.field_name)
+        self.assert_values({'2022-01-01': 47, '2022-01-01T00:00:00': 48})
+
+    def test_date_fields_to_datetime(self):
+        self.assert_values({'2022-01-01': 47, '2022-01-01T00:00:00': 48})
+        DjangoReversionDataFixHelper.date_fields_to_datetime(
+            ContentType,
+            Version,
+            PerOverview,
+            [self.field_name]
+        )
+        self.assert_values({'2022-01-01T00:00:00': 95})
+
+    def test_datetime_fields_to_date(self):
+        with self.assertRaises(RevertError):
+            self.confirm_version_data_serialization()
+        self.assert_values({'2022-01-01': 47, '2022-01-01T00:00:00': 48})
+        DjangoReversionDataFixHelper.datetime_fields_to_date(
+            ContentType,
+            Version,
+            PerOverview,
+            [self.field_name]
+        )
+        self.assert_values({'2022-01-01': 95})
+        self.confirm_version_data_serialization()