Merge remote-tracking branch 'vkf/exclude_fields'.

Author: ses4j

README.md

@@ -74,6 +74,7 @@ Combine bulk create, update, and delete. Make the DB match a set of in-memory ob
 -   `filters`: Q() filters specifying the subset of the database to work in. Use `None` or `[]` if you want to sync against the entire table.
 -   `batch_size`: (optional) passes through to Django `bulk_create.batch_size` and `bulk_update.batch_size`, and controls how many objects are created/updated per SQL query.
 -   `fields`: (optional) List of fields to update. If not set, will sync all fields that are editable and not auto-created.
+-   `exclude_fields`: (optional) List of fields to exclude from updates.
 -   `skip_creates`: (optional) If truthy, will not perform any object creations needed to fully sync. Defaults to not skip.
 -   `skip_updates`: (optional) If truthy, will not perform any object updates needed to fully sync. Defaults to not skip.
 -   `skip_deletes`: (optional) If truthy, will not perform any object deletions needed to fully sync. Defaults to not skip.

bulk_sync/__init__.py

@@ -1,6 +1,8 @@
 from collections import OrderedDict
 import logging
+
 from django.db import transaction, router
+from django.core.exceptions import FieldDoesNotExist
 
 logger = logging.getLogger(__name__)
 
@@ -11,6 +13,7 @@ def bulk_sync(
     filters,
     batch_size=None,
     fields=None,
+    exclude_fields=None,
     skip_creates=False,
     skip_updates=False,
     skip_deletes=False,
@@ -20,10 +23,13 @@ def bulk_sync(
     `new_models`: Django ORM objects that are the desired state.  They may or may not have `id` set.
     `key_fields`: Identifying attribute name(s) to match up `new_models` items with database rows.  If a foreign key
             is being used as a key field, be sure to pass the `fieldname_id` rather than the `fieldname`.
-    `filters`: Q() filters specifying the subset of the database to work in.  Use `None` or `[]` if you want to sync against the entire table.
+    `filters`: Q() filters specifying the subset of the database to work in. Use `None` or `[]` if you want to sync
+            against the entire table.
     `batch_size`: (optional) passes through to Django `bulk_create.batch_size` and `bulk_update.batch_size`, and controls
             how many objects are created/updated per SQL query.
-    `fields`: (optional) list of fields to update. If not set, will sync all fields that are editable and not auto-created.
+    `fields`: (optional) list of fields to update. If not set, will sync all fields that are editable and not
+            auto-created.
+    `exclude_fields`: (optional) list of fields to exclude from updates.
     `skip_creates`: If truthy, will not perform any object creations needed to fully sync. Defaults to not skip.
     `skip_updates`: If truthy, will not perform any object updates needed to fully sync. Defaults to not skip.
     `skip_deletes`: If truthy, will not perform any object deletions needed to fully sync. Defaults to not skip.
@@ -38,6 +44,17 @@ def bulk_sync(
             if not field.primary_key and not field.auto_created and field.editable
         ]
 
+    if exclude_fields is not None:
+        model_fields = set(field.name for field in db_class._meta.fields)
+        fields_to_update = set(fields)
+        fields_to_exclude = set(exclude_fields)
+
+        # Check that we're not attempting to exclude non-existent fields
+        if not fields_to_exclude <= model_fields:
+            raise FieldDoesNotExist(f'model "{db_class.__name__}" has no field(s) {fields_to_exclude - model_fields}')
+
+        fields = list(fields_to_update - fields_to_exclude)
+
     using = router.db_for_write(db_class)
     with transaction.atomic(using=using):
         objs = db_class.objects.all()

tests/tests.py

@@ -1,5 +1,6 @@
 from django.conf import settings
 from django.db import IntegrityError
+from django.core.exceptions import FieldDoesNotExist
 from django.db.models import Q
 from django.test import TestCase
 
@@ -78,18 +79,17 @@ def test_provided_pk_is_retained_but_raises_if_mismatch_with_keyfield(self):
             # Crashes because e1.id already exists in database, even though 'name' doesnt match so it tries to INSERT.
             ret = bulk_sync(new_models=new_objs, filters=Q(company_id=c1.id), key_fields=("name",))
 
-        unique_pk = Employee.objects.values_list('id', flat=True).order_by('-id').first() + 1
+        unique_pk = Employee.objects.values_list("id", flat=True).order_by("-id").first() + 1
         new_objs = [Employee(id=unique_pk, name="Notscott", age=41, company=c1)]
         ret = bulk_sync(new_models=new_objs, filters=Q(company_id=c1.id), key_fields=("name",))
 
         self.assertEqual(0, ret["stats"]["updated"])
-        self.assertEqual(1, ret["stats"]["created"]) # Added 'Notscott'
-        self.assertEqual(1, ret["stats"]["deleted"]) # Deleted 'Scott'
+        self.assertEqual(1, ret["stats"]["created"])  # Added 'Notscott'
+        self.assertEqual(1, ret["stats"]["deleted"])  # Deleted 'Scott'
 
         # Make sure we retained the PK
         self.assertEqual(Employee.objects.filter(id=unique_pk).count(), 1)
 
-
     def test_fields_parameter(self):
         c1 = Company.objects.create(name="Foo Products, Ltd.")
         c2 = Company.objects.create(name="Bar Microcontrollers, Inc.")
@@ -98,12 +98,39 @@ def test_fields_parameter(self):
         e2 = Employee.objects.create(name="Isaac", age=9, company=c2)
 
         # We should update Scott's age, and not touch company.
-        new_objs = [
-            Employee(name="Scott", age=41, company=c1),
-            Employee(name="Isaac", age=9, company=c1),
-        ]
+        new_objs = [Employee(name="Scott", age=41, company=c1), Employee(name="Isaac", age=9, company=c1)]
 
-        ret = bulk_sync(new_models=new_objs, filters=None, key_fields=("name",), fields=['age'])
+        ret = bulk_sync(new_models=new_objs, filters=None, key_fields=("name",), fields=["age"])
+
+        new_e1 = Employee.objects.get(id=e1.id)
+        self.assertEqual("Scott", new_e1.name)
+        self.assertEqual(41, new_e1.age)
+        self.assertEqual(c1, new_e1.company)
+
+        new_e2 = Employee.objects.get(id=e2.id)
+        self.assertEqual("Isaac", new_e2.name)
+        self.assertEqual(9, new_e2.age)
+        self.assertEqual(c2, new_e2.company)
+
+        self.assertEqual(2, ret["stats"]["updated"])
+        self.assertEqual(0, ret["stats"]["created"])
+        self.assertEqual(0, ret["stats"]["deleted"])
+
+    def test_exclude_fields_parameter(self):
+        c1 = Company.objects.create(name="Foo Products, Ltd.")
+        c2 = Company.objects.create(name="Bar Microcontrollers, Inc.")
+
+        e1 = Employee.objects.create(name="Scott", age=40, company=c1)
+        e2 = Employee.objects.create(name="Isaac", age=9, company=c2)
+
+        # We should update Scott's age, and not touch company.
+        new_objs = [Employee(name="Scott", age=41, company=c1), Employee(name="Isaac", age=9, company=c1)]
+
+        with self.assertRaises(FieldDoesNotExist):
+            # Crashes because we attempted to exclude a field that does not exist
+            bulk_sync(new_models=new_objs, filters=None, key_fields=("name",), exclude_fields=["missing_field"])
+
+        ret = bulk_sync(new_models=new_objs, filters=None, key_fields=("name",), exclude_fields=["company"])
 
         new_e1 = Employee.objects.get(id=e1.id)
         self.assertEqual("Scott", new_e1.name)
@@ -126,14 +153,12 @@ def test_skip_deletes(self):
         e2 = Employee.objects.create(name="Isaac", age=9, company=c1)
 
         # update Scott - this makes Isaac is the "stale object" that would be deleted if skip_deletes were False
-        new_objs = [
-            Employee(name="Scott", age=41, company=c1),
-        ]
+        new_objs = [Employee(name="Scott", age=41, company=c1)]
 
         # but Isaac should remain when the skip_deletes flag is True
         ret = bulk_sync(new_models=new_objs, filters=None, key_fields=("name",), skip_deletes=True)
 
-        self.assertEqual(["Scott", "Isaac"], [x.name for x in Employee.objects.all().order_by('id')])
+        self.assertEqual(["Scott", "Isaac"], [x.name for x in Employee.objects.all().order_by("id")])
 
         new_e1 = Employee.objects.get(id=e1.id)
         self.assertEqual(41, new_e1.age)
@@ -151,14 +176,12 @@ def test_skip_creates(self):
         e2 = Employee.objects.create(name="Isaac", age=9, company=c1)
 
         # create a new employee that will be ignored
-        new_objs = [
-            Employee(name="John", age=52, company=c1)
-        ]
+        new_objs = [Employee(name="John", age=52, company=c1)]
 
         ret = bulk_sync(new_models=new_objs, filters=None, key_fields=("name",), skip_creates=True, skip_deletes=True)
 
         self.assertEqual(2, Employee.objects.count())
-        self.assertEqual(["Scott", "Isaac"], [x.name for x in Employee.objects.all().order_by('id')])
+        self.assertEqual(["Scott", "Isaac"], [x.name for x in Employee.objects.all().order_by("id")])
 
         self.assertEqual(0, ret["stats"]["updated"])
         self.assertEqual(0, ret["stats"]["created"])
@@ -171,10 +194,7 @@ def test_skip_updates(self):
         e2 = Employee.objects.create(name="Isaac", age=9, company=c1)
 
         # update employee that will be ignored, create a new one
-        new_objs = [
-            Employee(name="Scott", age=100, company=c1),
-            Employee(name="Alice", age=36, company=c1)
-        ]
+        new_objs = [Employee(name="Scott", age=100, company=c1), Employee(name="Alice", age=36, company=c1)]
 
         ret = bulk_sync(new_models=new_objs, filters=None, key_fields=("name",), skip_updates=True)
 
@@ -184,15 +204,13 @@ def test_skip_updates(self):
 
         # Isaac is "stale" object - was deleted, Alice was created
         self.assertEqual(2, Employee.objects.count())
-        self.assertEqual(["Scott", "Alice"], [x.name for x in Employee.objects.all().order_by('id')])
-
+        self.assertEqual(["Scott", "Alice"], [x.name for x in Employee.objects.all().order_by("id")])
 
         self.assertEqual(0, ret["stats"]["updated"])
         self.assertEqual(1, ret["stats"]["created"])
         self.assertEqual(1, ret["stats"]["deleted"])
 
 
-
 class BulkCompareTests(TestCase):
     """ Test `bulk_compare` method """
 
@@ -261,5 +279,5 @@ def test_bulk_compare_with_ignore_relation_field(self):
         self.assertEqual([new_objs[2], new_objs[3]], ret["added"])
         self.assertEqual([e3], list(ret["removed"]))
         self.assertEqual([new_objs[0]], ret["updated"])
-        self.assertEqual({new_objs[0]: {'age': (40, 41)}}, ret["updated_details"])
+        self.assertEqual({new_objs[0]: {"age": (40, 41)}}, ret["updated_details"])
         self.assertEqual([new_objs[1]], ret["unchanged"])