GH-380: Save history on bulk_create (#412)

* GH-380: Added bulk_create_with_history util

* Updated documentation

* Fixed linting issues

* Added blawson's changes

Author: Ross Mechanic

CHANGES.rst

@@ -3,7 +3,8 @@ Changes
 
 Unreleased
 ----------
-- Add ability to specify alternative user_model for tracking
+- Add ability to specify alternative user_model for tracking (gh-371)
+- Add util function ``bulk_create_with_history`` to allow bulk_create with history saved
 
 2.1.1 (2018-06-15)
 ------------------

docs/usage.rst

@@ -311,3 +311,33 @@ And to revert to that ``HistoricalPoll`` instance, we can do:
 This will change the ``poll`` instance to have the data from the
 ``HistoricalPoll`` object and it will create a new row in the
 ``HistoricalPoll`` table indicating that a new change has been made.
+
+Bulk Creating and Queryset Updating
+-----------------------------------
+Django Simple History functions by saving history using a ``post_save`` signal
+every time that an object with history is saved. However, for certain bulk
+operations, such as bulk_create_ and `queryset updates <https://docs.djangoproject.com/en/2.0/ref/models/querysets/#update>`_,
+signals are not sent, and the history is not saved automatically. However,
+Django Simple History provides utility functions to work around this.
+
+Bulk Creating a Model with History
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ As of Django Simple History 2.2.0, we can use the utility function
+``bulk_create_with_history`` in order to bulk create objects while saving their
+history:
+
+.. _bulk_create: https://docs.djangoproject.com/en/2.0/ref/models/querysets/#bulk-create
+
+
+.. code-block:: pycon
+
+    >>> from simple_history.utils import bulk_create_with_history
+    >>> from simple_history.tests.models import Poll
+    >>> from django.utils.timezone import now
+    >>>
+    >>> data = [Poll(id=x, question='Question ' + str(x), pub_date=now()) for x in range(1000)]
+    >>> objs = bulk_create_with_history(data, Poll, batch_size=500)
+    >>> Poll.objects.count()
+    1000
+    >>> Poll.history.count()
+    1000

simple_history/tests/tests/test_utils.py

@@ -0,0 +1,110 @@
+from django.db import IntegrityError
+from django.test import TestCase, TransactionTestCase
+from django.utils.timezone import now
+from mock import Mock, patch
+
+from simple_history.exceptions import NotHistoricalModelError
+from simple_history.tests.models import (
+    Document,
+    Place,
+    Poll,
+    PollWithExcludeFields
+)
+from simple_history.utils import bulk_create_with_history
+
+
+class BulkCreateWithHistoryTestCase(TestCase):
+    def setUp(self):
+        self.data = [
+            Poll(id=1, question='Question 1', pub_date=now()),
+            Poll(id=2, question='Question 2', pub_date=now()),
+            Poll(id=3, question='Question 3', pub_date=now()),
+            Poll(id=4, question='Question 4', pub_date=now()),
+            Poll(id=5, question='Question 5', pub_date=now()),
+        ]
+
+    def test_bulk_create_history(self):
+        bulk_create_with_history(self.data, Poll)
+
+        self.assertEqual(Poll.objects.count(), 5)
+        self.assertEqual(Poll.history.count(), 5)
+
+    def test_bulk_create_history_num_queries_is_two(self):
+        with self.assertNumQueries(2):
+            bulk_create_with_history(self.data, Poll)
+
+    def test_bulk_create_history_on_model_without_history_raises_error(self):
+        self.data = [
+            Place(id=1, name='Place 1'),
+            Place(id=2, name='Place 2'),
+            Place(id=3, name='Place 3'),
+        ]
+        with self.assertRaises(NotHistoricalModelError):
+            bulk_create_with_history(self.data, Place)
+
+    def test_num_queries_when_batch_size_is_less_than_total(self):
+        with self.assertNumQueries(6):
+            bulk_create_with_history(self.data, Poll, batch_size=2)
+
+    def test_bulk_create_history_with_batch_size(self):
+        bulk_create_with_history(self.data, Poll, batch_size=2)
+
+        self.assertEqual(Poll.objects.count(), 5)
+        self.assertEqual(Poll.history.count(), 5)
+
+    def test_bulk_create_works_with_excluded_fields(self):
+        bulk_create_with_history(self.data, PollWithExcludeFields)
+
+        self.assertEqual(Poll.objects.count(), 0)
+        self.assertEqual(Poll.history.count(), 0)
+
+        self.assertEqual(PollWithExcludeFields.objects.count(), 5)
+        self.assertEqual(PollWithExcludeFields.history.count(), 5)
+
+
+class BulkCreateWithHistoryTransactionTestCase(TransactionTestCase):
+    def setUp(self):
+        self.data = [
+            Poll(id=1, question='Question 1', pub_date=now()),
+            Poll(id=2, question='Question 2', pub_date=now()),
+            Poll(id=3, question='Question 3', pub_date=now()),
+            Poll(id=4, question='Question 4', pub_date=now()),
+            Poll(id=5, question='Question 5', pub_date=now()),
+        ]
+
+    @patch('simple_history.manager.HistoryManager.bulk_history_create',
+           Mock(side_effect=Exception))
+    def test_transaction_rolls_back_if_bulk_history_create_fails(self):
+        with self.assertRaises(Exception):
+            bulk_create_with_history(self.data, Poll)
+
+        self.assertEqual(Poll.objects.count(), 0)
+        self.assertEqual(Poll.history.count(), 0)
+
+    def test_bulk_create_history_on_objects_that_already_exist(self):
+        Poll.objects.bulk_create(self.data)
+
+        with self.assertRaises(IntegrityError):
+            bulk_create_with_history(self.data, Poll)
+
+        self.assertEqual(Poll.objects.count(), 5)
+        self.assertEqual(Poll.history.count(), 0)
+
+    def test_bulk_create_history_rolls_back_when_last_exists(self):
+        Poll.objects.create(id=5, question='Question 5', pub_date=now())
+
+        self.assertEqual(Poll.objects.count(), 1)
+        self.assertEqual(Poll.history.count(), 1)
+
+        with self.assertRaises(IntegrityError):
+            bulk_create_with_history(self.data, Poll, batch_size=1)
+
+        self.assertEqual(Poll.objects.count(), 1)
+        self.assertEqual(Poll.history.count(), 1)
+
+    def test_bulk_create_fails_with_wrong_model(self):
+        with self.assertRaises(AttributeError):
+            bulk_create_with_history(self.data, Document)
+
+        self.assertEqual(Poll.objects.count(), 0)
+        self.assertEqual(Poll.history.count(), 0)

simple_history/utils.py

@@ -1,3 +1,5 @@
+from django.db import transaction
+
 from simple_history.exceptions import NotHistoricalModelError
 
 
@@ -31,3 +33,23 @@ def get_history_manager_for_model(model):
 def get_history_model_for_model(model):
     """Return the history model for a given app model."""
     return get_history_manager_for_model(model).model
+
+
+def bulk_create_with_history(objs, model, batch_size=None):
+    """
+    Bulk create the objects specified by objs while also bulk creating
+    their history (all in one transaction).
+    :param objs: List of objs (not yet saved to the db) of type model
+    :param model: Model class that should be created
+    :param batch_size: Number of objects that should be created in each batch
+    :return: List of objs with IDs
+    """
+
+    history_manager = get_history_manager_for_model(model)
+
+    with transaction.atomic(savepoint=False):
+        objs_with_id = model.objects.bulk_create(objs, batch_size=batch_size)
+        history_manager.bulk_history_create(objs_with_id,
+                                            batch_size=batch_size)
+
+    return objs_with_id