Update fixture generator and fixture for Django 2.x

Author: rogerdahl

conftest.py

@@ -58,18 +58,14 @@
 D1_SKIP_LIST = 'skip_passed/list'
 D1_SKIP_COUNT = 'skip_passed/count'
 
+# Dict lookup keys matching the keys in settings_test.DATABASE
 TEMPLATE_DB_KEY = 'template'
 TEST_DB_KEY = 'default'
 
-GMN_DB_FIXTURE_LABEL = 'db_fixture'
-
 # Allow redefinition of functions. Pytest allows multiple hooks with the same
 # name.
 # flake8: noqa: F811
 
-# template_db_lock = threading.Lock()
-template_db_lock = multiprocessing.Lock()
-
 # Hack to get access to print and logging output when running under pytest-xdist
 # and pytest-catchlog. Without this, only output from failed tests is displayed.
 sys.stdout = sys.stderr
@@ -112,7 +108,7 @@ def pytest_addoption(parser):
 
   parser.addoption(
     '--fixture-refresh', action='store_true',
-    help='Drop and create GMN template database from JSON fixture file'
+    help='Drop and recreate GMN template database from JSON fixture file'
   )
 
   # Skip passed tests
@@ -417,7 +413,8 @@ def enable_db_access(db):
 
 
 @pytest.yield_fixture(scope='session', autouse=True)
-def django_db_setup(request):
+@pytest.mark.django_db
+def django_db_setup(request, django_db_blocker):
   """Set up DB fixture
   When running in parallel with xdist, this is called once for each worker,
   causing a separate database to be set up for each worker.
@@ -426,46 +423,47 @@ def django_db_setup(request):
 
   db_set_unique_db_name(request)
 
-  # Regular multiprocessing.Lock() context manager did not work here. Also
-  # tried creating the lock at module scope, and also directly calling
-  # acquire() and release(). It's probably related to how the worker processes
-  # relate to each other when launched by pytest-xdist as compared to what the
-  # multiprocessing module expects.
-  with posix_ipc.Semaphore(
-      '/{}'.format(__name__), flags=posix_ipc.O_CREAT, initial_value=1
-  ):
-    logging.warning(
-      'LOCK BEGIN {} {}'.format(
-        db_get_name_by_key(TEMPLATE_DB_KEY), d1_common.date_time.utc_now()
+  with django_db_blocker.unblock():
+    # Regular multiprocessing.Lock() context manager did not work here. Also
+    # tried creating the lock at module scope, and also directly calling
+    # acquire() and release(). It's probably related to how the worker processes
+    # relate to each other when launched by pytest-xdist as compared to what the
+    # multiprocessing module expects.
+    with posix_ipc.Semaphore(
+        '/{}'.format(__name__), flags=posix_ipc.O_CREAT, initial_value=1
+    ):
+      logging.warning(
+        'LOCK BEGIN {} {}'.format(
+          db_get_name_by_key(TEMPLATE_DB_KEY), d1_common.date_time.utc_now()
+        )
       )
-    )
 
-    if not db_exists(TEMPLATE_DB_KEY):
-      db_create_blank(TEMPLATE_DB_KEY)
-      db_migrate(TEMPLATE_DB_KEY)
-      db_populate_by_json(TEMPLATE_DB_KEY)
-      db_migrate(TEMPLATE_DB_KEY)
+      if not db_exists(TEMPLATE_DB_KEY):
+        db_create_blank(TEMPLATE_DB_KEY)
+        db_migrate(TEMPLATE_DB_KEY)
+        db_populate_by_json(TEMPLATE_DB_KEY)
+        db_migrate(TEMPLATE_DB_KEY)
 
-    logging.warning(
-      'LOCK END {} {}'.format(
-        db_get_name_by_key(TEMPLATE_DB_KEY), d1_common.date_time.utc_now()
+      logging.warning(
+        'LOCK END {} {}'.format(
+          db_get_name_by_key(TEMPLATE_DB_KEY), d1_common.date_time.utc_now()
+        )
       )
-    )
 
-  db_drop(TEST_DB_KEY)
-  db_create_from_template()
-  # db_migrate(TEST_DB_KEY)
+    db_drop(TEST_DB_KEY)
+    db_create_from_template()
+    # db_migrate(TEST_DB_KEY)
 
-  # Haven't found out how to prevent transactions from being started. so
-  # closing the implicit transaction here so that template fixture remains
-  # available.
-  # django.db.connections[test_db_key].commit()
+    # Haven't found out how to prevent transactions from being started. so
+    # closing the implicit transaction here so that template fixture remains
+    # available.
+    # django.db.connections[TEST_DB_KEY].commit()
 
-  logging.debug(django.conf.settings)
+    # print(django.conf.settings)
 
-  yield
+    yield
 
-  db_drop(TEST_DB_KEY)
+    db_drop(TEST_DB_KEY)
 
 
 def db_get_name_by_key(db_key):
@@ -501,8 +499,10 @@ def db_populate_by_json(db_key):
   logging.debug('db_populate_by_json() {}'.format(db_key))
   django.core.management.call_command(
     'loaddata',
-    GMN_DB_FIXTURE_LABEL,
-    database=db_key #, commit=True
+    d1_test.sample.get_path('db_fixture.json.bz2'),
+    database=db_key,
+    # verbosity=0,
+    # commit=False,
   )
   db_commit_and_close(db_key)
 

gmn/src/d1_gmn/app/fixtures/db_fixture.json.bz2

@@ -28,7 +28,6 @@
 import d1_gmn.app.util
 import d1_gmn.app.views.slice
 import d1_gmn.app.views.util
-import d1_gmn.app.xslt
 
 import d1_common.const
 import d1_common.date_time

gmn/src/d1_gmn/app/fixtures/db_fixture_gmn1.gz

@@ -15,21 +15,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Test settings for GMN
+"""Test and debug settings for GMN
 - These settings are in effect when GMN is called through unit tests.
 """
 
 # noinspection PyUnresolvedReferences
 # flake8: noqa: F403,F401
 
 import logging
-# When running tests, turn Django's RuntimeWarning into exception
 import warnings
+import d1_common.util
 
 from d1_gmn.app.settings_default import *
 
-import d1_common.util
-
+# When running tests, turn Django's RuntimeWarning into exception
 warnings.filterwarnings(
   'error', r"DateTimeField .* received a naive datetime", RuntimeWarning,
   r'django\.db\.models\.fields'
@@ -79,6 +78,17 @@
 NUM_CHUNK_BYTES = 1024**2
 MAX_SLICE_ITEMS = 5000
 
+# mk_db_fixture:
+# - Uses DATABASES.default
+# - The default database is flushed then populated with test data by
+# mk_db_fixture
+#
+# Unit tests:
+# - Use DATABASES.default.NAME as a base name for temporary databases created
+# from the template database.
+# - Use DATABASES.template.NAME as the source database name when creating
+# databases from template.
+
 DATABASES = {
   'default': {
     'ENGINE': 'django.db.backends.postgresql_psycopg2',
@@ -87,15 +97,8 @@
     'PASSWORD': '',
     'HOST': '',
     'PORT': '',
-    # Transactions
-    #
-    # ATOMIC_REQUESTS is always True when running in production as implicit
-    # transactions form the basis of concurrency control in GMN. However, during
-    # debugging, uncommitted changes are hidden inside the transaction, making
-    # it impossible (?) to see the changes made so far by the code being
-    # debugged.
     'ATOMIC_REQUESTS': False,
-    'AUTOCOMMIT': False,
+    # 'AUTOCOMMIT': False,
   },
   'template': {
     'ENGINE': 'django.db.backends.postgresql_psycopg2',
@@ -105,7 +108,7 @@
     'HOST': '',
     'PORT': '',
     'ATOMIC_REQUESTS': False,
-    'AUTOCOMMIT': False,
+    # 'AUTOCOMMIT': False,
   },
 }
 

gmn/src/d1_gmn/app/middleware/response_handler.py

@@ -18,10 +18,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Create database entries for a set of test objects
+"""Create database fixture JSON file
 
-This creates the db entries by calling the GMN D1 APIs, then uses Django to dump
-the database to JSON.
+This creates the db entries for a set of test objects by calling the GMN D1
+APIs, then uses Django to dump the database to JSON.
 
 Objects are randomly distributed between categories:
   - Standalone, no SID
@@ -32,7 +32,7 @@
 Though object bytes are also created, they are not captured in the db fixture.
 See the README.md for more info on the fixtures.
 
-The django init needs to occur before the django and gmn_test_case imports, so
+The Django init needs to occur before the django and gmn_test_case imports, so
 we're stuck with a bit of a messy import section that isort and flake8 don't
 like.
 
@@ -69,7 +69,9 @@
 import d1_test.instance_generator.system_metadata
 import d1_test.instance_generator.user_agent
 
+# Dict lookup key matching the default key in settings_test.DATABASE
 TEST_DB_KEY = 'default'
+
 N_OBJECTS = 1000
 N_READ_EVENTS = 2 * N_OBJECTS
 
@@ -170,10 +172,19 @@ def save_compressed_db_fixture(self):
     with bz2.BZ2File(
         fixture_file_path, 'w', buffering=1024, compresslevel=9
     ) as bz2_file:
-      django.core.management.call_command('dumpdata', stdout=bz2_file)
+      django.core.management.call_command(
+        'dumpdata',
+        exclude=['auth.permission', 'contenttypes'],
+        database=TEST_DB_KEY,
+        stdout=io.TextIOWrapper(bz2_file),
+      )
 
   def save_pid_list_sample(self, chunk_size=500, **list_objects_kwargs):
-    """Get list of all PIDs in the DB fixture"""
+    """Get list of all PIDs in the DB fixture
+
+    These are for use in any tests that need to know which PIDs and SIDs are
+    available in the DB.
+    """
     client = self.client_v2
     with d1_gmn.tests.gmn_mock.disable_auth():
       start_idx = 0

gmn/src/d1_gmn/settings_test.py

@@ -70,7 +70,8 @@ def test_1020(self, gmn_client_v1_v2):
     """getLogRecords(): Slicing: Retrieve middle section
     """
     with d1_gmn.tests.gmn_mock.disable_auth():
-      log = gmn_client_v1_v2.getLogRecords(start=2000, count=7)
+      n_events = self.get_total_log_records(gmn_client_v1_v2)
+      log = gmn_client_v1_v2.getLogRecords(start=n_events // 2, count=7)
       self.norm_entry_id(log)
       self.sample.assert_equals(log, 'middle_section', gmn_client_v1_v2)
 

gmn/src/d1_gmn/tests/mk_db_fixture.py

@@ -46,7 +46,6 @@ class UTC(datetime.tzinfo):
   A naive Python datetime can be fixed to UTC by attaching it to this
   timezoneinfo based class.
   """
-
   def __repr__(self):
     return self.tzname(0)
 

gmn/src/d1_gmn/tests/test_get_log_records.py

@@ -342,8 +342,8 @@ def dt_from_ts(ts, tz=None):
     """Convert POSIX timestamp to a datetime
     - If {tz} supplied: The dt is adjusted to that tz before being returned.
     - If {tz} not supplied: the dt is returned as naive.
-    - Keeping this function out of d1_common.date_time since we only need
-    naive datetimes for testing.
+    - Keeping this function out of d1_common.date_time since naive datetimes
+    are only needed for testing.
     """
     return datetime.datetime.fromtimestamp(ts, tz)