Merge pull request #1945 from ivanistheone/bugfixed/1818

Cleaned up version of fixes for #1818

Author: rtibbles

kolibri/content/api.py

@@ -7,7 +7,7 @@
 from django.db.models import Q, Sum
 from django.db.models.aggregates import Count
 from kolibri.content import models, serializers
-from kolibri.content.content_db_router import get_active_content_database
+from kolibri.content.content_db_router import get_active_content_database, using_content_database
 from kolibri.logger.models import ContentSessionLog, ContentSummaryLog
 from le_utils.constants import content_kinds
 from rest_framework import filters, pagination, viewsets
@@ -359,10 +359,11 @@ def get_queryset(self):
 
 class ChannelFileSummaryViewSet(viewsets.ViewSet):
     def list(self, request, **kwargs):
-        file_summary = models.File.objects.aggregate(
-            total_files=Count('pk'),
-            total_file_size=Sum('file_size')
-        )
-        file_summary['channel_id'] = get_active_content_database()
-        # Need to wrap in an array to be fetchable as a Collection on client
-        return Response([file_summary])
+        with using_content_database(kwargs['channel_id']):
+            file_summary = models.File.objects.aggregate(
+                total_files=Count('pk'),
+                total_file_size=Sum('file_size')
+            )
+            file_summary['channel_id'] = get_active_content_database()
+            # Need to wrap in an array to be fetchable as a Collection on client
+            return Response([file_summary])

kolibri/content/middleware.py

@@ -9,9 +9,12 @@ class ContentDBRoutingMiddleware(object):
     """
 
     def process_view(self, request, view_func, view_args, view_kwargs):
-        request.PREVIOUSLY_ACTIVE_CONTENT_DATABASE = get_active_content_database(return_none_if_not_set=True)
-        if "channel_id" in view_kwargs:
-            set_active_content_database(view_kwargs["channel_id"])
+        if view_func.__name__ == 'TasksViewSet':
+            pass  # Fix #1818.1: skip get_active_content_database for Task workers to avoid locking DB
+        else:
+            request.PREVIOUSLY_ACTIVE_CONTENT_DATABASE = get_active_content_database(return_none_if_not_set=True)
+            if "channel_id" in view_kwargs:
+                set_active_content_database(view_kwargs["channel_id"])
 
     def process_response(self, request, response):
         set_active_content_database(getattr(request, "PREVIOUSLY_ACTIVE_CONTENT_DATABASE", None))

kolibri/content/utils/annotation.py

@@ -32,3 +32,7 @@ def update_channel_metadata_cache():
         if ch_metadata_obj.last_updated is None:
             ch_metadata_obj.last_updated = local_now()
             ch_metadata_obj.save()
+
+    # Fix #1818.1 content database files get locked (this is called on startup)
+    from django.db import connections
+    connections.close_all()

kolibri/content/utils/channels.py

@@ -53,9 +53,15 @@ def enumerate_content_database_file_paths(content_database_dir):
 def read_channel_metadata_from_db_file(channeldbpath):
     # import here to avoid circular imports whenever kolibri.content.models imports utils too
     from kolibri.content.models import ChannelMetadata
-
     with using_content_database(channeldbpath):
-        return ChannelMetadata.objects.first()
+        channel_metadata = ChannelMetadata.objects.first()
+
+    # FIX for #1818.2: DB file on removable media remains locked after import
+    from django.db import connections
+    connections.close_all()
+
+    return channel_metadata
+
 
 def get_channels_for_data_folder(datafolder):
     channels = []

kolibri/tasks/api.py

@@ -14,6 +14,7 @@
 import requests
 from django.core.management import call_command
 from django.http import Http404
+from django.db import connections
 from django.utils.translation import ugettext as _
 from kolibri.content.models import ChannelMetadataCache
 from kolibri.content.utils.channels import get_mounted_drives_with_channel_info
@@ -143,37 +144,53 @@ def localdrive(self, request):
 
 
 def _networkimport(channel_id, update_progress=None):
-    call_command("importchannel", "network", channel_id)
+    call_command(
+        "importchannel",
+        "network",
+        channel_id,
+        update_progress=update_progress)
     call_command(
         "importcontent",
         "network",
         channel_id,
         update_progress=update_progress)
-
+    connections.close_all()  # close all DB connections (FIX for #1818)
 
 def _localimport(drive_id, update_progress=None):
     drives = get_mounted_drives_with_channel_info()
     drive = drives[drive_id]
+    # copy channel's db file then copy all the content files from sorage dir
     for channel in drive.metadata["channels"]:
-        call_command("importchannel", "local", channel["id"], drive.datafolder)
+        call_command(
+            "importchannel",
+            "local",
+            channel["id"],
+            drive.datafolder,
+            update_progress=update_progress)
         call_command(
             "importcontent",
             "local",
             channel["id"],
             drive.datafolder,
             update_progress=update_progress)
+    connections.close_all()  # close all DB connections (FIX for #1818)
 
 
 def _localexport(drive_id, update_progress=None):
     drives = get_mounted_drives_with_channel_info()
     drive = drives[drive_id]
     for channel in ChannelMetadataCache.objects.all():
-        call_command("exportchannel", channel.id, drive.datafolder)
+        call_command(
+            "exportchannel",
+            channel.id,
+            drive.datafolder,
+            update_progress=update_progress)
         call_command(
             "exportcontent",
             channel.id,
             drive.datafolder,
             update_progress=update_progress)
+    connections.close_all()  # close all DB connections (FIX for #1818)
 
 
 def _job_to_response(job):