Merge branch 'develop' into textboxes

Author: rtibbles

.eslintrc

@@ -1,6 +1,8 @@
 parserOptions:
   ecmaVersion: 7
   sourceType: module
+globals:
+  __version: true
 extends:
   - eslint:recommended
   - prettier
@@ -23,4 +25,4 @@ rules:
   vue/html-quotes: [2, 'double']
   vue/order-in-components: 2
   # ignores the 'kolibri.*' imports
-  import/no-unresolved: [2, { ignore: ['kolibri\.'] }]
+  import/no-unresolved: [2, { ignore: ['kolibri[_.]'] }]

frontend_build/src/parse_bundle_plugin.js

@@ -92,6 +92,22 @@ var parseBundlePlugin = function(data, base_dir) {
     ],
   };
 
+  // Calculate these paths here, so that we can export __publicPath as a variable in the webpack define plugin
+  var publicPath, outputPath;
+
+  if (process.env.DEV_SERVER) {
+    var devServerConfig = require('./webpackdevserverconfig');
+    // If running webpack dev server point to that endpoint.
+    publicPath = devServerConfig.publicPath;
+    // Set output path to base dir, as no files will be written - all built files are cached in memory.
+    outputPath = devServerConfig.basePath
+      ? path.resolve(path.join(base_dir, devServerConfig.basePath))
+      : path.resolve(base_dir);
+  } else {
+    publicPath = path.join('/', data.static_url_root, data.name, '/');
+    outputPath = path.join(data.static_dir, data.name);
+  }
+
   bundle.plugins = bundle.plugins.concat([
     new ExtractTextPlugin('[name]' + data.version + '.css'),
     new WebpackRTLPlugin(),
@@ -110,27 +126,14 @@ var parseBundlePlugin = function(data, base_dir) {
       __events: JSON.stringify(data.events || {}),
       __once: JSON.stringify(data.once || {}),
       __version: JSON.stringify(data.version),
+      // This is necessary to allow modules that use service workers to fetch their service worker code
+      __publicPath: JSON.stringify(publicPath),
     }),
     new extract$trs(data.locale_data_folder, data.name),
   ]);
 
   bundle = merge.smart(bundle, local_config);
 
-  var publicPath, outputPath;
-
-  if (process.env.DEV_SERVER) {
-    var devServerConfig = require('./webpackdevserverconfig');
-    // If running webpack dev server point to that endpoint.
-    publicPath = devServerConfig.publicPath;
-    // Set output path to base dir, as no files will be written - all built files are cached in memory.
-    outputPath = devServerConfig.basePath
-      ? path.resolve(path.join(base_dir, devServerConfig.basePath))
-      : path.resolve(base_dir);
-  } else {
-    publicPath = path.join('/', data.static_url_root, data.name, '/');
-    outputPath = path.join(data.static_dir, data.name);
-  }
-
   bundle.core_name = data.core_name;
   bundle.name = data.name;
   bundle.context = base_dir;

kolibri/content/api.py

@@ -7,11 +7,12 @@
 from django.db.models import Q, Sum
 from django.db.models.aggregates import Count
 from kolibri.content import models, serializers
-from kolibri.content.content_db_router import get_active_content_database
+from kolibri.content.content_db_router import get_active_content_database, using_content_database
 from kolibri.logger.models import ContentSessionLog, ContentSummaryLog
 from le_utils.constants import content_kinds
 from rest_framework import filters, pagination, viewsets
 from rest_framework.decorators import detail_route, list_route
+from rest_framework.generics import get_object_or_404
 from rest_framework.response import Response
 from six.moves.urllib.parse import parse_qs, urlparse
 
@@ -250,15 +251,48 @@ class ContentNodeViewset(viewsets.ModelViewSet):
     filter_class = ContentNodeFilter
     pagination_class = OptionalPageNumberPagination
 
-    def get_queryset(self):
-        return models.ContentNode.objects.all().prefetch_related(
+    def prefetch_related(self, queryset):
+        return queryset.prefetch_related(
             'assessmentmetadata',
             'files',
         ).select_related('license')
 
+    def get_queryset(self, prefetch=True):
+        queryset = models.ContentNode.objects.all()
+        if prefetch:
+            return self.prefetch_related(queryset)
+        return queryset
+
+    def get_object(self, prefetch=True):
+        """
+        Returns the object the view is displaying.
+        You may want to override this if you need to provide non-standard
+        queryset lookups.  Eg if objects are referenced using multiple
+        keyword arguments in the url conf.
+        """
+        queryset = self.filter_queryset(self.get_queryset(prefetch=prefetch))
+
+        # Perform the lookup filtering.
+        lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
+
+        assert lookup_url_kwarg in self.kwargs, (
+            'Expected view %s to be called with a URL keyword argument '
+            'named "%s". Fix your URL conf, or set the `.lookup_field` '
+            'attribute on the view correctly.' %
+            (self.__class__.__name__, lookup_url_kwarg)
+        )
+
+        filter_kwargs = {self.lookup_field: self.kwargs[lookup_url_kwarg]}
+        obj = get_object_or_404(queryset, **filter_kwargs)
+
+        # May raise a permission denied
+        self.check_object_permissions(self.request, obj)
+
+        return obj
+
     @detail_route(methods=['get'])
     def descendants(self, request, **kwargs):
-        node = self.get_object()
+        node = self.get_object(prefetch=False)
         kind = self.request.query_params.get('descendant_kind', None)
         descendants = node.get_descendants()
         if kind:
@@ -269,7 +303,16 @@ def descendants(self, request, **kwargs):
 
     @detail_route(methods=['get'])
     def ancestors(self, request, **kwargs):
-        return Response(self.get_object().get_ancestors().values('pk', 'title'))
+        cache_key = 'contentnode_ancestors_{db}_{pk}'.format(db=get_active_content_database(), pk=kwargs.get('pk'))
+
+        if cache.get(cache_key) is not None:
+            return Response(cache.get(cache_key))
+
+        ancestors = list(self.get_object(prefetch=False).get_ancestors().values('pk', 'title'))
+
+        cache.set(cache_key, ancestors, 60 * 10)
+
+        return Response(ancestors)
 
     @detail_route(methods=['get'])
     def next_content(self, request, **kwargs):
@@ -316,10 +359,11 @@ def get_queryset(self):
 
 class ChannelFileSummaryViewSet(viewsets.ViewSet):
     def list(self, request, **kwargs):
-        file_summary = models.File.objects.aggregate(
-            total_files=Count('pk'),
-            total_file_size=Sum('file_size')
-        )
-        file_summary['channel_id'] = get_active_content_database()
-        # Need to wrap in an array to be fetchable as a Collection on client
-        return Response([file_summary])
+        with using_content_database(kwargs['channel_id']):
+            file_summary = models.File.objects.aggregate(
+                total_files=Count('pk'),
+                total_file_size=Sum('file_size')
+            )
+            file_summary['channel_id'] = get_active_content_database()
+            # Need to wrap in an array to be fetchable as a Collection on client
+            return Response([file_summary])

kolibri/content/fixtures/content_test.json

@@ -159,23 +159,29 @@
   {
     "fields": {
       "lang_code": "en",
-      "lang_subcode": "01"
+      "lang_subcode": "01",
+      "lang_name": "English-Test",
+      "lang_direction": "ltr"
     },
     "pk": 1,
     "model": "content.language"
   },
   {
     "fields": {
       "lang_code": "zh",
-      "lang_subcode": "02"
+      "lang_subcode": "02",
+      "lang_name": "Chinese-Test",
+      "lang_direction": "ltr"
     },
     "pk": 2,
     "model": "content.language"
   },
   {
     "fields": {
       "lang_code": "es",
-      "lang_subcode": "03"
+      "lang_subcode": "03",
+      "lang_name": "Spanish-Test",
+      "lang_direction": "ltr"
     },
     "pk": 3,
     "model": "content.language"

kolibri/content/middleware.py

@@ -9,9 +9,12 @@ class ContentDBRoutingMiddleware(object):
     """
 
     def process_view(self, request, view_func, view_args, view_kwargs):
-        request.PREVIOUSLY_ACTIVE_CONTENT_DATABASE = get_active_content_database(return_none_if_not_set=True)
-        if "channel_id" in view_kwargs:
-            set_active_content_database(view_kwargs["channel_id"])
+        if view_func.__name__ == 'TasksViewSet':
+            pass  # Fix #1818.1: skip get_active_content_database for Task workers to avoid locking DB
+        else:
+            request.PREVIOUSLY_ACTIVE_CONTENT_DATABASE = get_active_content_database(return_none_if_not_set=True)
+            if "channel_id" in view_kwargs:
+                set_active_content_database(view_kwargs["channel_id"])
 
     def process_response(self, request, response):
         set_active_content_database(getattr(request, "PREVIOUSLY_ACTIVE_CONTENT_DATABASE", None))

kolibri/content/migrations/0004_auto_20170801_1250.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.7 on 2017-08-01 19:50
+from __future__ import unicode_literals
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('content', '0003_auto_20170607_1212'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='contentnode',
+            name='lang',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='content.Language'),
+        ),
+        migrations.AddField(
+            model_name='language',
+            name='lang_direction',
+            field=models.CharField(choices=[('ltr', 'Left to Right'), ('rtl', 'Right to Left')], default='ltr', max_length=3),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='language',
+            name='lang_name',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='language',
+            name='id',
+            field=models.CharField(max_length=14, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='language',
+            name='lang_subcode',
+            field=models.CharField(blank=True, db_index=True, max_length=10, null=True),
+        ),
+    ]

kolibri/content/models.py

@@ -16,6 +16,7 @@
 from jsonfield import JSONField
 from kolibri.core.fields import DateTimeTzField
 from le_utils.constants import content_kinds, file_formats, format_presets
+from le_utils.constants.languages import LANGUAGE_DIRECTIONS
 from mptt.models import MPTTModel, TreeForeignKey
 from mptt.querysets import TreeQuerySet
 
@@ -123,6 +124,7 @@ class ContentNode(MPTTModel, ContentDatabaseModel):
     kind = models.CharField(max_length=200, choices=content_kinds.choices, blank=True)
     available = models.BooleanField(default=False)
     stemmed_metaphone = models.CharField(max_length=1800, blank=True)  # for fuzzy search in title and description
+    lang = models.ForeignKey('Language', blank=True, null=True)
 
     objects = ContentQuerySet.as_manager()
 
@@ -158,14 +160,17 @@ def get_descendant_kind_counts(self):
 
 @python_2_unicode_compatible
 class Language(ContentDatabaseModel):
-    id = models.CharField(max_length=7, primary_key=True)
+    id = models.CharField(max_length=14, primary_key=True)
     lang_code = models.CharField(max_length=3, db_index=True)
-    lang_subcode = models.CharField(max_length=3, db_index=True, blank=True, null=True)
+    lang_subcode = models.CharField(max_length=10, db_index=True, blank=True, null=True)
+    # Localized name
+    lang_name = models.CharField(max_length=100, blank=True, null=True)
+    lang_direction = models.CharField(max_length=3, choices=LANGUAGE_DIRECTIONS)
 
     objects = ContentQuerySet.as_manager()
 
     def __str__(self):
-        return self.lang_code
+        return self.lang_name
 
 
 @python_2_unicode_compatible

kolibri/content/serializers.py

@@ -1,3 +1,4 @@
+from django.core.cache import cache
 from django.db.models import Manager, Sum
 from django.db.models.query import RawQuerySet
 from kolibri.content.models import AssessmentMetaData, ChannelMetadataCache, ContentNode, File
@@ -115,6 +116,20 @@ class ContentNodeListSerializer(serializers.ListSerializer):
 
     def to_representation(self, data):
 
+        # Dealing with nested relationships, data can be a Manager,
+        # so, first get a queryset from the Manager if needed
+        data = data.all() if isinstance(data, Manager) else data
+
+        cache_key = None
+        # Cache parent look ups only
+        if "parent" in self.context['request'].GET:
+            cache_key = 'contentnode_list_{db}_{parent}'.format(
+                db=get_active_content_database(),
+                parent=self.context['request'].GET.get('parent'))
+
+            if cache.get(cache_key):
+                return cache.get(cache_key)
+
         if not data:
             return data
 
@@ -125,17 +140,25 @@ def to_representation(self, data):
             # Don't annotate topic progress as too expensive
             progress_dict = get_content_progress_fractions(data, user)
 
-        # Dealing with nested relationships, data can be a Manager,
-        # so, first get a queryset from the Manager if needed
-        iterable = data.all() if isinstance(data, Manager) else data
-
-        return [
-            self.child.to_representation(
+        result = []
+        topic_only = True
+        for item in data:
+            obj = self.child.to_representation(
                 item,
                 progress_fraction=progress_dict.get(item.content_id),
                 annotate_progress_fraction=False
-            ) for item in iterable
-        ]
+            )
+            topic_only = topic_only and obj.get('kind') == content_kinds.TOPIC
+            result.append(obj)
+
+        # Only store if all nodes are topics, because we don't annotate progress on them
+        # This has the happy side effect of not caching our dynamically calculated
+        # recommendation queries, which might change for the same user over time
+        # because they do not return topics
+        if topic_only and cache_key:
+            cache.set(cache_key, result, 60 * 10)
+
+        return result
 
 
 class ContentNodeSerializer(serializers.ModelSerializer):
@@ -165,7 +188,8 @@ def to_representation(self, instance, progress_fraction=None, annotate_progress_
                 progress_fraction = 0.0
             else:
                 user = self.context["request"].user
-                progress_fraction = get_content_progress_fraction(instance, user)
+                if instance.kind != content_kinds.TOPIC:
+                    progress_fraction = get_content_progress_fraction(instance, user)
         value = super(ContentNodeSerializer, self).to_representation(instance)
         value['progress_fraction'] = progress_fraction
         return value

kolibri/content/test/test_content_app.py

@@ -135,7 +135,7 @@ def test_all_str(self):
         self.assertEqual(str(p), 'tag_2')
         # test for Language __str__
         p = content.Language.objects.get(lang_code="en")
-        self.assertEqual(str(p), 'en')
+        self.assertEqual(str(p), 'English-Test')
         # test for ChannelMetadata __str__
         p = content.ChannelMetadata.objects.get(name="testing")
         self.assertEqual(str(p), 'testing')

kolibri/content/utils/annotation.py

@@ -32,3 +32,7 @@ def update_channel_metadata_cache():
         if ch_metadata_obj.last_updated is None:
             ch_metadata_obj.last_updated = local_now()
             ch_metadata_obj.save()
+
+    # Fix #1818.1 content database files get locked (this is called on startup)
+    from django.db import connections
+    connections.close_all()