Merge pull request #5545 from benjaoming/bug/video_url

Harden download robustness

Author: benjaoming

docs/installguide/release_notes.rst

@@ -12,10 +12,19 @@ to read the release notes.
 0.17.4 (unreleased)
 -------------------
 
+Added
+^^^^^
+
+* Progress displayed while downloading content packs :url-issue:`5356`
+
 Bug fixes
 ^^^^^^^^^
 
-* Video download retry upon connection timeouts/errors :url-issue:`5528`
+* Video download retry upon connection establishment errors :url-issue:`5528`
+* Resume download if unplugged connection while downloading :url-issue:`5545`
+* Blank videos produced when ``kalite.learningequality.org`` server down :url-issue:`5538`
+* Video download jobs hanging indefinitely after pressing "Cancel" :url-issue:`5545`
+* Also delete content database when deleting a content pack :url-issue:`5545`
 * Simplified login is now working when there are 1,000 or more users registered in a facility. :url-issue:`5523`
 
 New Features
@@ -26,12 +35,25 @@ New Features
 Developers
 ^^^^^^^^^^
 
- * Do not use `npm clean`, now requires npm>=5 for building on unclean systems :url-issue:`5519`
+* Do not use `npm clean`, now requires npm>=5 for building on unclean systems :url-issue:`5519`
 
 Contents
 ^^^^^^^^
 
- * Resized video torrent set for English rebuilt with missing videos
+* Resized video torrent set for English rebuilt with missing videos
+
+
+Known issues
+^^^^^^^^^^^^
+
+* It isn't possible to cancel video downloads if a video is downloading while
+  the connection is switched off.
+* **Chrome 55-56** has issues scrolling the menus on touch devices. Upgrading to Chrome 57 fixes this. :url-issue:`5407`
+* **Windows** needs at least Python 2.7.11. The Windows installer for KA Lite will install the latest version of Python. If you installed KA Lite in another way, and your Python installation is more than a year old, you probably have to upgrade Python - you can fetch the latest 2.7.12 version `here <https://www.python.org/downloads/windows/>`__.
+* **Windows** installer tray application option "Run on start" does not work, see `learningequality/installers#106 <https://github.com/learningequality/installers/issues/106>`__ (also contains `a work-around <https://github.com/learningequality/installers/issues/106#issuecomment-237729680>`__)
+* **Windows + IE9** One-Click device registration is broken. Work-around: Use a different browser or use manual device registration. :url-issue:`5409`
+* **Firefox 47**: Subtitles are misaligned in the video player. This is fixed by upgrading Firefox.
+* A limited number of exercises with radio buttons have problems displaying :url-issue:`5172`
 
 
 0.17.3

kalite/distributed/management/commands/retrievecontentpack.py

@@ -1,5 +1,7 @@
+import logging
 import os
 import shutil
+import socket
 import tempfile
 import zipfile
 
@@ -20,8 +22,10 @@
 from kalite.updates.management.utils import UpdatesStaticCommand
 from peewee import SqliteDatabase
 from kalite.topic_tools.content_models import Item, AssessmentItem
+from requests.exceptions import ConnectionError, HTTPError
 
-logging = django_settings.LOG
+
+logger = logging.getLogger(__name__)
 
 
 class Command(UpdatesStaticCommand):
@@ -143,8 +147,22 @@ def download(self, *args, **options):
 
         lang = args[1]
 
+        def download_callback(fraction):
+            percent = int(fraction * 100)
+            self.update_stage(
+                fraction,
+                stage_status=_("Downloaded {pct}% of {lang} content pack").format(
+                    pct=percent,
+                    lang=lang,
+                )
+            )
+
         with tempfile.NamedTemporaryFile() as f:
-            zf = download_content_pack(f, lang)
+            try:
+                zf = download_content_pack(f, lang, callback=download_callback)
+            except (socket.error, ConnectionError, HTTPError) as e:
+                self.cancel("Could not download content pack, unable to connect", str(e))
+                return
             self.process_content_pack(zf, lang)
             zf.close()
 
@@ -194,7 +212,7 @@ def extract_catalog_files(zf, lang):
     for zipmo, djangomo in filename_mapping.items():
         zipmof = zf.open(zipmo)
         mopath = os.path.join(modir, djangomo)
-        logging.debug("writing to %s" % mopath)
+        logger.debug("writing to %s" % mopath)
         with open(mopath, "wb") as djangomof:
             shutil.copyfileobj(zipmof, djangomof)
 

kalite/distributed/static/js/distributed/perseus/ke/package.json

@@ -23,7 +23,7 @@
     "devDependencies": {
         "jison" : "0.4.13",
         "cheerio": "0.19.0",
-        "uglify-js": "2.4.20",
+        "uglify-js": "2.6.0",
         "jshint": "2.7.0"
     }
 }

kalite/i18n/base.py

@@ -1,8 +1,8 @@
 import errno
+import logging
 import os
 import re
 import shutil
-import urllib
 import zipfile
 from distutils.version import LooseVersion
 from fle_utils.internet.webcache import invalidate_web_cache
@@ -19,17 +19,24 @@
 
 from fle_utils.config.models import Settings
 from fle_utils.general import ensure_dir, softload_json
+from fle_utils.internet.download import download_file
+import sys
+from kalite.topic_tools.settings import CONTENT_DATABASE_PATH
 
 
-CONTENT_PACK_URL_TEMPLATE = ("http://pantry.learningequality.org/downloads"
-                             "/ka-lite/{version}/content/contentpacks/{langcode}{suffix}.zip")
+CONTENT_PACK_URL_TEMPLATE = (
+    "http://pantry.learningequality.org/downloads"
+    "/ka-lite/{version}/content/contentpacks/{langcode}.zip"
+)
 
 
 CACHE_VARS = []
 
 
 from django.conf import settings
-logging = settings.LOG
+
+
+logger = logging.getLogger(__name__)
 
 
 class LanguageNotFoundError(Exception):
@@ -98,7 +105,7 @@ def get_code2lang_map(lang_code=None, force=False):
     global CODE2LANG_MAP
 
     if force or not CODE2LANG_MAP:
-        lmap = softload_json(settings.LANG_LOOKUP_FILEPATH, logger=logging.debug)
+        lmap = softload_json(settings.LANG_LOOKUP_FILEPATH, logger=logger.debug)
 
         CODE2LANG_MAP = {}
         for lc, entry in lmap.iteritems():
@@ -228,12 +235,12 @@ def _get_installed_language_packs():
                 metadata_filepath = os.path.join(locale_dir, django_disk_code, "%s_metadata.json" % lcode_to_ietf(django_disk_code))
                 lang_meta = softload_json(metadata_filepath, raises=True)
 
-                logging.debug("Found language pack %s" % (django_disk_code))
+                logger.debug("Found language pack %s" % (django_disk_code))
             except IOError as e:
                 if e.errno == errno.ENOENT:
-                    logging.info("Ignoring non-language pack %s in %s" % (django_disk_code, locale_dir))
+                    logger.info("Ignoring non-language pack %s in %s" % (django_disk_code, locale_dir))
                 else:
-                    logging.error("Error reading %s metadata (%s): %s" % (django_disk_code, metadata_filepath, e))
+                    logger.error("Error reading %s metadata (%s): %s" % (django_disk_code, metadata_filepath, e))
                 continue
 
             installed_language_packs.append(lang_meta)
@@ -273,9 +280,9 @@ def update_jsi18n_file(code="en"):
         try:
             icu_js = open(os.path.join(path, code, "%s_icu.js" % code), "r").read()
         except IOError:
-            logging.warn("No {code}_icu.js file found in locale_path {path}".format(code=code, path=path))
+            logger.warn("No {code}_icu.js file found in locale_path {path}".format(code=code, path=path))
     output_js = response.content + "\n" + icu_js
-    logging.info("Writing i18nized js file to {0}".format(output_file))
+    logger.info("Writing i18nized js file to {0}".format(output_file))
     with open(output_file, "w") as fp:
         fp.write(output_js)
     translation.deactivate()
@@ -309,7 +316,7 @@ def select_best_available_language(target_code, available_codes=None):
             store_cache = True
             available_codes = get_installed_language_packs().keys()
 
-    # logging.debug("choosing best language among %s" % (available_codes))
+    # logger.debug("choosing best language among %s" % (available_codes))
 
     # Make it a tuple so we can hash it
     available_codes = [lcode_to_django_lang(lc) for lc in available_codes if lc]
@@ -329,7 +336,7 @@ def select_best_available_language(target_code, available_codes=None):
         raise RuntimeError("No languages found")
 
     # if actual_code != target_code:
-    #    logging.debug("Requested code %s, got code %s" % (target_code, actual_code))
+    #    logger.debug("Requested code %s, got code %s" % (target_code, actual_code))
 
     # Store in cache when available_codes are not set
     if store_cache:
@@ -345,12 +352,21 @@ def delete_language(lang_code):
     for langpack_resource_path in langpack_resource_paths:
         try:
             shutil.rmtree(langpack_resource_path)
-            logging.info("Deleted language pack resource path: %s" % langpack_resource_path)
+            logger.info("Deleted language pack resource path: %s" % langpack_resource_path)
         except OSError as e:
             if e.errno != 2:    # Only ignore error: No Such File or Directory
                 raise
             else:
-                logging.debug("Not deleting missing language pack resource path: %s" % langpack_resource_path)
+                logger.debug("Not deleting missing language pack resource path: %s" % langpack_resource_path)
+
+    # Delete the content db of particular language
+    content_db = CONTENT_DATABASE_PATH.format(channel="khan", language=lang_code)
+    if os.path.isfile(content_db):
+        try:
+            os.unlink(content_db)
+        except OSError as e:
+            if e.errno != 2:    # Only ignore error: No Such File or Directory
+                raise
 
     invalidate_web_cache()
 
@@ -361,7 +377,7 @@ def set_request_language(request, lang_code):
     lang_code = select_best_available_language(lang_code)  # output is in django_lang format
 
     if lang_code != request.session.get(settings.LANGUAGE_COOKIE_NAME):
-        logging.debug("setting request language to %s (session language %s), from %s" % (lang_code, request.session.get("default_language"), request.session.get(settings.LANGUAGE_COOKIE_NAME)))
+        logger.debug("setting request language to %s (session language %s), from %s" % (lang_code, request.session.get("default_language"), request.session.get(settings.LANGUAGE_COOKIE_NAME)))
         # Just in case we have a db-backed session, don't write unless we have to.
         request.session[settings.LANGUAGE_COOKIE_NAME] = lang_code
 
@@ -376,29 +392,33 @@ def translate_block(language):
     translation.deactivate()
 
 
-def download_content_pack(fobj, lang, minimal=False):
+def get_content_pack_url(lang):
+    return CONTENT_PACK_URL_TEMPLATE.format(
+        version=SHORTVERSION,
+        langcode=lang,
+    )
+
+def download_content_pack(fobj, lang, callback=None):
     """Given a file object where the content pack lang will be stored, return a
     zipfile object pointing to the content pack.
+    """
+    url = get_content_pack_url(lang)
 
-    If minimal is set to True, append the "-minimal" flag when downloading the
-    contentpack.
+    logger.info("Downloading content pack from {}".format(url))
 
-    """
-    url = CONTENT_PACK_URL_TEMPLATE.format(
-        version=SHORTVERSION,
-        langcode=lang,
-        suffix="-minimal" if minimal else "",
-    )
+    def _callback(fraction):
+        sys.stdout.write("\rProgress: [{}{}] ({}%)".format(
+            "=" * (int(fraction * 100) / 2),
+            " " * (50 - int(fraction * 100) / 2),
+            int(fraction * 100),
+        ))
+        if callback:
+            callback(fraction)
 
-    logging.info("Downloading content pack from {}".format(url))
-    httpf = urllib.urlopen(url)  # returns a file-like object not exactly to zipfile's liking, so save first
+    download_file(url, fp=fobj, callback=_callback)
 
-    shutil.copyfileobj(httpf, fobj)
-    fobj.seek(0)
     zf = zipfile.ZipFile(fobj)
 
-    httpf.close()
-
     return zf