Raise DownloadCancelled when download has been cancelled and stop the outer loop

Author: Benjamin Bach

kalite/i18n/base.py

@@ -401,10 +401,6 @@ def get_content_pack_url(lang):
 def download_content_pack(fobj, lang, callback=None):
     """Given a file object where the content pack lang will be stored, return a
     zipfile object pointing to the content pack.
-
-    If minimal is set to True, append the "-minimal" flag when downloading the
-    contentpack.
-
     """
     url = get_content_pack_url(lang)
 

kalite/packages/bundled/fle_utils/internet/download.py

@@ -6,10 +6,10 @@
 
 
 from requests.packages.urllib3.util.retry import Retry
-from requests.exceptions import HTTPError
+from kalite.updates.settings import DOWNLOAD_SOCKET_TIMEOUT
 
 
-socket.setdefaulttimeout(20)
+socket.setdefaulttimeout(DOWNLOAD_SOCKET_TIMEOUT)
 
 
 logger = logging.getLogger(__name__)
@@ -58,16 +58,21 @@ def download_file(url, dst=None, callback=None, fp=None, max_retries=5):
     # Assuming the KA Lite version is included in user agent because of an
     # intention to create stats on learningequality.org
     from kalite.version import user_agent
+    
+    # Notice that we deliberate aren't using the ``timeout`` kwarg here, we
+    # will allow the stream to hang forever when a connection is disrupted
+    # but a download has already started. This is to not have to write "resume"
+    # logic on top of our retry logic.
     response = s.get(
         url,
         allow_redirects=True,
         stream=True,
+        # timeout=DOWNLOAD_SOCKET_TIMEOUT,
         headers={"user-agent": user_agent()}
     )
 
     response.raise_for_status()
 
-    
     # Don't do this things until passed the raise_for_status() point
     # If not called with a file pointer or destination, create a new temporary
     # file
@@ -95,6 +100,7 @@ def download_file(url, dst=None, callback=None, fp=None, max_retries=5):
     fp.seek(0)
 
     # Verify file existence
+    dst = fp.name or None
     if dst:
         if os.path.isfile(dst):
             size_on_disk = os.path.getsize(dst)

kalite/updates/management/commands/videodownload.py

@@ -152,11 +152,16 @@ def handle(self, *args, **options):
                                     failcnt=DOWNLOAD_MAX_RETRIES,
                                     retries=retries,
                                 )
-                                self.update_stage(
-                                    stage_name=video.get("youtube_id"),
-                                    stage_percent=0.,
-                                    notes=msg
-                                )
+                                try:
+                                    self.update_stage(
+                                        stage_name=video.get("youtube_id"),
+                                        stage_percent=0.,
+                                        notes=msg
+                                    )
+                                except AssertionError:
+                                    # Raised by update_stage when the video
+                                    # download job has ended
+                                    raise DownloadCancelled()
                                 logger.info(msg)
                                 time.sleep(30)
                                 continue
@@ -171,9 +176,9 @@ def handle(self, *args, **options):
                     annotate_content_models_by_youtube_id(youtube_ids=[video.get("youtube_id")], language=video.get("language"))
 
                 except DownloadCancelled:
-                    # Cancellation event
                     video_queue.clear()
                     failed_youtube_ids.append(video.get("youtube_id"))
+                    break
 
                 except (HTTPError, Exception) as e:
                     # Rather than getting stuck on one video,

kalite/updates/settings.py

@@ -17,3 +17,5 @@
 #: backoff rate defined as:
 #: {backoff factor} * (2 ^ ({number of total retries} - 1))
 DOWNLOAD_MAX_RETRIES = getattr(settings, "KALITE_DOWNLOAD_RETRIES", 5)
+
+DOWNLOAD_SOCKET_TIMEOUT = getattr(settings, "KALITE_DOWNLOAD_TIMEOUT", 20)

kalite/updates/static/js/updates/bundle_modules/update_videos.js

@@ -204,6 +204,7 @@ $(function() {
                 if(server_is_online){
                     base.updatesStart("videodownload", 5000, video_callbacks);
                 } else {
+                    base.updatesStart("videodownload", 10000, video_callbacks);
                     messages.show_message("error", gettext("Could not connect to the central server; videos cannot be downloaded at this time."));
                 }
             });

kalite/updates/templates/updates/update_videos.html

@@ -73,7 +73,7 @@ <h4 class="videos">
 
     <div>
         <!-- note: id must match the "updates" app process name -->
-        <div id="videodownload-progressbar">
+        <div id="videodownload-progressbar" style="margin-top: 20px">
             {% include "updates/progress-bar.html" %}
         </div>