fix for #865!! (#959)

Differentiate between when LiveWebLoader is used for
fully remote loading (eg. remote CDX + pywb) vs
just remote index (remote CDX + local WARCs)
by checking if 'archive_paths' has been explicitly set.
If it has, then skip LiveWebLoader when filename/offset
are provided (to fallback to revisit)

Author: ikreymer

pywb/warcserver/handlers.py

@@ -175,9 +175,10 @@ def __call__(self, params):
 #=============================================================================
 class DefaultResourceHandler(ResourceHandler):
     def __init__(self, index_source, warc_paths='', forward_proxy_prefix='',
+                 use_local_file_load=False,
                  **kwargs):
         loaders = [WARCPathLoader(warc_paths, index_source),
-                   LiveWebLoader(forward_proxy_prefix),
+                   LiveWebLoader(forward_proxy_prefix, use_local_file_load=use_local_file_load),
                    VideoLoader()
                   ]
         super(DefaultResourceHandler, self).__init__(index_source, loaders, **kwargs)

pywb/warcserver/resource/responseloader.py

@@ -274,9 +274,12 @@ class LiveWebLoader(BaseLoader):
                    'application/vnd.apple.mpegurl',
                    'application/dash+xml')
 
-    def __init__(self, forward_proxy_prefix=None, adapter=None):
+    def __init__(self, forward_proxy_prefix=None, adapter=None, use_local_file_load=False):
         self.forward_proxy_prefix = forward_proxy_prefix
 
+        # indicates if WARCs can be loaded locally, even if we get to this fallback
+        self.use_local_file_load = use_local_file_load
+
         socks_host = os.environ.get('SOCKS_HOST')
         socks_port = os.environ.get('SOCKS_PORT', 9050)
         if socks_host and socks_port:
@@ -285,8 +288,11 @@ def __init__(self, forward_proxy_prefix=None, adapter=None):
             self.socks_proxy = None
 
     def load_resource(self, cdx, params):
-        #if cdx.get('filename') and cdx.get('offset') is not None:
-        #    return None
+        if cdx.get('filename') and cdx.get('offset') is not None:
+            # if loading locally, skip here so can retry
+            # in case of revisit
+            if self.use_local_file_load:
+                return None
 
         load_url = cdx.get('load_url')
         if not load_url:

pywb/warcserver/warcserver.py

@@ -244,6 +244,9 @@ def load_coll(self, name, coll_config):
         # ARCHIVE CONFIG
         if not archive_paths:
             archive_paths = self.config.get('archive_paths')
+            use_local_file_load = False
+        else:
+            use_local_file_load = True
 
         # ACCESS CONFIG
         access_checker = None
@@ -252,7 +255,8 @@ def load_coll(self, name, coll_config):
 
         return DefaultResourceHandler(agg, archive_paths,
                                       rules_file=self.rules_file,
-                                      access_checker=access_checker)
+                                      access_checker=access_checker,
+                                      use_local_file_load=use_local_file_load)
 
     def init_sequence(self, coll_name, seq_config):
         if not isinstance(seq_config, list):