additional caching functionality

Author: Clara Brasseur

astroquery/__init__.py

@@ -12,7 +12,6 @@
 from ._astropy_init import *
 # ----------------------------------------------------------------------------
 
-
 import os
 import logging
 
@@ -55,9 +54,16 @@ def _get_bibtex():
 class Conf(_config.ConfigNamespace):
 
     default_cache_timeout = _config.ConfigItem(
-          60.0*60.0*24.0,
+          86400, # 24 hours
           'Astroquery-wide default cache timeout (seconds).'
           )
-
+    cache_location = _config.ConfigItem(
+          os.path.join(_config.paths.get_cache_dir(), 'astroquery'),
+          'Astroquery default cache location (within astropy cache).'
+          )
+    use_cache = _config.ConfigItem(
+        True,
+        "Astroquery global cache usage, False turns off all caching."
+        )
 
 conf = Conf()

astroquery/query.py

@@ -119,9 +119,12 @@ def request_file(self, cache_location):
     def from_cache(self, cache_location):
         request_file = self.request_file(cache_location)
         try:
-            current_time = datetime.utcnow()
-            cache_time = datetime.utcfromtimestamp(os.path.getmtime(request_file))
-            expired = ((current_time-cache_time) > timedelta(seconds=conf.default_cache_timeout))
+            if conf.default_cache_timeout is None:
+                expired = False
+            else:
+                current_time = datetime.utcnow()
+                cache_time = datetime.utcfromtimestamp(os.path.getmtime(request_file))
+                expired = ((current_time-cache_time) > timedelta(seconds=conf.default_cache_timeout))
             if not expired:
                 with open(request_file, "rb") as f:
                     response = pickle.load(f)
@@ -195,7 +198,10 @@ def __init__(self):
             paths.get_cache_dir(), 'astroquery',
             self.__class__.__name__.split("Class")[0])
         os.makedirs(self.cache_location, exist_ok=True)
-        self._cache_active = True
+
+        self.name = self.__class__.__name__.split("Class")[0]
+        self._cache_active = conf.use_cache
+
 
     def __call__(self, *args, **kwargs):
         """ init a fresh copy of self """
@@ -271,6 +277,7 @@ def _request(self, method, url,
             somewhere other than `BaseQuery.cache_location`
         timeout : int
         cache : bool
+            Override global cache settings.
         verify : bool
             Verify the server's TLS certificate?
             (see http://docs.python-requests.org/en/master/_modules/requests/sessions/?highlight=verify)
@@ -303,12 +310,22 @@ def _request(self, method, url,
             timeout=timeout,
             json=json
         )
+
+        # Set up cache
+        if (cache is True) or ((cache is not False) and conf.use_cache):
+            cache_location = os.path.join(conf.cache_location, self.name)
+            cache = True
+        else:
+            cache_location = None
+            cache = False
+            
         if save:
             local_filename = url.split('/')[-1]
             if os.name == 'nt':
                 # Windows doesn't allow special characters in filenames like
                 # ":" so replace them with an underscore
                 local_filename = local_filename.replace(':', '_')
+
             local_filepath = os.path.join(savedir or self.cache_location or '.', local_filename)
 
             response = self._download_file(url, local_filepath, cache=cache,
@@ -328,16 +345,17 @@ def _request(self, method, url,
                                              allow_redirects=allow_redirects,
                                              json=json)
             else:
-                response = query.from_cache(self.cache_location)
+                response = query.from_cache(cache_location)
                 if not response:
                     response = query.request(self._session,
-                                             self.cache_location,
+                                             cache_location,
                                              stream=stream,
                                              auth=auth,
                                              allow_redirects=allow_redirects,
                                              verify=verify,
                                              json=json)
                     to_cache(response, query.request_file(self.cache_location))
+
             self._last_query = query
             return response
 
@@ -359,6 +377,7 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None,
             supports HTTP "range" requests, the download will be continued
             where it left off.
         cache : bool
+            Cache downloaded file. Defaults to False.
         method : "GET" or "POST"
         head_safe : bool
         """

astroquery/setup_package.py

@@ -3,3 +3,4 @@
 
 def get_package_data():
     return {'astroquery': ['CITATION']}
+