From a1fd9180edf2b5675a0770a5b92fca30984ad0aa Mon Sep 17 00:00:00 2001 From: Julien Woillez Date: Tue, 23 Sep 2014 09:18:12 +0200 Subject: [PATCH 01/26] Add default_cache_timeout to astroquery package root. --- astroquery/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index 5513075fc9..312e1c8654 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -16,6 +16,7 @@ import logging from .logger import _init_log +from astropy import config as _config __all__ = ["__version__", "__githash__", "__citation__", "__bibtex__", "test", "log"] @@ -38,3 +39,13 @@ def _get_bibtex(): logging.addLevelName(5, "TRACE") log = logging.getLogger() log = _init_log() + +# Set up cache configuration +class Conf(_config.ConfigNamespace): + + default_cache_timeout = _config.ConfigItem( + 60.0*60.0*24.0, + 'Astroquery-wide default cache timeout (seconds).' + ) + +conf = Conf() From 3aa7612e301371ab017bbfa7939c6193c45a7995 Mon Sep 17 00:00:00 2001 From: Julien Woillez Date: Tue, 23 Sep 2014 09:18:52 +0200 Subject: [PATCH 02/26] Add cache timeout to AstroQuery --- astroquery/query.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index ddc3dc4fa9..f490e1075c 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -8,9 +8,12 @@ import keyring import io import os +import warnings import requests import textwrap +from datetime import datetime, timedelta + from astropy.config import paths from astroquery import log import astropy.units as u @@ -19,6 +22,7 @@ from . import version from .utils import system_tools +from astroquery import conf __all__ = ['BaseQuery', 'QueryWithLogin'] @@ -108,14 +112,21 @@ def request_file(self, cache_location): def from_cache(self, cache_location): request_file = self.request_file(cache_location) try: - with open(request_file, "rb") as f: - response = pickle.load(f) - if not isinstance(response, requests.Response): + current_time = datetime.utcnow() + cache_time = datetime.utcfromtimestamp(os.path.getmtime(request_file)) + expired = ((current_time-cache_time) > timedelta(seconds=conf.default_cache_timeout)) + if not expired: + with open(request_file, "rb") as f: + response = pickle.load(f) + if not isinstance(response, requests.Response): + response = None + else: + log.debug("Cache expired for {0}...".format(request_file)) response = None except FileNotFoundError: response = None if response: - log.debug("Retrieving data from {0}".format(request_file)) + log.debug("Retrieved data from {0}".format(request_file)) return response def remove_cache_file(self, cache_location): From 70b396e56d9de4a4a424426b2ba4ebd111ad2cf9 Mon Sep 17 00:00:00 2001 From: Julien Woillez Date: Tue, 23 Sep 2014 22:06:55 +0200 Subject: [PATCH 03/26] setup.py egg_info fix --- astroquery/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index 312e1c8654..49cf247758 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -12,6 +12,7 @@ from ._astropy_init import __version__, __githash__, test # ---------------------------------------------------------------------------- + import os import logging @@ -48,4 +49,5 @@ class Conf(_config.ConfigNamespace): 'Astroquery-wide default cache timeout (seconds).' ) + conf = Conf() From 2e9e278f44193b04e2ead7307b2784ba27cb8236 Mon Sep 17 00:00:00 2001 From: Clara Brasseur Date: Tue, 4 Feb 2020 14:19:51 -0500 Subject: [PATCH 04/26] making sure cache directory exists --- astroquery/query.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/astroquery/query.py b/astroquery/query.py index f490e1075c..335b150396 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -13,6 +13,7 @@ import textwrap from datetime import datetime, timedelta +from pathlib import Path from astropy.config import paths from astroquery import log @@ -29,10 +30,14 @@ def to_cache(response, cache_file): log.debug("Caching data to {0}".format(cache_file)) + response = copy.deepcopy(response) if hasattr(response, 'request'): for key in tuple(response.request.hooks.keys()): del response.request.hooks[key] + + chache_dir, _ = os.path.split(cache_file) + Path(chache_dir).mkdir(parents=True, exist_ok=True) with open(cache_file, "wb") as f: pickle.dump(response, f) From e7aa58f08d4c6a9bfd8e9b7e7b4d439c2bca7ef7 Mon Sep 17 00:00:00 2001 From: Clara Brasseur Date: Tue, 4 Feb 2020 15:28:16 -0500 Subject: [PATCH 05/26] additional caching functionality --- astroquery/__init__.py | 12 +++++++++--- astroquery/query.py | 30 ++++++++++++++++++++++++------ astroquery/setup_package.py | 1 + 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index 49cf247758..7b6d7839cc 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -12,7 +12,6 @@ from ._astropy_init import __version__, __githash__, test # ---------------------------------------------------------------------------- - import os import logging @@ -45,9 +44,16 @@ def _get_bibtex(): class Conf(_config.ConfigNamespace): default_cache_timeout = _config.ConfigItem( - 60.0*60.0*24.0, + 86400, # 24 hours 'Astroquery-wide default cache timeout (seconds).' ) - + cache_location = _config.ConfigItem( + os.path.join(_config.paths.get_cache_dir(), 'astroquery'), + 'Astroquery default cache location (within astropy cache).' + ) + use_cache = _config.ConfigItem( + True, + "Astroquery global cache usage, False turns off all caching." + ) conf = Conf() diff --git a/astroquery/query.py b/astroquery/query.py index 335b150396..c45a1ca4ba 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -117,9 +117,12 @@ def request_file(self, cache_location): def from_cache(self, cache_location): request_file = self.request_file(cache_location) try: - current_time = datetime.utcnow() - cache_time = datetime.utcfromtimestamp(os.path.getmtime(request_file)) - expired = ((current_time-cache_time) > timedelta(seconds=conf.default_cache_timeout)) + if conf.default_cache_timeout is None: + expired = False + else: + current_time = datetime.utcnow() + cache_time = datetime.utcfromtimestamp(os.path.getmtime(request_file)) + expired = ((current_time-cache_time) > timedelta(seconds=conf.default_cache_timeout)) if not expired: with open(request_file, "rb") as f: response = pickle.load(f) @@ -193,7 +196,10 @@ def __init__(self): paths.get_cache_dir(), 'astroquery', self.__class__.__name__.split("Class")[0]) os.makedirs(self.cache_location, exist_ok=True) - self._cache_active = True + + self.name = self.__class__.__name__.split("Class")[0] + self._cache_active = conf.use_cache + def __call__(self, *args, **kwargs): """ init a fresh copy of self """ @@ -269,6 +275,7 @@ def _request(self, method, url, somewhere other than `BaseQuery.cache_location` timeout : int cache : bool + Override global cache settings. verify : bool Verify the server's TLS certificate? (see http://docs.python-requests.org/en/master/_modules/requests/sessions/?highlight=verify) @@ -294,12 +301,21 @@ def _request(self, method, url, is True. """ + # Set up cache + if (cache is True) or ((cache is not False) and conf.use_cache): + cache_location = os.path.join(conf.cache_location, self.name) + cache = True + else: + cache_location = None + cache = False + if save: local_filename = url.split('/')[-1] if os.name == 'nt': # Windows doesn't allow special characters in filenames like # ":" so replace them with an underscore local_filename = local_filename.replace(':', '_') + local_filepath = os.path.join(savedir or self.cache_location or '.', local_filename) response = self._download_file(url, local_filepath, cache=cache, timeout=timeout, @@ -321,16 +337,17 @@ def _request(self, method, url, allow_redirects=allow_redirects, json=json) else: - response = query.from_cache(self.cache_location) + response = query.from_cache(cache_location) if not response: response = query.request(self._session, - self.cache_location, + cache_location, stream=stream, auth=auth, allow_redirects=allow_redirects, verify=verify, json=json) to_cache(response, query.request_file(self.cache_location)) + self._last_query = query return response @@ -352,6 +369,7 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None, supports HTTP "range" requests, the download will be continued where it left off. cache : bool + Cache downloaded file. Defaults to False. method : "GET" or "POST" head_safe : bool """ diff --git a/astroquery/setup_package.py b/astroquery/setup_package.py index 8a297455bd..e62302cef3 100644 --- a/astroquery/setup_package.py +++ b/astroquery/setup_package.py @@ -3,3 +3,4 @@ def get_package_data(): return {'astroquery': ['CITATION']} + From 0deed64652bf46fe4ec03cc190c69fab8b7f9924 Mon Sep 17 00:00:00 2001 From: Clara Brasseur Date: Tue, 4 Feb 2020 18:27:22 -0500 Subject: [PATCH 06/26] rethinking some of the caching --- astroquery/query.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index c45a1ca4ba..14c52da0a4 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -114,15 +114,15 @@ def request_file(self, cache_location): fn = os.path.join(cache_location, self.hash() + ".pickle") return fn - def from_cache(self, cache_location): + def from_cache(self, cache_location, cache_timeout): request_file = self.request_file(cache_location) try: - if conf.default_cache_timeout is None: + if cache_timeout is None: expired = False else: current_time = datetime.utcnow() cache_time = datetime.utcfromtimestamp(os.path.getmtime(request_file)) - expired = ((current_time-cache_time) > timedelta(seconds=conf.default_cache_timeout)) + expired = ((current_time-cache_time) > timedelta(seconds=cache_timeout)) if not expired: with open(request_file, "rb") as f: response = pickle.load(f) @@ -199,6 +199,8 @@ def __init__(self): self.name = self.__class__.__name__.split("Class")[0] self._cache_active = conf.use_cache + self.use_cache = conf.use_cache + self.cache_timeout = conf.default_cache_timeout def __call__(self, *args, **kwargs): @@ -238,6 +240,23 @@ def _response_hook(self, response, *args, **kwargs): f"{response.text}\n" f"-----------------------------------------", '\t') log.log(5, f"HTTP response\n{response_log}") + + def clear_cache(): + """Removes all cache files.""" + + cache_files = [x for x in os.listdir(self.cache_location) if x.endswidth("pickle")] + for fle in cache_files: + os.remove(fle) + + def reset_cache_preferences(): + """Resets cache preferences to default values""" + + self.cache_location = os.path.join( + conf.cache_location, + self.__class__.__name__.split("Class")[0]) + + self.use_cache = conf.use_cache + self.cache_timeout = conf.default_cache_timeout def _request(self, method, url, params=None, data=None, headers=None, @@ -300,13 +319,10 @@ def _request(self, method, url, and the server response object, if ``save`` is True and ``return_response_on_save`` is True. """ - - # Set up cache - if (cache is True) or ((cache is not False) and conf.use_cache): - cache_location = os.path.join(conf.cache_location, self.name) + + if (cache is not False) and self.use_cache: cache = True else: - cache_location = None cache = False if save: @@ -337,10 +353,10 @@ def _request(self, method, url, allow_redirects=allow_redirects, json=json) else: - response = query.from_cache(cache_location) + response = query.from_cache(self.cache_location, self.cache_timeout) if not response: response = query.request(self._session, - cache_location, + self.cache_location, stream=stream, auth=auth, allow_redirects=allow_redirects, From 04b41c96ceae80ea4c686b19bbec7ccd7c5938ee Mon Sep 17 00:00:00 2001 From: Clara Brasseur Date: Tue, 4 Feb 2020 19:27:30 -0500 Subject: [PATCH 07/26] fixing bugs --- astroquery/query.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index 14c52da0a4..f47eb112aa 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -241,14 +241,14 @@ def _response_hook(self, response, *args, **kwargs): f"-----------------------------------------", '\t') log.log(5, f"HTTP response\n{response_log}") - def clear_cache(): + def clear_cache(self): """Removes all cache files.""" - cache_files = [x for x in os.listdir(self.cache_location) if x.endswidth("pickle")] + cache_files = [x for x in os.listdir(self.cache_location) if x.endswith("pickle")] for fle in cache_files: - os.remove(fle) + os.remove(os.path.join(self.cache_location,fle)) - def reset_cache_preferences(): + def reset_cache_preferences(self): """Resets cache preferences to default values""" self.cache_location = os.path.join( From 5a3fde854a7e3ab0aa19e18a5cbaed780131845d Mon Sep 17 00:00:00 2001 From: Clara Brasseur Date: Wed, 5 Feb 2020 10:10:45 -0500 Subject: [PATCH 08/26] pep8 --- astroquery/__init__.py | 5 +++-- astroquery/query.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index 7b6d7839cc..59f7b5b2df 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -42,9 +42,9 @@ def _get_bibtex(): # Set up cache configuration class Conf(_config.ConfigNamespace): - + default_cache_timeout = _config.ConfigItem( - 86400, # 24 hours + 86400, # 24 hours 'Astroquery-wide default cache timeout (seconds).' ) cache_location = _config.ConfigItem( @@ -56,4 +56,5 @@ class Conf(_config.ConfigNamespace): "Astroquery global cache usage, False turns off all caching." ) + conf = Conf() diff --git a/astroquery/query.py b/astroquery/query.py index f47eb112aa..e18c1f43e1 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -246,7 +246,7 @@ def clear_cache(self): cache_files = [x for x in os.listdir(self.cache_location) if x.endswith("pickle")] for fle in cache_files: - os.remove(os.path.join(self.cache_location,fle)) + os.remove(os.path.join(self.cache_location, fle)) def reset_cache_preferences(self): """Resets cache preferences to default values""" @@ -254,7 +254,7 @@ def reset_cache_preferences(self): self.cache_location = os.path.join( conf.cache_location, self.__class__.__name__.split("Class")[0]) - + self.use_cache = conf.use_cache self.cache_timeout = conf.default_cache_timeout @@ -319,12 +319,12 @@ def _request(self, method, url, and the server response object, if ``save`` is True and ``return_response_on_save`` is True. """ - + if (cache is not False) and self.use_cache: cache = True else: cache = False - + if save: local_filename = url.split('/')[-1] if os.name == 'nt': From a43adb628e7817ad648b57fe1b89e86911f2bb2f Mon Sep 17 00:00:00 2001 From: "C.E. Brasseur" Date: Tue, 22 Mar 2022 22:14:32 +0000 Subject: [PATCH 09/26] cleanup after rebase --- astroquery/__init__.py | 2 +- astroquery/query.py | 13 +++++-------- astroquery/setup_package.py | 1 - 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index 59f7b5b2df..b6a6d81821 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -44,7 +44,7 @@ def _get_bibtex(): class Conf(_config.ConfigNamespace): default_cache_timeout = _config.ConfigItem( - 86400, # 24 hours + 604800 , # 1 week 'Astroquery-wide default cache timeout (seconds).' ) cache_location = _config.ConfigItem( diff --git a/astroquery/query.py b/astroquery/query.py index e18c1f43e1..020cc80dc9 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -8,7 +8,6 @@ import keyring import io import os -import warnings import requests import textwrap @@ -16,14 +15,13 @@ from pathlib import Path from astropy.config import paths -from astroquery import log import astropy.units as u from astropy.utils.console import ProgressBarOrSpinner import astropy.utils.data -from . import version -from .utils import system_tools -from astroquery import conf +from astroquery import version, log, conf +from astroquery.utils import system_tools + __all__ = ['BaseQuery', 'QueryWithLogin'] @@ -199,7 +197,6 @@ def __init__(self): self.name = self.__class__.__name__.split("Class")[0] self._cache_active = conf.use_cache - self.use_cache = conf.use_cache self.cache_timeout = conf.default_cache_timeout @@ -255,7 +252,7 @@ def reset_cache_preferences(self): conf.cache_location, self.__class__.__name__.split("Class")[0]) - self.use_cache = conf.use_cache + self._cache_active = conf.use_cache self.cache_timeout = conf.default_cache_timeout def _request(self, method, url, @@ -320,7 +317,7 @@ def _request(self, method, url, is True. """ - if (cache is not False) and self.use_cache: + if (cache is not False) and self._cache_active: cache = True else: cache = False diff --git a/astroquery/setup_package.py b/astroquery/setup_package.py index e62302cef3..8a297455bd 100644 --- a/astroquery/setup_package.py +++ b/astroquery/setup_package.py @@ -3,4 +3,3 @@ def get_package_data(): return {'astroquery': ['CITATION']} - From 2edc86e54d3e56851f151da4061f472b5c17a4c0 Mon Sep 17 00:00:00 2001 From: "C.E. Brasseur" Date: Tue, 22 Mar 2022 22:29:34 +0000 Subject: [PATCH 10/26] pep8 --- astroquery/__init__.py | 3 ++- astroquery/query.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index b6a6d81821..f08b531e3b 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -40,11 +40,12 @@ def _get_bibtex(): log = logging.getLogger() log = _init_log() + # Set up cache configuration class Conf(_config.ConfigNamespace): default_cache_timeout = _config.ConfigItem( - 604800 , # 1 week + 604800, # 1 week 'Astroquery-wide default cache timeout (seconds).' ) cache_location = _config.ConfigItem( diff --git a/astroquery/query.py b/astroquery/query.py index 020cc80dc9..118594ffa0 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -199,7 +199,6 @@ def __init__(self): self._cache_active = conf.use_cache self.cache_timeout = conf.default_cache_timeout - def __call__(self, *args, **kwargs): """ init a fresh copy of self """ return self.__class__(*args, **kwargs) @@ -237,7 +236,7 @@ def _response_hook(self, response, *args, **kwargs): f"{response.text}\n" f"-----------------------------------------", '\t') log.log(5, f"HTTP response\n{response_log}") - + def clear_cache(self): """Removes all cache files.""" From 3eb39ffe6b3975013a76b85e938b142626e6691c Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Wed, 3 Aug 2022 17:28:43 +0100 Subject: [PATCH 11/26] adding cache tests --- astroquery/query.py | 3 +- astroquery/tests/test_cache.py | 138 +++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 astroquery/tests/test_cache.py diff --git a/astroquery/query.py b/astroquery/query.py index 118594ffa0..bff2a551a0 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -248,8 +248,9 @@ def reset_cache_preferences(self): """Resets cache preferences to default values""" self.cache_location = os.path.join( - conf.cache_location, + paths.get_cache_dir(), 'astroquery', self.__class__.__name__.split("Class")[0]) + os.makedirs(self.cache_location, exist_ok=True) self._cache_active = conf.use_cache self.cache_timeout = conf.default_cache_timeout diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py new file mode 100644 index 0000000000..ea90aac887 --- /dev/null +++ b/astroquery/tests/test_cache.py @@ -0,0 +1,138 @@ +import pytest +import requests +import os + +from time import sleep + +from astroquery.query import QueryWithLogin + +URL1 = "http://fakeurl.edu" +URL2 = "http://fakeurl.ac.uk" + +TEXT1 = "Penguin" +TEXT2 = "Walrus" + +def set_response(resp_text, resp_status=200): + """Function that allows us to set a specific mock response for cache testing""" + + def get_mockreturn(url, *args, **kwargs): + """Generate a mock return to a requests call""" + + myresp = requests.Response() + myresp._content = resp_text + myresp.request = requests.PreparedRequest() + myresp.status_code = resp_status + + return myresp + + requests.Session.request = get_mockreturn + + +class TestClass(QueryWithLogin): + """Bare bones class for testing caching""" + + def test_func(self, requrl): + + return self._request(method="GET", url=requrl) + + def _login(self, username): + + resp = self._request(method="GET", url=username) + + if resp.content == "Penguin": + return True + else: + return False + + +def test_cache_reset(): + mytest = TestClass() + assert mytest._cache_active + + default_timeout = mytest.cache_timeout + default_loc = mytest.cache_location + + mytest.cache_timeout = 5 + mytest.cache_location = "new/location" + + mytest.reset_cache_preferences() + + assert mytest.cache_timeout == default_timeout + assert mytest.cache_location == default_loc + + +def test_basic_caching(): + + mytest = TestClass() + assert mytest._cache_active + + mytest.clear_cache() + assert len(os.listdir(mytest.cache_location)) == 0 + + set_response(TEXT1) + + resp = mytest.test_func(URL1) + assert resp.content == TEXT1 + assert len(os.listdir(mytest.cache_location)) == 1 + + set_response(TEXT2) + + resp = mytest.test_func(URL2) # query that has not been cached + assert resp.content == TEXT2 + assert len(os.listdir(mytest.cache_location)) == 2 + + resp = mytest.test_func(URL1) + assert resp.content == TEXT1 # query that was cached + assert len(os.listdir(mytest.cache_location)) == 2 # no new cache file + + mytest.clear_cache() + assert len(os.listdir(mytest.cache_location)) == 0 + + resp = mytest.test_func(URL1) + assert resp.content == TEXT2 # Now get new response + + +def test_login(): + + mytest = TestClass() + assert mytest._cache_active + + mytest.clear_cache() + assert len(os.listdir(mytest.cache_location)) == 0 + + set_response(TEXT1) # Text 1 is set as the approved password + + mytest.login("ceb") + assert mytest.authenticated() + assert len(os.listdir(mytest.cache_location)) == 0 # request should not be cached + + set_response(TEXT2) # Text 1 is not the approved password + + mytest.login("ceb") + assert not mytest.authenticated() # Should not be accessing cache + + +def test_timeout(): + + mytest = TestClass() + assert mytest._cache_active + + mytest.clear_cache() + assert len(os.listdir(mytest.cache_location)) == 0 + + mytest.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily + + set_response(TEXT1) # setting the response + + resp = mytest.test_func(URL1) # should be cached + assert resp.content == TEXT1 + + set_response(TEXT2) # changing the respont + + resp = mytest.test_func(URL1) # should access cached value + assert resp.content == TEXT1 + + sleep(2) # run out cache time + resp = mytest.test_func(URL1) + assert resp.content == TEXT2 # no see the new response + From 0aeb98da61027e88a68e3973c61e79c8a44ffa8c Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Wed, 3 Aug 2022 17:45:49 +0100 Subject: [PATCH 12/26] using config for cache location --- astroquery/query.py | 4 ++-- astroquery/tests/test_cache.py | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index bff2a551a0..9c04f94cfc 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -191,7 +191,7 @@ def __init__(self): olduseragent=S.headers['User-Agent'])) self.cache_location = os.path.join( - paths.get_cache_dir(), 'astroquery', + conf.cache_location, self.__class__.__name__.split("Class")[0]) os.makedirs(self.cache_location, exist_ok=True) @@ -248,7 +248,7 @@ def reset_cache_preferences(self): """Resets cache preferences to default values""" self.cache_location = os.path.join( - paths.get_cache_dir(), 'astroquery', + conf.cache_location, self.__class__.__name__.split("Class")[0]) os.makedirs(self.cache_location, exist_ok=True) diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index ea90aac887..b64541af4d 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -12,38 +12,39 @@ TEXT1 = "Penguin" TEXT2 = "Walrus" + def set_response(resp_text, resp_status=200): """Function that allows us to set a specific mock response for cache testing""" def get_mockreturn(url, *args, **kwargs): """Generate a mock return to a requests call""" - + myresp = requests.Response() myresp._content = resp_text myresp.request = requests.PreparedRequest() myresp.status_code = resp_status - + return myresp - + requests.Session.request = get_mockreturn class TestClass(QueryWithLogin): """Bare bones class for testing caching""" - + def test_func(self, requrl): - + return self._request(method="GET", url=requrl) - + def _login(self, username): - + resp = self._request(method="GET", url=username) - + if resp.content == "Penguin": return True else: return False - + def test_cache_reset(): mytest = TestClass() @@ -60,7 +61,7 @@ def test_cache_reset(): assert mytest.cache_timeout == default_timeout assert mytest.cache_location == default_loc - + def test_basic_caching(): mytest = TestClass() @@ -68,7 +69,7 @@ def test_basic_caching(): mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 - + set_response(TEXT1) resp = mytest.test_func(URL1) @@ -120,19 +121,18 @@ def test_timeout(): mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 - mytest.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily - + mytest.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily + set_response(TEXT1) # setting the response - + resp = mytest.test_func(URL1) # should be cached assert resp.content == TEXT1 set_response(TEXT2) # changing the respont - + resp = mytest.test_func(URL1) # should access cached value assert resp.content == TEXT1 sleep(2) # run out cache time resp = mytest.test_func(URL1) assert resp.content == TEXT2 # no see the new response - From 6c03dd06bd13f6093b7b680f5a25dd92b06be194 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Wed, 3 Aug 2022 17:50:12 +0100 Subject: [PATCH 13/26] pep8 --- astroquery/tests/test_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index b64541af4d..db9554011b 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -28,7 +28,7 @@ def get_mockreturn(url, *args, **kwargs): requests.Session.request = get_mockreturn - + class TestClass(QueryWithLogin): """Bare bones class for testing caching""" @@ -81,7 +81,7 @@ def test_basic_caching(): resp = mytest.test_func(URL2) # query that has not been cached assert resp.content == TEXT2 assert len(os.listdir(mytest.cache_location)) == 2 - + resp = mytest.test_func(URL1) assert resp.content == TEXT1 # query that was cached assert len(os.listdir(mytest.cache_location)) == 2 # no new cache file From d92d7b803decc568687b38b909ec79aa5be97502 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Thu, 4 Aug 2022 17:37:20 +0100 Subject: [PATCH 14/26] regularizing conf usage ande fixing suspend cache function --- astroquery/__init__.py | 4 ++-- astroquery/query.py | 23 ++++++++++------------- astroquery/tests/test_cache.py | 29 +++++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index f08b531e3b..f370871004 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -48,11 +48,11 @@ class Conf(_config.ConfigNamespace): 604800, # 1 week 'Astroquery-wide default cache timeout (seconds).' ) - cache_location = _config.ConfigItem( + default_cache_location = _config.ConfigItem( os.path.join(_config.paths.get_cache_dir(), 'astroquery'), 'Astroquery default cache location (within astropy cache).' ) - use_cache = _config.ConfigItem( + default_cache_active = _config.ConfigItem( True, "Astroquery global cache usage, False turns off all caching." ) diff --git a/astroquery/query.py b/astroquery/query.py index 9c04f94cfc..175b3f6750 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -190,14 +190,10 @@ def __init__(self): .format(vers=version.version, olduseragent=S.headers['User-Agent'])) - self.cache_location = os.path.join( - conf.cache_location, - self.__class__.__name__.split("Class")[0]) - os.makedirs(self.cache_location, exist_ok=True) - self.name = self.__class__.__name__.split("Class")[0] - self._cache_active = conf.use_cache - self.cache_timeout = conf.default_cache_timeout + + + self.reset_cache_preferences() def __call__(self, *args, **kwargs): """ init a fresh copy of self """ @@ -248,11 +244,11 @@ def reset_cache_preferences(self): """Resets cache preferences to default values""" self.cache_location = os.path.join( - conf.cache_location, + conf.default_cache_location, self.__class__.__name__.split("Class")[0]) os.makedirs(self.cache_location, exist_ok=True) - self._cache_active = conf.use_cache + self.cache_active = conf.default_cache_active self.cache_timeout = conf.default_cache_timeout def _request(self, method, url, @@ -317,7 +313,7 @@ def _request(self, method, url, is True. """ - if (cache is not False) and self._cache_active: + if (cache is not False) and self.cache_active: cache = True else: cache = False @@ -343,7 +339,7 @@ def _request(self, method, url, else: query = AstroQuery(method, url, params=params, data=data, headers=headers, files=files, timeout=timeout, json=json) - if ((self.cache_location is None) or (not self._cache_active) or (not cache)): + if ((self.cache_location is None) or (not self.cache_active) or (not cache)): with suspend_cache(self): response = query.request(self._session, stream=stream, auth=auth, verify=verify, @@ -493,12 +489,13 @@ class suspend_cache: def __init__(self, obj): self.obj = obj + self.original_cache_setting = self.obj.cache_active def __enter__(self): - self.obj._cache_active = False + self.obj.cache_active = False def __exit__(self, exc_type, exc_value, traceback): - self.obj._cache_active = True + self.obj.cache_active = self.original_cache_setting return False diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index db9554011b..1b7f0bc658 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -48,7 +48,7 @@ def _login(self, username): def test_cache_reset(): mytest = TestClass() - assert mytest._cache_active + assert mytest.cache_active default_timeout = mytest.cache_timeout default_loc = mytest.cache_location @@ -65,7 +65,7 @@ def test_cache_reset(): def test_basic_caching(): mytest = TestClass() - assert mytest._cache_active + assert mytest.cache_active mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 @@ -96,7 +96,7 @@ def test_basic_caching(): def test_login(): mytest = TestClass() - assert mytest._cache_active + assert mytest.cache_active mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 @@ -116,7 +116,7 @@ def test_login(): def test_timeout(): mytest = TestClass() - assert mytest._cache_active + assert mytest.cache_active mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 @@ -136,3 +136,24 @@ def test_timeout(): sleep(2) # run out cache time resp = mytest.test_func(URL1) assert resp.content == TEXT2 # no see the new response + + +def test_deactivate(): + + mytest = TestClass() + mytest.cache_active = False + + mytest.clear_cache() + assert len(os.listdir(mytest.cache_location)) == 0 + + set_response(TEXT1) + + resp = mytest.test_func(URL1) + assert resp.content == TEXT1 + assert len(os.listdir(mytest.cache_location)) == 0 + + set_response(TEXT2) + + resp = mytest.test_func(URL1) + assert resp.content == TEXT2 + assert len(os.listdir(mytest.cache_location)) == 0 From 49c7f3872106c5d9fbfb09bea59280a12ac286b6 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Thu, 4 Aug 2022 17:43:51 +0100 Subject: [PATCH 15/26] pep8 --- astroquery/query.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index 175b3f6750..afcb350d20 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -191,8 +191,6 @@ def __init__(self): olduseragent=S.headers['User-Agent'])) self.name = self.__class__.__name__.split("Class")[0] - - self.reset_cache_preferences() def __call__(self, *args, **kwargs): From 8e8430f30b0103007ec2001c2115823f39c00317 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Wed, 10 Aug 2022 21:22:31 +0100 Subject: [PATCH 16/26] adding cache docs --- docs/index.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index fefa351d1d..809fcd501e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -172,6 +172,36 @@ uncomment the relevant configuration item(s), and insert your desired value(s). configuration + +Caching +------- + +By default Astroquery employs query caching with a timeout of 1 week, although individual services +may have different settings. The user can clear their cache at any time, as well as suspend cache usage, +and change the cache location. Caching persists between Astroquery sessions. +If you know the service you are using has released new data recently, or if you believe you are +not recieving the newest data, try clearing the cache. + +The Astroquery cache is divided by service, so each service's cache should be managed invidually. +Shown here are the cache properties, using `~astroquery.simbad.Simbad` as an example. + +.. code-block:: python + + >>> from astroquery.simbad import Simbad + + >>> # Is the cache active? + >>> print(Simbad.cache_active) + True + + >>> # Cache location + >>> print(Simbad.cache_location) + /Users/username/.astropy/cache/astroquery/Simbad + + >>> # Cache timout in seconds + >>> print(Simbad.cache_timeout) + 604800 + + Available Services ================== From 41f77dd11ff80348d13bc9b094b2409d99f293cf Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Thu, 11 Aug 2022 16:23:22 +0100 Subject: [PATCH 17/26] fixing docs failure --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 809fcd501e..8f54e1671c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -183,7 +183,7 @@ If you know the service you are using has released new data recently, or if you not recieving the newest data, try clearing the cache. The Astroquery cache is divided by service, so each service's cache should be managed invidually. -Shown here are the cache properties, using `~astroquery.simbad.Simbad` as an example. +Shown here are the cache properties, using Simbad as an example. .. code-block:: python From 732b25315a15187d33d7d268f044781cfff0d947 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Mon, 12 Sep 2022 16:30:07 +0100 Subject: [PATCH 18/26] integrating better with astropy config framework --- astroquery/__init__.py | 21 ++++---- astroquery/query.py | 65 +++++++++---------------- astroquery/tests/test_cache.py | 89 +++++++++++++++++++++++----------- 3 files changed, 95 insertions(+), 80 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index f370871004..776e4cba65 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -44,18 +44,17 @@ def _get_bibtex(): # Set up cache configuration class Conf(_config.ConfigNamespace): - default_cache_timeout = _config.ConfigItem( - 604800, # 1 week - 'Astroquery-wide default cache timeout (seconds).' - ) - default_cache_location = _config.ConfigItem( - os.path.join(_config.paths.get_cache_dir(), 'astroquery'), - 'Astroquery default cache location (within astropy cache).' - ) - default_cache_active = _config.ConfigItem( + cache_timeout = _config.ConfigItem( + 604800, # 1 week + 'Astroquery-wide cache timeout (seconds).', + cfgtype='integer' + ) + + cache_active = _config.ConfigItem( True, - "Astroquery global cache usage, False turns off all caching." - ) + "Astroquery global cache usage, False turns off all caching.", + cfgtype='boolean' + ) conf = Conf() diff --git a/astroquery/query.py b/astroquery/query.py index afcb350d20..7d7b12404b 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -33,9 +33,6 @@ def to_cache(response, cache_file): if hasattr(response, 'request'): for key in tuple(response.request.hooks.keys()): del response.request.hooks[key] - - chache_dir, _ = os.path.split(cache_file) - Path(chache_dir).mkdir(parents=True, exist_ok=True) with open(cache_file, "wb") as f: pickle.dump(response, f) @@ -191,7 +188,7 @@ def __init__(self): olduseragent=S.headers['User-Agent'])) self.name = self.__class__.__name__.split("Class")[0] - self.reset_cache_preferences() + self.cache_location = None def __call__(self, *args, **kwargs): """ init a fresh copy of self """ @@ -231,23 +228,24 @@ def _response_hook(self, response, *args, **kwargs): f"-----------------------------------------", '\t') log.log(5, f"HTTP response\n{response_log}") - def clear_cache(self): - """Removes all cache files.""" + @property + def _cache_location(self): + cl = self.cache_location or os.path.join(paths.get_cache_dir(), 'astroquery', self.name) + Path(cl).mkdir(parents=True, exist_ok=True) + return cl - cache_files = [x for x in os.listdir(self.cache_location) if x.endswith("pickle")] - for fle in cache_files: - os.remove(os.path.join(self.cache_location, fle)) + def get_cache_location(self): + return self._cache_location - def reset_cache_preferences(self): - """Resets cache preferences to default values""" + def reset_cache_location(self): + self.cache_location = None - self.cache_location = os.path.join( - conf.default_cache_location, - self.__class__.__name__.split("Class")[0]) - os.makedirs(self.cache_location, exist_ok=True) + def clear_cache(self): + """Removes all cache files.""" - self.cache_active = conf.default_cache_active - self.cache_timeout = conf.default_cache_timeout + cache_files = [x for x in os.listdir(self._cache_location) if x.endswith("pickle")] + for fle in cache_files: + os.remove(os.path.join(self._cache_location, fle)) def _request(self, method, url, params=None, data=None, headers=None, @@ -311,7 +309,7 @@ def _request(self, method, url, is True. """ - if (cache is not False) and self.cache_active: + if (cache is not False) and conf.cache_active: cache = True else: cache = False @@ -323,7 +321,7 @@ def _request(self, method, url, # ":" so replace them with an underscore local_filename = local_filename.replace(':', '_') - local_filepath = os.path.join(savedir or self.cache_location or '.', local_filename) + local_filepath = os.path.join(savedir or self._cache_location or '.', local_filename) response = self._download_file(url, local_filepath, cache=cache, timeout=timeout, continuation=continuation, method=method, @@ -337,23 +335,23 @@ def _request(self, method, url, else: query = AstroQuery(method, url, params=params, data=data, headers=headers, files=files, timeout=timeout, json=json) - if ((self.cache_location is None) or (not self.cache_active) or (not cache)): - with suspend_cache(self): + if ((self._cache_location is None) or (not cache)): + with conf.set_temp("cache_active", False): response = query.request(self._session, stream=stream, auth=auth, verify=verify, allow_redirects=allow_redirects, json=json) else: - response = query.from_cache(self.cache_location, self.cache_timeout) + response = query.from_cache(self._cache_location, conf.cache_timeout) if not response: response = query.request(self._session, - self.cache_location, + self._cache_location, stream=stream, auth=auth, allow_redirects=allow_redirects, verify=verify, json=json) - to_cache(response, query.request_file(self.cache_location)) + to_cache(response, query.request_file(self._cache_location)) self._last_query = query return response @@ -480,23 +478,6 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None, return response -class suspend_cache: - """ - A context manager that suspends caching. - """ - - def __init__(self, obj): - self.obj = obj - self.original_cache_setting = self.obj.cache_active - - def __enter__(self): - self.obj.cache_active = False - - def __exit__(self, exc_type, exc_value, traceback): - self.obj.cache_active = self.original_cache_setting - return False - - class QueryWithLogin(BaseQuery): """ This is the base class for all the query classes which are required to @@ -549,7 +530,7 @@ def _login(self, *args, **kwargs): pass def login(self, *args, **kwargs): - with suspend_cache(self): + with conf.set_temp("cache_active", False): self._authenticated = self._login(*args, **kwargs) return self._authenticated diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index 1b7f0bc658..222d66dc58 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -3,8 +3,12 @@ import os from time import sleep +from pathlib import Path + +from astropy.config import paths from astroquery.query import QueryWithLogin +from astroquery import conf URL1 = "http://fakeurl.edu" URL2 = "http://fakeurl.ac.uk" @@ -46,68 +50,99 @@ def _login(self, username): return False -def test_cache_reset(): - mytest = TestClass() - assert mytest.cache_active +def test_conf(): + default_timeout = conf.cache_timeout + default_active = conf.cache_active + + assert default_timeout == 604800 + assert default_active is True - default_timeout = mytest.cache_timeout - default_loc = mytest.cache_location + with conf.set_temp("cache_timeout", 5): + assert conf.cache_timeout == 5 - mytest.cache_timeout = 5 - mytest.cache_location = "new/location" + with conf.set_temp("cache_active", False): + assert conf.cache_active is False - mytest.reset_cache_preferences() + assert conf.cache_timeout == default_timeout + assert conf.cache_active == default_active - assert mytest.cache_timeout == default_timeout - assert mytest.cache_location == default_loc + conf.cache_timeout = 5 + conf.cache_active = False + conf.reset() + + assert conf.cache_timeout == default_timeout + assert conf.cache_active == default_active def test_basic_caching(): mytest = TestClass() - assert mytest.cache_active + assert conf.cache_active mytest.clear_cache() - assert len(os.listdir(mytest.cache_location)) == 0 + assert len(os.listdir(mytest.get_cache_location())) == 0 set_response(TEXT1) resp = mytest.test_func(URL1) assert resp.content == TEXT1 - assert len(os.listdir(mytest.cache_location)) == 1 + assert len(os.listdir(mytest.get_cache_location())) == 1 set_response(TEXT2) resp = mytest.test_func(URL2) # query that has not been cached assert resp.content == TEXT2 - assert len(os.listdir(mytest.cache_location)) == 2 + assert len(os.listdir(mytest.get_cache_location())) == 2 resp = mytest.test_func(URL1) assert resp.content == TEXT1 # query that was cached - assert len(os.listdir(mytest.cache_location)) == 2 # no new cache file + assert len(os.listdir(mytest.get_cache_location())) == 2 # no new cache file mytest.clear_cache() - assert len(os.listdir(mytest.cache_location)) == 0 + assert len(os.listdir(mytest.get_cache_location())) == 0 resp = mytest.test_func(URL1) assert resp.content == TEXT2 # Now get new response +def test_change_location(tmpdir): + + mytest = TestClass() + default_cache_location = mytest.get_cache_location() + + assert paths.get_cache_dir() in default_cache_location + assert "astroquery" in mytest.get_cache_location() + assert mytest.name in mytest.get_cache_location() + + new_loc = os.path.join(tmpdir, "new_dir") + mytest.cache_location = new_loc + assert mytest.get_cache_location() == new_loc + + mytest.reset_cache_location() + assert mytest.get_cache_location() == default_cache_location + + Path(new_loc).mkdir(parents=True, exist_ok=True) + with paths.set_temp_cache(new_loc): + assert new_loc in mytest.get_cache_location() + assert "astroquery" in mytest.get_cache_location() + assert mytest.name in mytest.get_cache_location() + + def test_login(): mytest = TestClass() - assert mytest.cache_active + assert conf.cache_active mytest.clear_cache() - assert len(os.listdir(mytest.cache_location)) == 0 + assert len(os.listdir(mytest.get_cache_location())) == 0 set_response(TEXT1) # Text 1 is set as the approved password mytest.login("ceb") assert mytest.authenticated() - assert len(os.listdir(mytest.cache_location)) == 0 # request should not be cached + assert len(os.listdir(mytest.get_cache_location())) == 0 # request should not be cached - set_response(TEXT2) # Text 1 is not the approved password + set_response(TEXT2) # Text 2 is not the approved password mytest.login("ceb") assert not mytest.authenticated() # Should not be accessing cache @@ -116,12 +151,12 @@ def test_login(): def test_timeout(): mytest = TestClass() - assert mytest.cache_active + assert conf.cache_active mytest.clear_cache() - assert len(os.listdir(mytest.cache_location)) == 0 + assert len(os.listdir(mytest.get_cache_location())) == 0 - mytest.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily + conf.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily set_response(TEXT1) # setting the response @@ -141,19 +176,19 @@ def test_timeout(): def test_deactivate(): mytest = TestClass() - mytest.cache_active = False + conf.cache_active = False mytest.clear_cache() - assert len(os.listdir(mytest.cache_location)) == 0 + assert len(os.listdir(mytest.get_cache_location())) == 0 set_response(TEXT1) resp = mytest.test_func(URL1) assert resp.content == TEXT1 - assert len(os.listdir(mytest.cache_location)) == 0 + assert len(os.listdir(mytest.get_cache_location())) == 0 set_response(TEXT2) resp = mytest.test_func(URL1) assert resp.content == TEXT2 - assert len(os.listdir(mytest.cache_location)) == 0 + assert len(os.listdir(mytest.get_cache_location())) == 0 From 93242ba53d13ce2a34a6ff7b8c892e8afe86962d Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Mon, 12 Sep 2022 16:47:14 +0100 Subject: [PATCH 19/26] adding back suspend cache for backward compatibility --- astroquery/query.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/astroquery/query.py b/astroquery/query.py index 7d7b12404b..8c4660546e 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -478,6 +478,22 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None, return response +class suspend_cache: + """ + A context manager that suspends caching. + """ + + def __init__(self, obj=None): + self.original_cache_setting = conf.cache_active + + def __enter__(self): + conf.cache_active = False + + def __exit__(self, exc_type, exc_value, traceback): + conf.cache_active = self.original_cache_setting + return False + + class QueryWithLogin(BaseQuery): """ This is the base class for all the query classes which are required to From 12eae52b533dc8db44df1276b6631c90a387280d Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Mon, 19 Sep 2022 15:01:30 +0100 Subject: [PATCH 20/26] making new caching backwards compatible --- astroquery/query.py | 49 ++++++++++++------------ astroquery/tests/test_cache.py | 70 ++++++++++++++++++++++++---------- 2 files changed, 74 insertions(+), 45 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index 8c4660546e..9e80b5129c 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -18,6 +18,8 @@ import astropy.units as u from astropy.utils.console import ProgressBarOrSpinner import astropy.utils.data +from astropy.utils import deprecated +from astropy.utils.exceptions import AstropyDeprecationWarning from astroquery import version, log, conf from astroquery.utils import system_tools @@ -106,7 +108,7 @@ def hash(self): return self._hash def request_file(self, cache_location): - fn = os.path.join(cache_location, self.hash() + ".pickle") + fn = cache_location.joinpath(self.hash() + ".pickle") return fn def from_cache(self, cache_location, cache_timeout): @@ -116,7 +118,7 @@ def from_cache(self, cache_location, cache_timeout): expired = False else: current_time = datetime.utcnow() - cache_time = datetime.utcfromtimestamp(os.path.getmtime(request_file)) + cache_time = datetime.utcfromtimestamp(request_file.stat().st_mtime) expired = ((current_time-cache_time) > timedelta(seconds=cache_timeout)) if not expired: with open(request_file, "rb") as f: @@ -139,8 +141,8 @@ def remove_cache_file(self, cache_location): """ request_file = self.request_file(cache_location) - if os.path.exists(request_file): - os.remove(request_file) + if request_file.exists: + request_file.unlink() else: raise FileNotFoundError(f"Tried to remove cache file {request_file} but " "it does not exist") @@ -188,7 +190,7 @@ def __init__(self): olduseragent=S.headers['User-Agent'])) self.name = self.__class__.__name__.split("Class")[0] - self.cache_location = None + self._cache_location = None def __call__(self, *args, **kwargs): """ init a fresh copy of self """ @@ -229,23 +231,23 @@ def _response_hook(self, response, *args, **kwargs): log.log(5, f"HTTP response\n{response_log}") @property - def _cache_location(self): - cl = self.cache_location or os.path.join(paths.get_cache_dir(), 'astroquery', self.name) - Path(cl).mkdir(parents=True, exist_ok=True) + def cache_location(self): + cl = self._cache_location or Path(paths.get_cache_dir(), 'astroquery', self.name) + cl.mkdir(parents=True, exist_ok=True) return cl - def get_cache_location(self): - return self._cache_location + @cache_location.setter + def cache_location(self, loc): + self._cache_location = Path(loc) def reset_cache_location(self): - self.cache_location = None + """Resets the cache location to the default astropy cache""" + self._cache_location = None def clear_cache(self): """Removes all cache files.""" - - cache_files = [x for x in os.listdir(self._cache_location) if x.endswith("pickle")] - for fle in cache_files: - os.remove(os.path.join(self._cache_location, fle)) + for fle in self.cache_location.glob("*.pickle"): + fle.unlink() def _request(self, method, url, params=None, data=None, headers=None, @@ -309,10 +311,7 @@ def _request(self, method, url, is True. """ - if (cache is not False) and conf.cache_active: - cache = True - else: - cache = False + cache &= conf.cache_active if save: local_filename = url.split('/')[-1] @@ -321,7 +320,7 @@ def _request(self, method, url, # ":" so replace them with an underscore local_filename = local_filename.replace(':', '_') - local_filepath = os.path.join(savedir or self._cache_location or '.', local_filename) + local_filepath = os.path.join(savedir or self.cache_location or '.', local_filename) response = self._download_file(url, local_filepath, cache=cache, timeout=timeout, continuation=continuation, method=method, @@ -335,23 +334,23 @@ def _request(self, method, url, else: query = AstroQuery(method, url, params=params, data=data, headers=headers, files=files, timeout=timeout, json=json) - if ((self._cache_location is None) or (not cache)): + if (self.cache_location is None) or (not cache): with conf.set_temp("cache_active", False): response = query.request(self._session, stream=stream, auth=auth, verify=verify, allow_redirects=allow_redirects, json=json) else: - response = query.from_cache(self._cache_location, conf.cache_timeout) + response = query.from_cache(self.cache_location, conf.cache_timeout) if not response: response = query.request(self._session, - self._cache_location, + self.cache_location, stream=stream, auth=auth, allow_redirects=allow_redirects, verify=verify, json=json) - to_cache(response, query.request_file(self._cache_location)) + to_cache(response, query.request_file(self.cache_location)) self._last_query = query return response @@ -478,6 +477,8 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None, return response +@deprecated(since="v0.4.7", message=("The suspend_cache function is deprecated," + "Use the conf set_temp function instead.")) class suspend_cache: """ A context manager that suspends caching. diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index 222d66dc58..0b5e00afee 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -51,6 +51,8 @@ def _login(self, username): def test_conf(): + conf.reset() + default_timeout = conf.cache_timeout default_active = conf.cache_active @@ -75,72 +77,75 @@ def test_conf(): def test_basic_caching(): + conf.reset() mytest = TestClass() assert conf.cache_active mytest.clear_cache() - assert len(os.listdir(mytest.get_cache_location())) == 0 + assert len(os.listdir(mytest.cache_location)) == 0 set_response(TEXT1) resp = mytest.test_func(URL1) assert resp.content == TEXT1 - assert len(os.listdir(mytest.get_cache_location())) == 1 + assert len(os.listdir(mytest.cache_location)) == 1 set_response(TEXT2) resp = mytest.test_func(URL2) # query that has not been cached assert resp.content == TEXT2 - assert len(os.listdir(mytest.get_cache_location())) == 2 + assert len(os.listdir(mytest.cache_location)) == 2 resp = mytest.test_func(URL1) assert resp.content == TEXT1 # query that was cached - assert len(os.listdir(mytest.get_cache_location())) == 2 # no new cache file + assert len(os.listdir(mytest.cache_location)) == 2 # no new cache file mytest.clear_cache() - assert len(os.listdir(mytest.get_cache_location())) == 0 + assert len(os.listdir(mytest.cache_location)) == 0 resp = mytest.test_func(URL1) assert resp.content == TEXT2 # Now get new response def test_change_location(tmpdir): + conf.reset() mytest = TestClass() - default_cache_location = mytest.get_cache_location() + default_cache_location = mytest.cache_location - assert paths.get_cache_dir() in default_cache_location - assert "astroquery" in mytest.get_cache_location() - assert mytest.name in mytest.get_cache_location() + assert paths.get_cache_dir() in str(default_cache_location) + assert "astroquery" in mytest.cache_location.parts + assert mytest.name in mytest.cache_location.parts - new_loc = os.path.join(tmpdir, "new_dir") + new_loc = "new_dir" mytest.cache_location = new_loc - assert mytest.get_cache_location() == new_loc + assert str(mytest.cache_location) == new_loc mytest.reset_cache_location() - assert mytest.get_cache_location() == default_cache_location + assert mytest.cache_location == default_cache_location Path(new_loc).mkdir(parents=True, exist_ok=True) with paths.set_temp_cache(new_loc): - assert new_loc in mytest.get_cache_location() - assert "astroquery" in mytest.get_cache_location() - assert mytest.name in mytest.get_cache_location() + assert new_loc in mytest.cache_location.parts + assert "astroquery" in mytest.cache_location.parts + assert mytest.name in mytest.cache_location.parts def test_login(): + conf.reset() mytest = TestClass() assert conf.cache_active mytest.clear_cache() - assert len(os.listdir(mytest.get_cache_location())) == 0 + assert len(os.listdir(mytest.cache_location)) == 0 set_response(TEXT1) # Text 1 is set as the approved password mytest.login("ceb") assert mytest.authenticated() - assert len(os.listdir(mytest.get_cache_location())) == 0 # request should not be cached + assert len(os.listdir(mytest.cache_location)) == 0 # request should not be cached set_response(TEXT2) # Text 2 is not the approved password @@ -149,12 +154,13 @@ def test_login(): def test_timeout(): + conf.reset() mytest = TestClass() assert conf.cache_active mytest.clear_cache() - assert len(os.listdir(mytest.get_cache_location())) == 0 + assert len(os.listdir(mytest.cache_location)) == 0 conf.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily @@ -174,21 +180,43 @@ def test_timeout(): def test_deactivate(): + conf.reset() mytest = TestClass() conf.cache_active = False mytest.clear_cache() - assert len(os.listdir(mytest.get_cache_location())) == 0 + assert len(os.listdir(mytest.cache_location)) == 0 set_response(TEXT1) resp = mytest.test_func(URL1) assert resp.content == TEXT1 - assert len(os.listdir(mytest.get_cache_location())) == 0 + assert len(os.listdir(mytest.cache_location)) == 0 set_response(TEXT2) resp = mytest.test_func(URL1) assert resp.content == TEXT2 - assert len(os.listdir(mytest.get_cache_location())) == 0 + assert len(os.listdir(mytest.cache_location)) == 0 + + conf.reset() + assert conf.cache_active is True + + with conf.set_temp('cache_active', False): + mytest.clear_cache() + assert len(os.listdir(mytest.cache_location)) == 0 + + set_response(TEXT1) + + resp = mytest.test_func(URL1) + assert resp.content == TEXT1 + assert len(os.listdir(mytest.cache_location)) == 0 + + set_response(TEXT2) + + resp = mytest.test_func(URL1) + assert resp.content == TEXT2 + assert len(os.listdir(mytest.cache_location)) == 0 + + assert conf.cache_active is True From d992ecaad86faeb647ebc1ecd93e9ecd50904d25 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Mon, 19 Sep 2022 15:57:36 +0100 Subject: [PATCH 21/26] updating docs to reflect changes --- docs/index.rst | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8f54e1671c..d1047e8708 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -182,24 +182,32 @@ and change the cache location. Caching persists between Astroquery sessions. If you know the service you are using has released new data recently, or if you believe you are not recieving the newest data, try clearing the cache. -The Astroquery cache is divided by service, so each service's cache should be managed invidually. -Shown here are the cache properties, using Simbad as an example. + +The Astroquery cache location is divided by service, so each service's cache should be managed invidually, +however whether the cache is active and the expiration time are controlled centrally through the +astroquery ``conf`` module. Astroquery uses the Astropy configuration infrastructure, information about +temporarily or permanently changing configuration values can be found +`here `_. + +Shown here are the cache properties, using Simbad as an example: .. code-block:: python + >>> from astroquery import conf >>> from astroquery.simbad import Simbad >>> # Is the cache active? - >>> print(Simbad.cache_active) + >>> print(conf.cache_active) True + >>> # Cache timout in seconds + >>> print(conf.cache_timeout) + 604800 + >>> # Cache location >>> print(Simbad.cache_location) /Users/username/.astropy/cache/astroquery/Simbad - >>> # Cache timout in seconds - >>> print(Simbad.cache_timeout) - 604800 Available Services From ad6b910cc1a5149d6ac08477351056a6c840ec90 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Mon, 3 Oct 2022 16:40:04 +0100 Subject: [PATCH 22/26] renaming cache config class --- astroquery/__init__.py | 4 ++-- astroquery/query.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index 776e4cba65..f8cc7c62a1 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -42,7 +42,7 @@ def _get_bibtex(): # Set up cache configuration -class Conf(_config.ConfigNamespace): +class Cache_Conf(_config.ConfigNamespace): cache_timeout = _config.ConfigItem( 604800, # 1 week @@ -57,4 +57,4 @@ class Conf(_config.ConfigNamespace): ) -conf = Conf() +cache_conf = Cache_Conf() diff --git a/astroquery/query.py b/astroquery/query.py index 9e80b5129c..19d44fc46d 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -21,7 +21,7 @@ from astropy.utils import deprecated from astropy.utils.exceptions import AstropyDeprecationWarning -from astroquery import version, log, conf +from astroquery import version, log, cache_conf from astroquery.utils import system_tools @@ -311,7 +311,7 @@ def _request(self, method, url, is True. """ - cache &= conf.cache_active + cache &= cache_conf.cache_active if save: local_filename = url.split('/')[-1] @@ -335,13 +335,13 @@ def _request(self, method, url, query = AstroQuery(method, url, params=params, data=data, headers=headers, files=files, timeout=timeout, json=json) if (self.cache_location is None) or (not cache): - with conf.set_temp("cache_active", False): + with cache_conf.set_temp("cache_active", False): response = query.request(self._session, stream=stream, auth=auth, verify=verify, allow_redirects=allow_redirects, json=json) else: - response = query.from_cache(self.cache_location, conf.cache_timeout) + response = query.from_cache(self.cache_location, cache_conf.cache_timeout) if not response: response = query.request(self._session, self.cache_location, @@ -485,7 +485,7 @@ class suspend_cache: """ def __init__(self, obj=None): - self.original_cache_setting = conf.cache_active + self.original_cache_setting = cache_conf.cache_active def __enter__(self): conf.cache_active = False From 373081bdb6c03e10c48f1fc07ff31b3767335b40 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Mon, 24 Oct 2022 17:56:34 +0100 Subject: [PATCH 23/26] implementing review suggestions --- astroquery/__init__.py | 4 +- astroquery/query.py | 17 +++---- astroquery/tests/test_cache.py | 87 ++++++++++++++++------------------ docs/index.rst | 20 +++++--- 4 files changed, 66 insertions(+), 62 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index f8cc7c62a1..ae87b0b77c 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -45,8 +45,8 @@ def _get_bibtex(): class Cache_Conf(_config.ConfigNamespace): cache_timeout = _config.ConfigItem( - 604800, # 1 week - 'Astroquery-wide cache timeout (seconds).', + 604800, + 'Astroquery-wide cache timeout (seconds). Default is 1 week (604800).', cfgtype='integer' ) diff --git a/astroquery/query.py b/astroquery/query.py index 19d44fc46d..25ed76316e 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -119,14 +119,14 @@ def from_cache(self, cache_location, cache_timeout): else: current_time = datetime.utcnow() cache_time = datetime.utcfromtimestamp(request_file.stat().st_mtime) - expired = ((current_time-cache_time) > timedelta(seconds=cache_timeout)) + expired = current_time-cache_time > timedelta(seconds=cache_timeout) if not expired: with open(request_file, "rb") as f: response = pickle.load(f) if not isinstance(response, requests.Response): response = None else: - log.debug("Cache expired for {0}...".format(request_file)) + log.debug(f"Cache expired for {request_file}...") response = None except FileNotFoundError: response = None @@ -251,7 +251,7 @@ def clear_cache(self): def _request(self, method, url, params=None, data=None, headers=None, - files=None, save=False, savedir='', timeout=None, cache=True, + files=None, save=False, savedir='', timeout=None, cache=None, stream=False, auth=None, continuation=True, verify=True, allow_redirects=True, json=None, return_response_on_save=False): @@ -311,7 +311,8 @@ def _request(self, method, url, is True. """ - cache &= cache_conf.cache_active + if cache is None: # Global caching not overridden + cache = cache_conf.cache_active if save: local_filename = url.split('/')[-1] @@ -334,7 +335,7 @@ def _request(self, method, url, else: query = AstroQuery(method, url, params=params, data=data, headers=headers, files=files, timeout=timeout, json=json) - if (self.cache_location is None) or (not cache): + if not cache: with cache_conf.set_temp("cache_active", False): response = query.request(self._session, stream=stream, auth=auth, verify=verify, @@ -488,10 +489,10 @@ def __init__(self, obj=None): self.original_cache_setting = cache_conf.cache_active def __enter__(self): - conf.cache_active = False + cache_conf.cache_active = False def __exit__(self, exc_type, exc_value, traceback): - conf.cache_active = self.original_cache_setting + cache_conf.cache_active = self.original_cache_setting return False @@ -547,7 +548,7 @@ def _login(self, *args, **kwargs): pass def login(self, *args, **kwargs): - with conf.set_temp("cache_active", False): + with cache_conf.set_temp("cache_active", False): self._authenticated = self._login(*args, **kwargs) return self._authenticated diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index 0b5e00afee..d9c7d87b8d 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -8,7 +8,7 @@ from astropy.config import paths from astroquery.query import QueryWithLogin -from astroquery import conf +from astroquery import cache_conf URL1 = "http://fakeurl.edu" URL2 = "http://fakeurl.ac.uk" @@ -33,7 +33,7 @@ def get_mockreturn(url, *args, **kwargs): requests.Session.request = get_mockreturn -class TestClass(QueryWithLogin): +class CacheTestClass(QueryWithLogin): """Bare bones class for testing caching""" def test_func(self, requrl): @@ -42,45 +42,40 @@ def test_func(self, requrl): def _login(self, username): - resp = self._request(method="GET", url=username) - - if resp.content == "Penguin": - return True - else: - return False + return self._request(method="GET", url=username).content == "Penguin" def test_conf(): - conf.reset() + cache_conf.reset() - default_timeout = conf.cache_timeout - default_active = conf.cache_active + default_timeout = cache_conf.cache_timeout + default_active = cache_conf.cache_active assert default_timeout == 604800 assert default_active is True - with conf.set_temp("cache_timeout", 5): - assert conf.cache_timeout == 5 + with cache_conf.set_temp("cache_timeout", 5): + assert cache_conf.cache_timeout == 5 - with conf.set_temp("cache_active", False): - assert conf.cache_active is False + with cache_conf.set_temp("cache_active", False): + assert cache_conf.cache_active is False - assert conf.cache_timeout == default_timeout - assert conf.cache_active == default_active + assert cache_conf.cache_timeout == default_timeout + assert cache_conf.cache_active == default_active - conf.cache_timeout = 5 - conf.cache_active = False - conf.reset() + cache_conf.cache_timeout = 5 + cache_conf.cache_active = False + cache_conf.reset() - assert conf.cache_timeout == default_timeout - assert conf.cache_active == default_active + assert cache_conf.cache_timeout == default_timeout + assert cache_conf.cache_active == default_active def test_basic_caching(): - conf.reset() + cache_conf.reset() - mytest = TestClass() - assert conf.cache_active + mytest = CacheTestClass() + assert cache_conf.cache_active mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 @@ -108,35 +103,35 @@ def test_basic_caching(): assert resp.content == TEXT2 # Now get new response -def test_change_location(tmpdir): - conf.reset() +def test_change_location(tmp_path): + cache_conf.reset() - mytest = TestClass() + mytest = CacheTestClass() default_cache_location = mytest.cache_location assert paths.get_cache_dir() in str(default_cache_location) assert "astroquery" in mytest.cache_location.parts assert mytest.name in mytest.cache_location.parts - new_loc = "new_dir" + new_loc = tmp_path.joinpath("new_dir") mytest.cache_location = new_loc - assert str(mytest.cache_location) == new_loc + assert mytest.cache_location == new_loc mytest.reset_cache_location() assert mytest.cache_location == default_cache_location - Path(new_loc).mkdir(parents=True, exist_ok=True) + new_loc.mkdir(parents=True, exist_ok=True) with paths.set_temp_cache(new_loc): - assert new_loc in mytest.cache_location.parts + assert str(new_loc) in str(mytest.cache_location) assert "astroquery" in mytest.cache_location.parts assert mytest.name in mytest.cache_location.parts def test_login(): - conf.reset() + cache_conf.reset() - mytest = TestClass() - assert conf.cache_active + mytest = CacheTestClass() + assert cache_conf.cache_active mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 @@ -154,15 +149,15 @@ def test_login(): def test_timeout(): - conf.reset() + cache_conf.reset() - mytest = TestClass() - assert conf.cache_active + mytest = CacheTestClass() + assert cache_conf.cache_active mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 - conf.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily + cache_conf.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily set_response(TEXT1) # setting the response @@ -180,10 +175,10 @@ def test_timeout(): def test_deactivate(): - conf.reset() + cache_conf.reset() - mytest = TestClass() - conf.cache_active = False + mytest = CacheTestClass() + cache_conf.cache_active = False mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 @@ -200,10 +195,10 @@ def test_deactivate(): assert resp.content == TEXT2 assert len(os.listdir(mytest.cache_location)) == 0 - conf.reset() - assert conf.cache_active is True + cache_conf.reset() + assert cache_conf.cache_active is True - with conf.set_temp('cache_active', False): + with cache_conf.set_temp('cache_active', False): mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 @@ -219,4 +214,4 @@ def test_deactivate(): assert resp.content == TEXT2 assert len(os.listdir(mytest.cache_location)) == 0 - assert conf.cache_active is True + assert cache_conf.cache_active is True diff --git a/docs/index.rst b/docs/index.rst index d1047e8708..7cb43a41b9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -176,8 +176,8 @@ uncomment the relevant configuration item(s), and insert your desired value(s). Caching ------- -By default Astroquery employs query caching with a timeout of 1 week, although individual services -may have different settings. The user can clear their cache at any time, as well as suspend cache usage, +By default Astroquery employs query caching with a timeout of 1 week. +The user can clear their cache at any time, as well as suspend cache usage, and change the cache location. Caching persists between Astroquery sessions. If you know the service you are using has released new data recently, or if you believe you are not recieving the newest data, try clearing the cache. @@ -185,7 +185,7 @@ not recieving the newest data, try clearing the cache. The Astroquery cache location is divided by service, so each service's cache should be managed invidually, however whether the cache is active and the expiration time are controlled centrally through the -astroquery ``conf`` module. Astroquery uses the Astropy configuration infrastructure, information about +astroquery ``cache_conf`` module. Astroquery uses the Astropy configuration infrastructure, information about temporarily or permanently changing configuration values can be found `here `_. @@ -193,15 +193,15 @@ Shown here are the cache properties, using Simbad as an example: .. code-block:: python - >>> from astroquery import conf + >>> from astroquery import cache_conf >>> from astroquery.simbad import Simbad >>> # Is the cache active? - >>> print(conf.cache_active) + >>> print(cache_conf.cache_active) True >>> # Cache timout in seconds - >>> print(conf.cache_timeout) + >>> print(cache_conf.cache_timeout) 604800 >>> # Cache location @@ -209,6 +209,14 @@ Shown here are the cache properties, using Simbad as an example: /Users/username/.astropy/cache/astroquery/Simbad +To clear the cache: + +.. code-block:: python + + >>> Simbad.clear_cache() + + + Available Services ================== From 5b55499f9f557f888e80dfc3954962684e602ca7 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Mon, 24 Oct 2022 18:14:18 +0100 Subject: [PATCH 24/26] pep8 --- astroquery/query.py | 1 - astroquery/tests/test_cache.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index 25ed76316e..9b454bc87b 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -19,7 +19,6 @@ from astropy.utils.console import ProgressBarOrSpinner import astropy.utils.data from astropy.utils import deprecated -from astropy.utils.exceptions import AstropyDeprecationWarning from astroquery import version, log, cache_conf from astroquery.utils import system_tools diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index d9c7d87b8d..18c484a061 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -1,9 +1,7 @@ -import pytest import requests import os from time import sleep -from pathlib import Path from astropy.config import paths From 6dc09dd48630c31a8b8f677a12e64277111383c6 Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Thu, 27 Oct 2022 15:56:17 +0100 Subject: [PATCH 25/26] avoiding sleep --- astroquery/tests/test_cache.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index 18c484a061..cca83fa5e5 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -1,7 +1,8 @@ import requests import os -from time import sleep +from time import mktime +from datetime import datetime from astropy.config import paths @@ -146,7 +147,7 @@ def test_login(): assert not mytest.authenticated() # Should not be accessing cache -def test_timeout(): +def test_timeout(monkeypatch): cache_conf.reset() mytest = CacheTestClass() @@ -155,21 +156,23 @@ def test_timeout(): mytest.clear_cache() assert len(os.listdir(mytest.cache_location)) == 0 - cache_conf.cache_timeout = 2 # Set to 2 sec so we can reach timeout easily - set_response(TEXT1) # setting the response resp = mytest.test_func(URL1) # should be cached assert resp.content == TEXT1 - set_response(TEXT2) # changing the respont + set_response(TEXT2) # changing the response resp = mytest.test_func(URL1) # should access cached value assert resp.content == TEXT1 - sleep(2) # run out cache time + # Changing the file date so the cache will consider it expired + cache_file = list(mytest.cache_location.iterdir())[0] + modTime = mktime(datetime(1970, 1, 1).timetuple()) + os.utime(cache_file, (modTime, modTime)) + resp = mytest.test_func(URL1) - assert resp.content == TEXT2 # no see the new response + assert resp.content == TEXT2 # now see the new response def test_deactivate(): From 016f6b4da62495fef4314ad6f74ef0f0b56ad87f Mon Sep 17 00:00:00 2001 From: "C. E. Brasseur" Date: Thu, 10 Nov 2022 14:32:56 +0000 Subject: [PATCH 26/26] updates due to feedback --- astroquery/__init__.py | 3 ++- astroquery/query.py | 4 ++-- astroquery/tests/test_cache.py | 11 +++++++++-- docs/index.rst | 6 ++---- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/astroquery/__init__.py b/astroquery/__init__.py index ae87b0b77c..0c11541b2e 100644 --- a/astroquery/__init__.py +++ b/astroquery/__init__.py @@ -46,7 +46,8 @@ class Cache_Conf(_config.ConfigNamespace): cache_timeout = _config.ConfigItem( 604800, - 'Astroquery-wide cache timeout (seconds). Default is 1 week (604800).', + ('Astroquery-wide cache timeout (seconds). Default is 1 week (604800). ' + 'Setting to None prevents the cache from expiring (not recommended).'), cfgtype='integer' ) diff --git a/astroquery/query.py b/astroquery/query.py index 9b454bc87b..998d1a1cdd 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -35,7 +35,7 @@ def to_cache(response, cache_file): for key in tuple(response.request.hooks.keys()): del response.request.hooks[key] with open(cache_file, "wb") as f: - pickle.dump(response, f) + pickle.dump(response, f, protocol=4) def _replace_none_iterable(iterable): @@ -284,7 +284,7 @@ def _request(self, method, url, somewhere other than `BaseQuery.cache_location` timeout : int cache : bool - Override global cache settings. + Optional, if specified, overrides global cache settings. verify : bool Verify the server's TLS certificate? (see http://docs.python-requests.org/en/master/_modules/requests/sessions/?highlight=verify) diff --git a/astroquery/tests/test_cache.py b/astroquery/tests/test_cache.py index cca83fa5e5..d04d7782b9 100644 --- a/astroquery/tests/test_cache.py +++ b/astroquery/tests/test_cache.py @@ -167,13 +167,20 @@ def test_timeout(monkeypatch): assert resp.content == TEXT1 # Changing the file date so the cache will consider it expired - cache_file = list(mytest.cache_location.iterdir())[0] + cache_file = next(mytest.cache_location.iterdir()) modTime = mktime(datetime(1970, 1, 1).timetuple()) os.utime(cache_file, (modTime, modTime)) - + resp = mytest.test_func(URL1) assert resp.content == TEXT2 # now see the new response + # Testing a cache timeout of "none" + cache_conf.cache_timeout = None + set_response(TEXT1) + + resp = mytest.test_func(URL1) + assert resp.content == TEXT2 # cache is accessed + def test_deactivate(): cache_conf.reset() diff --git a/docs/index.rst b/docs/index.rst index 7cb43a41b9..9050e00bec 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -195,17 +195,15 @@ Shown here are the cache properties, using Simbad as an example: >>> from astroquery import cache_conf >>> from astroquery.simbad import Simbad - + ... >>> # Is the cache active? >>> print(cache_conf.cache_active) True - >>> # Cache timout in seconds >>> print(cache_conf.cache_timeout) 604800 - >>> # Cache location - >>> print(Simbad.cache_location) + >>> print(Simbad.cache_location) # doctest: +IGNORE_OUTPUT /Users/username/.astropy/cache/astroquery/Simbad