Merge pull request #1992 from jwoillez/logging

Added HTTP logging capability

Author: C. E. Brasseur

CHANGES.rst

@@ -128,6 +128,15 @@ esa.xmm_newton
 
 - Added new function to download EPIC sources metadate. [#1814]
 
+
+Infrastructure, Utility and Other Changes and Additions
+-------------------------------------------------------
+
+- HTTP requests and responses can now be logged when the astropy
+  logger is set to level "DEBUG" and "TRACE" respectively. [#1992]
+- Astroquery and all its modules now uses a logger similar to Astropy's. [#1992]
+
+
 0.4.1 (2020-06-19)
 ==================
 

astroquery/__init__.py

@@ -13,6 +13,9 @@
 # ----------------------------------------------------------------------------
 
 import os
+import logging
+
+from .logger import _init_log
 
 
 # Set the bibtex entry to the article referenced in CITATION.
@@ -39,3 +42,9 @@ def _get_bibtex():
 except ImportError:
     # TODO: Issue a warning using the logging framework
     __githash__ = ''
+
+
+# Setup logging for astroquery
+logging.addLevelName(5, "TRACE")
+log = logging.getLogger()
+log = _init_log()

astroquery/alma/core.py

@@ -16,7 +16,7 @@
 from six.moves.urllib_parse import urljoin
 import six
 from astropy.table import Table, Column, vstack
-from astropy import log
+from astroquery import log
 from astropy.utils import deprecated
 from astropy.utils.console import ProgressBar
 from astropy.utils.exceptions import AstropyDeprecationWarning

astroquery/alma/utils.py

@@ -7,7 +7,7 @@
 import numpy as np
 
 from astropy import wcs
-from astropy import log
+from astroquery import log
 from astropy import units as u
 from astropy.io import fits
 from astroquery.utils.commons import ASTROPY_LT_4_1

astroquery/astrometry_net/core.py

@@ -6,7 +6,7 @@
 import six
 from six import string_types
 from astropy.io import fits
-from astropy import log
+from astroquery import log
 from astropy.stats import sigma_clipped_stats
 from astropy.coordinates import SkyCoord
 

astroquery/cadc/core.py

@@ -7,7 +7,7 @@
 Module to query the Canadian Astronomy Data Centre (CADC).
 """
 
-import logging
+from astroquery import log
 import warnings
 import requests
 from numpy import ma
@@ -38,8 +38,6 @@
 
 CADC_COOKIE_PREFIX = 'CADC_SSO'
 
-logger = logging.getLogger(__name__)
-
 # TODO figure out what to do if anything about them. Some might require
 # fixes on the CADC servers
 warnings.filterwarnings('ignore', module='astropy.io.votable')
@@ -192,7 +190,7 @@ def login(self, user=None, password=None, certificate_file=None):
             try:
                 response.raise_for_status()
             except Exception as e:
-                logger.error('Logging error: {}'.format(e))
+                log.error('Logging error: {}'.format(e))
                 raise e
             # extract cookie
             cookie = '"{}"'.format(response.text)
@@ -357,7 +355,7 @@ def get_images(self, coordinates, radius,
                 images.append(fn.get_fits())
             except requests.exceptions.HTTPError as err:
                 # Catch HTTPError if user is unauthorized to access file
-                logger.debug(
+                log.debug(
                     "{} - Problem retrieving the file: {}".
                     format(str(err), str(err.url)))
                 pass
@@ -811,7 +809,7 @@ def get_access_url(service, capability=None):
                 response = requests.get(conf.CADC_REGISTRY_URL)
                 response.raise_for_status()
             except requests.exceptions.HTTPError as err:
-                logger.debug(
+                log.debug(
                     "ERROR getting the CADC registry: {}".format(str(err)))
                 raise err
             for line in response.text.splitlines():
@@ -835,7 +833,7 @@ def get_access_url(service, capability=None):
         response2 = requests.get(caps_url)
         response2.raise_for_status()
     except Exception as e:
-        logger.debug(
+        log.debug(
             "ERROR getting the service capabilities: {}".format(str(e)))
         raise e
 

astroquery/casda/core.py

@@ -12,7 +12,7 @@
 import astropy.coordinates as coord
 from astropy.table import Table
 from astropy.io.votable import parse
-from astropy import log
+from astroquery import log
 
 # 3. local imports - use relative imports
 # commonly required local imports shown below as example

astroquery/casda/tests/test_casda.py

@@ -11,7 +11,7 @@
 import astropy.units as u
 from astropy.table import Table, Column
 from astropy.io.votable import parse
-from astropy import log
+from astroquery import log
 
 from astroquery.casda import Casda
 

astroquery/cosmosim/core.py

@@ -19,7 +19,7 @@
 # Astropy imports
 from astropy.table import Table
 import astropy.io.votable as votable
-from astropy import log as logging
+from astroquery import log
 
 # Astroquery imports
 from ..query import QueryWithLogin
@@ -72,8 +72,8 @@ def _login(self, username=None, password=None, store_password=False,
 
         # login after login (interactive)
         if hasattr(self, 'username'):
-            logging.warning("Attempting to login while another user ({0}) "
-                            "is already logged in.".format(self.username))
+            log.warning("Attempting to login while another user ({0}) "
+                        "is already logged in.".format(self.username))
             self.check_login_status()
             return
 
@@ -144,7 +144,7 @@ def logout(self, deletepw=False):
             del self.username
             del self.password
         else:
-            logging.error("You must log in before attempting to logout.")
+            log.error("You must log in before attempting to logout.")
 
     def check_login_status(self):
         """
@@ -220,8 +220,8 @@ def run_sql_query(self, query_string, tablename=None, queue=None,
                 gen_tablename = "{0}".format(phase)
             else:
                 gen_tablename = str(soup.find(id="table").string)
-            logging.warning("Table name {0} is already taken."
-                            .format(tablename))
+            log.warning("Table name {0} is already taken."
+                        .format(tablename))
             warnings.warn("Generated table name: {0}".format(gen_tablename))
         elif tablename is None:
             result = self._request('POST', CosmoSim.QUERY_URL,
@@ -293,7 +293,7 @@ def check_job_status(self, jobid=None):
             auth=(self.username, self.password), data={'print': 'b'},
             cache=False)
 
-        logging.info("Job {}: {}".format(jobid, response.content))
+        log.info("Job {}: {}".format(jobid, response.content))
         return response.content
 
     def check_all_jobs(self, phase=None, regex=None, sortby=None):
@@ -586,7 +586,7 @@ def completed_job_info(self, jobid=None, output=False):
             elif len(dictkeys) == 1:
                 print(self.response_dict_current[dictkeys[0]]['content'])
             else:
-                logging.error('No completed jobs found.')
+                log.error('No completed jobs found.')
             return
         else:
             return
@@ -680,7 +680,7 @@ def general_job_info(self, jobid=None, output=False):
                 auth=(self.username, self.password), cache=False)]
 
             if response_list[0].ok is False:
-                logging.error('Must provide a valid jobid.')
+                log.error('Must provide a valid jobid.')
                 return
             else:
                 self.response_dict_current = {}
@@ -940,7 +940,7 @@ def explore_db(self, db=None, table=None, col=None):
                          for key in self.db_dict[db].keys()]
                 size2 = len(max(slist, key=np.size))
             except (KeyError, NameError):
-                logging.error("Must first specify a valid database.")
+                log.error("Must first specify a valid database.")
                 return
             # if col is specified, table must be specified, and I need to
             # check the max size of any given column in the structure
@@ -954,13 +954,13 @@ def explore_db(self, db=None, table=None, col=None):
                                                    [table]['columns']
                                                    [col].keys())))
                         except(KeyError, NameError):
-                            logging.error("Must first specify a valid column "
-                                          "of the `{0}` table within the `{1}`"
-                                          " db".format(table, db))
+                            log.error("Must first specify a valid column "
+                                      "of the `{0}` table within the `{1}`"
+                                      " db".format(table, db))
                             return
                 except (KeyError, NameError):
-                    logging.error("Must first specify a valid table within "
-                                  "the `{0}` db.".format(db))
+                    log.error("Must first specify a valid table within "
+                              "the `{0}` db.".format(db))
                     return
 
             t['Projects'] = (['--> @ {}:'.format(db)] +
@@ -1202,7 +1202,7 @@ def _check_phase(self, jobid):
         time.sleep(1)
 
         if jobid not in self.job_dict.keys():
-            logging.error("Job not present in job dictionary.")
+            log.error("Job not present in job dictionary.")
             return
 
         else:
@@ -1291,7 +1291,7 @@ def _initialize_alerting(self, jobid, mail=None, text=None):
             self._smspw = password_from_keyring
 
         if not password_from_keyring:
-            logging.warning("CosmoSim SMS alerting has not been initialized.")
+            log.warning("CosmoSim SMS alerting has not been initialized.")
             warnings.warn("Initializing SMS alerting.")
             keyring.set_password("astroquery:cosmosim.SMSAlert",
                                  self._smsaddress, "LambdaCDM")

astroquery/esa/hubble/core.py

@@ -28,7 +28,7 @@
 import os
 
 from . import conf
-from astropy import log
+from astroquery import log
 
 __all__ = ['ESAHubble', 'ESAHubbleClass']