Merge pull request #767 from keflavich/nrao_login

Upgrades & fixes to the NRAO archive tool

Author: keflavich

CHANGES

@@ -5,6 +5,7 @@
 - Add license statements to appropriate places (#776)
 - Correct HEASARC tool, which was sending incorrect data to the server (#774)
 - Fix NIST issue #714 which led to badly-parsed tables (#773)
+- NRAO archive tool allows user logins and HTML-based queries (#767)
 
 0.3.3 (2016-10-11)
 ------------------

astroquery/alma/core.py

@@ -305,7 +305,7 @@ def stage_data(self, uids):
         summary.raise_for_status()
         self._staging_log['json_data'] = json_data = summary.json()
 
-        username = self._username if hasattr(self, '_username') else 'anonymous'
+        username = self.USERNAME if self.USERNAME else 'anonymous'
 
         # templates:
         # https://almascience.eso.org/dataPortal/requests/keflavich/946895898/ALMA/
@@ -447,7 +447,7 @@ def _login(self, username=None, store_password=False,
         """
 
         if username is None:
-            if self.USERNAME == "":
+            if not self.USERNAME:
                 raise LoginError("If you do not pass a username to login(), "
                                  "you should configure a default one!")
             else:
@@ -496,7 +496,7 @@ def _login(self, username=None, store_password=False,
 
         if authenticated:
             log.info("Authentication successful!")
-            self._username = username
+            self.USERNAME = username
         else:
             log.exception("Authentication failed!")
         # When authenticated, save password in keyring if needed

astroquery/nrao/__init__.py

@@ -15,6 +15,9 @@ class Conf(_config.ConfigNamespace):
     timeout = _config.ConfigItem(
         60,
         'Time limit for connecting to NRAO server.')
+    username = _config.ConfigItem(
+        "",
+        'Optional default username for ALMA archive.')
 
 conf = Conf()
 

astroquery/nrao/core.py

@@ -2,18 +2,25 @@
 from __future__ import print_function
 
 import re
+import os
 import warnings
 import functools
+import getpass
+import keyring
 
+import numpy as np
 import astropy.units as u
 import astropy.io.votable as votable
 from astropy import coordinates
 from astropy.extern import six
+from astropy.table import Table
+from astropy import log
+from bs4 import BeautifulSoup
 
-from ..query import BaseQuery
-from ..utils import commons, async_to_sync
+from ..query import QueryWithLogin
+from ..utils import commons, async_to_sync, system_tools
 from ..utils.docstr_chompers import prepend_docstr_noreturns
-from ..exceptions import TableParseError
+from ..exceptions import TableParseError, LoginError
 
 from . import conf
 
@@ -44,10 +51,11 @@ def wrapper(*args, **kwargs):
 
 
 @async_to_sync
-class NraoClass(BaseQuery):
+class NraoClass(QueryWithLogin):
 
     DATA_URL = conf.server
     TIMEOUT = conf.timeout
+    USERNAME = conf.username
 
     # dicts and lists for data archive queries
     telescope_code = {
@@ -124,15 +132,27 @@ def _args_to_payload(self, **kwargs):
 
         querytype : str
             The type of query to perform.  "OBSSUMMARY" is the default, but
-            it is only valid for VLA/VLBA observations.  ARCHIVE will not
-            work at all because it relies on XML data.  OBSERVATION will
+            it is only valid for VLA/VLBA observations.  ARCHIVE will give
+            the list of files available for download.  OBSERVATION will
             provide full details of the sources observed and under what
             configurations.
         source_id : str, optional
             A source name (to be parsed by SIMBAD or NED)
+        protocol : 'VOTable-XML' or 'HTML'
+            The type of table to return.  In theory, this should not matter,
+            but in practice the different table types actually have different
+            content.  For ``querytype='ARCHIVE'``, the protocol will be force
+            to HTML because the archive doesn't support votable returns for
+            archive queries.
         get_query_payload : bool, optional
             if set to `True` then returns the dictionary sent as the HTTP
             request.  Defaults to `False`
+        cache : bool
+            Cache the query results
+        retry : bool or int
+            The number of times to retry querying the server if it doesn't
+            raise an exception but returns a null result (this sort of behavior
+            seems unique to the NRAO archive)
 
         Returns
         -------
@@ -149,7 +169,7 @@ def _args_to_payload(self, **kwargs):
 
         request_payload = dict(
             QUERYTYPE=kwargs.get('querytype', "OBSSUMMARY"),
-            PROTOCOL="VOTable-XML",
+            PROTOCOL=kwargs.get('protocol',"VOTable-XML"),
             MAX_ROWS="NO LIMIT",
             SORT_PARM="Starttime",
             SORT_ORDER="Asc",
@@ -187,6 +207,20 @@ def _args_to_payload(self, **kwargs):
             PASSWD="",  # TODO: implement login...
             SUBMIT="Submit Query")
 
+        if (request_payload['QUERYTYPE'] == "ARCHIVE" and
+            request_payload['PROTOCOL'] != 'HTML'):
+            warnings.warn("Changing protocol to HTML: ARCHIVE queries do not"
+                          " support votable returns")
+            request_payload['PROTOCOL'] = 'HTML'
+
+        if request_payload['PROTOCOL'] not in ('HTML','VOTable-XML'):
+            raise ValueError("Only HTML and VOTable-XML returns are supported")
+
+        if request_payload['QUERYTYPE'] not in ('ARCHIVE', 'OBSSUMMARY',
+                                                'OBSERVATION'):
+            raise ValueError("Only ARCHIVE, OBSSUMMARY, and OBSERVATION "
+                             "querytypes are supported")
+
         if 'coordinates' in kwargs:
             c = commons.parse_coordinates(
                 kwargs['coordinates']).transform_to(coordinates.ICRS)
@@ -195,10 +229,91 @@ def _args_to_payload(self, **kwargs):
 
         return request_payload
 
+    def _login(self, username=None, store_password=False,
+               reenter_password=False):
+        """
+        Login to the NRAO archive
+
+        Parameters
+        ----------
+        username : str, optional
+            Username to the NRAO archive. If not given, it should be specified
+            in the config file.
+        store_password : bool, optional
+            Stores the password securely in your keyring. Default is False.
+        reenter_password : bool, optional
+            Asks for the password even if it is already stored in the
+            keyring. This is the way to overwrite an already stored passwork
+            on the keyring. Default is False.
+        """
+
+        # Developer notes:
+        # Login via https://my.nrao.edu/cas/login
+        # # this can be added to auto-redirect back to the query tool: ?service=https://archive.nrao.edu/archive/advquery.jsp
+
+        if username is None:
+            if not self.USERNAME:
+                raise LoginError("If you do not pass a username to login(), "
+                                 "you should configure a default one!")
+            else:
+                username = self.USERNAME
+
+        # Check if already logged in
+        loginpage = self._request("GET", "https://my.nrao.edu/cas/login",
+                                  cache=False)
+        root = BeautifulSoup(loginpage.content, 'html5lib')
+        if root.find('div', class_='success'):
+            log.info("Already logged in.")
+            return True
+
+        # Get password from keyring or prompt
+        if reenter_password is False:
+            password_from_keyring = keyring.get_password(
+                "astroquery:my.nrao.edu", username)
+        else:
+            password_from_keyring = None
+
+        if password_from_keyring is None:
+            if system_tools.in_ipynb():
+                log.warning("You may be using an ipython notebook:"
+                            " the password form will appear in your terminal.")
+            password = getpass.getpass("{0}, enter your NRAO archive password:"
+                                       "\n".format(username))
+        else:
+            password = password_from_keyring
+        # Authenticate
+        log.info("Authenticating {0} on my.nrao.edu ...".format(username))
+        # Do not cache pieces of the login process
+        data = {kw: root.find('input', {'name': kw})['value']
+                for kw in ('lt', '_eventId', 'execution')}
+        data['username'] = username
+        data['password'] = password
+        data['execution'] = 'e1s1' # not sure if needed
+        data['_eventId'] = 'submit'
+        data['submit'] = 'LOGIN'
+
+        login_response = self._request("POST", "https://my.nrao.edu/cas/login",
+                                       data=data, cache=False)
+
+        authenticated = ('You have successfully logged in' in
+                         login_response.text)
+
+        if authenticated:
+            log.info("Authentication successful!")
+            self.USERNAME = username
+        else:
+            log.exception("Authentication failed!")
+        # When authenticated, save password in keyring if needed
+        if authenticated and password_from_keyring is None and store_password:
+            keyring.set_password("astroquery:my.nrao.edu", username, password)
+
+        return authenticated
+
     @prepend_docstr_noreturns(_args_to_payload.__doc__)
     def query_async(self,
                     get_query_payload=False,
                     cache=True,
+                    retry=False,
                     **kwargs):
         """
         Returns
@@ -213,14 +328,32 @@ def query_async(self,
             return request_payload
         response = self._request('POST', self.DATA_URL, params=request_payload,
                                  timeout=self.TIMEOUT, cache=cache)
+        self._last_response = response
+
+        response.raise_for_status()
+
+        if not response.content:
+            if cache:
+                last_pickle = self._last_query.hash()+".pickle"
+                cache_fn = os.path.join(self.cache_location, last_pickle)
+                os.remove(cache_fn)
+            if retry > 0:
+                self.query_async(cache=cache, retry=retry-1, **kwargs)
+            else:
+                raise ValueError("Query resulted in an empty result but "
+                                 "the server did not raise an error.")
+        
         return response
 
     @prepend_docstr_noreturns(_args_to_payload.__doc__)
     def query_region_async(self, coordinates, radius=1 * u.deg,
                            equinox='J2000', telescope='all', start_date="",
                            end_date="", freq_low=None, freq_up=None,
                            telescope_config='all', obs_band='all',
-                           sub_array='all', get_query_payload=False):
+                           querytype='OBSSUMMARY', sub_array='all',
+                           protocol='VOTable-XML',
+                           retry=False,
+                           get_query_payload=False, cache=True):
         """
         Returns
         -------
@@ -239,9 +372,22 @@ def query_region_async(self, coordinates, radius=1 * u.deg,
                                 telescope_config=telescope_config,
                                 obs_band=obs_band,
                                 sub_array=sub_array,
-                                get_query_payload=get_query_payload)
+                                querytype=querytype,
+                                protocol=protocol,
+                                get_query_payload=get_query_payload,
+                                retry=retry,
+                                cache=cache)
 
     def _parse_result(self, response, verbose=False):
+        if '<?xml' in response.text[:5]:
+            return self._parse_votable_result(response, verbose=verbose)
+        elif '<html>' in response.text[:6]:
+            return self._parse_html_result(response, verbose=verbose)
+        else:
+            raise ValueError("Unrecognized response type; it does not appear "
+                             "to be VO-XML or HTML")
+
+    def _parse_votable_result(self, response, verbose=False):
         if not verbose:
             commons.suppress_vo_warnings()
 
@@ -281,4 +427,26 @@ def _parse_result(self, response, verbose=False):
                                   "raw response can be found in self.response,"
                                   " and the error in self.table_parse_error.")
 
+    def _parse_html_result(self, response, verbose=False):
+        # pares the HTML return...
+        root = BeautifulSoup(response.content, 'html5lib')
+
+        htmltable = root.findAll('table')
+        #if len(htmltable) != 1:
+        #    raise ValueError("Found the wrong number of tables: {0}"
+        #                     .format(len(htmltable)))
+
+        string_to_parse = htmltable[-1].encode('ascii')
+
+        if six.PY2:
+            from astropy.io.ascii import html
+            from astropy.io.ascii.core import convert_numpy
+            htmlreader = html.HTML()
+            htmlreader.outputter.default_converters.append(convert_numpy(np.unicode))
+            table = htmlreader.read(string_to_parse)
+        else:
+            table = Table.read(string_to_parse.decode('utf-8'), format='ascii.html')
+
+        return table
+
 Nrao = NraoClass()