Merge pull request #484 from keflavich/skyview_docs_fixes

Skyview: Documentation, validation

Author: keflavich

astroquery/skyview/core.py

@@ -1,5 +1,5 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
-import requests
+import pprint
 from bs4 import BeautifulSoup
 from astropy.extern.six.moves.urllib import parse as urlparse
 from astropy.extern import six
@@ -23,9 +23,9 @@ def __init__(self):
         self._default_form_values = None
 
     def _get_default_form_values(self, form):
-        """Return the already selected values of a given form (a BeautifulSoup
+        """
+        Return the already selected values of a given form (a BeautifulSoup
         form node) as a dict.
-
         """
         res = []
         for elem in form.find_all(['input', 'select']):
@@ -48,20 +48,21 @@ def _get_default_form_values(self, form):
                     if option.get('selected') == '':
                         value = option.get('value', option.text.strip())
                         res.append((elem.get('name'), value))
-        return dict(
-            (k, v) for (k, v) in res if v not in [None, u'None', u'null'] and v
-        )
+        return {k:v 
+                for (k, v) in res
+                if v not in [None, u'None', u'null'] and v
+               }
 
-    def _submit_form(self, input=None):
-        """Fill out the form of the SkyView site and submit it with the
+    def _generate_payload(self, input=None):
+        """
+        Fill out the form of the SkyView site and submit it with the
         values given in `input` (a dictionary where the keys are the form
         element's names and the values are their respective values).
-
         """
         if input is None:
             input = {}
-        response = requests.get(self.URL)
-        bs = BeautifulSoup(response.content)
+        form_response = self._request('GET', self.URL)
+        bs = BeautifulSoup(form_response.content, "html.parser")
         form = bs.find('form')
         # cache the default values to save HTTP traffic
         if self._default_form_values is None:
@@ -73,14 +74,19 @@ def _submit_form(self, input=None):
             if v is not None:
                 payload[k] = v
         url = urlparse.urljoin(self.URL, form.get('action'))
-        response = requests.get(url, params=payload)
+        return url, payload
+
+    def _submit_form(self, input=None, cache=True):
+        url, payload = self._generate_payload(input=input)
+        response = self._request('GET', url, params=payload, cache=cache)
         return response
 
-    def get_images(
-            self, position, survey, coordinates=None, projection=None,
-            pixels=None, scaling=None, sampler=None, resolver=None,
-            deedger=None, lut=None, grid=None, gridlabels=None):
-        """Query the SkyView service, download the FITS file that will be
+    def get_images(self, position, survey, coordinates=None, projection=None,
+                   pixels=None, scaling=None, sampler=None, resolver=None,
+                   deedger=None, lut=None, grid=None, gridlabels=None,
+                   cache=True):
+        """
+        Query the SkyView service, download the FITS file that will be
         found and return a generator over the local paths to the
         downloaded FITS files.
 
@@ -177,33 +183,34 @@ def get_images(
         A list of `astropy.fits.HDUList` objects
 
         """
-        readable_objects = self.get_images_async(
-            position, survey, coordinates, projection,
-            pixels, scaling, sampler, resolver,
-            deedger, lut, grid, gridlabels)
+        readable_objects = self.get_images_async(position, survey, coordinates,
+                                                 projection, pixels, scaling,
+                                                 sampler, resolver, deedger,
+                                                 lut, grid, gridlabels,
+                                                 cache=cache)
         return [obj.get_fits() for obj in readable_objects]
 
     @prepend_docstr_noreturns(get_images.__doc__)
-    def get_images_async(
-            self, position, survey, coordinates=None, projection=None,
-            pixels=None, scaling=None, sampler=None, resolver=None,
-            deedger=None, lut=None, grid=None, gridlabels=None):
+    def get_images_async(self, position, survey, coordinates=None,
+                         projection=None, pixels=None, scaling=None,
+                         sampler=None, resolver=None, deedger=None, lut=None,
+                         grid=None, gridlabels=None, cache=True):
         """
         Returns
         -------
         A list of context-managers that yield readable file-like objects
         """
-        image_urls = self.get_image_list(
-            position, survey, coordinates, projection,
-            pixels, scaling, sampler, resolver,
-            deedger, lut, grid, gridlabels)
+        image_urls = self.get_image_list(position, survey, coordinates,
+                                         projection, pixels, scaling, sampler,
+                                         resolver, deedger, lut, grid,
+                                         gridlabels, cache=cache)
         return [commons.FileContainer(url) for url in image_urls]
 
     @prepend_docstr_noreturns(get_images.__doc__)
-    def get_image_list(
-            self, position, survey, coordinates=None, projection=None,
-            pixels=None, scaling=None, sampler=None, resolver=None,
-            deedger=None, lut=None, grid=None, gridlabels=None):
+    def get_image_list(self, position, survey, coordinates=None,
+                       projection=None, pixels=None, scaling=None,
+                       sampler=None, resolver=None, deedger=None, lut=None,
+                       grid=None, gridlabels=None, cache=True):
         """
         Returns
         -------
@@ -216,6 +223,9 @@ def get_image_list(
          u'http://skyview.gsfc.nasa.gov/tempspace/fits/skv6183161285798_2.fits',
          u'http://skyview.gsfc.nasa.gov/tempspace/fits/skv6183161285798_3.fits']
         """
+
+        self._validate_surveys(survey)
+
         input = {
             'Position': position,
             'survey': survey,
@@ -229,17 +239,50 @@ def get_image_list(
             'resolver': resolver,
             'Sampler': sampler,
             'pixels': pixels}
-        response = self._submit_form(input)
+        response = self._submit_form(input, cache=cache)
         urls = self._parse_response(response)
         return urls
 
     def _parse_response(self, response):
-        bs = BeautifulSoup(response.content)
+        bs = BeautifulSoup(response.content, "html.parser")
         urls = []
         for a in bs.find_all('a'):
             if a.text == 'FITS':
                 href = a.get('href')
                 urls.append(urlparse.urljoin(response.url, href))
         return urls
 
+    @property
+    def survey_dict(self):
+        if not hasattr(self, '_survey_dict'):
+
+            response = self._request('GET', self.URL)
+            page = BeautifulSoup(response.content, "html.parser")
+            surveys = page.findAll('select', {'name':'survey'})
+            
+            self._survey_dict = {sel['id']:[x.text for x in sel.findAll('option')]
+                                for sel in surveys}
+        return self._survey_dict
+
+    @property
+    def _valid_surveys(self):
+        # Return a flat list of all valid surveys
+        return [x for v in self.survey_dict.values() for x in v]
+
+    def _validate_surveys(self, surveys):
+        if not isinstance(surveys, list):
+            surveys = [surveys]
+
+        for sv in surveys:
+            if sv not in self._valid_surveys:
+                raise ValueError("Survey is not among the surveys hosted "
+                                 "at skyview.  See list_surveys or "
+                                 "survey_dict for valid surveys.")
+
+    def list_surveys(self):
+        """
+        Print out a formatted version of the survey dict
+        """
+        pprint.pprint(self.survey_dict)
+
 SkyView = SkyViewClass()

astroquery/skyview/setup_package.py

@@ -4,5 +4,6 @@
 
 def get_package_data():
     paths = [os.path.join('data', '*.html'),
+             os.path.join('data', '*.txt'),
              ]
     return {'astroquery.skyview.tests': paths}