Merge pull request #789 from keflavich/hash_files

xmatch: bug with caching

Author: keflavich

astroquery/query.py

@@ -64,8 +64,15 @@ def hash(self):
             request_key = (self.method, self.url)
             for k in (self.params, self.data, self.headers, self.files):
                 if isinstance(k, dict):
-                    request_key += (tuple(sorted(k.items(),
-                                                 key=_replace_none_iterable)),)
+                    entry = (tuple(sorted(k.items(),
+                                          key=_replace_none_iterable)))
+                    entry = tuple((k_,v_.read()) if hasattr(v_,'read')
+                                  else (k_,v_) for k_,v_ in entry)
+                    for k_,v_ in entry:
+                        if hasattr(v_,'read') and hasattr(v_,'seek'):
+                            v_.seek(0)
+
+                    request_key += entry
                 elif isinstance(k, tuple) or isinstance(k, list):
                     request_key += (tuple(sorted(k,
                                                  key=_replace_none_iterable)),)

astroquery/xmatch/core.py

@@ -18,7 +18,7 @@ class XMatchClass(BaseQuery):
     TIMEOUT = conf.timeout
 
     def query(self, cat1, cat2, max_distance, colRA1=None, colDec1=None,
-              colRA2=None, colDec2=None, cache=True):
+              colRA2=None, colDec2=None, cache=True, get_query_payload=False):
         """
         Query the `CDS cross-match service
         <http://cdsxmatch.u-strasbg.fr/xmatch>`_ by finding matches between
@@ -58,12 +58,16 @@ def query(self, cat1, cat2, max_distance, colRA1=None, colDec1=None,
             Query results table
         """
         response = self.query_async(cat1, cat2, max_distance, colRA1, colDec1,
-                                    colRA2, colDec2, cache=cache)
+                                    colRA2, colDec2, cache=cache,
+                                    get_query_payload=get_query_payload)
+        if get_query_payload:
+            return response
         return ascii.read(response.text, format='csv')
 
     @prepend_docstr_noreturns("\n" + query.__doc__)
     def query_async(self, cat1, cat2, max_distance, colRA1=None, colDec1=None,
-                    colRA2=None, colDec2=None, cache=True):
+                    colRA2=None, colDec2=None, cache=True,
+                    get_query_payload=False):
         """
         Returns
         -------
@@ -83,8 +87,13 @@ def query_async(self, cat1, cat2, max_distance, colRA1=None, colDec1=None,
         self._prepare_sending_table(1, payload, kwargs, cat1, colRA1, colDec1)
         self._prepare_sending_table(2, payload, kwargs, cat2, colRA2, colDec2)
 
+        if get_query_payload:
+            return payload, kwargs
+
         response = self._request(method='POST', url=self.URL, data=payload,
                                  timeout=self.TIMEOUT, cache=cache, **kwargs)
+        response.raise_for_status()
+
         return response
 
     def _prepare_sending_table(self, i, payload, kwargs, cat, colRA, colDec):
@@ -101,10 +110,10 @@ def _prepare_sending_table(self, i, payload, kwargs, cat, colRA, colDec):
             fp = six.StringIO()
             cat.write(fp, format='ascii.csv')
             fp.seek(0)
-            kwargs['files'] = {catstr: fp}
+            kwargs['files'] = {catstr: ('cat1.csv', fp.read())}
         else:
             # assume it's a file-like object, support duck-typing
-            kwargs['files'] = {catstr: cat}
+            kwargs['files'] = {catstr: ('cat1.csv', cat.read())}
         if not self.is_table_available(cat):
             if ((colRA is None) or (colDec is None)):
                 raise ValueError('Specify the name of the RA/Dec columns in' +

astroquery/xmatch/tests/test_xmatch_remote.py

@@ -1,9 +1,11 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 import os.path
+import os
 
 from astropy.tests.helper import pytest, remote_data
 from astropy.table import Table
 from astropy.units import arcsec
+from astropy.io import ascii
 
 from ...xmatch import XMatch
 
@@ -38,7 +40,7 @@ def test_xmatch_is_avail_table(xmatch):
 
 @remote_data
 def test_xmatch_query(xmatch):
-    with open(os.path.join(DATA_DIR, 'posList.csv')) as pos_list:
+    with open(os.path.join(DATA_DIR, 'posList.csv'), 'r') as pos_list:
         table = xmatch.query(
             cat1=pos_list, cat2='vizier:II/246/out', max_distance=5 * arcsec,
             colRA1='ra', colDec1='dec')
@@ -49,6 +51,8 @@ def test_xmatch_query(xmatch):
         'e_Jmag', 'e_Hmag', 'e_Kmag', 'Qfl', 'Rfl', 'X', 'MeasureJD']
     assert len(table) == 11
 
+    http_test_table = http_test()
+    assert all(table == http_test_table)
 
 @remote_data
 def test_xmatch_query_astropy_table(xmatch):
@@ -63,3 +67,16 @@ def test_xmatch_query_astropy_table(xmatch):
         'errHalfMaj', 'errHalfMin', 'errPosAng', 'Jmag', 'Hmag', 'Kmag',
         'e_Jmag', 'e_Hmag', 'e_Kmag', 'Qfl', 'Rfl', 'X', 'MeasureJD']
     assert len(table) == 11
+
+    http_test_table = http_test()
+    assert all(table == http_test_table)
+
+@remote_data
+def http_test():
+    # this can be used to check that the API is still functional & doing as expected
+    infile = os.path.join(DATA_DIR, 'posList.csv')
+    outfile = os.path.join(DATA_DIR, 'http_result.csv')
+    os.system('curl -X POST -F request=xmatch -F distMaxArcsec=5 -F RESPONSEFORMAT=csv -F cat1=@{1} -F colRA1=ra -F colDec1=dec -F cat2=vizier:II/246/out  http://cdsxmatch.u-strasbg.fr/xmatch/api/v1/sync > {0}'.format(outfile,
+                                                                                                                                                                                                                          infile))
+    table = ascii.read(outfile, format='csv')
+    return table