Merge pull request #2214 from weaverba137/sdss-spectra-url

Update Spectra URLs

Author: bsipocz

CHANGES.rst

@@ -8,6 +8,10 @@ New Tools and Services
 Service fixes and enhancements
 ------------------------------
 
+sdss
+^^^^
+
+- Fix URL for individual spectrum file download in recent data releases. [#2214]
 
 Infrastructure, Utility and Other Changes and Additions
 -------------------------------------------------------

astroquery/sdss/core.py

@@ -38,9 +38,9 @@ class SDSSClass(BaseQuery):
                           '{rerun}/{run}/{camcol}/'
                           'frame-{band}-{run:06d}-{camcol}-'
                           '{field:04d}.fits.bz2')
-    SPECTRA_URL_SUFFIX = ('{base}/dr{dr}/{instrument}/spectro/redux/'
-                          '{run2d}/spectra/{plate:04d}/'
-                          'spec-{plate:04d}-{mjd}-{fiber:04d}.fits')
+    SPECTRA_URL_SUFFIX = ('{base}/dr{dr}/sdss/spectro/redux/'
+                          '{run2d}/spectra/{plate:0>4d}/'
+                          'spec-{plate:0>4d}-{mjd}-{fiber:04d}.fits')
 
     TEMPLATES_URL = 'http://classic.sdss.org/dr7/algorithms/spectemplates/spDR2'
     # Cross-correlation templates from DR-7 - no clear way to look this up via
@@ -555,7 +555,7 @@ def get_spectra_async(self, coordinates=None, radius=2. * u.arcsec,
 
         if not matches:
             request_payload = self._args_to_payload(
-                specobj_fields=['instrument', 'run2d', 'plate',
+                specobj_fields=['run2d', 'plate',
                                 'mjd', 'fiberID'],
                 coordinates=coordinates, radius=radius, spectro=True,
                 plate=plate, mjd=mjd, fiberID=fiberID,
@@ -583,14 +583,16 @@ def get_spectra_async(self, coordinates=None, radius=2. * u.arcsec,
             # - run2d sometimes (#739)
             if isinstance(row['run2d'], bytes):
                 run2d = row['run2d'].decode()
+            elif isinstance(row['run2d'], (np.integer, int)):
+                run2d = str(row['run2d'])
             else:
                 run2d = row['run2d']
+            if data_release > 15 and run2d not in ('26', '103', '104'):
+                linkstr = linkstr.replace('/spectra/', '/spectra/full/')
             link = linkstr.format(
                 base=conf.sas_baseurl, dr=data_release,
-                instrument=row['instrument'].lower(),
                 run2d=run2d, plate=row['plate'],
                 fiber=row['fiberID'], mjd=row['mjd'])
-
             results.append(commons.FileContainer(link,
                                                  encoding='binary',
                                                  remote_timeout=timeout,
@@ -859,9 +861,12 @@ def _parse_result(self, response, verbose=False):
 
         if 'error_message' in io.BytesIO(response.content):
             raise RemoteServiceError(response.content)
+        skip_header = 0
+        if response.content.startswith(b'#Table'):
+            skip_header = 1
         arr = np.atleast_1d(np.genfromtxt(io.BytesIO(response.content),
                                           names=True, dtype=None,
-                                          delimiter=',', skip_header=1,
+                                          delimiter=',', skip_header=skip_header,
                                           comments='#'))
 
         if len(arr) == 0:

astroquery/sdss/field_names.py

@@ -16,8 +16,7 @@
 
 # Default photometric and spectroscopic quantities to retrieve.
 photoobj_defs = ['ra', 'dec', 'objid', 'run', 'rerun', 'camcol', 'field']
-specobj_defs = ['z', 'plate', 'mjd', 'fiberID', 'specobjid', 'run2d',
-                'instrument']
+specobj_defs = ['z', 'plate', 'mjd', 'fiberID', 'specobjid', 'run2d']
 crossid_defs = ['ra', 'dec', 'psfMag_u', 'psfMagerr_u', 'psfMag_g',
                 'psfMagerr_g', 'psfMag_r', 'psfMagerr_r', 'psfMag_i',
                 'psfMagerr_i', 'psfMag_z', 'psfMagerr_z']

astroquery/sdss/tests/test_sdss.py

@@ -125,7 +125,7 @@ def data_path(filename):
 coords_column = Column(coords_list, name='coordinates')
 
 # List of all data releases.
-dr_list = list(range(1, sdss.conf.default_release + 1))
+dr_list = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)
 
 
 # We are not testing queries for DR11 because it is not easily available to

astroquery/sdss/tests/test_sdss_remote.py

@@ -11,6 +11,9 @@
 from ... import sdss
 from ...exceptions import TimeoutError
 
+# DR11 is a quasi-internal data release that does not have SkyServer support.
+dr_list = (8, 9, 10, 12, 13, 14, 15, 16)
+
 
 @pytest.mark.remote_data
 class TestSDSSRemote:
@@ -34,10 +37,19 @@ def test_images_timeout(self):
                          "error with 'No route to host'.  We don't know a "
                          "workaround for this yet.")
 
-    def test_sdss_spectrum(self):
-        xid = sdss.SDSS.query_region(self.coords, spectro=True)
+    @pytest.mark.parametrize("dr", dr_list)
+    def test_sdss_spectrum(self, dr):
+        xid = sdss.SDSS.query_region(self.coords, spectro=True, data_release=dr)
         assert isinstance(xid, Table)
-        sp = sdss.SDSS.get_spectra(matches=xid)
+        sp = sdss.SDSS.get_spectra(matches=xid, data_release=dr)
+
+    def test_sdss_spectrum_plate_mjd_fiber(self):
+        """These plates are only available in recent data releases.
+        """
+        sp = sdss.SDSS.get_spectra(plate=9403, mjd=58018, fiberID=485,
+                                   data_release=16)
+        sp = sdss.SDSS.get_spectra(plate=10909, mjd=58280, fiberID=485,
+                                   data_release=16)
 
     def test_sdss_spectrum_mjd(self):
         sp = sdss.SDSS.get_spectra(plate=2345, fiberID=572)
@@ -75,25 +87,22 @@ def test_sdss_image_coord(self):
 
     def test_sdss_specobj(self):
         colnames = ['ra', 'dec', 'objid', 'run', 'rerun', 'camcol', 'field',
-                    'z', 'plate', 'mjd', 'fiberID', 'specobjid', 'run2d',
-                    'instrument']
+                    'z', 'plate', 'mjd', 'fiberID', 'specobjid', 'run2d']
         dtypes = [float, float, np.int64, int, int, int, int, float, int, int,
-                  int, np.int64, int, bytes]
+                  int, np.int64, int]
         data = [
             [46.8390680395307, 5.16972676625711, 1237670015125750016, 5714,
              301, 2, 185, -0.0006390358, 2340, 53733, 291, 2634685834112034816,
-             26, 'SDSS'],
+             26],
             [46.8705377929765, 5.42458826592292, 1237670015662621224, 5714,
-             301, 3, 185, 0, 2340, 53733, 3, 2634606669274834944, 26, 'SDSS'],
+             301, 3, 185, 0, 2340, 53733, 3, 2634606669274834944, 26],
             [46.8899751105478, 5.09432755808192, 1237670015125815346, 5714,
              301, 2, 186, -4.898809E-05, 2340, 53733, 287, 2634684734600407040,
-             26, 'SDSS'],
+             26],
             [46.8954031261838, 5.9739184644185, 1237670016199491831, 5714,
-             301, 4, 185, 0, 2340, 53733, 329, 2634696279472498688, 26,
-             'SDSS'],
+             301, 4, 185, 0, 2340, 53733, 329, 2634696279472498688, 26],
             [46.9155836662379, 5.50671723824944, 1237670015662686398, 5714,
-             301, 3, 186, 0, 2340, 53733, 420, 2634721293362030592, 26,
-             'SDSS']]
+             301, 3, 186, 0, 2340, 53733, 420, 2634721293362030592, 26]]
         table = Table(data=[x for x in zip(*data)],
                       names=colnames, dtype=dtypes)
         xid = sdss.SDSS.query_specobj(plate=2340)