Skip to content

Commit 842f2da

Browse files
lvaleromlvalerom
committed
review comments
1 parent c30e59e commit 842f2da

File tree

2 files changed

+95
-25
lines changed

2 files changed

+95
-25
lines changed

‎astroquery/esa/hsa/core.py

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from astropy import units as u
1010
from ...utils import commons
1111
from astroquery import log
12-
from astroquery.exceptions import LoginError, InputWarning
12+
from astroquery.exceptions import LoginError
1313
from astroquery.query import BaseQuery
1414
from astroquery.utils.tap.core import Tap
1515

@@ -32,7 +32,8 @@ def __init__(self, tap_handler=None):
3232
self._tap = tap_handler
3333

3434
def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None,
35-
instrument_name=None, filename=None, verbose=False,
35+
instrument_name=None, filename=None, observation_oid=None,
36+
instrument_oid=None, product_level=None, verbose=False,
3637
download_dir="", cache=True, **kwargs):
3738
"""
3839
Download data from Herschel
@@ -82,6 +83,15 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None,
8283
if instrument_name is not None:
8384
params['instrument_name'] = instrument_name
8485

86+
if observation_oid is not None:
87+
params['observation_oid'] = observation_oid
88+
89+
if instrument_oid is not None:
90+
params['instrument_oid'] = instrument_oid
91+
92+
if product_level is not None:
93+
params['product_level'] = product_level
94+
8595
link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items())
8696

8797
link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items())
@@ -90,22 +100,23 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None,
90100
log.info(link)
91101

92102
response = self._request('HEAD', link, save=False, cache=cache)
93-
response.raise_for_status()
94-
95-
if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']:
96-
_, params = cgi.parse_header(response.headers['Content-Disposition'])
97-
else:
103+
if response.status_code == 401:
98104
error = "Data protected by proprietary rights. Please check your credentials"
99105
raise LoginError(error)
100106

107+
response.raise_for_status()
108+
101109
if filename is None:
102110
if observation_id is not None:
103111
filename = observation_id
104112
else:
105113
error = "Please set either 'obervation_id' or 'filename' for the output"
106114
raise ValueError(error)
107115

108-
r_filename = params["filename"]
116+
_, res_params = cgi.parse_header(response.headers['Content-Disposition'])
117+
118+
119+
r_filename = res_params["filename"]
109120
suffixes = Path(r_filename).suffixes
110121

111122
if len(suffixes) > 1 and suffixes[-1] == ".jpg":
@@ -123,6 +134,7 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None,
123134
return filename
124135

125136
def get_observation(self, observation_id, instrument_name, *, filename=None,
137+
observation_oid=None, instrument_oid=None, product_level=None,
126138
verbose=False, download_dir="", cache=True, **kwargs):
127139
"""
128140
Download observation from Herschel.
@@ -171,6 +183,15 @@ def get_observation(self, observation_id, instrument_name, *, filename=None,
171183
'observation_id': observation_id,
172184
'instrument_name': instrument_name}
173185

186+
if observation_oid is not None:
187+
params['observation_oid'] = observation_oid
188+
189+
if instrument_oid is not None:
190+
params['instrument_oid'] = instrument_oid
191+
192+
if product_level is not None:
193+
params['product_level'] = product_level
194+
174195
link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items())
175196

176197
link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items())
@@ -179,15 +200,15 @@ def get_observation(self, observation_id, instrument_name, *, filename=None,
179200
log.info(link)
180201

181202
response = self._request('HEAD', link, save=False, cache=cache)
203+
if response.status_code == 401:
204+
error = "Data protected by proprietary rights. Please check your credentials"
205+
raise LoginError(error)
206+
182207
response.raise_for_status()
183208

184-
if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']:
185-
_, params = cgi.parse_header(response.headers['Content-Disposition'])
186-
else:
187-
error = "Data protected by propietary rights. Please check your credentials"
188-
raise LoginError(error)
209+
_, res_params = cgi.parse_header(response.headers['Content-Disposition'])
189210

190-
r_filename = params["filename"]
211+
r_filename = res_params["filename"]
191212
suffixes = Path(r_filename).suffixes
192213

193214
if filename is None:
@@ -359,7 +380,7 @@ def get_columns(self, table_name, *, only_names=True, verbose=False):
359380
else:
360381
return columns
361382

362-
def query_observations(self, coordinate, radius, n_obs=10):
383+
def query_observations(self, coordinate, radius, *, n_obs=10):
363384
"""
364385
Get the observation IDs from a given region
365386

‎astroquery/esa/hsa/tests/test_hsa_remote.py

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,44 @@
44

55
import os
66
import tarfile
7+
from requests.exceptions import ChunkedEncodingError
78

89
from ..core import HSAClass
910

10-
spire_chksum = [10243, 10772, 9029, 11095, 3875, 3893, 3903, 3868, 3888, 3951, 3905, 3875, 3887, 3908, 3871, 3892, 3894, 3900, 3870, 3892, 3938, 3904, 3893, 3866, 3893, 3930, 3901, 3867, 3893, 3927, 3903, 3868, 3892, 3926, 3904, 3869, 3889, 3937, 3899, 3868, 3887, 3928, 3912, 3865, 3887, 3937, 3899, 3865, 3891, 3925, 3911, 3868, 3883, 3937, 3902, 3870, 3893, 3922, 3914, 3869, 3886, 3924, 3903, 3870, 3889, 3922, 3903, 3867, 3889, 3932, 3902, 3869, 3892, 3930, 3900, 3868, 3892, 3916, 3910, 3868, 3889, 3932, 3907, 3869, 3892, 3914, 3900, 3871, 3884, 3918, 3903, 3894, 3874, 3892, 3906, 3872, 3889, 3895, 9724, 10495, 9293, 11299, 12132, 10602, 11283, 9582, 10568, 11354, 11228, 11812, 11006, 11224, 11382, 10232, 11049, 11011, 3862, 9671, 10844, 10899, 3946, 3941, 3953, 3954, 3948, 3940, 3954, 3948, 3945, 9286, 11197, 3902, 10137, 11664, 12038, 3963, 3948, 11970, 3867, 11452, 3908, 3908, 3894, 3908, 3908, 3894, 3885, 3914, 3876, 11820, 3958, 3905, 3963, 3925, 3974, 3923, 12241, 3897, 3900, 12328, 11911, 3893, 3885, 3895, 3895, 3891, 11970, 12252, 3899, 12503, 12135, 11826, 3899, 3899, 3912, 3912, 11530, 11625, 11850, 10981, 3872, 3890, 3896, 3895, 3867, 3891, 10561, 10991, 11209, 3853, 3871, 3869, 11372, 10826, 10995, 3887, 3882, 3883, 3879, 3884, 3883, 3889, 3883, 3887, 3876, 3885, 3883, 3882, 3888, 3886, 3882, 3882, 3882, 3887, 3885, 3885, 3884, 3880, 3881, 3878, 3882, 3884, 3879, 3887, 3882, 3884, 3877, 3883, 3882, 3881, 3882, 3876, 3879, 3881, 3881, 3884, 3882, 3882, 3882, 3888, 3883, 3880, 3880, 3875, 3880, 3884, 3883, 3887, 3880, 3879, 3874, 3882, 3879, 3887, 3876, 3888, 3888, 3885, 3885, 3876, 3880, 3887, 3877, 3884, 3886, 3879, 3879, 3877, 3880, 3876, 3889, 3885, 3886, 3885, 3879, 3879, 12424, 10112, 10885, 3895, 3922, 3888, 3900, 3898, 3919, 3925, 3930, 3923, 3932, 3931, 3917, 3922, 3917, 3919, 3935, 3929, 3923, 3929, 3924, 3906, 3918, 3895, 3892, 11322, 10879, 3944, 11338, 3921, 11009, 10969, 11352, 10984, 3944, 11345, 11333, 11088, 11331, 11099, 11324, 11118, 10591, 11502, 6291]
11+
spire_chksum = [10233, 10762, 9019, 10869, 3944, 11328, 3921, 10999, 10959, 11342, 10974, 3944, 11335, 11323, 11078, 11321, 11089, 11314, 11108, 6281]
1112

12-
pacs_chksum = [10205, 10752, 8914, 3869, 9736, 3925, 3942, 11201, 10454, 11393, 10520, 11080, 3979, 11121, 3961, 9745, 10492, 9317, 11304, 12025, 10584, 11295, 9564, 10585, 11367, 11229, 11808, 11039, 11225, 11411, 10246, 11066, 11013, 10158, 3939, 3927, 3941, 3936, 3942, 3925, 9691, 10381, 10944, 3990, 10970, 3969, 9453, 10802, 3881, 11926, 3896, 12528, 3908, 11898, 3883, 3894, 3878, 11231, 3910, 3885, 3876, 3913, 3967, 3970, 3908, 3903, 3901, 3958, 3898, 3870, 3915, 3878, 3895, 3901, 3888, 3880, 3922, 3881, 3926, 3883, 12229, 3958, 3949, 3936, 3962, 3949, 3947, 3868, 13210, 3885, 3901, 11463, 3939, 3967, 3878, 3900, 3911, 3947, 3917, 3940, 3953, 3904, 3869, 3874, 3880, 3893, 3964, 3980, 3889, 3871, 3908, 3882, 3888, 3895, 3905, 3889, 13163, 3879, 11036, 12469, 3886, 3887, 3863, 3921, 3881, 3911, 3878, 3921, 11149, 13032, 3880, 3881, 3883, 3883, 3895, 3899, 3919, 12498, 3896, 3942, 3894, 11407, 3866, 11716, 3889, 3890, 3888, 3873, 9870, 10738, 3978, 10759, 3949, 10720, 3921, 10390, 3938, 10365, 11182, 3905, 10724, 3943, 10742, 3942, 10025, 3924, 3935, 10388, 11185, 11206, 6288]
13+
pacs_chksum = [10208, 10755, 8917, 10028, 3924, 3935, 6291]
1314

1415

1516
@pytest.mark.remote_data
1617
class TestHSARemote:
1718
tmp_dir = tempfile.TemporaryDirectory()
19+
retries = 2
20+
21+
def access_archive_with_retries(self, f, params):
22+
for _ in range(self.retries):
23+
try:
24+
res = f(**params)
25+
return res
26+
except ChunkedEncodingError:
27+
pass
28+
return None
29+
1830

1931
def test_download_data_observation_pacs(self):
2032
obs_id = "1342191813"
2133
parameters = {'retrieval_type': "OBSERVATION",
2234
'observation_id': obs_id,
2335
'instrument_name': "PACS",
36+
'product_level': 'LEVEL3',
37+
'cache': False,
2438
'download_dir': self.tmp_dir.name}
2539
expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar")
2640
hsa = HSAClass()
27-
res = hsa.download_data(**parameters)
41+
#res = hsa.download_data(**parameters)
42+
res = self.access_archive_with_retries(hsa.download_data, parameters)
43+
if res == None:
44+
pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries))
2845
assert res == expected_res
2946
assert os.path.isfile(res)
3047
chksum = []
@@ -40,11 +57,16 @@ def test_download_data_observation_pacs_filename(self):
4057
parameters = {'retrieval_type': "OBSERVATION",
4158
'observation_id': obs_id,
4259
'instrument_name': "PACS",
60+
'product_level': 'LEVEL3',
4361
'filename': fname,
62+
'cache': False,
4463
'download_dir': self.tmp_dir.name}
4564
expected_res = os.path.join(self.tmp_dir.name, fname + ".tar")
4665
hsa = HSAClass()
47-
res = hsa.download_data(**parameters)
66+
#res = hsa.download_data(**parameters)
67+
res = self.access_archive_with_retries(hsa.download_data, parameters)
68+
if res == None:
69+
pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries))
4870
assert res == expected_res
4971
assert os.path.isfile(res)
5072
chksum = []
@@ -59,11 +81,16 @@ def test_download_data_observation_pacs_compressed(self):
5981
parameters = {'retrieval_type': "OBSERVATION",
6082
'observation_id': obs_id,
6183
'instrument_name': "PACS",
84+
'product_level': 'LEVEL3',
6285
'compress': 'true',
86+
'cache': False,
6387
'download_dir': self.tmp_dir.name}
6488
expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tgz")
6589
hsa = HSAClass()
66-
res = hsa.download_data(**parameters)
90+
#res = hsa.download_data(**parameters)
91+
res = self.access_archive_with_retries(hsa.download_data, parameters)
92+
if res == None:
93+
pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries))
6794
assert res == expected_res
6895
assert os.path.isfile(res)
6996
chksum = []
@@ -78,10 +105,15 @@ def test_download_data_observation_spire(self):
78105
parameters = {'retrieval_type': "OBSERVATION",
79106
'observation_id': obs_id,
80107
'instrument_name': "SPIRE",
108+
'product_level': 'LEVEL2',
109+
'cache': False,
81110
'download_dir': self.tmp_dir.name}
82111
expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar")
83112
hsa = HSAClass()
84-
res = hsa.download_data(**parameters)
113+
#res = hsa.download_data(**parameters)
114+
res = self.access_archive_with_retries(hsa.download_data, parameters)
115+
if res == None:
116+
pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries))
85117
assert res == expected_res
86118
assert os.path.isfile(res)
87119
chksum = []
@@ -96,10 +128,14 @@ def test_download_data_postcard_pacs(self):
96128
parameters = {'retrieval_type': "POSTCARD",
97129
'observation_id': obs_id,
98130
'instrument_name': "PACS",
131+
'cache': False,
99132
'download_dir': self.tmp_dir.name}
100133
expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg")
101134
hsa = HSAClass()
102-
res = hsa.download_data(**parameters)
135+
#res = hsa.download_data(**parameters)
136+
res = self.access_archive_with_retries(hsa.download_data, parameters)
137+
if res == None:
138+
pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries))
103139
assert res == expected_res
104140
assert os.path.isfile(res)
105141
os.remove(res)
@@ -111,10 +147,14 @@ def test_download_data_postcard_pacs_filename(self):
111147
'observation_id': obs_id,
112148
'instrument_name': "PACS",
113149
'filename': fname,
150+
'cache': False,
114151
'download_dir': self.tmp_dir.name}
115152
expected_res = os.path.join(self.tmp_dir.name, fname + ".jpg")
116153
hsa = HSAClass()
117-
res = hsa.download_data(**parameters)
154+
#res = hsa.download_data(**parameters)
155+
res = self.access_archive_with_retries(hsa.download_data, parameters)
156+
if res == None:
157+
pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries))
118158
assert res == expected_res
119159
assert os.path.isfile(res)
120160
os.remove(res)
@@ -123,10 +163,15 @@ def test_get_observation(self):
123163
obs_id = "1342191813"
124164
parameters = {'observation_id': obs_id,
125165
'instrument_name': "PACS",
166+
'product_level': 'LEVEL3',
167+
'cache': False,
126168
'download_dir': self.tmp_dir.name}
127169
expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar")
128170
hsa = HSAClass()
129-
res = hsa.get_observation(**parameters)
171+
#res = hsa.get_observation(**parameters)
172+
res = self.access_archive_with_retries(hsa.get_observation, parameters)
173+
if res == None:
174+
pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries))
130175
assert res == expected_res
131176
assert os.path.isfile(res)
132177
chksum = []
@@ -140,10 +185,14 @@ def test_get_postcard(self):
140185
obs_id = "1342191813"
141186
parameters = {'observation_id': obs_id,
142187
'instrument_name': "PACS",
188+
'cache': False,
143189
'download_dir': self.tmp_dir.name}
144190
expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg")
145191
hsa = HSAClass()
146-
res = hsa.get_postcard(**parameters)
192+
#res = hsa.get_postcard(**parameters)
193+
res = self.access_archive_with_retries(hsa.get_postcard, parameters)
194+
if res == None:
195+
pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries))
147196
assert res == expected_res
148197
assert os.path.isfile(res)
149198
os.remove(res)

0 commit comments

Comments
 (0)