diff --git a/CHANGES.rst b/CHANGES.rst index a4e04bfa1a..9ca9112009 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,11 @@ New Tools and Services API changes ----------- +esa.hubble +^^^^^^^^^^ + +- Migration to PyVO [#3367] + gaia ^^^^ diff --git a/astroquery/esa/hubble/__init__.py b/astroquery/esa/hubble/__init__.py index 427d0e0154..72c7b61609 100644 --- a/astroquery/esa/hubble/__init__.py +++ b/astroquery/esa/hubble/__init__.py @@ -9,16 +9,23 @@ """ + from astropy import config as _config from astropy.config import paths import os +EHST_COMMON_SERVER = "https://hst.esac.esa.int/tap-server/" +EHST_TAP_COMMON = "tap" + class Conf(_config.ConfigNamespace): """ Configuration parameters for `astroquery.esa.hubble`. """ - EHST_TAP_SERVER = _config.ConfigItem("https://hst.esac.esa.int/tap-server/tap", "eHST TAP Server") + EHST_DOMAIN_SERVER = _config.ConfigItem(EHST_COMMON_SERVER, "eHST TAP Common Server") + EHST_TAP_SERVER = _config.ConfigItem(EHST_COMMON_SERVER + EHST_TAP_COMMON, "eHST TAP Server") + EHST_DATA_SERVER = _config.ConfigItem(EHST_COMMON_SERVER + 'data?', "eHST Data Server") + EHST_TABLES_SERVER = _config.ConfigItem(EHST_COMMON_SERVER + EHST_TAP_COMMON + "/tables", "eHST TAP Common Server") EHST_TARGET_ACTION = _config.ConfigItem("servlet/target-resolver?", "eHST Target Resolver") EHST_MESSAGES = _config.ConfigItem("notification?action=GetNotifications", "eHST Messages") TIMEOUT = 60 diff --git a/astroquery/esa/hubble/core.py b/astroquery/esa/hubble/core.py index fdb554913a..2f1bf226d2 100644 --- a/astroquery/esa/hubble/core.py +++ b/astroquery/esa/hubble/core.py @@ -8,8 +8,9 @@ European Space Agency (ESA) """ + + import os -from urllib.parse import urlencode import astroquery.esa.utils.utils as esautils @@ -18,12 +19,10 @@ from astropy.coordinates import Angle from numpy.ma import MaskedArray -from astroquery.utils.tap import TapPlus -from astroquery.query import BaseQuery -import json +from astroquery.query import BaseQuery, BaseVOQuery import warnings +import pyvo from astropy.utils.exceptions import AstropyDeprecationWarning -from astropy.utils.decorators import deprecated from . import conf from astroquery import log @@ -31,7 +30,7 @@ __all__ = ['ESAHubble', 'ESAHubbleClass'] -class ESAHubbleClass(BaseQuery): +class ESAHubbleClass(BaseVOQuery, BaseQuery): """ Class to init ESA Hubble Module and communicate with eHST TAP """ @@ -41,15 +40,27 @@ class ESAHubbleClass(BaseQuery): product_types = ["SCIENCE", "PREVIEW", "THUMBNAIL", "AUXILIARY"] copying_string = "Copying file to {0}..." - def __init__(self, *, tap_handler=None, show_messages=True): - if tap_handler is None: - self._tap = TapPlus(url=conf.EHST_TAP_SERVER, - data_context='data', client_id="ASTROQUERY") + def __init__(self, *, show_messages=True, auth_session=None): + super().__init__() + + self._tap = None + # Checks if auth session has been defined. If not, create a new session + if auth_session: + self._auth_session = auth_session else: - self._tap = tap_handler + self._auth_session = esautils.ESAAuthSession() + if show_messages: self.get_status_messages() + @property + def tap(self) -> pyvo.dal.TAPService: + if self._tap is None: + self._tap = pyvo.dal.TAPService( + conf.EHST_TAP_SERVER, session=self._auth_session) + + return self._tap + def download_product(self, observation_id, *, calibration_level=None, filename=None, folder=None, verbose=False, product_type=None): """ @@ -95,7 +106,6 @@ def download_product(self, observation_id, *, calibration_level=None, AstropyDeprecationWarning) params = {"OBSERVATIONID": observation_id, - "TAPCLIENT": "ASTROQUERY", "RETRIEVAL_TYPE": "OBSERVATION"} if filename is None: @@ -112,8 +122,13 @@ def download_product(self, observation_id, *, calibration_level=None, filename = self._get_product_filename(product_type, filename) output_file = self.__get_download_path(folder, filename) - self._tap.load_data(params_dict=params, output_file=output_file, verbose=verbose) - + esautils.download_file( + url=conf.EHST_DATA_SERVER, + session=self.tap._session, + params=params, + verbose=verbose, + filename=output_file + ) return esautils.check_rename_to_gz(filename=output_file) def __get_download_path(self, folder, filename): @@ -417,12 +432,17 @@ def download_file(self, file, *, filename=None, folder=None, verbose=False): None. The file is associated """ - params = {"RETRIEVAL_TYPE": "PRODUCT", "ARTIFACTID": file, "TAPCLIENT": "ASTROQUERY"} + params = {"RETRIEVAL_TYPE": "PRODUCT", "ARTIFACTID": file} if filename is None: filename = file output_file = self.__get_download_path(folder, filename) - self._tap.load_data(params_dict=params, output_file=output_file, verbose=verbose) - + esautils.download_file( + url=conf.EHST_DATA_SERVER, + session=self.tap._session, + params=params, + verbose=verbose, + filename=output_file + ) return esautils.check_rename_to_gz(filename=output_file) def get_postcard(self, observation_id, *, calibration_level="RAW", @@ -458,8 +478,7 @@ def get_postcard(self, observation_id, *, calibration_level="RAW", params = {"RETRIEVAL_TYPE": "OBSERVATION", "OBSERVATIONID": observation_id, - "PRODUCTTYPE": "PREVIEW", - "TAPCLIENT": "ASTROQUERY"} + "PRODUCTTYPE": "PREVIEW"} if calibration_level: params["CALIBRATIONLEVEL"] = calibration_level @@ -469,8 +488,8 @@ def get_postcard(self, observation_id, *, calibration_level="RAW", if filename is None: filename = observation_id - self._tap.load_data(params_dict=params, output_file=filename, verbose=verbose) - + esautils.download_file(url=conf.EHST_DATA_SERVER, session=self.tap._session, params=params, verbose=verbose, + filename=filename) return filename def __get_product_type_by_resolution(self, resolution): @@ -670,18 +689,17 @@ def _query_tap_target(self, target): try: params = {"TARGET_NAME": target, "RESOLVER_TYPE": "ALL", - "FORMAT": "json", - "TAPCLIENT": "ASTROQUERY"} - - subContext = conf.EHST_TARGET_ACTION - connHandler = self._tap._TapPlus__getconnhandler() - data = urlencode(params) - target_response = connHandler.execute_secure(subContext, data, verbose=True) - for line in target_response: - target_result = json.loads(line.decode("utf-8")) - if target_result['objects']: - ra = target_result['objects'][0]['raDegrees'] - dec = target_result['objects'][0]['decDegrees'] + "FORMAT": "json"} + + target_response = esautils.execute_servlet_request( + tap=self.tap, + query_params=params, + url=conf.EHST_DOMAIN_SERVER + conf.EHST_TARGET_ACTION + ) + if target_response: + if target_response['objects']: + ra = target_response['objects'][0]['raDegrees'] + dec = target_response['objects'][0]['decDegrees'] return SkyCoord(ra=ra, dec=dec, unit="deg") except (ValueError, KeyError): raise ValueError("This target cannot be resolved") @@ -747,45 +765,15 @@ def query_tap(self, query, *, async_job=False, output_file=None, A table object """ if async_job: - job = self._tap.launch_job_async(query=query, - output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=output_file is not None) + query_result = self.tap.run_async(query) + result = query_result.to_table() else: - job = self._tap.launch_job(query=query, output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=output_file is not None) - table = job.get_results() - return table + result = self.tap.search(query).to_table() - @deprecated(since="0.4.7", alternative="query_tap") - def query_hst_tap(self, query, *, async_job=False, output_file=None, - output_format="votable", verbose=False): - """Launches a synchronous or asynchronous job to query the HST tap + if output_file: + esautils.download_table(result, output_file, output_format) - Parameters - ---------- - query : str, mandatory - query (adql) to be executed - async_job : bool, optional, default 'False' - executes the query (job) in asynchronous/synchronous mode (default - synchronous) - output_file : str, optional, default None - file name where the results are saved if dumpToFile is True. - If this parameter is not provided, the jobid is used instead - output_format : str, optional, default 'votable' - results format - verbose : bool, optional, default 'False' - flag to display information about the process - - Returns - ------- - A table object - """ - self.query_tap(query=query, async_job=False, output_file=None, - output_format="votable", verbose=False) + return result def query_criteria(self, *, calibration_level=None, data_product_type=None, intent=None, @@ -923,17 +911,11 @@ def get_tables(self, *, only_names=True, verbose=False): ------- A list of tables """ - - tables = self._tap.load_tables(only_names=only_names, - include_shared_tables=False, - verbose=verbose) - if only_names is True: - table_names = [] - for t in tables: - table_names.append(t.name) - return table_names + tables = self.tap.tables + if only_names: + return list(tables.keys()) else: - return tables + return list(tables.values()) def get_status_messages(self): """Retrieve the messages to inform users about @@ -941,16 +923,23 @@ def get_status_messages(self): """ try: - subContext = conf.EHST_MESSAGES - connHandler = self._tap._TapPlus__getconnhandler() - response = connHandler.execute_tapget(subContext, verbose=False) - if response.status == 200: - for line in response: - string_message = line.decode("utf-8") - print(string_message[string_message.index('=') + 1:]) + esautils.execute_servlet_request( + url=conf.EHST_TAP_SERVER + "/" + conf.EHST_MESSAGES, + tap=self.tap, + query_params={}, + parser_method=self.parse_messages_response + ) except OSError: print("Status messages could not be retrieved") + def parse_messages_response(self, response): + string_messages = [] + for line in response.iter_lines(): + string_message = line.decode("utf-8") + string_messages.append(string_message[string_message.index('=') + 1:]) + print(string_messages[len(string_messages)-1]) + return string_messages + def get_columns(self, table_name, *, only_names=True, verbose=False): """Get the available columns for a table in EHST TAP service @@ -968,9 +957,7 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): A list of columns """ - tables = self._tap.load_tables(only_names=False, - include_shared_tables=False, - verbose=verbose) + tables = self.get_tables(only_names=False) columns = None for t in tables: if str(t.name) == str(table_name): @@ -1061,4 +1048,5 @@ def get_datalabs_path(self, filename, default_volume=None): return full_path -ESAHubble = ESAHubbleClass() +# Need to be False in order to avoid reaching out to the remote server at import time +ESAHubble = ESAHubbleClass(show_messages=False) diff --git a/astroquery/esa/hubble/tests/data/cone_search.vot b/astroquery/esa/hubble/tests/data/cone_search.vot index 214cc99bdf..88d184a20b 100644 --- a/astroquery/esa/hubble/tests/data/cone_search.vot +++ b/astroquery/esa/hubble/tests/data/cone_search.vot @@ -1,6 +1,6 @@ ESA HST Archive Metadata Query Service - +
o58502w0q1999-07-24 14:14:57.271999-07-24 14:18:29.43351383.5103850694451383.5128406597220.12000-07-24 14:18:29.0w058502HSTscience0APERTURE=OV28X50LP|ASNID=NONE|DETECTOR=CCD|EXPEND=51383.51284065972|EXPSTART=51383.51038506944|EXPTIME=20.1|FILTER=MIRVIS|OBSMODE=ACQ|OBSTYPE=IMAGINGHST SingletonfalseM31BROAD_CATEGORY=GALAXY|TARGET_DESCRIP=GALAXY;SPIRAL8018Green, Richard F.Demographics of Nuclear Black HolesSTIS/CCDCRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2aimageCRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2a10.684572524500547241.26917617069089770.00197465062424587775MIRVIS
diff --git a/astroquery/esa/hubble/tests/test_esa_hubble.py b/astroquery/esa/hubble/tests/test_esa_hubble.py index c26d5ecdc6..f36e979575 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble.py @@ -10,22 +10,26 @@ """ import os -import shutil +import functools import gzip +from collections import Counter from pathlib import Path -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock from unittest.mock import patch +import xml.etree.ElementTree as ET +from requests.models import Response +import io + import numpy as np import pytest from astropy import coordinates from astropy.table.table import Table -from requests.models import Response +from pyvo.dal import DALQuery from astroquery.esa.hubble import ESAHubbleClass import astroquery.esa.utils.utils as esautils from astroquery.esa.hubble.tests.dummy_tap_handler import DummyHubbleTapHandler -from astropy.utils.exceptions import AstropyDeprecationWarning def data_path(filename): @@ -33,47 +37,16 @@ def data_path(filename): return os.path.join(data_dir, filename) -def get_mockreturn(method, request, url, params, *args, **kwargs): - file = 'm31.vot' - if 'OBSERVATION_ID' in params: - file = params['OBSERVATION_ID'] + ".vot" - response = data_path(file) - shutil.copy(response + '.test', response) - return response - - -@pytest.fixture(autouse=True) -def ehst_request(request): - try: - mp = request.getfixturevalue("monkeypatch") - except AttributeError: - mp = request.getfuncargvalue("monkeypatch") - mp.setattr(ESAHubbleClass, '_request', get_mockreturn) - return mp - - -def get_cone_mockreturn(params, *args, **kwargs): - file = data_path('cone_search_m31_5.vot') - if 'OBSERVATION_ID' in kwargs: - file = kwargs['OBSERVATION_ID'] + ".vot" - response = data_path(file) - shutil.copy(response + '.test', response) - return response - - -@pytest.fixture(autouse=True) -def ehst_cone_search(request): - mp = request.getfixturevalue("monkeypatch") - mp.setattr(ESAHubbleClass, 'cone_search', get_cone_mockreturn) - return mp - +class FakeHTTPResponse: + def __init__(self, data): + self._data = data + self._read_called = False -class MockResponse: - observation_id = 'test' - - @staticmethod - def pformat(): - return True + def read(self, decode_content=True, **kwargs): + if not self._read_called: + self._read_called = True + return self._data + return b"" # EOF after first read class TestESAHubble: @@ -89,108 +62,165 @@ def get_dummy_tap_handler(self, method='launch_job', query=None): return dummyTapHandler def test_download_product_errors(self): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) with pytest.raises(ValueError) as err: ehst.download_product(observation_id="J6FL25S4Q", product_type="DUMMY") assert "This product_type is not allowed" in err.value.args[0] - def test_download_product_by_calibration(self, tmp_path): + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') + @patch('astroquery.esa.utils.utils.check_rename_to_gz') + def test_download_product_by_calibration(self, rename_mock, download_mock, tmp_path): + path = Path(tmp_path, "J6FL25S4Q.vot.test") parameters = {'observation_id': "J6FL25S4Q", 'cal_level': "RAW", - 'filename': Path(tmp_path, "J6FL25S4Q.vot.test"), + 'filename': path, 'verbose': True} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.download_product(observation_id=parameters['observation_id'], - calibration_level=parameters['cal_level'], - filename=parameters['filename'], - verbose=parameters['verbose']) - - def test_download_product_by_product_type(self, tmp_path): + rename_mock.return_value = path + ehst = ESAHubbleClass(show_messages=False) + result = ehst.download_product( + observation_id=parameters['observation_id'], + calibration_level=parameters['cal_level'], + filename=parameters['filename'], + verbose=parameters['verbose']) + assert rename_mock.call_count == 1 + assert download_mock.call_count == 1 + assert result == path + + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') + @patch('astroquery.esa.utils.utils.check_rename_to_gz') + def test_download_product_by_product_type(self, rename_mock, download_mock, tmp_path): + path = Path(tmp_path, "J6FL25S4Q.vot.test") parameters = {'observation_id': "J6FL25S4Q", 'product_type': "SCIENCE", - 'filename': Path(tmp_path, "J6FL25S4Q.vot.test"), + 'filename': path, 'verbose': True} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.download_product(observation_id=parameters['observation_id'], - product_type=parameters['product_type'], - filename=parameters['filename'], - verbose=parameters['verbose']) + rename_mock.return_value = path + ehst = ESAHubbleClass(show_messages=False) + result = ehst.download_product( + observation_id=parameters['observation_id'], + product_type=parameters['product_type'], + filename=parameters['filename'], + verbose=parameters['verbose']) + assert rename_mock.call_count == 1 + assert download_mock.call_count == 1 + assert result == path + parameters['product_type'] = "SCIENCE" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.download_product(observation_id=parameters['observation_id'], - product_type=parameters['product_type'], - filename=parameters['filename'], - verbose=parameters['verbose']) - parameters['product_type'] = "PREVIEW" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.download_product(observation_id=parameters['observation_id'], - product_type=parameters['product_type'], - filename=parameters['filename'], - verbose=parameters['verbose']) - - def test_get_postcard(self, tmp_path): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.get_postcard(observation_id="X0MC5101T", - filename=Path(tmp_path, "X0MC5101T.vot"), - verbose=True) - ehst.get_postcard(observation_id="X0MC5101T", - filename=Path(tmp_path, "X0MC5101T.vot"), resolution=1024, - verbose=True) + ehst = ESAHubbleClass(show_messages=False) + result = ehst.download_product( + observation_id=parameters['observation_id'], + product_type=parameters['product_type'], + filename=parameters['filename'], + verbose=parameters['verbose']) + + assert rename_mock.call_count == 2 + assert download_mock.call_count == 2 + assert result == path + parameters['product_type'] = "PREVIEW" + ehst = ESAHubbleClass(show_messages=False) + result = ehst.download_product( + observation_id=parameters['observation_id'], + product_type=parameters['product_type'], + filename=parameters['filename'], + verbose=parameters['verbose']) + + assert rename_mock.call_count == 3 + assert download_mock.call_count == 3 + assert result == path + + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') + def test_get_postcard(self, download_mock, tmp_path): + path = Path(tmp_path, "X0MC5101T.vot") + ehst = ESAHubbleClass(show_messages=False) + observation_id = "X0MC5101T" + result = ehst.get_postcard( + observation_id=observation_id, + filename=path.__str__(), + verbose=True) + args, kwargs = download_mock.call_args + assert kwargs["params"]["PRODUCTTYPE"] == 'THUMBNAIL' + assert download_mock.call_count == 1 + assert result == path.__str__() + + result = ehst.get_postcard( + observation_id=observation_id, + filename=path.__str__(), resolution=1024, + verbose=True) + args, kwargs = download_mock.call_args + assert kwargs["params"]["PRODUCTTYPE"] == 'PREVIEW' + assert download_mock.call_count == 2 + assert result == path.__str__() + + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) @patch.object(ESAHubbleClass, 'cone_search') - @patch.object(ESAHubbleClass, '_query_tap_target') - def test_query_target(self, mock_query_tap_target, mock_cone_search): - mock_query_tap_target.return_value = 10, 10 + @patch('astroquery.esa.utils.utils.execute_servlet_request') + def test_query_target(self, mock_servlet_request, mock_cone_search): + mock_servlet_request.return_value = {'objects': [{"raDegrees": 90, "decDegrees": 90}]} mock_cone_search.return_value = "test" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) table = ehst.query_target(name="test") assert table == "test" + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) def test_cone_search(self): coords = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + # query_tap_mock.return_value = "test result" + ehst = ESAHubbleClass(show_messages=False) parameters = {'coordinates': coords, 'radius': 0.0, - 'filename': 'file_cone', 'output_format': 'votable', 'cache': True} target_file = data_path('cone_search.vot') with open(target_file, mode='rb') as file: target_obj = file.read() - response = Response() - response._content = target_obj - ehst._request = MagicMock(return_value=response) - ehst.cone_search(coordinates=parameters['coordinates'], - radius=parameters['radius'], - filename=parameters['filename'], - output_format=parameters['output_format'], - cache=parameters['cache']) - DummyHubbleTapHandler("cone_search", parameters) - - def test_cone_search_coords(self): + fake_response = FakeHTTPResponse(target_obj) + read_partial = functools.partial(fake_response.read) + mock_response = MagicMock() + mock_response.read = read_partial + with patch.object(DALQuery, 'execute_stream', return_value=mock_response): + + result = ehst.cone_search( + coordinates=parameters['coordinates'], + radius=parameters['radius'], + output_format=parameters['output_format'], + cache=parameters['cache']) + + # Trying to get number of elements read + # Define the namespace map + ns = {'v': 'http://www.ivoa.net/xml/VOTable/v1.2'} + # Parse XML string + root = ET.fromstring(target_obj.decode('utf8')) + # Find all TR elements inside TABLEDATA (with namespace) + trs = root.findall('.//v:TABLEDATA/v:TR', ns) + # Extract first and count occurrences + first_column_values = [tr.find('v:TD', ns).text for tr in trs if tr.find('v:TD', ns) is not None] + # Count unique values + counts = Counter(first_column_values) + unique_list = list(counts.keys()) + + assert len(unique_list) == len(result) + + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService') + def test_cone_search_coords(self, mock_tap): coords = "00h42m44.51s +41d16m08.45s" - parameterst = {'query': "select top 10 * from hsc_v2.hubble_sc2", - 'output_file': "test2.vot", - 'output_format': "votable", - 'verbose': True} - dummyTapHandler = DummyHubbleTapHandler("launch_job", parameterst) - parameters = {'coordinates': coords, 'radius': 0.0, - 'filename': 'file_cone', 'async_job': False, 'output_format': 'votable', 'cache': True, 'verbose': True} - ehst = ESAHubbleClass(tap_handler=dummyTapHandler, show_messages=False) + ehst = ESAHubbleClass(show_messages=False) ehst.cone_search(coordinates=parameters['coordinates'], radius=parameters['radius'], - filename=parameters['filename'], output_format=parameters['output_format'], async_job=parameters['async_job'], cache=parameters['cache'], @@ -200,100 +230,120 @@ def test_cone_search_coords(self): assert "Coordinates must be either a string or " \ "astropy.coordinates" in err.value.args[0] - def test_query_tap(self): + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.search') + def test_query_tap(self, mock_search): parameters = {'query': "select top 10 * from hsc_v2.hubble_sc2", 'async_job': False, 'output_file': "test2.vot", 'output_format': "votable", 'verbose': False} - parameters2 = {'query': "select top 10 * from hsc_v2.hubble_sc2", - 'output_file': "test2.vot", - 'output_format': "votable", - 'verbose': False} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) ehst.query_tap(query=parameters['query'], async_job=parameters['async_job'], output_file=parameters['output_file'], output_format=parameters['output_format'], verbose=parameters['verbose']) - self.get_dummy_tap_handler().check_call("launch_job", parameters2) - - def test_get_tables(self): - parameters = {'only_names': True, - 'verbose': True} - - DummyHubbleTapHandler("get_tables", parameters) - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.get_tables(only_names=True, verbose=True) - def test_get_artifact(self, tmp_path): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - path = Path(tmp_path, "w0ji0v01t_c2f.fits.gz") - ehst.get_artifact(artifact_id=path) + mock_search.assert_called_once_with(parameters['query']) - def test_download_file(self, tmp_path): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + def test_get_tables(self): + table_set = PropertyMock() + table_set.keys.return_value = ['caom2.harveststate', 'caom2.publications'] + table_set.values.return_value = ['caom2.harveststate', 'caom2.publications'] + with patch('astroquery.esa.integral.core.pyvo.dal.TAPService', autospec=True) as hubble_mock: + hubble_mock.return_value.tables = table_set + ehst = ESAHubbleClass(show_messages=False) + tables = ehst.get_tables() + assert len(tables) == 2 + + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') + @patch('astroquery.esa.utils.utils.check_rename_to_gz') + def test_get_artifact(self, rename_mock, download_mock, tmp_path): + filename = "w0ji0v01t_c2f.fits.gz" + ehst = ESAHubbleClass(show_messages=False) + path = Path(tmp_path, filename) + rename_mock.return_value = path + assert ehst.get_artifact(artifact_id=path) == path + + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') + @patch('astroquery.esa.utils.utils.check_rename_to_gz') + def test_download_file(self, rename_mock, download_mock, tmp_path): + ehst = ESAHubbleClass(show_messages=False) file = 'w0ji0v01t_c2f.fits' path = Path(tmp_path, file + '.gz') - ehst.download_file(file=path, filename=path) + rename_mock.return_value = path + assert ehst.download_file(file=path, filename=path) == path - def test_get_associated_files(self): + @patch.object(ESAHubbleClass, 'tap', new_callable=PropertyMock) + def test_get_associated_files(self, mock_tap_prop): observation_id = 'test' - query = (f"select art.artifact_id as filename, p.calibration_level, art.archive_class as type, " - f"pg_size_pretty(art.size_uncompr) as size_uncompressed from ehst.artifact art " - f"join ehst.plane p on p.plane_id = art.plane_id where " - f"art.observation_id = '{observation_id}'") - parameters = {'query': query, - 'output_file': 'test2.vot', - 'output_format': "votable", - 'verbose': False} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(query=query), show_messages=False) - ehst.get_associated_files(observation_id=observation_id) - self.get_dummy_tap_handler(query=query).check_call("launch_job", parameters) + # Mock the return of self.vo + mock_tap_service = MagicMock() + mock_tap_prop.return_value = mock_tap_service + # Mock the chain: search().to_table() + mock_table = "mocked table result" + mock_search_result = MagicMock() + mock_search_result.to_table.return_value = mock_table + mock_tap_service.search.return_value = mock_search_result + + ehst = ESAHubbleClass(show_messages=False) + result = ehst.get_associated_files(observation_id=observation_id) + assert result == mock_table + + @patch.object(ESAHubbleClass, 'download_file') @patch.object(ESAHubbleClass, 'get_associated_files') - def test_download_fits(self, mock_associated_files): + def test_download_fits(self, mock_associated_files, mock_download_file): observation_id = 'test' - query = (f"select art.artifact_id as filename, p.calibration_level, art.archive_class as type, " - f"pg_size_pretty(art.size_uncompr) as size_uncompressed from ehst.artifact art " - f"join ehst.plane p on p.plane_id = art.plane_id where " - f"art.observation_id = '{observation_id}'") - parameters = {'query': query, - 'output_file': 'test2.vot', - 'output_format': "votable", - 'verbose': False} - mock_associated_files.return_value = [{'filename': 'test.fits'}] - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(query=query), show_messages=False) + filename = "test.fits" + path = "/dummy/path/" + filename + mock_associated_files.return_value = [{'filename': filename}] + mock_download_file.return_value = path + ehst = ESAHubbleClass(show_messages=False) ehst.download_fits_files(observation_id=observation_id) - self.get_dummy_tap_handler(query=query).check_call("launch_job", parameters) + mock_download_file.assert_called_once_with(file=filename, filename=filename, folder=None, verbose=False) def test_is_not_gz(self, tmp_path): target_file = data_path('cone_search.vot') - ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ESAHubbleClass(show_messages=False) assert esautils.check_rename_to_gz(target_file) in target_file def test_is_gz(self, tmp_path): - ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - # test_file = data_path('m31.vot.test') + ESAHubbleClass(show_messages=False) temp_file = 'testgz' target_file = os.path.join(tmp_path, temp_file) with gzip.open(target_file, 'wb') as f: f.write(b'') - # with open(test_file, 'rb') as f_in, gzip.open(target_file, 'wb') as f_out: - # f_out.writelines(f_in) assert esautils.check_rename_to_gz(target_file) in f"{target_file}.fits.gz" - def test_get_columns(self): - parameters = {'table_name': "table", - 'only_names': True, - 'verbose': True} - - dummyTapHandler = DummyHubbleTapHandler("get_columns", parameters) - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.get_columns(table_name="table", only_names=True, verbose=True) - dummyTapHandler.check_call("get_columns", parameters) - + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch.object(ESAHubbleClass, 'get_tables') + def test_get_columns(self, get_tables_mock): + # Create a mock VOSITable object + mock_table1 = MagicMock() + mock_table1.name = "table1" + mock_table1_col1 = MagicMock() + mock_table1_col1.name = "column1" + mock_table1_col2 = MagicMock() + mock_table1_col2.name = "column2" + mock_table1.columns = [mock_table1_col1, mock_table1_col2] + + mock_table2 = MagicMock() + mock_table2.name = "table2" + mock_table2.columns = [MagicMock(name="column3"), MagicMock(name="column4")] + + get_tables_mock.return_value = [mock_table1, mock_table2] + ehst = ESAHubbleClass(show_messages=False) + result = ehst.get_columns(table_name="table1", only_names=True, verbose=True) + assert len(result) == 2 + assert result[0] == "column1" + assert result[1] == "column2" + + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) def test_query_criteria_proposal(self): parameters1 = {'proposal': 12345, 'async_job': False, @@ -301,26 +351,18 @@ def test_query_criteria_proposal(self): 'output_format': "votable", 'verbose': True, 'get_query': True} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) test_query = ehst.query_criteria(proposal=parameters1['proposal'], async_job=parameters1['async_job'], output_file=parameters1['output_file'], output_format=parameters1['output_format'], verbose=parameters1['verbose'], get_query=parameters1['get_query']) - parameters2 = {'query': test_query, - 'output_file': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': False} - parameters3 = {'query': "select * from ehst.archive where(" - "proposal_id = '12345')", - 'output_file': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': False} - dummy_tap_handler = DummyHubbleTapHandler("launch_job", parameters2) - dummy_tap_handler.check_call("launch_job", parameters3) + assert test_query == "select * from ehst.archive where(proposal_id = '12345')" - def test_retrieve_observations_from_proposal(self): + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch.object(ESAHubbleClass, 'query_tap') + def test_retrieve_observations_from_proposal(self, mock_query_tap): program = 12345 parameters1 = {'proposal': program, 'async_job': False, @@ -328,30 +370,35 @@ def test_retrieve_observations_from_proposal(self): 'output_format': "votable", 'verbose': True, 'get_query': True} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) ehst.get_observations_from_program(program=parameters1['proposal']) - dummy_tap_handler = DummyHubbleTapHandler("launch_job", None) - dummy_tap_handler.check_method("launch_job") + mock_query_tap.assert_called_once_with( + query="select * from ehst.archive where(proposal_id = '12345')", output_file=None, + output_format=parameters1['output_format'], + async_job=parameters1['async_job'], + verbose=False) + @patch.object(ESAHubbleClass, 'download_file') @patch.object(ESAHubbleClass, 'get_associated_files') @patch.object(ESAHubbleClass, 'query_criteria') - def test_download_fits_from_proposal(self, mock_observations, mock_files): + def test_download_fits_from_proposal(self, mock_observations, mock_files, mock_download_file): mock_observations.return_value = {'observation_id': ['test']} mock_files.return_value = [{'filename': 'test.fits'}] - tap_handler = self.get_dummy_tap_handler("load_data") - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler("load_data"), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) ehst.download_files_from_program(program=12345, only_fits=True) - tap_handler.check_method("load_data") + mock_download_file.assert_called_once_with(file='test.fits', filename='test.fits', folder=None, verbose=False) + @patch.object(ESAHubbleClass, 'download_file') @patch.object(ESAHubbleClass, 'get_associated_files') @patch.object(ESAHubbleClass, 'query_criteria') - def test_download_all_from_proposal(self, mock_observations, mock_files): + def test_download_all_from_proposal(self, mock_observations, mock_files, mock_download_file): mock_observations.return_value = {'observation_id': ['test']} - mock_files.return_value = {'filename': ['test.fits']} - tap_handler = self.get_dummy_tap_handler("load_data") - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler("load_data"), show_messages=False) + mock_files.return_value = {'filename': ['test.fits', 'test2.fits']} + ehst = ESAHubbleClass(show_messages=False) ehst.download_files_from_program(program=12345, only_fits=False) - tap_handler.check_method("load_data") + mock_download_file.assert_any_call(file='test.fits', folder=None) + mock_download_file.assert_any_call(file='test2.fits', folder=None) + assert mock_download_file.call_count == 2 def test_query_criteria(self): parameters1 = {'calibration_level': "PRODUCT", @@ -365,7 +412,7 @@ def test_query_criteria(self): 'output_format': "votable", 'verbose': True, 'get_query': True} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) test_query = ehst.query_criteria(calibration_level=parameters1['calibration_level'], data_product_type=parameters1['data_product_type'], intent=parameters1['intent'], @@ -377,22 +424,13 @@ def test_query_criteria(self): output_format=parameters1['output_format'], verbose=parameters1['verbose'], get_query=parameters1['get_query']) - parameters2 = {'query': test_query, - 'output_file': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': False} - parameters3 = {'query': "select * from ehst.archive where(" - "calibration_level=3 AND " - "data_product_type LIKE '%image%' AND " - "intent LIKE '%science%' AND (collection " - "LIKE '%HST%') AND (instrument_name LIKE " - "'%WFC3%') AND (filter " - "LIKE '%F555W%'))", - 'output_file': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': False} - dummy_tap_handler = DummyHubbleTapHandler("launch_job", parameters2) - dummy_tap_handler.check_call("launch_job", parameters3) + assert test_query == ("select * from ehst.archive where(" + "calibration_level=3 AND " + "data_product_type LIKE '%image%' AND " + "intent LIKE '%science%' AND (collection " + "LIKE '%HST%') AND (instrument_name LIKE " + "'%WFC3%') AND (filter " + "LIKE '%F555W%'))") def test_query_criteria_numeric_calibration(self): parameters1 = {'calibration_level': 1, @@ -406,7 +444,7 @@ def test_query_criteria_numeric_calibration(self): 'output_format': "votable", 'verbose': True, 'get_query': True} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) test_query = ehst.query_criteria(calibration_level=parameters1['calibration_level'], data_product_type=parameters1['data_product_type'], intent=parameters1['intent'], @@ -418,63 +456,45 @@ def test_query_criteria_numeric_calibration(self): output_format=parameters1['output_format'], verbose=parameters1['verbose'], get_query=parameters1['get_query']) - parameters2 = {'query': test_query, - 'output_file': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': False} - parameters3 = {'query': "select * from ehst.archive where(" - "calibration_level=1 AND " - "data_product_type LIKE '%image%' AND " - "intent LIKE '%science%' AND (collection " - "LIKE '%HST%') AND (instrument_name LIKE " - "'%WFC3%') AND (filter " - "LIKE '%F555W%'))", - 'output_file': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': False} - dummy_tap_handler = DummyHubbleTapHandler("launch_job", parameters2) - dummy_tap_handler.check_call("launch_job", parameters3) + assert test_query == ( + "select * from ehst.archive where(" + "calibration_level=1 AND " + "data_product_type LIKE '%image%' AND " + "intent LIKE '%science%' AND (collection " + "LIKE '%HST%') AND (instrument_name LIKE " + "'%WFC3%') AND (filter " + "LIKE '%F555W%'))") parameters1['calibration_level'] = 4 with pytest.raises(KeyError) as err: - ehst.query_criteria(calibration_level=parameters1['calibration_level'], - data_product_type=parameters1['data_product_type'], - intent=parameters1['intent'], - obs_collection=parameters1['obs_collection'], - instrument_name=parameters1['instrument_name'], - filters=parameters1['filters'], - async_job=parameters1['async_job'], - output_file=parameters1['output_file'], - output_format=parameters1['output_format'], - verbose=parameters1['verbose'], - get_query=parameters1['get_query']) + ehst.query_criteria( + calibration_level=parameters1['calibration_level'], + data_product_type=parameters1['data_product_type'], + intent=parameters1['intent'], + obs_collection=parameters1['obs_collection'], + instrument_name=parameters1['instrument_name'], + filters=parameters1['filters'], + async_job=parameters1['async_job'], + output_file=parameters1['output_file'], + output_format=parameters1['output_format'], + verbose=parameters1['verbose'], + get_query=parameters1['get_query']) assert "Calibration level must be between 0 and 3" in err.value.args[0] - def test_cone_search_criteria(self): - parameters1 = {'target': "m31", - 'radius': 7, - 'data_product_type': "image", - 'obs_collection': ['HST'], - 'instrument_name': ['ACS/WFC'], - 'filters': ['F435W'], - 'async_job': False, - 'filename': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': True} - test_query = "select o.*, p.calibration_level, p.data_product_type, " \ - "pos.ra, pos.dec from ehst.observation AS o JOIN " \ - "ehst.plane as p on o.observation_uuid=p.observation_" \ - "uuid JOIN ehst.position as pos on p.plane_id = " \ - "pos.plane_id where((o.collection LIKE '%HST%') AND " \ - "(o.instrument_name LIKE '%WFPC2%') AND " \ - "(o.filter LIKE '%F606W%') AND " \ - "1=CONTAINS(POINT('ICRS', pos.ra, pos.dec)," \ - "CIRCLE('ICRS', 10.6847083, 41.26875, " \ - "0.11666666666666667)))" - parameters3 = {'query': test_query, - 'output_file': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': False} - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + @patch.object(ESAHubbleClass, 'query_tap') + def test_cone_search_criteria(self, mock_query_tap): + parameters1 = { + 'target': "m31", + 'radius': 7, + 'data_product_type': "image", + 'obs_collection': ['HST'], + 'instrument_name': ['ACS/WFC'], + 'filters': ['F435W'], + 'async_job': False, + 'filename': "output_test_query_by_criteria.vot.gz", + 'output_format': "votable", + 'verbose': True + } + ehst = ESAHubbleClass(show_messages=False) query_criteria_query = "select o.*, p.calibration_level, " \ "p.data_product_type, pos.ra, pos.dec from " \ "ehst.observation AS o JOIN ehst.plane as p " \ @@ -499,8 +519,8 @@ def test_cone_search_criteria(self): filename=parameters1['filename'], output_format=parameters1['output_format'], verbose=parameters1['verbose']) - dummy_tap_handler = DummyHubbleTapHandler("launch_job", parameters3) - dummy_tap_handler.check_call("launch_job", parameters3) + mock_query_tap.assert_called_once() + c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') ehst.cone_search_criteria(coordinates=c, radius=parameters1['radius'], @@ -514,6 +534,7 @@ def test_cone_search_criteria(self): filename=parameters1['filename'], output_format=parameters1['output_format'], verbose=parameters1['verbose']) + assert mock_query_tap.call_count == 2 with pytest.raises(TypeError) as err: ehst.cone_search_criteria(target=parameters1['target'], coordinates=123, @@ -533,25 +554,21 @@ def test_cone_search_criteria(self): assert "Please use only target or coordinates as" \ "parameter." in err.value.args[0] - def test_query_criteria_no_params(self): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + @patch.object(ESAHubbleClass, 'query_tap') + def test_query_criteria_no_params(self, mock_query_tap): + ehst = ESAHubbleClass(show_messages=False) ehst.query_criteria(async_job=False, output_file="output_test_query_" "by_criteria.vot.gz", output_format="votable", verbose=True) - parameters = {'query': "select o.*, p.calibration_level, " - "p.data_product_type from ehst.observation " - "AS o LEFT JOIN ehst.plane as p on " - "o.observation_uuid=p.observation_uuid", - 'output_file': "output_test_query_by_criteria.vot.gz", - 'output_format': "votable", - 'verbose': False} - dummy_tap_handler = DummyHubbleTapHandler("launch_job", parameters) - dummy_tap_handler.check_call("launch_job", parameters) + mock_query_tap.assert_called_once_with(query='select * from ehst.archive', async_job=False, + output_file="output_test_query_by_criteria.vot.gz", + output_format="votable", + verbose=True) def test_empty_list(self): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) with pytest.raises(ValueError) as err: ehst.query_criteria(instrument_name=[1], async_job=False, @@ -563,19 +580,19 @@ def test_empty_list(self): "elements that are not strings" in err.value.args[0] def test__get_decoded_string(self): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) dummy = '\x74\x65\x73\x74' decoded_string = ehst._get_decoded_string(dummy) assert decoded_string == 'test' def test__get_decoded_string_unicodedecodeerror(self): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) dummy = '\xd0\x91' decoded_string = ehst._get_decoded_string(dummy) assert decoded_string == dummy def test__get_decoded_string_attributeerror(self): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) dummy = True decoded_string = ehst._get_decoded_string(dummy) assert decoded_string == dummy @@ -586,7 +603,7 @@ def test__select_related_composite(self, mock_query): 'b': [2.0, 5.0], 'observation_id': ['x', 'y']} data_table = Table(arr) - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" oids = ehst._select_related_composite(observation_id=dummy_obs_id) @@ -598,7 +615,7 @@ def test_select_related_members(self, mock_query): 'b': [2.0, 5.0], 'members': ['caom:HST/test', 'y']} data_table = Table(arr) - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" oids = ehst._select_related_members(observation_id=dummy_obs_id) @@ -610,7 +627,7 @@ def test_get_observation_type(self, mock_query): 'b': [2.0, 5.0], 'obs_type': ['HST Test', 'y']} data_table = Table(arr) - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" oids = ehst.get_observation_type(observation_id=dummy_obs_id) @@ -618,7 +635,7 @@ def test_get_observation_type(self, mock_query): def test_get_observation_type_obs_id_none_valueerror(self): with pytest.raises(ValueError): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) dummy_obs_id = None ehst.get_observation_type(observation_id=dummy_obs_id) @@ -629,7 +646,7 @@ def test_get_observation_type_invalid_obs_id_valueerror(self, mock_query): 'b': [], 'obs_type': []} data_table = Table(arr) - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) mock_query.return_value = data_table dummy_obs_id = '1234' ehst.get_observation_type(observation_id=dummy_obs_id) @@ -642,7 +659,7 @@ def test_get_hst_link(self, mock_observation_type, mock_query): 'b': [2.0], 'observation_id': ['1234']} data_table = Table(arr) - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) mock_query.return_value = data_table dummy_obs_id = "1234" oids = ehst.get_hap_hst_link(observation_id=dummy_obs_id) @@ -653,7 +670,7 @@ def test_get_hst_link(self, mock_observation_type, mock_query): def test_get_hap_link(self, mock_select_related_members, mock_observation_type): mock_select_related_members.return_value = 'test' mock_observation_type.return_value = "HAP" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) dummy_obs_id = "1234" oids = ehst.get_hap_hst_link(observation_id=dummy_obs_id) assert oids == 'test' @@ -662,7 +679,7 @@ def test_get_hap_link(self, mock_select_related_members, mock_observation_type): def test_get_hap_hst_link_invalid_id_valueerror(self, mock_observation_type): with pytest.raises(ValueError): mock_observation_type.return_value = "valueerror" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) dummy_obs_id = "1234" ehst.get_hap_hst_link(observation_id=dummy_obs_id) @@ -670,7 +687,7 @@ def test_get_hap_hst_link_invalid_id_valueerror(self, mock_observation_type): def test_get_hap_hst_link_compositeerror(self, mock_observation_type): with pytest.raises(ValueError): mock_observation_type.return_value = "HAP Composite" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) dummy_obs_id = "1234" ehst.get_hap_hst_link(observation_id=dummy_obs_id) @@ -678,7 +695,7 @@ def test_get_hap_hst_link_compositeerror(self, mock_observation_type): @patch.object(ESAHubbleClass, 'get_observation_type') def test_get_member_observations_composite(self, mock_observation_type, mock_select_related_members): mock_observation_type.return_value = "Composite" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) mock_select_related_members.return_value = 'test' dummy_obs_id = "1234" oids = ehst.get_member_observations(observation_id=dummy_obs_id) @@ -688,7 +705,7 @@ def test_get_member_observations_composite(self, mock_observation_type, mock_sel @patch.object(ESAHubbleClass, 'get_observation_type') def test_get_member_observations_simple(self, mock_observation_type, mock_select_related_composite): mock_observation_type.return_value = "Simple" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) mock_select_related_composite.return_value = 'test' dummy_obs_id = "1234" oids = ehst.get_member_observations(observation_id=dummy_obs_id) @@ -698,7 +715,7 @@ def test_get_member_observations_simple(self, mock_observation_type, mock_select def test_get_member_observations_invalid_id_valueerror(self, mock_observation_type): with pytest.raises(ValueError): mock_observation_type.return_value = "valueerror" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) dummy_obs_id = "1234" ehst.get_member_observations(observation_id=dummy_obs_id) @@ -709,7 +726,7 @@ def test_cone_search_criteria_only_target(self, mock_query_tap, mock__query_tap_ mock_query_criteria.return_value = "Simple query" mock__query_tap_target.return_value = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') mock_query_tap.return_value = "table" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) oids = ehst.cone_search_criteria(target="m11", radius=1) assert oids == 'table' @@ -718,7 +735,7 @@ def test_cone_search_criteria_only_target(self, mock_query_tap, mock__query_tap_ def test_cone_search_criteria_only_coordinates(self, mock_query_tap, mock_query_criteria): mock_query_criteria.return_value = "Simple query" mock_query_tap.return_value = "table" - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) oids = ehst.cone_search_criteria(coordinates="00h42m44.51s +41d16m08.45s", radius=1) assert oids == 'table' @@ -726,37 +743,57 @@ def test_cone_search_criteria_only_coordinates(self, mock_query_tap, mock_query_ def test_cone_search_criteria_typeerror(self, mock_query_criteria): mock_query_criteria.return_value = "Simple query" with pytest.raises(TypeError): - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) + ehst = ESAHubbleClass(show_messages=False) ehst.cone_search_criteria(coordinates="00h42m44.51s +41d16m08.45s", target="m11", radius=1) - def test_query_hst_tap(self): - parameters = {'query': "select top 10 * from hsc_v2.hubble_sc2", - 'async_job': False, - 'output_file': "test2.vot", - 'output_format': "votable", - 'verbose': False} - - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - with pytest.warns(AstropyDeprecationWarning): - ehst.query_hst_tap(query=parameters['query'], - async_job=parameters['async_job'], - output_file=parameters['output_file'], - output_format=parameters['output_format'], - verbose=parameters['verbose']) - - @patch("http.client.HTTPSConnection") - @patch("http.client.HTTPResponse") - def test_show_messages(self, mock_conn, mock_res): - mock_res.status = 400 - mock_conn.getresponse = MagicMock(return_value=mock_res) + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.execute_servlet_request') + def test_show_messages(self, mock_execute_servlet_request): ESAHubbleClass() - mock_res.assert_called() - - def test_get_datalabs_path(self): - parameters = {'filename': "ib4x04ivq_flt.jpg", - 'default_volume': None} - - dummyTapHandler = DummyHubbleTapHandler("get_datalabs_path", parameters) - ehst = ESAHubbleClass(tap_handler=self.get_dummy_tap_handler(), show_messages=False) - ehst.get_datalabs_path(filename="ib4x04ivq_flt.jpg", default_volume="") - dummyTapHandler.check_call("get_datalabs_path", parameters) + mock_execute_servlet_request.assert_called() + + def test_parse_messages_response(self): + ehst = ESAHubbleClass(show_messages=False) + response = Response() + response.status_code = 200 + plain_text = ( + "notification_id1[type: type1,subtype1]=msg1\n" + "notification_id2[type: type2,subtype2]=msg2\n" + "notification_idn[type: typen,subtypen]=msgn" + ) + response._content = plain_text.encode('utf-8') # encode to bytes + response.headers['Content-Type'] = 'text/plain' + response.raw = io.BytesIO(response._content) + messages = ehst.parse_messages_response(response) + assert len(messages) == 3 + assert messages == ["msg1", "msg2", "msgn"] + + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch.object(ESAHubbleClass, 'query_tap') + @patch.object(ESAHubbleClass, '_get_decoded_string') + def test_get_datalabs_path(self, mock_get_decoded_string, mock_query_tap): + ehst = ESAHubbleClass(show_messages=False) + + with patch('os.path.exists') as mock_exists: + # Set up the return values for the query_tap method + values = [ + {"file_path": ["path/to/file"]}, # First query result + {"observation_id": ["obs123"]}, # Second query result + {"instrument_name": ["instrumentXYZ"]} # Third query result + ] + mock_query_tap.side_effect = values + values + # Set up the return value for the _get_decoded_string method + mock_get_decoded_string.return_value = "/hstdata/hstdata_i/i/b4x/04" + # Set up the return value for os.path.exists + mock_exists.return_value = True + # Example usage + filename = "ib4x04ivq_flt.jpg" + default_volume = None + full_path = ehst.get_datalabs_path(filename=filename, default_volume=default_volume) + assert full_path == "/data/user/hub_hstdata_i/i/b4x/04/ib4x04ivq_flt.jpg" + + # Test with default_volume provided + default_volume = "myPath" + + full_path = ehst.get_datalabs_path(filename=filename, default_volume=default_volume) + assert full_path == "/data/user/myPath/i/b4x/04/ib4x04ivq_flt.jpg" diff --git a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py index abbe5ea050..813d0ab3b3 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py @@ -18,7 +18,7 @@ from astroquery.esa.hubble import ESAHubble from astropy import coordinates -esa_hubble = ESAHubble() +esa_hubble = ESAHubble(show_messages=False) def data_path(filename): @@ -31,9 +31,9 @@ def create_temp_folder(): def remove_last_job(): - jobs = esa_hubble._tap.list_async_jobs() + jobs = esa_hubble.tap.get_job_list() if len(jobs) > 0: - esa_hubble._tap.remove_jobs(jobs[-1].jobid) + jobs[-1].delete() @pytest.mark.remote_data @@ -51,7 +51,7 @@ class TestEsaHubbleRemoteData: temp_folder = create_temp_folder() temp_folder_for_fits = create_temp_folder() - def test_query_tap_async(self): + def test_query_tap_async(self, recwarn): result = esa_hubble.query_tap(query=self.top_obs_query, async_job=True) assert len(result) > 10 assert "observation_id" in result.keys() @@ -79,7 +79,7 @@ def test_get_artifact(self): os.path.exists(temp_file + 'fits.gz')] assert any([os.path.exists(f) for f in possible_values]) - def test_cone_search(self): + def test_cone_search(self, recwarn): c = coordinates.SkyCoord("00h42m44.51s +41d16m08.45s", frame='icrs') compressed_temp_file = os.path.join(self.temp_folder.name, "cone_search_m31_5.vot.gz") # open & extracting the file @@ -132,10 +132,14 @@ def test_retrieve_fits_from_program(self): folder=str(self.temp_folder_for_fits.name)) assert len(os.listdir(self.temp_folder_for_fits.name)) > 0 - def test_get_datalabs_path_image(self): + def test_get_datalabs_path_image(self, recwarn): result = esa_hubble.get_datalabs_path(filename='ib4x04ivq_flt.jpg', default_volume=None) + assert len(recwarn) == 1 + assert "ib4x04ivq_flt.jpg" in str(recwarn[0].message) assert result == '/data/user/hub_hstdata_i/i/b4x/04/ib4x04ivq_flt.jpg' - def test_get_datalabs_path_fits(self): + def test_get_datalabs_path_fits(self, recwarn): result = esa_hubble.get_datalabs_path(filename='ib4x04ivq_flt.fits', default_volume=None) + assert len(recwarn) == 1 + assert "ib4x04ivq_flt.fits" in str(recwarn[0].message) assert result == '/data/user/hub_hstdata_i/i/b4x/04/ib4x04ivq_flt.fits.gz' diff --git a/astroquery/esa/utils/tests/test_utils.py b/astroquery/esa/utils/tests/test_utils.py index c0e15bf427..e425dfaf9d 100644 --- a/astroquery/esa/utils/tests/test_utils.py +++ b/astroquery/esa/utils/tests/test_utils.py @@ -157,6 +157,26 @@ def test_execute_servlet_request(self, mock_tap): mock_tap._session.get.assert_called_once_with(url='https://dummyurl.com/service', params={'test': 'dummy', 'TAPCLIENT': 'ASTROQUERY'}) + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService') + def test_execute_servlet_request_with_parser_method(self, mock_tap): + mock_parser_method = Mock() + mock_response = Mock() + mock_response.status_code = 200 + mock_tap._session.get.return_value = mock_response + query_params = {'test': 'dummy'} + esautils.execute_servlet_request( + url='https://dummyurl.com/service', + tap=mock_tap, + query_params=query_params, + parser_method=mock_parser_method + ) + + mock_tap._session.get.assert_called_once_with(url='https://dummyurl.com/service', + params={'test': 'dummy', 'TAPCLIENT': 'ASTROQUERY'}) + + mock_parser_method.assert_called_once() + @patch('astroquery.esa.integral.core.pyvo.dal.TAPService.capabilities', []) @patch('astroquery.esa.integral.core.pyvo.dal.TAPService') def test_execute_servlet_request_error(self, mock_tap): diff --git a/astroquery/esa/utils/utils.py b/astroquery/esa/utils/utils.py index 6cf0cf097b..0f92330ff9 100644 --- a/astroquery/esa/utils/utils.py +++ b/astroquery/esa/utils/utils.py @@ -175,7 +175,7 @@ def download_table(astropy_table, output_file=None, output_format=None): astropy_table.write(output_file, format=output_format, overwrite=True) -def execute_servlet_request(url, tap, *, query_params=None): +def execute_servlet_request(url, tap, *, query_params=None, parser_method=None): """ Method to execute requests to the servlets on a server @@ -187,6 +187,8 @@ def execute_servlet_request(url, tap, *, query_params=None): TAP instance from where the session will be extracted query_params: dict, optional Parameters to be included in the request + parser_method: dict, optional + Method to parse the response Returns ------- @@ -199,7 +201,10 @@ def execute_servlet_request(url, tap, *, query_params=None): # Use the TAPService session to perform a custom GET request response = tap._session.get(url=url, params=query_params) if response.status_code == 200: - return response.json() + if parser_method is None: + return response.json() + else: + return parser_method(response) else: response.raise_for_status()
o58502w0q1999-07-24 14:14:57.271999-07-24 14:18:29.43351383.5103850694451383.5128406597220.12000-07-24 14:18:29.0w058502HSTscience0APERTURE=OV28X50LP|ASNID=NONE|DETECTOR=CCD|EXPEND=51383.51284065972|EXPSTART=51383.51038506944|EXPTIME=20.1|FILTER=MIRVIS|OBSMODE=ACQ|OBSTYPE=IMAGINGHST SingletonfalseM31BROAD_CATEGORY=GALAXY|TARGET_DESCRIP=GALAXY;SPIRAL8018Green, Richard F.Demographics of Nuclear Black HolesSTIS/CCDCRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2aimageCRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2a10.684572524500547241.26917617069089770.00197465062424587775MIRVIS
o585020101999-07-24 15:23:05.2071999-07-24 15:23:57.20751383.5576991550951383.5583010069468.02000-07-24 15:23:57.00158502HSTscience0APERTURE=52X0.1|DETECTOR=CCD|EXPEND=51383.558301006946|EXPSTART=51383.55769915509|EXPTIME=8.0|FILTER=MIRVIS|OBSMODE=ACCUM|OBSTYPE=IMAGINGHST CompositefalseM31BROAD_CATEGORY=GALAXY|TARGET_DESCRIP=GALAXY;SPIRAL8018Green, Richard F.Demographics of Nuclear Black HolesSTIS/CCDCAL_VER=3.4.2 (19-Jan-2018)|CRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2aimageCAL_VER=3.4.2 (19-Jan-2018)|CRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2a10.684571244359037941.26924947607619030.0219188618172586891MIRVIS
o58501skq1999-07-23 14:04:17.271999-07-23 14:07:49.43351382.50297766203651382.50543325231420.12007-01-04 15:44:24.0sk58501HSTscience0APERTURE=OV28X50LP|ASNID=NONE|DETECTOR=CCD|EXPEND=51382.505433252314|EXPSTART=51382.502977662036|EXPTIME=20.1|FILTER=MIRVIS|OBSMODE=ACQ|OBSTYPE=IMAGINGHST SingletonfalseM31BROAD_CATEGORY=GALAXY|TARGET_DESCRIP=GALAXY;SPIRAL8018Green, Richard F.Demographics of Nuclear Black HolesSTIS/CCDCRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2aimageCRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2a10.684572524500547241.26917617069089770.00197465062424587775MIRVIS
o585010101999-07-23 15:12:56.2131999-07-23 15:13:48.21351382.55065061342651382.551252465288.02007-01-11 08:56:27.00158501HSTscience0APERTURE=52X0.1|DETECTOR=CCD|EXPEND=51382.55125246528|EXPSTART=51382.550650613426|EXPTIME=8.0|FILTER=MIRVIS|OBSMODE=ACCUM|OBSTYPE=IMAGINGHST CompositefalseM31BROAD_CATEGORY=GALAXY|TARGET_DESCRIP=GALAXY;SPIRAL8018Green, Richard F.Demographics of Nuclear Black HolesSTIS/CCDCAL_VER=3.4.2 (19-Jan-2018)|CRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2aimageCAL_VER=3.4.2 (19-Jan-2018)|CRDS_VER=7.1.5, 7.1.5, 3548bc1|CSYS_VER=hstdp-2018.1|OPUS_VER=HSTDP 2018_2a10.684571244359037941.26924947607619030.0219188618172586891MIRVIS
from each