Skip to content

Commit 97b7b8f

Browse files
bsipoczbsipocz
authored
Merge pull request #3295 from esdc-esac-esa-int/ESA_euclid_EUCLIDPCR-1971_upload_user_table_from_fits_file
Euclid: upgrade the TAP method upload_table to upload tables in fits format
2 parents 7ac08b8 + 868c135 commit 97b7b8f

File tree

14 files changed

+467
-103
lines changed

14 files changed

+467
-103
lines changed

‎CHANGES.rst

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,14 @@ heasarc
3131
- Heasarc.locate_data returns empty rows with an error in the error_message column if there are
3232
no data associated with that row rather than filtering it out. [#3275]
3333

34-
3534
utils.tap
3635
^^^^^^^^^
3736

3837
- Get the cookie associated to the keys JSESSIONID or SESSION due to the tap library release at ESAC. [#3289]
3938

39+
- The method ``upload_table`` accepts file formats accepted by astropy's
40+
``Table.read()``. [#3295]
41+
4042

4143
Infrastructure, Utility and Other Changes and Additions
4244
-------------------------------------------------------

‎astroquery/esa/euclid/__init__.py

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,6 @@ class Conf(_config.ConfigNamespace):
1515
Configuration parameters for `astroquery.esa.euclid`.
1616
"""
1717

18-
URL_BASE = _config.ConfigItem('https://eas.esac.esa.int/', 'Euclid base URL')
19-
20-
EUCLID_TAP_SERVER = _config.ConfigItem('https://easidr.esac.esa.int/tap-server/tap', 'Euclid TAP Server')
21-
EUCLID_DATALINK_SERVER = _config.ConfigItem("https://easidr.esac.esa.int/sas-dd/data?", "Euclid DataLink Server")
22-
EUCLID_CUTOUT_SERVER = _config.ConfigItem("https://easidr.esac.esa.int/sas-cutout/cutout?", "Euclid Cutout Server")
23-
2418
ROW_LIMIT = _config.ConfigItem(50,
2519
"Number of rows to return from database query (set to -1 for unlimited).")
2620

‎astroquery/utils/tap/conn/tests/DummyConnHandler.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,11 @@
1414
1515
1616
"""
17-
from astroquery.utils.tap import taputils
18-
1917
import requests
2018

19+
from astroquery.utils.tap import taputils
20+
from astroquery.utils.tap.conn.tapconn import TapConn
21+
2122

2223
class DummyConnHandler:
2324

@@ -158,3 +159,12 @@ def execute_secure(self, subcontext=None, data=None, verbose=False):
158159

159160
def get_host_url(self):
160161
return "my fake object"
162+
163+
def encode_multipart(self, fields, files):
164+
tap = TapConn(ishttps=False, host='host')
165+
return tap.encode_multipart(fields, files)
166+
167+
def execute_upload(self, data,
168+
content_type="application/x-www-form-urlencoded", *,
169+
verbose=False):
170+
return self.defaultResponse

‎astroquery/utils/tap/core.py

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@
1515
"""
1616
import getpass
1717
import os
18-
import requests
1918
import tempfile
20-
from astropy.table.table import Table
2119
from urllib.parse import urlencode
2220

21+
import requests
22+
from astropy.table.table import Table
23+
2324
from astroquery import log
2425
from astroquery.utils.tap import taputils
2526
from astroquery.utils.tap.conn.tapconn import TapConn
@@ -1341,8 +1342,9 @@ def upload_table(self, *, upload_resource=None, table_name=None, table_descripti
13411342
resource temporary table name associated to the uploaded resource
13421343
table_description : str, optional, default None
13431344
table description
1344-
format : str, optional, default 'VOTable'
1345-
resource format
1345+
format : str, optional, default 'votable'
1346+
resource format. Only formats described in
1347+
https://docs.astropy.org/en/stable/io/unified.html#built-in-table-readers-writers are accepted.
13461348
verbose : bool, optional, default 'False'
13471349
flag to display information about the process
13481350
"""
@@ -1381,9 +1383,7 @@ def upload_table(self, *, upload_resource=None, table_name=None, table_descripti
13811383
log.info(f"Uploaded table '{table_name}'.")
13821384
return None
13831385

1384-
def __uploadTableMultipart(self, resource, *, table_name=None,
1385-
table_description=None,
1386-
resource_format="VOTable",
1386+
def __uploadTableMultipart(self, resource, *, table_name=None, table_description=None, resource_format="votable",
13871387
verbose=False):
13881388
connHandler = self.__getconnhandler()
13891389
if isinstance(resource, Table):
@@ -1397,24 +1397,38 @@ def __uploadTableMultipart(self, resource, *, table_name=None,
13971397
fh = tempfile.NamedTemporaryFile(delete=False)
13981398
resource.write(fh, format='votable')
13991399
fh.close()
1400-
f = open(fh.name, "r")
1401-
chunk = f.read()
1402-
f.close()
1400+
1401+
with open(fh.name, "r") as f:
1402+
chunk = f.read()
1403+
14031404
os.unlink(fh.name)
14041405
files = [['FILE', 'pytable', chunk]]
1405-
contentType, body = connHandler.encode_multipart(args, files)
1406+
content_type, body = connHandler.encode_multipart(args, files)
14061407
else:
14071408
if not (str(resource).startswith("http")): # upload from file
14081409
args = {
14091410
"TASKID": str(-1),
14101411
"TABLE_NAME": str(table_name),
14111412
"TABLE_DESC": str(table_description),
1412-
"FORMAT": str(resource_format)}
1413+
"FORMAT": 'votable'}
14131414
log.info(f"Sending file: {resource}")
1414-
with open(resource, "r") as f:
1415-
chunk = f.read()
1416-
files = [['FILE', os.path.basename(resource), chunk]]
1417-
contentType, body = connHandler.encode_multipart(args, files)
1415+
if resource_format.lower() == 'votable':
1416+
with open(resource, "r") as f:
1417+
chunk = f.read()
1418+
files = [['FILE', os.path.basename(resource), chunk]]
1419+
else:
1420+
table = Table.read(str(resource), format=resource_format)
1421+
fh = tempfile.NamedTemporaryFile(delete=False)
1422+
table.write(fh, format='votable')
1423+
fh.close()
1424+
1425+
with open(fh.name, "r") as f:
1426+
chunk = f.read()
1427+
1428+
os.unlink(fh.name)
1429+
files = [['FILE', 'pytable', chunk]]
1430+
1431+
content_type, body = connHandler.encode_multipart(args, files)
14181432
else: # upload from URL
14191433
args = {
14201434
"TASKID": str(-1),
@@ -1423,8 +1437,8 @@ def __uploadTableMultipart(self, resource, *, table_name=None,
14231437
"FORMAT": str(resource_format),
14241438
"URL": str(resource)}
14251439
files = [['FILE', "", ""]]
1426-
contentType, body = connHandler.encode_multipart(args, files)
1427-
response = connHandler.execute_upload(body, contentType)
1440+
content_type, body = connHandler.encode_multipart(args, files)
1441+
response = connHandler.execute_upload(body, content_type)
14281442
if verbose:
14291443
print(response.status, response.reason)
14301444
print(response.getheaders())

‎astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.csv

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
source_id,ra,dec
2+
3834447128563320320,149.8678677871318,1.12773018116361
3+
3834447162923057280,149.8953864417848,1.1345712682426434

‎astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.ecsv

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# %ECSV 1.0
2+
# ---
3+
# delimiter: ','
4+
# datatype:
5+
# -
6+
# name: source_id
7+
# datatype: int64
8+
# description: Unique source identifier (unique within a particular Data Release)
9+
# meta:
10+
# ucd: meta.id
11+
# -
12+
# name: ra
13+
# datatype: float64
14+
# unit: deg
15+
# description: Right ascension
16+
# meta:
17+
# ucd: pos.eq.ra;meta.main
18+
# utype: stc:AstroCoords.Position3D.Value3.C1
19+
# CoosysSystem: ICRS
20+
# CoosysEpoch: J2016.0
21+
# -
22+
# name: dec
23+
# datatype: float64
24+
# unit: deg
25+
# description: Declination
26+
# meta:
27+
# ucd: pos.eq.dec;meta.main
28+
# utype: stc:AstroCoords.Position3D.Value3.C2
29+
# CoosysSystem: ICRS
30+
# CoosysEpoch: J2016.0
31+
# meta:
32+
# name: votable
33+
# QUERY_STATUS: OK
34+
# QUERY: 'SELECT TOP 2 source_id, ra, dec FROM gaiadr3.gaia_source '
35+
# CAPTION: 'How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html'
36+
# CITATION: 'How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html'
37+
# JOBID: 1744351221317O
38+
# RELEASE: Gaia DR3
39+
source_id,ra,dec
40+
3834447128563320320,149.8678677871318,1.12773018116361
41+
3834447162923057280,149.8953864417848,1.1345712682426434

‎astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.fits

8.44 KB
Binary file not shown.

‎astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.json

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
{
2+
"VOTABLE": {
3+
"RESOURCE": {
4+
"INFO": [
5+
{
6+
"_name": "QUERY_STATUS",
7+
"_value": "OK"
8+
},
9+
{
10+
"_name": "QUERY",
11+
"_value": "SELECT TOP 20 * FROM user_jferna01.my_votable_fits ",
12+
"__cdata": "SELECT TOP 20 *\nFROM user_jferna01.my_votable_fits "
13+
},
14+
{
15+
"_name": "CAPTION",
16+
"_value": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html",
17+
"__cdata": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html"
18+
},
19+
{
20+
"_name": "CITATION",
21+
"_value": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html",
22+
"_ucd": "meta.bib",
23+
"__cdata": "How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html"
24+
},
25+
{
26+
"_name": "PAGE",
27+
"_value": ""
28+
},
29+
{
30+
"_name": "PAGE_SIZE",
31+
"_value": ""
32+
},
33+
{
34+
"_name": "JOBID",
35+
"_value": "1744360074103O",
36+
"__cdata": "1744360074103O"
37+
},
38+
{
39+
"_name": "JOBNAME",
40+
"_value": ""
41+
}
42+
],
43+
"TABLE": {
44+
"FIELD": [
45+
{
46+
"DESCRIPTION": "Object Identifier",
47+
"_datatype": "int",
48+
"_name": "my_votable_fits_oid"
49+
},
50+
{
51+
"_datatype": "long",
52+
"_name": "source_id"
53+
},
54+
{
55+
"_datatype": "double",
56+
"_name": "ra",
57+
"_unit": "deg"
58+
},
59+
{
60+
"_datatype": "double",
61+
"_name": "dec",
62+
"_unit": "deg"
63+
}
64+
],
65+
"DATA": {
66+
"TABLEDATA": {
67+
"TR": [
68+
{
69+
"TD": [
70+
"1",
71+
"3834447128563320320",
72+
"149.8678677871318",
73+
"1.12773018116361"
74+
]
75+
},
76+
{
77+
"TD": [
78+
"2",
79+
"3834447162923057280",
80+
"149.8953864417848",
81+
"1.1345712682426434"
82+
]
83+
}
84+
]
85+
}
86+
}
87+
},
88+
"_type": "results"
89+
},
90+
"_version": "1.4",
91+
"_xmlns": "http://www.ivoa.net/xml/VOTable/v1.3",
92+
"_xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
93+
"_xsi:schemaLocation": "http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/votable-1.4.xsd"
94+
}
95+
}

‎astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result.vot

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/votable-1.4.xsd">
3+
<RESOURCE type="results">
4+
<INFO name="QUERY_STATUS" value="OK"/>
5+
6+
<INFO name="QUERY" value="SELECT TOP 20 *
7+
FROM user_jferna01.my_votable_fits "><![CDATA[SELECT TOP 20 *
8+
FROM user_jferna01.my_votable_fits ]]></INFO>
9+
<INFO name="CAPTION" value="How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html"><![CDATA[How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
10+
<INFO name="CITATION" value="How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html" ucd="meta.bib"><![CDATA[How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
11+
<INFO name="PAGE" value=""/>
12+
<INFO name="PAGE_SIZE" value=""/>
13+
<INFO name="JOBID" value="1744360074103O"><![CDATA[1744360074103O]]></INFO>
14+
<INFO name="JOBNAME" value=""></INFO>
15+
16+
<TABLE>
17+
<FIELD datatype="int" name="my_votable_fits_oid">
18+
<DESCRIPTION>Object Identifier</DESCRIPTION>
19+
</FIELD>
20+
<FIELD datatype="long" name="source_id"/>
21+
<FIELD datatype="double" name="ra" unit="deg"/>
22+
<FIELD datatype="double" name="dec" unit="deg"/>
23+
<DATA>
24+
<BINARY2>
25+
<STREAM encoding='base64'>
26+
AAAAAAE1NrFZAAiGAEBiu8WSql90P/ILLs1s9TAAAAAAAjU2sWEACICAQGK8pwF3
27+
l+w/8ic0M8FVmA==
28+
</STREAM>
29+
</BINARY2>
30+
</DATA>
31+
</TABLE>
32+
</RESOURCE>
33+
</VOTABLE>

‎astroquery/utils/tap/tests/data/test_upload_file/1744351221317O-result_plain.vot

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/votable-1.4.xsd">
3+
<RESOURCE type="results">
4+
<INFO name="QUERY_STATUS" value="OK"/>
5+
6+
<INFO name="QUERY" value="SELECT TOP 20 * FROM user_jferna01.my_votable_fits "><![CDATA[SELECT TOP 20 *
7+
FROM user_jferna01.my_votable_fits ]]></INFO>
8+
<INFO name="CAPTION" value="How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html"><![CDATA[How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
9+
<INFO name="CITATION" value="How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html" ucd="meta.bib"><![CDATA[How to cite and acknowledge Gaia: https://gea.esac.esa.int/archive/documentation/credits.html]]></INFO>
10+
<INFO name="PAGE" value=""/>
11+
<INFO name="PAGE_SIZE" value=""/>
12+
<INFO name="JOBID" value="1744360074103O"><![CDATA[1744360074103O]]></INFO>
13+
<INFO name="JOBNAME" value=""/>
14+
15+
<TABLE>
16+
<FIELD datatype="int" name="my_votable_fits_oid">
17+
<DESCRIPTION>Object Identifier</DESCRIPTION>
18+
</FIELD>
19+
<FIELD datatype="long" name="source_id"/>
20+
<FIELD datatype="double" name="ra" unit="deg"/>
21+
<FIELD datatype="double" name="dec" unit="deg"/>
22+
<DATA>
23+
<TABLEDATA>
24+
<TR><TD>1</TD><TD>3834447128563320320</TD><TD>149.8678677871318</TD><TD>1.12773018116361</TD></TR>
25+
<TR><TD>2</TD><TD>3834447162923057280</TD><TD>149.8953864417848</TD><TD>1.1345712682426434</TD></TR>
26+
</TABLEDATA>
27+
</DATA>
28+
29+
</TABLE>
30+
</RESOURCE>
31+
</VOTABLE>

0 commit comments

Comments
 (0)