Skip to content

Commit 1be9420

Browse files
authored
Merge pull request #3052 from cds-astro/fix-add-votable-fields-fluxes
[SIMBAD] Add filter names as possible votable fields
2 parents 091e6b3 + 9670d00 commit 1be9420

File tree

6 files changed

+277
-114
lines changed

6 files changed

+277
-114
lines changed

astroquery/simbad/core.py

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -217,13 +217,14 @@ def list_wildcards():
217217
def list_votable_fields(self):
218218
"""List all options to add columns to SIMBAD's output.
219219
220-
They are of three types:
220+
They are of four types:
221221
222222
- "column of basic": a column of the basic table. There fields can also be explored with
223223
`~astroquery.simbad.SimbadClass.list_columns`.
224224
- "table": a table other than basic that has a declared direct join
225225
- "bundle of basic columns": a pre-selected bundle of columns of basic. Ex: "parallax" will add all
226226
columns relevant to parallax
227+
- "filter name": an optical filter name
227228
228229
Examples
229230
--------
@@ -266,8 +267,11 @@ def list_votable_fields(self):
266267
"description": [value["description"] for _, value in bundle_entries.items()],
267268
"type": ["bundle of basic columns"] * len(bundle_entries)},
268269
dtype=["object", "object", "object"])
269-
# vstack the three types of options
270-
return vstack([tables, basic_columns, bundles], metadata_conflicts="silent")
270+
# get the filter names
271+
filters = self.query_tap("SELECT filtername AS name, description FROM filter")
272+
filters["type"] = Column(["filter name"] * len(filters), dtype="object")
273+
# vstack the four types of options
274+
return vstack([tables, basic_columns, bundles, filters], metadata_conflicts="silent")
271275

272276
def _get_bundle_columns(self, bundle_name):
273277
"""Get the list of columns in the preselected bundles.
@@ -324,11 +328,17 @@ def _add_table_to_output(self, table):
324328
"query to be written and called with 'SimbadClass.query_tap'.")
325329

326330
columns = list(self.list_columns(table)["column_name"])
327-
columns = [column.casefold() for column in columns if column not in {"oidref", "oidbibref"}]
328331

329-
# the alias is mandatory to be able to distinguish between duplicates like
330-
# mesDistance.bibcode and mesDiameter.bibcode.
331-
alias = [f'"{table}.{column}"' if not column.startswith(table) else None for column in columns]
332+
# allfluxes is the only case-dependent table
333+
if table == "allfluxes":
334+
columns = [column for column in columns if column not in {"oidref", "oidbibref"}]
335+
alias = [column.replace("_", "") if "_" in column else None
336+
for column in columns]
337+
else:
338+
columns = [column.casefold() for column in columns if column not in {"oidref", "oidbibref"}]
339+
# the alias is mandatory to be able to distinguish between duplicates like
340+
# mesDistance.bibcode and mesDiameter.bibcode.
341+
alias = [f"{table}.{column}" if not column.startswith(table) else None for column in columns]
332342

333343
# modify the attributes here
334344
self.columns_in_output += [_Column(table, column, alias)
@@ -366,27 +376,43 @@ def add_votable_fields(self, *args):
366376
['basic.main_id', 'basic.ra', 'basic.dec', 'basic.coo_err_maj', 'basic.coo_err_min', ...
367377
"""
368378

369-
# the legacy way of adding fluxes is the only case-dependant option
379+
# the legacy way of adding fluxes
370380
args = list(args)
381+
fluxes_to_add = []
371382
for arg in args:
383+
if arg.startswith("flux_"):
384+
raise ValueError("The votable fields 'flux_***(filtername)' are removed and replaced "
385+
"by 'flux' that will add all information for every filters. "
386+
"See section on filters in "
387+
"https://astroquery.readthedocs.io/en/latest/simbad/simbad_evolution.html"
388+
" to see the new ways to interact with SIMBAD's fluxes.")
372389
if re.match(r"^flux.*\(.+\)$", arg):
373-
warnings.warn("The notation 'flux(X)' is deprecated since 0.4.8. "
390+
warnings.warn("The notation 'flux(U)' is deprecated since 0.4.8 in favor of 'U'. "
374391
"See section on filters in "
375392
"https://astroquery.readthedocs.io/en/latest/simbad/simbad_evolution.html "
376-
"to see how it can be replaced.", DeprecationWarning, stacklevel=2)
377-
flux_filter = re.findall(r"\((\w+)\)", arg)[0]
378-
if len(flux_filter) == 1 and flux_filter.islower():
379-
flux_filter = flux_filter + "_"
380-
self.joins.append(_Join("allfluxes", _Column("basic", "oid"),
381-
_Column("allfluxes", "oidref")))
382-
self.columns_in_output.append(_Column("allfluxes", flux_filter))
393+
"to see the new ways to interact with SIMBAD's fluxes.", DeprecationWarning, stacklevel=2)
394+
fluxes_to_add.append(re.findall(r"\((\w+)\)", arg)[0])
383395
args.remove(arg)
384396

385-
# casefold args
386-
args = set(map(str.casefold, args))
387-
388397
# output options
389398
output_options = self.list_votable_fields()
399+
# fluxes are case-dependant
400+
fluxes = output_options[output_options["type"] == "filter name"]["name"]
401+
# add fluxes
402+
fluxes_to_add += [flux for flux in args if flux in fluxes]
403+
if fluxes_to_add:
404+
self.joins.append(_Join("allfluxes", _Column("basic", "oid"),
405+
_Column("allfluxes", "oidref")))
406+
for flux in fluxes_to_add:
407+
if len(flux) == 1 and flux.islower():
408+
# the name in the allfluxes view has a trailing underscore. This is not
409+
# the case in the list of filter names, so we homogenize with an alias
410+
self.columns_in_output.append(_Column("allfluxes", flux + "_", flux))
411+
else:
412+
self.columns_in_output.append(_Column("allfluxes", flux))
413+
414+
# casefold args because we allow case difference for every other argument (legacy behavior)
415+
args = set(map(str.casefold, args))
390416
output_options["name"] = list(map(str.casefold, list(output_options["name"])))
391417
basic_columns = output_options[output_options["type"] == "column of basic"]["name"]
392418
all_tables = output_options[output_options["type"] == "table"]["name"]
@@ -1374,7 +1400,7 @@ def _query(self, top, columns, joins, criteria, from_table="basic",
13741400
top = f" TOP {top}" if top != -1 else ""
13751401

13761402
# columns
1377-
input_columns = [f'{column.table}."{column.name}" AS {column.alias}' if column.alias is not None
1403+
input_columns = [f'{column.table}."{column.name}" AS "{column.alias}"' if column.alias is not None
13781404
else f'{column.table}."{column.name}"' for column in columns]
13791405
# remove possible duplicates
13801406
unique_columns = []
@@ -1391,7 +1417,7 @@ def _query(self, top, columns, joins, criteria, from_table="basic",
13911417
[unique_joins.append(join) for join in joins if join not in unique_joins]
13921418
join = " " + " ".join([(f'{join.join_type} {join.table} ON {join.column_left.table}."'
13931419
f'{join.column_left.name}" = {join.column_right.table}."'
1394-
f'{join.column_right.name}"') for join in joins])
1420+
f'{join.column_right.name}"') for join in unique_joins])
13951421

13961422
# criteria
13971423
if criteria != []:

astroquery/simbad/tests/data/simbad_output_options.xml

Lines changed: 91 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
<?xml version="1.0" encoding="utf-8"?>
2-
<!-- Produced with astropy.io.votable version 6.0.0
2+
<!-- Produced with astropy.io.votable version 6.1.0
33
http://www.astropy.org/ -->
44
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
55
<RESOURCE type="results">
6-
<TABLE ID="result_S1711533763224" name="result_S1711533763224">
7-
<FIELD ID="name" arraysize="*" datatype="char" name="name">
6+
<TABLE ID="result_S1719407661907" name="result_S1719407661907">
7+
<FIELD ID="name" arraysize="*" datatype="char" name="name" ucd="instr.filter">
88
<DESCRIPTION>
9-
column name
9+
flux filter name
1010
</DESCRIPTION>
1111
</FIELD>
12-
<FIELD ID="description" arraysize="*" datatype="char" name="description">
12+
<FIELD ID="description" arraysize="*" datatype="unicodeChar" name="description" ucd="meta.note;instr.filter">
1313
<DESCRIPTION>
14-
brief description of column
14+
flux filter description
1515
</DESCRIPTION>
1616
</FIELD>
1717
<FIELD ID="type" arraysize="*" datatype="unicodeChar" name="type"/>
@@ -507,6 +507,91 @@
507507
<TD>all fields related with radial velocity and redshift</TD>
508508
<TD>bundle of basic columns</TD>
509509
</TR>
510+
<TR>
511+
<TD>U</TD>
512+
<TD>Magnitude U</TD>
513+
<TD>filter name</TD>
514+
</TR>
515+
<TR>
516+
<TD>B</TD>
517+
<TD>Magnitude B</TD>
518+
<TD>filter name</TD>
519+
</TR>
520+
<TR>
521+
<TD>V</TD>
522+
<TD>Magnitude V</TD>
523+
<TD>filter name</TD>
524+
</TR>
525+
<TR>
526+
<TD>R</TD>
527+
<TD>Magnitude R</TD>
528+
<TD>filter name</TD>
529+
</TR>
530+
<TR>
531+
<TD>I</TD>
532+
<TD>Magnitude I</TD>
533+
<TD>filter name</TD>
534+
</TR>
535+
<TR>
536+
<TD>J</TD>
537+
<TD>Magnitude J</TD>
538+
<TD>filter name</TD>
539+
</TR>
540+
<TR>
541+
<TD>H</TD>
542+
<TD>Magnitude H</TD>
543+
<TD>filter name</TD>
544+
</TR>
545+
<TR>
546+
<TD>K</TD>
547+
<TD>Magnitude K</TD>
548+
<TD>filter name</TD>
549+
</TR>
550+
<TR>
551+
<TD>u</TD>
552+
<TD>Magnitude SDSS u</TD>
553+
<TD>filter name</TD>
554+
</TR>
555+
<TR>
556+
<TD>g</TD>
557+
<TD>Magnitude SDSS g</TD>
558+
<TD>filter name</TD>
559+
</TR>
560+
<TR>
561+
<TD>r</TD>
562+
<TD>Magnitude SDSS r</TD>
563+
<TD>filter name</TD>
564+
</TR>
565+
<TR>
566+
<TD>i</TD>
567+
<TD>Magnitude SDSS i</TD>
568+
<TD>filter name</TD>
569+
</TR>
570+
<TR>
571+
<TD>z</TD>
572+
<TD>Magnitude SDSS z</TD>
573+
<TD>filter name</TD>
574+
</TR>
575+
<TR>
576+
<TD>G</TD>
577+
<TD>Magnitude Gaia G</TD>
578+
<TD>filter name</TD>
579+
</TR>
580+
<TR>
581+
<TD>F150W</TD>
582+
<TD>JWST NIRCam F150W</TD>
583+
<TD>filter name</TD>
584+
</TR>
585+
<TR>
586+
<TD>F200W</TD>
587+
<TD>JWST NIRCam F200W</TD>
588+
<TD>filter name</TD>
589+
</TR>
590+
<TR>
591+
<TD>F444W</TD>
592+
<TD>JWST NIRCan F444W</TD>
593+
<TD>filter name</TD>
594+
</TR>
510595
</TABLEDATA>
511596
</DATA>
512597
</TABLE>

astroquery/simbad/tests/test_simbad.py

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ def test_mocked_simbad():
148148
simbad_instance = simbad.Simbad()
149149
# this mocks the list_votable_fields
150150
options = simbad_instance.list_votable_fields()
151-
assert len(options) > 90
151+
assert len(options) >= 115
152152
# this mocks the hardlimit
153153
assert simbad_instance.hardlimit == 2000000
154154

@@ -157,16 +157,13 @@ def test_mocked_simbad():
157157
# ----------------------------
158158

159159

160-
@pytest.mark.usefixtures("_mock_basic_columns")
161-
def test_votable_fields_utils(monkeypatch):
162-
monkeypatch.setattr(simbad.SimbadClass, "query_tap",
163-
lambda self, _: Table([["biblio"], ["biblio description"]],
164-
names=["name", "description"],
165-
dtype=["object", "object"]))
160+
@pytest.mark.usefixtures("_mock_simbad_class")
161+
def test_votable_fields_utils():
166162
options = simbad.SimbadClass().list_votable_fields()
167163
assert set(options.group_by("type").groups.keys["type"]) == {"table",
168164
"column of basic",
169-
"bundle of basic columns"}
165+
"bundle of basic columns",
166+
"filter name"}
170167

171168
description = simbad.SimbadClass().get_field_description("velocity")
172169
assert description == 'all fields related with radial velocity and redshift'
@@ -201,6 +198,7 @@ def test_get_bundle_columns(bundle_name, column):
201198
assert column in simbad.SimbadClass()._get_bundle_columns(bundle_name)
202199

203200

201+
@pytest.mark.usefixtures("_mock_simbad_class")
204202
@pytest.mark.usefixtures("_mock_linked_to_basic")
205203
def test_add_table_to_output(monkeypatch):
206204
# if table = basic, no need to add a join
@@ -218,10 +216,16 @@ def test_add_table_to_output(monkeypatch):
218216
simbad.core._Column("basic", "oid"),
219217
simbad.core._Column("mesdiameter", "oidref")
220218
) in simbad_instance.joins
221-
assert simbad.core._Column("mesdiameter", "bibcode", '"mesdiameter.bibcode"'
219+
assert simbad.core._Column("mesdiameter", "bibcode", "mesdiameter.bibcode"
222220
) in simbad_instance.columns_in_output
223-
assert simbad.core._Column("mesdiameter", "oidref", '"mesdiameter.oidref"'
221+
assert simbad.core._Column("mesdiameter", "oidref", "mesdiameter.oidref"
224222
) not in simbad_instance.columns_in_output
223+
# add allfluxes to test the special case
224+
monkeypatch.setattr(simbad.SimbadClass, "list_columns", lambda self, _: Table([["U", "u_"]],
225+
names=["column_name"]))
226+
simbad_instance._add_table_to_output("allfluxes")
227+
assert simbad.core._Column("allfluxes", "U") in simbad_instance.columns_in_output
228+
assert simbad.core._Column("allfluxes", "u_", "u") in simbad_instance.columns_in_output
225229

226230

227231
@pytest.mark.usefixtures("_mock_simbad_class")
@@ -244,6 +248,9 @@ def test_add_votable_fields():
244248
# add a bundle
245249
simbad_instance.add_votable_fields("dimensions")
246250
assert simbad.core._Column("basic", "galdim_majaxis") in simbad_instance.columns_in_output
251+
# add filter name
252+
simbad_instance.add_votable_fields("u")
253+
assert "allfluxes.u_" in simbad_instance.get_votable_fields()
247254
# a column which name has changed should raise a warning but still
248255
# be added under its new name
249256
simbad_instance.columns_in_output = []
@@ -256,7 +263,9 @@ def test_add_votable_fields():
256263
simbad_instance.add_votable_fields("distance")
257264
# errors are raised for the deprecated fields with options
258265
simbad_instance = simbad.SimbadClass()
259-
with pytest.warns(DeprecationWarning, match=r"The notation \'flux\(X\)\' is deprecated since 0.4.8. *"):
266+
with pytest.raises(ValueError, match=r"The votable fields \'flux_\*\*\*\(filtername\)\' are removed *"):
267+
simbad_instance.add_votable_fields("flux_error(u)")
268+
with pytest.warns(DeprecationWarning, match=r"The notation \'flux\(U\)\' is deprecated since 0.4.8 *"):
260269
simbad_instance.add_votable_fields("flux(u)")
261270
assert "u_" in str(simbad_instance.columns_in_output)
262271
with pytest.raises(ValueError, match="Coordinates conversion and formatting is no longer supported*"):
@@ -525,7 +534,7 @@ def test_list_linked_tables():
525534
def test_query():
526535
column = simbad.core._Column("basic", "*")
527536
# bare minimum with an alias
528-
expected = 'SELECT basic."main_id" AS my_id FROM basic'
537+
expected = 'SELECT basic."main_id" AS "my_id" FROM basic'
529538
assert simbad.Simbad._query(-1, [simbad.core._Column("basic", "main_id", "my_id")], [],
530539
[], get_query_payload=True)["QUERY"] == expected
531540
# with top

astroquery/simbad/tests/test_simbad_remote.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ def test_query_tap(self):
129129
" c 3")
130130
assert expect == str(result)
131131
# Test query_tap raised errors
132-
with pytest.raises(DALOverflowWarning, match="Partial result set *"):
132+
with pytest.warns(DALOverflowWarning, match="Partial result set *"):
133133
truncated_result = Simbad.query_tap("SELECT * from basic", maxrec=2)
134134
assert len(truncated_result) == 2
135135
with pytest.raises(ValueError, match="The maximum number of records cannot exceed 2000000."):
@@ -173,12 +173,12 @@ def test_add_bundle_to_output(self):
173173
assert len(simbad_instance.columns_in_output) == 8
174174
assert _Column("basic", "galdim_majaxis") in simbad_instance.columns_in_output
175175

176-
def test_add_table_to_output(self):
176+
def test_add_votable_fields(self):
177177
simbad_instance = Simbad()
178178
# empty before the test
179179
simbad_instance.columns_in_output = []
180180
simbad_instance.add_votable_fields("otypes")
181-
assert _Column("otypes", "otype", '"otypes.otype"') in simbad_instance.columns_in_output
181+
assert _Column("otypes", "otype", 'otypes.otype') in simbad_instance.columns_in_output
182182
# tables also require a join
183183
assert _Join("otypes",
184184
_Column("basic", "oid"),
@@ -191,3 +191,10 @@ def test_add_table_to_output(self):
191191
# mixed columns bundles and tables
192192
simbad_instance.add_votable_fields("flux", "velocity", "update_date")
193193
assert len(simbad_instance.columns_in_output) == 19
194+
195+
# add fluxes by their filter names
196+
simbad_instance = Simbad()
197+
simbad_instance.add_votable_fields("U", "V")
198+
simbad_instance.add_votable_fields("u")
199+
result = simbad_instance.query_object("HD 147933")
200+
assert all(filtername in result.colnames for filtername in {"u", "U", "V"})

0 commit comments

Comments
 (0)