Upated iter method to use iter_datalinks, added documentation and tests

Author: d-giles

docs/dal/index.rst

@@ -869,6 +869,15 @@ DatalinkResults using
     >>> next(datalink.bysemantics("#this")).content_type
     'application/fits'
 
+As data becomes available from different cloud providers, some services are
+including cloud access information in the associated Datalink in JSON format.
+The ``get_cloud_params`` and ``iter_get_cloud_params`` fucntions for
+Records and Results respectively return an ``astropy.Table`` with the
+parameters specified to access data via the cloud service provider specified.
+
+More generic functions, ``parse_json_params`` and ``iter_parse_json_params``,
+act directly on the Record or Results object and can parse JSON columns with
+a given column name, key, and optionally parameters to match.
 
 Server-side processing
 ----------------------

pyvo/dal/adhoc.py

@@ -403,7 +403,28 @@ def iter_parse_json_params(
         verbose: bool=False,
         **match_params
     ):
+        """
+        Iterate over all Records in a DalResult and return parsed json parameters.
+
+        Parameters
+        ----------
+        colname : str, optional
+            The column containing JSON to be parsed, by default "cloud_access"
+        key : str, optional
+            The key to filter JSON results by, by default "aws"
+        verbose : bool, optional
+            Whether to print progress and errors, by default False
+        **match_params : str, optional
+            Any further parameters to match on.
 
+        Returns
+        -------
+        astropy.Table
+            A table containing the JSON parameters separated into columns, each
+            row corresponding to a matching JSON entry for each DataLinkRecord
+            for each row of the original DalResult.
+
+        """
         for irow, record in enumerate(self):
             access_points = record.parse_json_params(
                 colname=colname,
@@ -427,9 +448,55 @@ def iter_get_cloud_params(
         verbose: bool=False,
         **match_params
     ):
-        for irow, record in enumerate(self):
+        """
+        Iterate over all Records in a DalResult and return parsed cloud parameters.
+
+        Parameters
+        ----------
+        colname : str, optional
+            The column containing JSON to be parsed, by default "cloud_access"
+        provider : str, optional
+            The key to filter JSON results by, by default "aws"
+        verbose : bool, optional
+            Whether to print progress and errors, by default False
+        **match_params : str, optional
+            Any further parameters to match on.
+
+        Returns
+        -------
+        astropy.Table
+            A table containing the JSON parameters separated into columns, each
+            row corresponding to matching JSON entries from each Record.
+
+        """
+        for irow, dl_results in enumerate(self.iter_datalinks()):
+
+            products = dl_results.bysemantics("#this")
+
+            for jrow, row in enumerate(products):
+                # if no colname column, there is nothing to do    
+                try:
+                    access_points = row.parse_json_params(
+                        colname=colname,
+                        key=provider,
+                        verbose=verbose,
+                        **match_params
+                        )
+                    access_points.add_column([jrow]*len(access_points), name="datalink_row", index=0)
+                    if jrow == 0:
+                        new_dl_table = access_points
+                    else:
+                        for row in access_points.iterrows():
+                            new_dl_table.add_row(row)
+                except KeyError:
+                    # no json column, continue
+                    if verbose:
+                        print(f'No column {colname} found for row {irow}, datalink {jrow}')
+                    new_dl_table = TableElement(VOTableFile()).to_table()
+                    continue
+
             # do the json parsing
-            cloud_params = record.get_cloud_params(colname, provider, verbose, **match_params)
+            cloud_params = access_points
             cloud_params.add_column([irow]*len(cloud_params), name="record_row", index=0)
             if irow == 0:
                 new_table = cloud_params
@@ -498,16 +565,20 @@ def parse_json_params(
         
         Parameters
         ----------
-        colname: str
-            name of column to search in
-        provider: str, optional
-            name of data provider: only 'aws' is presently supported.
-        verbose: bool
-            If True, print progress and debug text.
-            
-        Return
-        ------
-        A dict or a list of dict of parameters for every row in products
+        colname : str, optional
+            The column containing JSON to be parsed, by default "cloud_access"
+        key : str, optional
+            The key to filter JSON results by, by default "aws"
+        verbose : bool, optional
+            Whether to print progress and errors, by default False
+        **match_params : str, optional
+            Any further parameters to match on.
+
+        Returns
+        -------
+        astropy.Table
+            A table containing the JSON parameters separated into columns, each
+            row representing a matching JSON entry.
 
         """
         import json
@@ -548,7 +619,7 @@ def parse_json_params(
         return new_table
 
     def get_cloud_params(self, colname="cloud_access", provider="aws", verbose=False, **match_params):
-        """Parse information stored as JSON by key
+        """Parse cloud information stored as JSON by provider
         
         Parameters
         ----------
@@ -558,10 +629,13 @@ def get_cloud_params(self, colname="cloud_access", provider="aws", verbose=False
             name of data provider: only 'aws' is presently supported.
         verbose: bool
             If True, print progress and debug text.
+        **match_params
             
         Return
         ------
-        An astropy Table with parameters for every row in the datalinks
+        astropy.Table
+            A table containing the JSON parameters separated into columns,
+            each row being a unique JSON entry and/or from a different DatalinkRecord.
 
         """
         dl_results = self.getdatalink()

pyvo/dal/tests/test_datalink.py

@@ -4,10 +4,12 @@
 Tests for pyvo.dal.datalink
 """
 from functools import partial
+from io import BytesIO
 import re
 
 import pytest
 
+from astropy.io.votable import parse, writeto, from_table
 import pyvo as vo
 from pyvo.dal.adhoc import DatalinkResults, DALServiceError
 from pyvo.dal.sia2 import SIA2Results
@@ -74,6 +76,21 @@ def callback(request, context):
     ) as matcher:
         yield matcher
 
+@pytest.fixture()
+def datalink_cloud(mocker):
+    def callback(request, context): 
+        dl_base = parse('pyvo/dal/tests/data/datalink/datalink.xml')
+        dl_base_table = dl_base.get_first_table().to_table()
+        cloud_access_str = '{"aws": {"bucket_name": "test", "key":"path/to/cloudfile.fits", "region": "us-west-2"}}'
+        dl_base_table.add_column([cloud_access_str]*4, name='cloud_access')
+        out = BytesIO()
+        writeto(from_table(dl_base_table), out)
+        return out.getvalue()
+
+    with mocker.register_uri(
+        'GET', 'http://example.com/datalink-cloud.xml', content=callback
+    ) as matcher:
+        yield matcher
 
 @pytest.fixture()
 def obscore_datalink(mocker):
@@ -336,3 +353,59 @@ def test_no_datalink():
     result = results[0]
     with pytest.raises(DALServiceError, match="No datalink found for record."):
         result.getdatalink()
+
+@pytest.mark.filterwarnings("ignore::astropy.io.votable.exceptions.E02")
+@pytest.mark.usefixtures('datalink_cloud')
+class TestJsonColumns:
+    """Tests for producing datalinks from tables containing links to
+    datalink documents.
+    """
+
+    res = testing.create_dalresults([
+            {"name": "access_url", "datatype": "char", "arraysize": "*",
+                "ucd": "meta.ref.url"},
+            {"name": "access_format", "datatype": "char", "arraysize": "*",
+                "utype": "meta.code.mime"},
+            {"name": "cloud_access", "datatype": "char", "arraysize": "*"},],
+            [("http://example.com/datalink-cloud.xml",
+                "application/x-votable+xml;content=datalink",
+                '{"aws": {"bucket_name": "test", "key":"path/to/file1.fits", "region": "us-west-2"}}',),
+                ("http://example.com/datalink-cloud.xml",
+                "application/x-votable+xml;content=datalink",
+                '{"aws": {"bucket_name": "test", "key":"path/to/file2.fits", "region": "us-west-2"}}',)],
+            resultsClass=SIA2Results
+        )
+    def test_record_w_json(self):
+
+
+        parsed_json_matches = self.res[0].parse_json_params("cloud_access", "aws")
+        assert parsed_json_matches[0]["bucket_name"] == "test"
+        assert parsed_json_matches[0]["key"] == "path/to/file1.fits"
+        assert parsed_json_matches[0]["region"] == "us-west-2"
+
+    def test_iter_json(self):
+
+        parsed_json_matches = self.res.iter_parse_json_params("cloud_access", "aws")
+        assert parsed_json_matches[0]["record_row"] == 0
+        assert parsed_json_matches[0]["bucket_name"] == "test"
+        assert parsed_json_matches[0]["key"] == "path/to/file1.fits"
+        assert parsed_json_matches[0]["region"] == "us-west-2"
+        assert parsed_json_matches[1]["record_row"] == 1
+        assert parsed_json_matches[1]["key"] == "path/to/file2.fits"
+
+    def test_datalink_json(self):
+        parsed_cloud_params = self.res[0].get_cloud_params("cloud_access", "aws")
+        assert parsed_cloud_params[0]["bucket_name"] == "test"
+        assert parsed_cloud_params[0]["key"] == "path/to/cloudfile.fits"
+        assert parsed_cloud_params[0]["region"] == "us-west-2"
+
+    def test_iter_datalink_json(self):
+        parsed_json_matches = self.res.iter_get_cloud_params("cloud_access", "aws")
+        assert parsed_json_matches[0]["record_row"] == 0
+        assert parsed_json_matches[0]["datalink_row"] == 0
+        assert parsed_json_matches[0]["bucket_name"] == "test"
+        assert parsed_json_matches[0]["key"] == "path/to/cloudfile.fits"
+        assert parsed_json_matches[0]["region"] == "us-west-2"
+        assert parsed_json_matches[1]["record_row"] == 1
+        assert parsed_json_matches[1]["datalink_row"] == 0
+        assert parsed_json_matches[1]["key"] == "path/to/cloudfile.fits"