readers.stac & readers.tindex: forward HTTP headers & queries to readers (PDAL#4738)

* initial filespec support for tindex reader_args

* move into setReaderOptions; add docs

Author: ibell13

doc/stages/readers.stac.md

@@ -112,6 +112,14 @@ reader_args
 ```bash
 --readers.stac.reader_args \
 '[{"type": "readers.ept", "resolution": 100}, {"type": "readers.las", "nosrs": true}]'
+```
+
+  HTTP headers & queries can be forwarded to the reader using a partial {ref}`filespec`
+  JSON object, omitting the `path` component:
+
+```bash
+--readers.stac.reader_args \
+'{"type": "readers.copc", "filespec": {"headers": {"your_header_key": "header_val"}, "query": {"your_query_key": "query_val"}}}'
 ```
 
 catalog_schema_url

doc/stages/readers.tindex.md

@@ -67,8 +67,16 @@ reader_args
   Exmaple:
 
 ```bash
---readers.stac.reader_args \
+--readers.tindex.reader_args \
 '[{"type": "readers.ept", "resolution": 100}, {"type": "readers.las", "nosrs": true}]'
+```
+
+  HTTP headers & queries can be forwarded to the reader using a partial {ref}`filespec`
+  JSON object, omitting the `path` component:
+
+```bash
+--readers.tindex.reader_args \
+'{"type": "readers.copc", "filespec": {"headers": {"your_header_key": "header_val"}, "query": {"your_query_key": "query_val"}}}'
 ```
 
 srs_column

io/TIndexReader.cpp

@@ -119,13 +119,30 @@ NL::json handleReaderArgs(NL::json rawReaderArgs)
     return readerArgs;
 }
 
-Options setReaderOptions(const NL::json& readerArgs, const std::string& driver)
+Options setReaderOptions(const NL::json& readerArgs, const std::string& driver,
+    const std::string& filename)
 {
     Options readerOptions;
+    readerOptions.add("filename", filename);
     if (readerArgs.contains(driver)) {
         NL::json args = readerArgs.at(driver).get<NL::json>();
         for (auto& arg : args.items()) {
             NL::detail::value_t type = readerArgs.at(driver).at(arg.key()).type();
+
+            // We treat the partial FileSpec as a special case
+            if (arg.key() == "file_spec" || arg.key() == "filespec")
+            {
+                NL::json filespecArg = arg.value();
+                if (!filespecArg.is_object())
+                    throw pdal_error("value for " + driver + "'file_spec' argument " +
+                        " must be a valid JSON object.");
+                filespecArg += {"path", filename};
+
+                // This doesn't check if the driver supports headers/queries: if not,
+                // the reader will only use the filename
+                readerOptions.replace("filename", filespecArg.dump());
+                continue;
+            }
             switch(type)
             {
                 case NL::detail::value_t::string:
@@ -354,9 +371,9 @@ void TIndexReader::initialize()
             throwError("Unable to create reader for file '" + f.m_filename +
                 "'.");
         reader->setLog(log());
-        Options readerOptions = setReaderOptions(m_args->m_readerArgs, driver);
 
-        readerOptions.add("filename", f.m_filename);
+        Options readerOptions = setReaderOptions(m_args->m_readerArgs, driver, f.m_filename);
+
         reader->setOptions(readerOptions);
         Stage *premerge = reader;
 

io/private/stac/Item.cpp

@@ -75,8 +75,7 @@ bool Item::init(const Filters& filters, NL::json rawReaderArgs,
         validate();
 
     NL::json readerArgs = handleReaderArgs(rawReaderArgs);
-    m_readerOptions = setReaderOptions(readerArgs, m_driver);
-    m_readerOptions.add("filename", m_assetPath);
+    m_readerOptions = setReaderOptions(readerArgs, m_driver, m_assetPath);
     return true;
 }
 
@@ -137,16 +136,30 @@ NL::json Item::handleReaderArgs(NL::json rawReaderArgs)
 }
 
 Options Item::setReaderOptions(const NL::json& readerArgs,
-    const std::string& driver) const
+    const std::string& driver, const std::string& filename) const
 {
     Options readerOptions;
+    readerOptions.add("filename", filename);
     if (readerArgs.contains(driver)) {
         NL::json args = jsonValue(readerArgs, driver);
         for (auto& arg : args.items())
         {
             std::string key = arg.key();
             NL::json val = arg.value();
             NL::detail::value_t type = val.type();
+            // We treat a partial FileSpec as a special case
+            if (key == "file_spec" || key == "filespec")
+            {
+                if (!val.is_object())
+                    throw pdal_error("value for " + driver + "'file_spec' argument " +
+                        " must be a valid JSON object.");
+                val += {"path", filename};
+
+                // This doesn't check if the driver supports headers/queries: if not,
+                // the reader will only use the filename
+                readerOptions.replace("filename", val.dump());
+                continue;
+            }
 
             // if value is of type string, dump() returns string with
             // escaped string inside and kills pdal program args

io/private/stac/Item.hpp

@@ -103,7 +103,8 @@ class Item
     std::string m_id;
 
     std::string extractDriverFromItem(const NL::json& asset) const;
-    Options setReaderOptions(const NL::json& readerArgs, const std::string& driver) const;
+    Options setReaderOptions(const NL::json& readerArgs, const std::string& driver,
+        const std::string& filename) const;
 
     NL::json handleReaderArgs(NL::json rawReaderArgs);
     void validate();