Merge branch 'main' into issues/355-query-in-get-request

Author: gadomski

.github/workflows/continuous-integration.yml

@@ -20,26 +20,41 @@ jobs:
           - "3.11"
         os:
           - ubuntu-latest
-#          - windows-latest
+          - windows-latest
+          - macos-latest
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: 'setup.py'
+          cache: "pip"
+          cache-dependency-path: |
+            setup.py
+            requirements-dev.txt
 
-      - name: Execute linters and test suites
-        run: ./scripts/cibuild
+      - name: Install package
+        run: pip install .
+
+      - name: Install dev requirements
+        run: pip install -r requirements-dev.txt
+
+      - name: Run pre-commit
+        run: pre-commit run --all-files
+
+      - name: Run pytest
+        run: pytest -Werror -s --block-network --cov pystac_client --cov-report term-missing
+
+      - name: Run coverage
+        run: coverage xml
 
       - name: Upload All coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.xml
-          fail_ci_if_error: false 
+          fail_ci_if_error: false
 
   min-versions:
     name: min-versions
@@ -49,8 +64,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
-          cache: 'pip'
-          cache-dependency-path: 'requirements-min.txt'
+          cache: "pip"
+          cache-dependency-path: "requirements-min.txt"
       - name: Install minimum requirements
         run: pip install -r requirements-min.txt
       - name: Install
@@ -68,8 +83,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
-          cache: 'pip'
-          cache-dependency-path: 'requirements-docs.txt'
+          cache: "pip"
+          cache-dependency-path: "requirements-docs.txt"
       - name: Install pandoc
         run: sudo apt-get update && sudo apt-get -y install pandoc
       - name: Install docs requirements
@@ -87,8 +102,8 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
-          cache: 'pip'
-          cache-dependency-path: 'setup.py'
+          cache: "pip"
+          cache-dependency-path: "setup.py"
       - name: Install
         run: pip install .
       - name: Install any pre-releases of pystac
@@ -97,3 +112,18 @@ jobs:
         run: pip install -r requirements-dev.txt
       - name: Test
         run: ./scripts/test
+
+  dev-and-docs-requirements:
+    name: dev and docs requirements check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          cache: "pip"
+          cache-dependency-path: "setup.py"
+      - name: Install
+        run: pip install .
+      - name: Install dev and docs requirements
+        run: pip install -r requirements-dev.txt -r requirements-docs.txt

.pre-commit-config.yaml

@@ -3,34 +3,38 @@
 
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.12.0
     hooks:
       - id: black
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.1.0
+    rev: v2.2.2
     hooks:
       - id: codespell
         args: [--ignore-words=.codespellignore]
         types_or: [jupyter, markdown, python, shell]
   - repo: https://github.com/PyCQA/doc8
-    rev: 0.11.1
+    rev: v1.1.1
     hooks:
       - id: doc8
+        args: [--ignore=D004]
+        additional_dependencies:
+          - importlib_metadata < 5; python_version == "3.7"
   - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
+    rev: 6.0.0
     hooks:
       - id: flake8
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.11.4
     hooks:
       - id: isort
         name: isort (python)
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.960
+    rev: v0.991
     hooks:
       - id: mypy
         files: ".*\\.py$"
         additional_dependencies:
           - pystac
+          - pytest-vcr
           - types-requests
           - types-python-dateutil

CHANGELOG.md

@@ -10,13 +10,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - Python 3.11 support [#347](https://github.com/stac-utils/pystac-client/pull/347)
+- `request_modifier` to `StacApiIO` to allow for additional authentication mechanisms (e.g. AWS SigV4) [#371](https://github.com/stac-utils/pystac-client/issues/371)
+- *Authentication* tutorial, demonstrating how to use to the provided hooks to use both basic and AWS SigV4 authentication [#371](https://github.com/stac-utils/pystac-client/issues/371)
+- CI checks for Windows and MacOS [#378](https://github.com/stac-utils/pystac-client/pull/378)
 
 ### Changed
 
 ### Fixed
 
 - Some mishandled cases for datetime intervals [#363](https://github.com/stac-utils/pystac-client/pull/363)
 - `query` parameter in GET requests [#362](https://github.com/stac-utils/pystac-client/pull/362)
+- Collection requests when the Client's url ends in a '/' [#373](https://github.com/stac-utils/pystac-client/pull/373), [#405](https://github.com/stac-utils/pystac-client/pull/405)
+- Parse datetimes more strictly [#364](https://github.com/stac-utils/pystac-client/pull/364)
 
 ### Removed
 

docs/tutorials.rst

@@ -47,3 +47,13 @@ Calculating Coverage Percentage of the AOI by an Item
 This tutorial demonstrates the use of pystac-client to calculate the
 percentage an Item's geometry that intesects with the area of interest
 (AOI) specified in the search by the `intersects` parameter.
+
+Authentication
+--------------
+
+- :tutorial:`GitHub version <authentication.md>`
+- :ref:`Docs version </tutorials/authentication.md>`
+
+This tutorial demontrates different ways the pystac-client can be
+used to access a private stac api, when protected with various
+authentication mechanisms.

docs/tutorials/authentication.md

@@ -0,0 +1,83 @@
+# Authentication
+
+While not integrated into this library directly, pystac-client provides a series of hooks that support a wide variety of authentication mechanisms. These can be used when interacting with stac API implementations behind various authorization walls.
+
+## Basic auth
+
+Pystac-client supports HTTP basic authentication by simply exposing the ability to define headers to be used when sending requests.  Simply encode the token and provide the header.
+
+```python
+import base64
+import pystac_client
+
+# encode credentials
+user_name = "yellowbeard"
+password = "yaarg"
+userpass = f"{user_name}:{password}"
+b64_userpass = base64.b64encode(userpass.encode()).decode()
+
+# create the client
+client = pystac_client.Client.open(
+    url="https://planetarycomputer.microsoft.com/api/stac/v1",
+    headers={
+        'Authorization': f"Basic {b64_userpass}"
+    }
+)
+```
+
+## Token auth
+
+Providing a authentication token can be accomplished using the same mechanism as described above for [basic auth](#basic-auth). Simply provide the token in the `Authorization` header to the client in the same manner.
+
+## AWS SigV4
+
+Accessing a stac api protected by AWS IAM often requires signing the request using [AWS SigV4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html). Unlike basic and token authentication, the entire request is part of the signing process. Thus the `Authorization` header cannot be added when the client is created, rather it must be generated and added after the request is fully formed.
+
+Pystac-client provides a lower-level hook, the `request_modifier` parameter, which can mutate the request, adding the necessary header after the request has been generated but before it is sent.
+
+The code cell below demonstrates this, using the `boto3` module.
+
+```python
+import boto3
+import botocore.auth
+import botocore.awsrequest
+import pystac_client
+import requests
+
+# Details regarding the private stac api
+region = "us-east-1"
+service_name = "execute-api"
+endpoint_id = "xxxxxxxx"
+deployment_stage = "dev"
+stac_api_url = f"https://{endpoint_id}.{service_name}.{region}.amazonaws.com/{deployment_stage}"
+
+# load AWS credentials
+credentials = boto3.Session(region_name=region).get_credentials()
+signer = botocore.auth.SigV4Auth(credentials, service_name, region)
+
+def sign_request(request: requests.Request) -> requests.Request:
+    """Sign the request using AWS SigV4.
+
+    Args:
+        request (requests.Request): The fully populated request to sign.
+
+    Returns:
+        requests.Request: The provided request object, with auth header added.
+    """
+    aws_request = botocore.awsrequest.AWSRequest(
+        method=request.method,
+        url=request.url,
+        params=request.params,
+        data=request.data,
+        headers=request.headers
+    )
+    signer.add_auth(aws_request)
+    request.headers = aws_request.headers
+    return request
+
+# create the client
+client = pystac_client.Client.open(
+    url=stac_api_url,
+    request_modifier=sign_request
+)
+```

pystac_client/client.py

@@ -12,8 +12,10 @@
 )
 
 import pystac
+import pystac.utils
 import pystac.validation
 from pystac import CatalogType, Collection
+from requests import Request
 
 from pystac_client._utils import Modifiable, call_modifier
 from pystac_client.collection_client import CollectionClient
@@ -93,6 +95,8 @@ def open(
         parameters: Optional[Dict[str, Any]] = None,
         ignore_conformance: bool = False,
         modifier: Optional[Callable[[Modifiable], None]] = None,
+        request_modifier: Optional[Callable[[Request], Union[Request, None]]] = None,
+        stac_io: Optional[StacApiIO] = None,
     ) -> "Client":
         """Opens a STAC Catalog or API
         This function will read the root catalog of a STAC Catalog or API
@@ -128,12 +132,31 @@ def open(
                 After getting a child collection with, e.g.
                 :meth:`Client.get_collection`, the child items of that collection
                 will still be signed with ``modifier``.
+            request_modifier: A callable that either modifies a `Request` instance or
+                returns a new one. This can be useful for injecting Authentication
+                headers and/or signing fully-formed requests (e.g. signing requests
+                using AWS SigV4).
+
+                The callable should expect a single argument, which will be an instance
+                of :class:`requests.Request`.
+
+                If the callable returns a `requests.Request`, that will be used.
+                Alternately, the callable may simply modify the provided request object
+                and return `None`.
+            stac_io: A `StacApiIO` object to use for I/O requests. Generally, leave
+                this to the default. However in cases where customized I/O processing
+                is required, a custom instance can be provided here.
 
         Return:
             catalog : A :class:`Client` instance for this Catalog/API
         """
         client: Client = cls.from_file(
-            url, headers=headers, parameters=parameters, modifier=modifier
+            url,
+            headers=headers,
+            parameters=parameters,
+            modifier=modifier,
+            request_modifier=request_modifier,
+            stac_io=stac_io,
         )
         search_link = client.get_search_link()
         # if there is a search link, but no conformsTo advertised, ignore
@@ -161,14 +184,19 @@ def from_file(  # type: ignore
         headers: Optional[Dict[str, str]] = None,
         parameters: Optional[Dict[str, Any]] = None,
         modifier: Optional[Callable[[Modifiable], None]] = None,
+        request_modifier: Optional[Callable[[Request], Union[Request, None]]] = None,
     ) -> "Client":
         """Open a STAC Catalog/API
 
         Returns:
             Client: A Client (PySTAC Catalog) of the root Catalog for this Catalog/API
         """
         if stac_io is None:
-            stac_io = StacApiIO(headers=headers, parameters=parameters)
+            stac_io = StacApiIO(
+                headers=headers,
+                parameters=parameters,
+                request_modifier=request_modifier,
+            )
 
         client: Client = super().from_file(href, stac_io)  # type: ignore
 
@@ -226,7 +254,7 @@ def get_collection(self, collection_id: str) -> Optional[Collection]:
             CollectionClient: A STAC Collection
         """
         if self._supports_collections() and self._stac_io:
-            url = f"{self.get_self_href()}/collections/{collection_id}"
+            url = self._get_collections_href(collection_id)
             collection = CollectionClient.from_dict(
                 self._stac_io.read_json(url),
                 root=self,
@@ -253,9 +281,9 @@ def get_collections(self) -> Iterator[Collection]:
         """
         collection: Union[Collection, CollectionClient]
 
-        if self._supports_collections() and self.get_self_href() is not None:
-            url = f"{self.get_self_href()}/collections"
-            for page in self._stac_io.get_pages(url):  # type: ignore
+        if self._supports_collections() and self._stac_io:
+            url = self._get_collections_href()
+            for page in self._stac_io.get_pages(url):
                 if "collections" not in page:
                     raise APIError("Invalid response from /collections")
                 for col in page["collections"]:
@@ -476,3 +504,36 @@ def get_search_link(self) -> Optional[pystac.Link]:
             ),
             None,
         )
+
+    def _get_collections_href(self, collection_id: Optional[str] = None) -> str:
+        self_href = self.get_self_href()
+        if self_href is None:
+            data_link = self.get_single_link("data")
+            if data_link is None:
+                raise ValueError(
+                    "cannot build a collections href without a self href or a data link"
+                )
+            else:
+                collections_href = data_link.href
+        else:
+            collections_href = f"{self_href.rstrip('/')}/collections"
+
+        if not pystac.utils.is_absolute_href(collections_href):
+            collections_href = self._make_absolute_href(collections_href)
+
+        if collection_id is None:
+            return collections_href
+        else:
+            return f"{collections_href.rstrip('/')}/{collection_id}"
+
+    def _make_absolute_href(self, href: str) -> str:
+        self_link = self.get_single_link("self")
+        if self_link is None:
+            raise ValueError("cannot build an absolute href without a self link")
+        elif not pystac.utils.is_absolute_href(self_link.href):
+            raise ValueError(
+                "cannot build an absolute href from "
+                f"a relative self link: {self_link.href}"
+            )
+        else:
+            return pystac.utils.make_absolute_href(href, self_link.href)