Use same margin0 = timedelta(seconds=T_ORBIT + 60) for ASF as CDSE (#71)

* Fix bad typing and docstring on `download_eofs`

* Use same `margin0 = timedelta(seconds=T_ORBIT + 60)` for ASF as CDSE

Fixes #68

* Add tests verifying #68 fix

* Add cassettes

Author: scottstanie

eof/_select_orbit.py

@@ -1,4 +1,5 @@
 """Module for filtering/selecting from orbit query"""
+
 from __future__ import annotations
 
 import operator
@@ -24,7 +25,7 @@ def last_valid_orbit(
     t1: datetime,
     data: Sequence[SentinelOrbit],
     margin0=timedelta(seconds=T_ORBIT + 60),
-    margin1=timedelta(minutes=5),
+    margin1=timedelta(seconds=60),
 ) -> str:
     # Using a start margin of > 1 orbit so that the start of the orbit file will
     # cover the ascending node crossing of the acquisition

eof/asf_client.py

@@ -6,14 +6,14 @@
 from __future__ import annotations
 
 import os
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
 from typing import Literal
 
 import requests
 
 from ._asf_s3 import get_orbit_files, ASF_BUCKET_NAME
-from ._select_orbit import T_ORBIT, ValidityError, last_valid_orbit
+from ._select_orbit import ValidityError, last_valid_orbit
 from ._types import Filename
 from .log import logger
 from .products import SentinelOrbit
@@ -102,19 +102,12 @@ def get_download_urls(
             "S1B": [eof for eof in eof_list if eof.mission == "S1B"],
             "S1C": [eof for eof in eof_list if eof.mission == "S1C"],
         }
-        # For precise orbits, use a larger front margin to ensure coverage
-        if orbit_type == "precise":
-            margin0 = timedelta(seconds=T_ORBIT + 60)
-        else:
-            margin0 = timedelta(seconds=60)
 
         remaining_orbits = []
         urls = []
         for dt, mission in zip(orbit_dts, missions):
             try:
-                filename = last_valid_orbit(
-                    dt, dt, mission_to_eof_list[mission], margin0=margin0
-                )
+                filename = last_valid_orbit(dt, dt, mission_to_eof_list[mission])
                 # Construct the full download URL using the bucket name from _asf_s3
                 url = f"https://{ASF_BUCKET_NAME}.s3.amazonaws.com/{filename}"
                 urls.append(url)

eof/dataspace_client.py

@@ -1,4 +1,5 @@
 """Client to get orbit files from dataspace.copernicus.eu ."""
+
 from __future__ import annotations
 
 from concurrent.futures import ThreadPoolExecutor
@@ -91,7 +92,7 @@ def query_orbit(
 
     @staticmethod
     def query_orbit_for_product(
-        product,
+        product: str | S1Product,
         orbit_type: str = "precise",
         t0_margin: timedelta = T0,
         t1_margin: timedelta = T1,

eof/download.py

@@ -27,9 +27,11 @@
 import glob
 import itertools
 import os
+from collections.abc import Sequence
+from datetime import datetime
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
-from typing import Optional
+from typing import Literal, Optional
 
 from dateutil.parser import parse
 from requests.exceptions import HTTPError
@@ -44,11 +46,11 @@
 
 
 def download_eofs(
-    orbit_dts=None,
-    missions=None,
-    sentinel_file=None,
-    save_dir=".",
-    orbit_type="precise",
+    orbit_dts: Sequence[datetime | str] | None = None,
+    missions: Sequence[str] | None = None,
+    sentinel_file: str | None = None,
+    save_dir: str = ".",
+    orbit_type: Literal["precise", "restituted"] = "precise",
     force_asf: bool = False,
     asf_user: str = "",
     asf_password: str = "",
@@ -59,29 +61,65 @@ def download_eofs(
     netrc_file: Optional[Filename] = None,
     max_workers: int = MAX_WORKERS,
 ) -> list[Path]:
-    """Downloads and saves EOF files for specific dates
-
-    Args:
-        orbit_dts (list[str] or list[datetime.datetime]): datetime for orbit coverage
-        missions (list[str]): optional, to specify S1A, S1B or S1C
-            No input downloads both, must be same len as orbit_dts
-        sentinel_file (str): path to Sentinel-1 filename to download one .EOF for
-        save_dir (str): directory to save the EOF files into
-        orbit_type (str): precise or restituted
-
-    Returns:
-        list[str]: all filenames of saved orbit files
-
-    Raises:
-        ValueError - for missions argument not being one of 'S1A', 'S1B', 'S1C',
-            having different lengths, or `sentinel_file` being invalid
+    """Downloads and saves Sentinel precise or restituted orbit files (EOF).
+
+    By default, it tries to download from Copernicus Data Space Ecosystem first,
+    then falls back to ASF if the first source fails (unless `force_asf` is True).
+
+    Parameters
+    ----------
+    orbit_dts : list[datetime] or list[str] or None
+        List of datetimes for orbit coverage.
+        If strings are provided, they will be parsed into datetime objects.
+    missions : list[str] or None
+        List of mission identifiers ('S1A', 'S1B', or 'S1C'). If None,
+        orbits for all missions will be attempted. Must be same length as orbit_dts.
+    sentinel_file : str or None
+        Path to a Sentinel-1 file to download a matching EOF for.
+        If provided, orbit_dts and missions are ignored.
+    save_dir : str, default="."
+        Directory to save the downloaded EOF files into.
+    orbit_type : {'precise', 'restituted'}, default='precise'
+        Type of orbit file to download (precise=POEORB or restituted=RESORB).
+    force_asf : bool, default=False
+        If True, skip trying Copernicus Data Space and download directly from ASF.
+    asf_user : str, default=""
+        ASF username (deprecated, ASF orbits are now publicly available).
+    asf_password : str, default=""
+        ASF password (deprecated, ASF orbits are now publicly available).
+    cdse_access_token : str or None, default=None
+        Copernicus Data Space Ecosystem access token.
+    cdse_user : str, default=""
+        Copernicus Data Space Ecosystem username.
+    cdse_password : str, default=""
+        Copernicus Data Space Ecosystem password.
+    cdse_2fa_token : str, default=""
+        Copernicus Data Space Ecosystem two-factor authentication token.
+    netrc_file : str or None, default=None
+        Path to .netrc file for authentication credentials.
+    max_workers : int, default=MAX_WORKERS
+        Number of parallel downloads to run.
+
+    Returns
+    -------
+    list[Path]
+        Paths to all successfully downloaded orbit files.
+
+    Raises
+    ------
+    ValueError
+        If missions argument contains values other than 'S1A', 'S1B', 'S1C',
+        if missions and orbit_dts have different lengths, or if sentinel_file is invalid.
+    HTTPError
+        If there's an HTTP error during download that's not a rate limit error.
     """
     # TODO: condense list of same dates, different hours?
     if missions and all(m not in ("S1A", "S1B", "S1C") for m in missions):
         raise ValueError('missions argument must be "S1A", "S1B" or "S1C"')
     if sentinel_file:
         sent = Sentinel(sentinel_file)
         orbit_dts, missions = [sent.start_time], [sent.mission]
+    assert orbit_dts is not None and missions is not None
     if missions and len(missions) != len(orbit_dts):
         raise ValueError("missions arg must be same length as orbit_dts")
     if not missions:
@@ -114,7 +152,7 @@ def download_eofs(
                 )
 
             if query:
-                logger.info("Attempting download from SciHub")
+                logger.info("Attempting download from Copernicus Data Space Ecosystem")
                 try:
                     results = client.download_all(
                         query, output_directory=save_dir, max_workers=max_workers

eof/tests/cassettes/test_asf_client/test_query_resorb_s1_reader_issue68.yaml

@@ -0,0 +1,97 @@
+interactions:
+- request:
+    body: client_id=cdse-public&username=scott.stanie%40gmail.com&password=1Z8Xmfl2KbSx%21&grant_type=password
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, br, zstd
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '100'
+      Content-Type:
+      - application/x-www-form-urlencoded
+      User-Agent:
+      - python-requests/2.32.3
+    method: POST
+    uri: https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token
+  response:
+    body:
+      string: '{"access_token":"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJYVUh3VWZKaHVDVWo0X3k4ZF8xM0hxWXBYMFdwdDd2anhob2FPLUxzREZFIn0.eyJleHAiOjE3NDQwNTA3MTMsImlhdCI6MTc0NDA1MDExMywianRpIjoiNGVhZWI1MzAtYjIxNS00MWFkLTkzMWUtYTk4MjRhMTQ0Njg1IiwiaXNzIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwiYXVkIjpbIkNMT1VERkVSUk9fUFVCTElDIiwiYWNjb3VudCJdLCJzdWIiOiI2YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJjZHNlLXB1YmxpYyIsInNlc3Npb25fc3RhdGUiOiI2NmI1YTc2ZS02MzVhLTQ2ZDItYjczOC0zNzVmYmU1NDRkOGMiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cHM6Ly9sb2NhbGhvc3Q6NDIwMCIsIioiLCJodHRwczovL3dvcmtzcGFjZS5zdGFnaW5nLWNkc2UtZGF0YS1leHBsb3Jlci5hcHBzLnN0YWdpbmcuaW50cmEuY2xvdWRmZXJyby5jb20iXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iLCJkZWZhdWx0LXJvbGVzLWNkYXMiLCJjb3Blcm5pY3VzLWdlbmVyYWwiXX0sInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6IkFVRElFTkNFX1BVQkxJQyBvcGVuaWQgZW1haWwgcHJvZmlsZSBvbmRlbWFuZF9wcm9jZXNzaW5nIHVzZXItY29udGV4dCIsInNpZCI6IjY2YjVhNzZlLTYzNWEtNDZkMi1iNzM4LTM3NWZiZTU0NGQ4YyIsImdyb3VwX21lbWJlcnNoaXAiOlsiL2FjY2Vzc19ncm91cHMvdXNlcl90eXBvbG9neS9jb3Blcm5pY3VzX2dlbmVyYWwiLCIvb3JnYW5pemF0aW9ucy9kZWZhdWx0LTZiNTI5NDJlLTIzMTAtNGRkYi1iOGRlLTA2NTUyMTI5NDA2MS9yZWd1bGFyX3VzZXIiXSwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJTY290dCBTdGFuaWV3aWN6Iiwib3JnYW5pemF0aW9ucyI6WyJkZWZhdWx0LTZiNTI5NDJlLTIzMTAtNGRkYi1iOGRlLTA2NTUyMTI5NDA2MSJdLCJ1c2VyX2NvbnRleHRfaWQiOiJjN2JlMjE3Yi04Mzc4LTRkOTQtYmFmNi0yN2RmNWYxMGY3NmMiLCJjb250ZXh0X3JvbGVzIjp7fSwiY29udGV4dF9ncm91cHMiOlsiL2FjY2Vzc19ncm91cHMvdXNlcl90eXBvbG9neS9jb3Blcm5pY3VzX2dlbmVyYWwvIiwiL29yZ2FuaXphdGlvbnMvZGVmYXVsdC02YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEvcmVndWxhcl91c2VyLyJdLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJzY290dC5zdGFuaWVAZ21haWwuY29tIiwiZ2l2ZW5fbmFtZSI6IlNjb3R0IiwiZmFtaWx5X25hbWUiOiJTdGFuaWV3aWN6IiwidXNlcl9jb250ZXh0IjoiZGVmYXVsdC02YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEiLCJlbWFpbCI6InNjb3R0LnN0YW5pZUBnbWFpbC5jb20ifQ.MX_4QR1SLWjqRY_wib_01mMnmlpHVbUVagadHj9iQ1f8h6TuXavmiCGD0coI4i66nxLVTC5yIMrdq6w92MUFvNpIbxwSZPfCXgATsWKTfBB8oeMpYecrDOWpyzyChfnUwPwmMNDjibUqS4KONjY9QDhJGmtyFkjcqwY4h4ssKX_0A1dBG_DuwNbsvRDPB2yga3D24PsrwYUFF-DKbGZACGmE35plekLkq4WDPIQLxeOSGOx04SXZzv4MYlaQzO1QkJkKxML2zm3vj2LyYWoVVa5Ik-SnDdpiYWjtbian7lelzTQ6ahRKlNO7qtHKiSHgoPKOzqOpU__Q6t5veDKjfg","expires_in":600,"refresh_expires_in":3600,"refresh_token":"eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhZmFlZTU2Zi1iNWZiLTRiMzMtODRlYS0zMWY2NzMyMzNhNzgifQ.eyJleHAiOjE3NDQwNTM3MTMsImlhdCI6MTc0NDA1MDExMywianRpIjoiODQ2NTE0ZDctZGM0Mi00M2JkLTgwM2ItNjRlMjhjYTQ4MWJiIiwiaXNzIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwiYXVkIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwic3ViIjoiNmI1Mjk0MmUtMjMxMC00ZGRiLWI4ZGUtMDY1NTIxMjk0MDYxIiwidHlwIjoiUmVmcmVzaCIsImF6cCI6ImNkc2UtcHVibGljIiwic2Vzc2lvbl9zdGF0ZSI6IjY2YjVhNzZlLTYzNWEtNDZkMi1iNzM4LTM3NWZiZTU0NGQ4YyIsInNjb3BlIjoiQVVESUVOQ0VfUFVCTElDIG9wZW5pZCBlbWFpbCBwcm9maWxlIG9uZGVtYW5kX3Byb2Nlc3NpbmcgdXNlci1jb250ZXh0Iiwic2lkIjoiNjZiNWE3NmUtNjM1YS00NmQyLWI3MzgtMzc1ZmJlNTQ0ZDhjIn0.gTNW81DWt5Nje-dW4F9mfi-KoH4PgHXD0qmR9y0tgzo","token_type":"Bearer","not-before-policy":0,"session_state":"66b5a76e-635a-46d2-b738-375fbe544d8c","scope":"AUDIENCE_PUBLIC
+        openid email profile ondemand_processing user-context"}'
+    headers:
+      cache-control:
+      - no-store
+      content-length:
+      - '3423'
+      content-type:
+      - application/json
+      pragma:
+      - no-cache
+      referrer-policy:
+      - no-referrer
+      set-cookie:
+      - KEYCLOAK_LOCALE=; Version=1; Comment=Expiring cookie; Expires=Thu, 01-Jan-1970
+        00:00:10 GMT; Max-Age=0; Path=/auth/realms/CDSE/; Secure; HttpOnly
+      - KC_RESTART=; Version=1; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Max-Age=0;
+        Path=/auth/realms/CDSE/; Secure; HttpOnly
+      - KC_AUTH_STATE=; Version=1; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Max-Age=0;
+        Path=/auth/realms/CDSE/; Secure
+      - SERVERID=keycloak001.waw2; path=/
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - ALLOW-FROM https://www.google.com
+      x-xss-protection:
+      - 1; mode=block
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate, br, zstd
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: https://catalogue.dataspace.copernicus.eu/odata/v1/Products?%24filter=startswith%28Name%2C%27S1A%27%29+and+contains%28Name%2C%27AUX_RESORB%27%29+and+ContentDate%2FStart+lt+%272025-03-10T19%3A02%3A43.428571Z%27+and+ContentDate%2FEnd+gt+%272025-03-10T20%3A43%3A28.000000Z%27&%24orderby=ContentDate%2FStart+asc&%24top=1
+  response:
+    body:
+      string: '{"@odata.context":"$metadata#Products","value":[{"@odata.mediaContentType":"application/octet-stream","Id":"6f5734c8-fdfc-11ef-a200-0242ac120002","Name":"S1A_OPER_AUX_RESORB_OPOD_20250310T220905_V20250310T180852_20250310T212622.EOF","ContentType":"application/octet-stream","ContentLength":590749,"OriginDate":"2025-03-10T22:10:11.570000Z","PublicationDate":"2025-03-10T22:10:47.026472Z","ModificationDate":"2025-03-10T22:10:47.026472Z","Online":true,"EvictionDate":"9999-12-31T23:59:59.999999Z","S3Path":"/eodata/Sentinel-1/AUX/AUX_RESORB/2025/03/10/S1A_OPER_AUX_RESORB_OPOD_20250310T220905_V20250310T180852_20250310T212622.EOF","Checksum":[{"Value":"ead3d2760014f518fe0521c6930d5fc5","Algorithm":"MD5","ChecksumDate":"2025-03-10T22:10:45.527687Z"},{"Value":"485c5afb96399347b4e75dcdb0532d6b438a9cd2bb0ae0d9c0fbb29382777ad3","Algorithm":"BLAKE3","ChecksumDate":"2025-03-10T22:10:45.546864Z"}],"ContentDate":{"Start":"2025-03-10T18:08:52.000000Z","End":"2025-03-10T21:26:22.000000Z"},"Footprint":null,"GeoFootprint":null}],"@odata.nextLink":"https://catalogue.dataspace.copernicus.eu/odata/v1/Products?%24filter=startswith%28Name%2C%27S1A%27%29+and+contains%28Name%2C%27AUX_RESORB%27%29+and+ContentDate%2FStart+lt+%272025-03-10T19%3A02%3A43.428571Z%27+and+ContentDate%2FEnd+gt+%272025-03-10T20%3A43%3A28.000000Z%27&%24orderby=ContentDate%2FStart+asc&%24top=1&%24skip=1"}'
+    headers:
+      Access-Control-Allow-Credentials:
+      - 'true'
+      Access-Control-Allow-Headers:
+      - DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization
+      Access-Control-Allow-Methods:
+      - GET, PUT, POST, DELETE, PATCH, OPTIONS
+      Access-Control-Allow-Origin:
+      - '*'
+      Access-Control-Max-Age:
+      - '1728000'
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '1370'
+      Content-Type:
+      - application/json
+      Date:
+      - Mon, 07 Apr 2025 18:21:54 GMT
+      Strict-Transport-Security:
+      - max-age=15724800; includeSubDomains
+      request-id:
+      - 44b4898c-19e8-44c4-89af-2db17409d0f9
+      x-envoy-upstream-service-time:
+      - '394'
+    status:
+      code: 200
+      message: OK
+version: 1

eof/tests/cassettes/test_dataspace_client/test_query_resorb_s1_reader_issue68.yaml

@@ -3,6 +3,7 @@
 import pytest
 
 from eof.asf_client import ASFClient
+from eof.products import Sentinel
 from eof._asf_s3 import ASF_BUCKET_NAME, list_public_bucket
 
 
@@ -52,3 +53,19 @@ def test_list_public_bucket_poeorb():
         precise[0]
         == "AUX_POEORB/S1A_OPER_AUX_POEORB_OPOD_20210203T122423_V20210113T225942_20210115T005942.EOF"
     )
+
+
+@pytest.mark.vcr
+def test_query_resorb_s1_reader_issue68():
+    f = "S1A_IW_SLC__1SDV_20250310T204228_20250310T204253_058247_0732D8_1AA3"
+    sent = Sentinel(f)
+    orbit_dts, missions = [sent.start_time], [sent.mission]
+
+    client = ASFClient()
+
+    urls = client.get_download_urls(orbit_dts, missions, orbit_type="restituted")
+    assert len(urls) == 1
+    expected = (
+        "S1A_OPER_AUX_RESORB_OPOD_20250310T220905_V20250310T180852_20250310T212622.EOF"
+    )
+    assert urls[0].split("/")[-1] == expected

eof/tests/test_asf_client.py

@@ -39,3 +39,16 @@ def test_query_resorb_edge_case():
         r["title"]
         == "S1A_OPER_AUX_RESORB_OPOD_20230823T174849_V20230823T141024_20230823T172754"
     )
+
+
+@pytest.mark.vcr
+def test_query_resorb_s1_reader_issue68():
+    f = "S1A_IW_SLC__1SDV_20250310T204228_20250310T204253_058247_0732D8_1AA3"
+
+    client = DataspaceClient()
+    query = client.query_orbit_for_product(f, orbit_type="restituted")
+    assert len(query) == 1
+    expected = (
+        "S1A_OPER_AUX_RESORB_OPOD_20250310T220905_V20250310T180852_20250310T212622.EOF"
+    )
+    assert query[0]["Name"] == expected