Merge pull request #15 from terrafloww/patchfix/aws_creds_issues

patch fix aws creds and add uv package manger

Author: print-sid8

.github/workflows/ci.yaml

@@ -4,14 +4,16 @@ on:
     branches:
       - main
     paths:
-      - requirements.txt
+      - pyproject.toml
+      - uv.lock
       - 'src/**'
       - 'examples/**'
   pull_request:
     branches:
       - main
     paths:
-      - requirements.txt
+      - pyproject.toml
+      - uv.lock
       - 'src/**'
       - 'examples/**'
 
@@ -21,32 +23,33 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.10", "3.11"]
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
 
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install uv
-        uv pip install -r requirements.txt --system
-        pip install -e .[dev]
+        uv sync --all-extras
+        uv build
 
     - name: Lint with ruff
       run: |
-        ruff check .
+        uvx ruff check .
 
     - name: Format with black
       run: |
-        black --check .
+        uvx black --check .
 
     - name: Run tests
       run: |
-        pytest --cov=rasteret
+        uv run pytest --cov=rasteret

examples/basic_workflow_xarray.py …es/basic_workflow_xarray_landsat_paid.pyexamples/basic_workflow_xarray.py renamed to examples/basic_workflow_xarray_landsat_paid.py examples/basic_workflow_xarray.py renamed to examples/basic_workflow_xarray_landsat_paid.py

@@ -17,8 +17,8 @@ def main():
     workspace_dir = Path.home() / "rasteret_workspace"
     workspace_dir.mkdir(exist_ok=True)
 
-    custom_name = "bangalore"
-    date_range = ("2024-01-01", "2024-03-31")
+    custom_name = "bangalore_landsat"
+    date_range = ("2024-01-01", "2024-01-31")
     data_source = DataSources.LANDSAT
 
     # Define area and time of interest
@@ -37,6 +37,7 @@ def main():
     # "bangalore_202401-03_landsat", if date_range spans across years, it will be "bangalore_202401-202503_landsat"
     collections = Rasteret.list_collections(workspace_dir=workspace_dir)
     for c in collections:
+        print(f"Existing collection in workspace dir {workspace_dir} :")
         print(
             f"- {c['name']}: {c['data_source']}, {c['date_range']}, {c['size']} scenes"
         )

examples/basic_workflow_xarray_sentinel.py

@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2025 Terrafloww Labs, Inc.
+
+from pathlib import Path
+from shapely.geometry import Polygon
+
+from rasteret import Rasteret
+from rasteret.constants import DataSources
+from rasteret.core.utils import save_per_geometry
+
+import xarray as xr
+
+aoi1_polygon = Polygon(
+    [(77.55, 13.01), (77.58, 13.01), (77.58, 13.08), (77.55, 13.08), (77.55, 13.01)]
+)
+
+
+# Get the total bounds of all polygons above
+bbox = aoi1_polygon.bounds
+# OR
+# give even larger AOI bounds that covers all your future analysis areas
+# eg., Polygon of a State or a Country
+# bbox = country_polygon.bounds
+
+# Collection configuration
+
+# give your custom name for local collection, it will be attached to the
+# beginning of the collection name for eg., bangalore_202401-12_landsat
+custom_name = "bangalore"
+
+# here we are aiming to write 1 year worth of STAC metadata and COG file headers to local disk
+date_range = ("2024-01-01", "2024-01-31")
+
+# choose from LANDSAT / SENTINEL2
+data_source = DataSources.SENTINEL2
+
+# Set up workspace folder as you wish
+workspace_dir = Path.home() / "rasteret_workspace"
+workspace_dir.mkdir(exist_ok=True)
+
+# List existing collections if there are any in the workspace folder
+collections = Rasteret.list_collections(workspace_dir=workspace_dir)
+for c in collections:
+    print(f"\nExisting Collection in workspace dir {workspace_dir}:")
+    print(f"- {c['name']}: {c['data_source']}, {c['date_range']}, {c['size']} scenes")
+
+# Try loading existing collection
+try:
+    # example name given here
+    processor = Rasteret.load_collection(
+        "bangalore_202401-12_landsat", workspace_dir=workspace_dir
+    )
+except ValueError:
+
+    # Instantiate the Class
+    processor = Rasteret(
+        workspace_dir=workspace_dir,
+        custom_name=custom_name,
+        data_source=data_source,
+        date_range=date_range,
+    )
+
+    # and create a new collection
+
+    # we are giving the BBOX for which STAC items and their COG headers will be fetched
+    # and also filtering using PySTAC filters for LANDSAT 8 platform specifically
+    # from LANDSAT USGS STAC, and giving a scene level cloud-cover filter
+    processor.create_collection(
+        bbox=bbox,
+        cloud_cover_lt=20,
+    )
+
+# Now we can query the collection created above, to get the data we want
+# in this case 2 geometries, 2 bands, and a few PySTAC search filters are provided
+ds = processor.get_xarray(
+    geometries=[aoi1_polygon],
+    bands=["B04", "B08"],
+    cloud_cover_lt=20,
+    date_range=["2024-01-10", "2024-01-30"],
+)
+# this returns an xarray dataset variable "ds" with the data for the geometries and bands specified
+# behind the scenes, the library is efficiently filtering the local STAC geoparquet,
+# for the LANDSAT scenes that pass the filters and dates provided
+# then its getting the tif urls of the requested bands
+# then grabbing COG tiles only for the geometries from those tif files
+# and creating a xarray dataset for each geometry and its time series data
+
+# Calculate NDVI
+ndvi = (ds.B08 - ds.B04) / (ds.B08 + ds.B04)
+
+# for LANDSAT satellite
+# ndvi_ds = (ds.B5 - ds.B4) / (ds.B5 + ds.B4)
+
+# give a data variable name for NDVI array
+ndvi_ds = xr.Dataset(
+    {"NDVI": ndvi},
+    coords=ds.coords,
+    attrs=ds.attrs,
+)
+
+# create a output folder if you wish to
+output_dir = Path(f"ndvi_results_{custom_name}")
+output_dir.mkdir(exist_ok=True)
+
+# Save results from xarray to geotiff files, each geometry's data will be stored in
+# its own folder. We can also give file-name prefix
+# and also mention which Xarray varible to save as geotiffs
+output_files = save_per_geometry(
+    ndvi_ds, output_dir, file_prefix="ndvi", data_var="NDVI"
+)
+
+for geom_id, filepath in output_files.items():
+    print(f"Geometry {geom_id}: {filepath}")

examples/example_xarray_nb.ipynb

@@ -29,7 +29,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "d6cf7929-caed-44c2-825a-b58d5b7bec94",
    "metadata": {},
    "outputs": [],
@@ -39,22 +39,18 @@
     "workspace_dir.mkdir(exist_ok=True)\n",
     "\n",
     "# Define a custom name for the stac index\n",
-    "custom_name = \"bangalore-v2\"\n",
+    "custom_name = \"bangalore_sentinel2_nb\"\n",
     "\n",
     "# Define area, time range and data source required in stac search\n",
-    "date_range = (\"2024-12-01\", \"2025-01-30\")\n",
-    "data_source = DataSources.LANDSAT  # or SENTINEL2\n",
+    "date_range = (\"2025-01-01\", \"2025-01-30\")\n",
+    "data_source = DataSources.SENTINEL2  # or SENTINEL2\n",
     "\n",
     "aoi1_polygon = Polygon(\n",
     "    [(77.55, 13.01), (77.58, 13.01), (77.58, 13.08), (77.55, 13.08), (77.55, 13.01)]\n",
     ")\n",
     "\n",
-    "aoi2_polygon = Polygon(\n",
-    "    [(77.56, 13.02), (77.59, 13.02), (77.59, 13.09), (77.56, 13.09), (77.56, 13.02)]\n",
-    ")\n",
-    "\n",
-    "# get total bounds of all polygons above for stac search and stac index creation\n",
-    "bbox = aoi1_polygon.union(aoi2_polygon).bounds\n",
+    "# get total bounds of all polygons above for STAC search and STAC index creation\n",
+    "bbox = aoi1_polygon.bounds\n",
     "\n"
    ]
   },
@@ -70,6 +66,7 @@
     "print(\"--------------------------\")\n",
     "collections = Rasteret.list_collections(workspace_dir=workspace_dir)\n",
     "for c in collections:\n",
+    "    print(f\"Existing collection in workspace dir {workspace_dir}:\")\n",
     "    print(f\"- {c['name']}: {c['data_source']}, {c['date_range']}, {c['size']} scenes\")"
    ]
   },
@@ -81,7 +78,7 @@
    "outputs": [],
    "source": [
     "# You can load any existing collection in any folder\n",
-    "processor = Rasteret.load_collection(\"bangalore_202401-01_landsat\", workspace_dir=workspace_dir)\n"
+    "#processor = Rasteret.load_collection(\"bangalore_202401-01_landsat\", workspace_dir=workspace_dir)\n"
    ]
   },
   {
@@ -102,8 +99,7 @@
     "processor.create_collection(\n",
     "    bbox=bbox,\n",
     "    date_range=date_range,\n",
-    "    cloud_cover_lt=20,\n",
-    "    platform={\"in\": [\"LANDSAT_8\"]}\n",
+    "    cloud_cover_lt=50,\n",
     ")"
    ]
   },
@@ -120,8 +116,9 @@
     "print(\"\\n3. Processing Data\")\n",
     "print(\"-----------------\")\n",
     "ds = processor.get_xarray(\n",
-    "    geometries=[aoi1_polygon, aoi2_polygon],\n",
-    "    bands=[\"B4\", \"B5\"],\n",
+    "    geometries=[aoi1_polygon],\n",
+    "    bands=[\"B04\", \"B08\"],\n",
+    "    # give more STAC specific filters if requried\n",
     "    cloud_cover_lt=20\n",
     ")\n",
     "\n",
@@ -130,13 +127,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "aa8f4c96-5c5f-4847-a6f3-1a1a06257312",
    "metadata": {},
    "outputs": [],
    "source": [
     "# Calculate NDVI\n",
-    "ndvi = (ds.B5 - ds.B4) / (ds.B5 + ds.B4)\n",
+    "ndvi = (ds.B08 - ds.B04) / (ds.B08 + ds.B04)\n",
     "\n",
     "# Create a new dataset with NDVI as a variable\n",
     "ndvi_ds = xr.Dataset(\n",

pyproject.toml

@@ -1,37 +1,45 @@
-[build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"]
-build-backend = "setuptools.build_meta"
+[project]
+name = "rasteret"
+version = "0.2.1"
+description = "Python library that gives you faster querying of Cloud-Optimized GeoTIFFs (COGs), and lowers S3 HTTP requests in your workflows."
+readme = "README.md"
+requires-python = ">=3.10,<3.12"
+authors = [
+    { name = "Sidharth Subramaniam", email = "sid@terrafloww.com" }
+]
+dependencies = [
+    "numpy>=1.24.0",
+    "pyarrow>=14.0.1",
+    "pystac-client>=0.7.5",
+    "stac-geoparquet==0.6.0",
+    "aiohttp>=3.9.1",
+    "aiofiles>=23.2.1",
+    "cachetools>=5.3.2",
+    "geoarrow-pyarrow>=0.1.0",
+    "geoarrow-pandas>=0.1.0",
+    "boto3==1.18.32",
+    "xarray==2025.01.0",
+    "rioxarray>=0.7.0",
+    "imagecodecs>=2023.9.18",
+    "zstandard>=0.22.0",
+    "pyproj>=3.6.1",
+    "shapely>=2.0.2",
+    "affine>=2.4.0",
+    "rasterio>=1.4.0",
+    "rich>=13.7.0",
+    "tqdm>=4.66.1",
+]
 
-[tool.black]
-line-length = 88
-include = '\.pyi?$'
-exclude = '''
-/(
-    \.eggs
-  | \.git
-  | \.hg
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | _build
-  | buck-out
-  | build
-  | dist
-)/
-'''
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.4.2",
+    "pytest-cov>=7.0.0",
+    "black>=23.12.1",
+]
 
-[tool.isort]
-profile = "black"
-multi_line_output = 3
-include_trailing_comma = true
-force_grid_wrap = 0
-use_parentheses = true
-ensure_newline_before_comments = true
-line_length = 88
+[project.scripts]
+rasteret = "rasteret:main"
 
-[tool.pytest.ini_options]
-minversion = "6.0"
-addopts = "-ra -q --cov=rasteret"
-testpaths = [
-    "tests",
-]
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"