Merge pull request #15 from terrafloww/patchfix/aws_creds_issues

patch fix aws creds and add uv package manger

Author: print-sid8

.github/workflows/ci.yaml

@@ -4,14 +4,16 @@ on:
     branches:
       - main
     paths:
-      - requirements.txt
+      - pyproject.toml
+      - uv.lock
       - 'src/**'
       - 'examples/**'
   pull_request:
     branches:
       - main
     paths:
-      - requirements.txt
+      - pyproject.toml
+      - uv.lock
       - 'src/**'
       - 'examples/**'
 
@@ -21,32 +23,33 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.10", "3.11"]
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
 
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install uv
-        uv pip install -r requirements.txt --system
-        pip install -e .[dev]
+        uv sync --all-extras
+        uv build
 
     - name: Lint with ruff
       run: |
-        ruff check .
+        uvx ruff check .
 
     - name: Format with black
       run: |
-        black --check .
+        uvx black --check .
 
     - name: Run tests
       run: |
-        pytest --cov=rasteret
+        uv run pytest --cov=rasteret

README.md

@@ -167,7 +167,8 @@ export AWS_SECRET_ACCESS_KEY='your_secret_key'
 
 ## 🔧 Installation
 ```bash
-pip install rasteret
+# once cloned
+uv sync
 ```
 
 <br>
@@ -186,6 +187,8 @@ from rasteret import Rasteret
 from rasteret.constants import DataSources
 from rasteret.core.utils import save_per_geometry
 
+import xarray as xr
+
 aoi1_polygon = Polygon([
     (77.55, 13.01),
     (77.58, 13.01),
@@ -194,16 +197,8 @@ aoi1_polygon = Polygon([
     (77.55, 13.01)
 ])
 
-aoi2_polygon = Polygon([
-    (77.56, 13.02),
-    (77.59, 13.02),
-    (77.59, 13.09),
-    (77.56, 13.09),
-    (77.56, 13.02)
-])
-
 # Get the total bounds of all polygons above
-bbox = aoi1_polygon.union(aoi2_polygon).bounds
+bbox = aoi1_polygon.bounds
 # OR
 # give even larger AOI bounds that covers all your future analysis areas
 # eg., Polygon of a State or a Country
@@ -223,10 +218,10 @@ in your workspace directory, if they were created earlier.
 custom_name = "bangalore"
 
 # here we are aiming to write 1 year worth of STAC metadata and COG file headers to local disk
-date_range = ("2024-01-01", "2024-12-31")
+date_range = ("2025-01-01", "2025-01-31")
 
 # choose from LANDSAT / SENTINEL2
-data_source = DataSources.LANDSAT
+data_source = DataSources.SENTINEL2
 
 # Set up workspace folder as you wish
 workspace_dir = Path.home() / "rasteret_workspace"
@@ -235,6 +230,7 @@ workspace_dir.mkdir(exist_ok=True)
 # List existing collections if there are any in the workspace folder
 collections = Rasteret.list_collections(workspace_dir=workspace_dir)
 for c in collections:
+    print(f"\nExisting Collection in workspace dir {workspace_dir}:")
     print(f"- {c['name']}: {c['data_source']}, {c['date_range']}, {c['size']} scenes")
 ```
 
@@ -250,9 +246,9 @@ except ValueError:
     # Instantiate the Class
     processor = Rasteret(
         workspace_dir=workspace_dir,
-        custom_name="bangalore",
-        data_source=DataSources.LANDSAT,
-        date_range=("2024-01-01", "2024-01-31")
+        custom_name=custom_name,
+        data_source=data_source,
+        date_range=date_range
     )
 
     # and create a new collection
@@ -263,7 +259,6 @@ except ValueError:
     processor.create_collection(
         bbox=bbox,
         cloud_cover_lt=20,
-        platform={"in": ["LANDSAT_8"]}
     )
 ```
 
@@ -273,10 +268,9 @@ except ValueError:
 # Now we can query the collection created above, to get the data we want
 # in this case 2 geometries, 2 bands, and a few PySTAC search filters are provided
 ds = processor.get_xarray(
-    geometries=[aoi1_polygon,aoi2_polygon],
-    bands=["B4", "B5"],
+    geometries=[aoi1_polygon],
+    bands=["B04", "B08"],
     cloud_cover_lt=20,
-    date_range=["2024-01-10", "2024-01-30"]
 )
 # this returns an xarray dataset variable "ds" with the data for the geometries and bands specified
 # behind the scenes, the library is efficiently filtering the local STAC geoparquet,
@@ -286,7 +280,10 @@ ds = processor.get_xarray(
 # and creating a xarray dataset for each geometry and its time series data
 
 # Calculate NDVI
-ndvi_ds = (ds.B5 - ds.B4) / (ds.B5 + ds.B4)
+ndvi = (ds.B08 - ds.B04) / (ds.B08 + ds.B04)
+
+# for LANDSAT satellite
+# ndvi = (ds.B5 - ds.B4) / (ds.B5 + ds.B4)
 
 # give a data variable name for NDVI array
 ndvi_ds = xr.Dataset(

examples/basic_workflow_xarray.py …es/basic_workflow_xarray_landsat_paid.pyexamples/basic_workflow_xarray.py renamed to examples/basic_workflow_xarray_landsat_paid.py examples/basic_workflow_xarray.py renamed to examples/basic_workflow_xarray_landsat_paid.py

@@ -14,8 +14,8 @@ def main():
     workspace_dir = Path.home() / "rasteret_workspace"
     workspace_dir.mkdir(exist_ok=True)
 
-    custom_name = "bangalore"
-    date_range = ("2024-01-01", "2024-03-31")
+    custom_name = "bangalore_landsat"
+    date_range = ("2024-01-01", "2024-01-31")
     data_source = DataSources.LANDSAT
 
     # Define area and time of interest
@@ -34,6 +34,7 @@ def main():
     # "bangalore_202401-03_landsat", if date_range spans across years, it will be "bangalore_202401-202503_landsat"
     collections = Rasteret.list_collections(workspace_dir=workspace_dir)
     for c in collections:
+        print(f"Existing collection in workspace dir {workspace_dir} :")
         print(
             f"- {c['name']}: {c['data_source']}, {c['date_range']}, {c['size']} scenes"
         )

examples/basic_workflow_xarray_sentinel.py

@@ -0,0 +1,110 @@
+from pathlib import Path
+from shapely.geometry import Polygon
+
+from rasteret import Rasteret
+from rasteret.constants import DataSources
+from rasteret.core.utils import save_per_geometry
+
+import xarray as xr
+
+aoi1_polygon = Polygon(
+    [(77.55, 13.01), (77.58, 13.01), (77.58, 13.08), (77.55, 13.08), (77.55, 13.01)]
+)
+
+
+# Get the total bounds of all polygons above
+bbox = aoi1_polygon.bounds
+# OR
+# give even larger AOI bounds that covers all your future analysis areas
+# eg., Polygon of a State or a Country
+# bbox = country_polygon.bounds
+
+# Collection configuration
+
+# give your custom name for local collection, it will be attached to the
+# beginning of the collection name for eg., bangalore_202401-12_landsat
+custom_name = "bangalore"
+
+# here we are aiming to write 1 year worth of STAC metadata and COG file headers to local disk
+date_range = ("2024-01-01", "2024-01-31")
+
+# choose from LANDSAT / SENTINEL2
+data_source = DataSources.SENTINEL2
+
+# Set up workspace folder as you wish
+workspace_dir = Path.home() / "rasteret_workspace"
+workspace_dir.mkdir(exist_ok=True)
+
+# List existing collections if there are any in the workspace folder
+collections = Rasteret.list_collections(workspace_dir=workspace_dir)
+for c in collections:
+    print(f"\nExisting Collection in workspace dir {workspace_dir}:")
+    print(f"- {c['name']}: {c['data_source']}, {c['date_range']}, {c['size']} scenes")
+
+# Try loading existing collection
+try:
+    # example name given here
+    processor = Rasteret.load_collection(
+        "bangalore_202401-12_landsat", workspace_dir=workspace_dir
+    )
+except ValueError:
+
+    # Instantiate the Class
+    processor = Rasteret(
+        workspace_dir=workspace_dir,
+        custom_name=custom_name,
+        data_source=data_source,
+        date_range=date_range,
+    )
+
+    # and create a new collection
+
+    # we are giving the BBOX for which STAC items and their COG headers will be fetched
+    # and also filtering using PySTAC filters for LANDSAT 8 platform specifically
+    # from LANDSAT USGS STAC, and giving a scene level cloud-cover filter
+    processor.create_collection(
+        bbox=bbox,
+        cloud_cover_lt=20,
+    )
+
+# Now we can query the collection created above, to get the data we want
+# in this case 2 geometries, 2 bands, and a few PySTAC search filters are provided
+ds = processor.get_xarray(
+    geometries=[aoi1_polygon],
+    bands=["B04", "B08"],
+    cloud_cover_lt=20,
+    date_range=["2024-01-10", "2024-01-30"],
+)
+# this returns an xarray dataset variable "ds" with the data for the geometries and bands specified
+# behind the scenes, the library is efficiently filtering the local STAC geoparquet,
+# for the LANDSAT scenes that pass the filters and dates provided
+# then its getting the tif urls of the requested bands
+# then grabbing COG tiles only for the geometries from those tif files
+# and creating a xarray dataset for each geometry and its time series data
+
+# Calculate NDVI
+ndvi = (ds.B08 - ds.B04) / (ds.B08 + ds.B04)
+
+# for LANDSAT satellite
+# ndvi_ds = (ds.B5 - ds.B4) / (ds.B5 + ds.B4)
+
+# give a data variable name for NDVI array
+ndvi_ds = xr.Dataset(
+    {"NDVI": ndvi},
+    coords=ds.coords,
+    attrs=ds.attrs,
+)
+
+# create a output folder if you wish to
+output_dir = Path(f"ndvi_results_{custom_name}")
+output_dir.mkdir(exist_ok=True)
+
+# Save results from xarray to geotiff files, each geometry's data will be stored in
+# its own folder. We can also give file-name prefix
+# and also mention which Xarray varible to save as geotiffs
+output_files = save_per_geometry(
+    ndvi_ds, output_dir, file_prefix="ndvi", data_var="NDVI"
+)
+
+for geom_id, filepath in output_files.items():
+    print(f"Geometry {geom_id}: {filepath}")

examples/example_xarray_nb.ipynb

@@ -29,7 +29,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "d6cf7929-caed-44c2-825a-b58d5b7bec94",
    "metadata": {},
    "outputs": [],
@@ -39,22 +39,18 @@
     "workspace_dir.mkdir(exist_ok=True)\n",
     "\n",
     "# Define a custom name for the stac index\n",
-    "custom_name = \"bangalore-v2\"\n",
+    "custom_name = \"bangalore_sentinel2_nb\"\n",
     "\n",
     "# Define area, time range and data source required in stac search\n",
-    "date_range = (\"2024-12-01\", \"2025-01-30\")\n",
-    "data_source = DataSources.LANDSAT  # or SENTINEL2\n",
+    "date_range = (\"2025-01-01\", \"2025-01-30\")\n",
+    "data_source = DataSources.SENTINEL2  # or SENTINEL2\n",
     "\n",
     "aoi1_polygon = Polygon(\n",
     "    [(77.55, 13.01), (77.58, 13.01), (77.58, 13.08), (77.55, 13.08), (77.55, 13.01)]\n",
     ")\n",
     "\n",
-    "aoi2_polygon = Polygon(\n",
-    "    [(77.56, 13.02), (77.59, 13.02), (77.59, 13.09), (77.56, 13.09), (77.56, 13.02)]\n",
-    ")\n",
-    "\n",
-    "# get total bounds of all polygons above for stac search and stac index creation\n",
-    "bbox = aoi1_polygon.union(aoi2_polygon).bounds\n",
+    "# get total bounds of all polygons above for STAC search and STAC index creation\n",
+    "bbox = aoi1_polygon.bounds\n",
     "\n"
    ]
   },
@@ -70,6 +66,7 @@
     "print(\"--------------------------\")\n",
     "collections = Rasteret.list_collections(workspace_dir=workspace_dir)\n",
     "for c in collections:\n",
+    "    print(f\"Existing collection in workspace dir {workspace_dir}:\")\n",
     "    print(f\"- {c['name']}: {c['data_source']}, {c['date_range']}, {c['size']} scenes\")"
    ]
   },
@@ -81,7 +78,7 @@
    "outputs": [],
    "source": [
     "# You can load any existing collection in any folder\n",
-    "processor = Rasteret.load_collection(\"bangalore_202401-01_landsat\", workspace_dir=workspace_dir)\n"
+    "#processor = Rasteret.load_collection(\"bangalore_202401-01_landsat\", workspace_dir=workspace_dir)\n"
    ]
   },
   {
@@ -102,8 +99,7 @@
     "processor.create_collection(\n",
     "    bbox=bbox,\n",
     "    date_range=date_range,\n",
-    "    cloud_cover_lt=20,\n",
-    "    platform={\"in\": [\"LANDSAT_8\"]}\n",
+    "    cloud_cover_lt=50,\n",
     ")"
    ]
   },
@@ -120,8 +116,9 @@
     "print(\"\\n3. Processing Data\")\n",
     "print(\"-----------------\")\n",
     "ds = processor.get_xarray(\n",
-    "    geometries=[aoi1_polygon, aoi2_polygon],\n",
-    "    bands=[\"B4\", \"B5\"],\n",
+    "    geometries=[aoi1_polygon],\n",
+    "    bands=[\"B04\", \"B08\"],\n",
+    "    # give more STAC specific filters if requried\n",
     "    cloud_cover_lt=20\n",
     ")\n",
     "\n",
@@ -130,13 +127,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "aa8f4c96-5c5f-4847-a6f3-1a1a06257312",
    "metadata": {},
    "outputs": [],
    "source": [
     "# Calculate NDVI\n",
-    "ndvi = (ds.B5 - ds.B4) / (ds.B5 + ds.B4)\n",
+    "ndvi = (ds.B08 - ds.B04) / (ds.B08 + ds.B04)\n",
     "\n",
     "# Create a new dataset with NDVI as a variable\n",
     "ndvi_ds = xr.Dataset(\n",