feat: added export all collections.

Author: pantierra

charts/pgstac-geoparquet-exporter/README.md

@@ -25,46 +25,62 @@ helm install exporter ./charts/pgstac-geoparquet-exporter \
   --create-namespace \
   --set database.existingSecret=pgstac-db \
   --set storage.existingSecret=s3-creds \
-  --set storage.outputPath=s3://my-bucket/exports
+  --set storage.outputPath=s3://my-bucket/exports \
+  --set stacApiUrl=https://example.com/stac/v1
 ```
 
 ## Configuration
 
 Edit `values.yaml` or create a custom values file:
 
 ```yaml
+# STAC API URL (required for link injection)
+stacApiUrl: "https://example.com/stac/v1"
+
 # Collections to export
 exportConfig:
+  # Export all collections from database (ignores collections list below)
+  exportAll: false
+  
+  # Specify individual collections (or leave empty and set exportAll: true)
   collections:
     - name: sentinel-2
       partition_by: year
       start_year: 2015
     - name: landsat-8
       partition_by: null  # Single file
 
-# Schedules
-completeExport:
-  schedule: "0 2 1 * *"  # Monthly
-incrementalExport:
-  schedule: "0 3 * * *"  # Daily
-
-# Resources
-incrementalExport:
-  resources:
-    requests:
-      memory: "1Gi"
-      cpu: "250m"
+# Job schedules and resources
+jobs:
+  complete:
+    schedule: "0 2 1 * *"  # Monthly
+    resources:
+      requests:
+        memory: "2Gi"
+        cpu: "500m"
+
+  incremental:
+    schedule: "0 3 * * *"  # Daily
+    resources:
+      requests:
+        memory: "1Gi"
+        cpu: "250m"
 ```
 
 ## Key Parameters
 
 | Parameter | Description | Default |
 |-----------|-------------|---------|
-| `database.existingSecret` | DB credentials secret | `default-pguser-eoapi` |
-| `storage.outputPath` | Output path (s3:// or local) | `s3://eoapi-geoparquet/geoparquet` |
-| `storage.existingSecret` | S3 credentials secret | `data-access` |
-| `completeExport.schedule` | Complete export cron | `"0 2 1 * *"` |
-| `incrementalExport.schedule` | Incremental export cron | `"0 3 * * *"` |
+| `stacApiUrl` | **Required**. STAC API URL for link injection | `""` |
+| `database.existingSecret` | DB credentials secret name | `default-pguser-eoapi` |
+| `storage.outputPath` | Output path (s3:// or local) | `s3://some-bucket/geoparquet` |
+| `storage.existingSecret` | S3 credentials secret name | `""` |
+| `storage.endpoint` | Custom S3 endpoint URL | `""` |
+| `storage.region` | AWS region | `""` |
+| `jobs.complete.schedule` | Complete export cron schedule | `"0 2 1 * *"` |
+| `jobs.incremental.schedule` | Incremental export cron schedule | `"0 3 * * *"` |
+| `exportConfig.exportAll` | Export all collections from database | `false` |
+| `exportConfig.collections[].partition_by` | Partitioning: `null`, `year`, `month` | `year` |
 
 ## Usage
 
@@ -83,3 +99,16 @@ helm upgrade exporter ./charts/pgstac-geoparquet-exporter -n data-access -f valu
 
 - **Complete**: Full export with optional yearly/monthly partitioning
 - **Incremental**: Only changed items since last run (state in `{OUTPUT_PATH}/.last_sync`)
+
+### Export All Collections
+
+Set `exportConfig.exportAll: true` to automatically export all collections from the pgSTAC database instead of manually specifying them. Useful when collections are added dynamically.
+
+## Advanced Options
+
+Additional configuration available in `values.yaml`:
+- `exportConfig.settings` - chunk_size, statement_timeout
+- `jobs` - successfulJobsHistoryLimit, failedJobsHistoryLimit, concurrencyPolicy
+- `extraEnv`, `extraVolumes`, `extraVolumeMounts` - Additional resources
+- `nodeSelector`, `tolerations`, `affinity` - Pod scheduling
+- `podSecurityContext`, `securityContext` - Security settings

charts/pgstac-geoparquet-exporter/templates/configmap.yaml

@@ -11,6 +11,8 @@ metadata:
   {{- end }}
 data:
   export-config.yaml: |
+    exportAll: {{ .Values.exportConfig.exportAll }}
+    
     collections:
     {{- range .Values.exportConfig.collections }}
       - name: {{ .name }}

charts/pgstac-geoparquet-exporter/values.yaml

@@ -67,10 +67,11 @@ storage:
 
 # Export configuration
 exportConfig:
-  collections:
-    - name: ""
-      partition_by: year  # Options: null, year, month
-      start_year: 2015
+  # Set to true to export all collections from the database
+  # When true, collections list below is ignored
+  exportAll: false
+  
+  collections: []
 
   settings:
     chunk_size: 10000

src/pgstac_geoparquet_exporter/__main__.py

@@ -6,6 +6,7 @@
 from pathlib import Path
 from typing import Any, Callable
 
+import psycopg
 import pyarrow.fs as pafs  # type: ignore
 import yaml
 from stac_geoparquet.pgstac_reader import pgstac_to_parquet, sync_pgstac_to_parquet
@@ -31,6 +32,18 @@ def add_links(item: dict[str, Any]) -> dict[str, Any]:
     return add_links
 
 
+def get_all_collections(conninfo: str) -> list[dict[str, Any]]:
+    """Fetch all collection IDs from pgSTAC database."""
+    conn = psycopg.connect(conninfo)
+    try:
+        with conn.cursor() as cur:
+            cur.execute("SELECT id FROM pgstac.collections")
+            collection_ids = [row[0] for row in cur.fetchall()]
+            return [{"name": coll_id} for coll_id in collection_ids]
+    finally:
+        conn.close()
+
+
 def main() -> int:
     mode = os.environ.get("EXPORT_MODE", "complete")
     config_path = os.environ.get("CONFIG_PATH", "/config/export-config.yaml")
@@ -56,6 +69,19 @@ def main() -> int:
     with open(config_path) as f:
         config = yaml.safe_load(f)
 
+    # Determine which collections to export
+    export_all = config.get("exportAll", False)
+    if export_all:
+        print("Fetching all collections from database...")
+        collections = get_all_collections(conninfo)
+        print(f"Found {len(collections)} collections")
+        # Apply default settings from config
+        default_settings = config.get("exportConfig", {}).get("settings", {})
+        for coll in collections:
+            coll.update(default_settings)
+    else:
+        collections = config.get("collections", [])
+
     # Configure S3 filesystem if using S3/MinIO
     filesystem = None
     if output_base.startswith("s3://"):
@@ -77,7 +103,7 @@ def main() -> int:
 
     if mode == "complete":
         # Complete export
-        for coll in config.get("collections", []):
+        for coll in collections:
             collection_id = coll["name"]
             partition_frequency = coll.get("partition_frequency")
 
@@ -126,7 +152,7 @@ def main() -> int:
         # Incremental mode - export only updated items
         print("Using incremental mode with sync_pgstac_to_parquet")
 
-        for coll in config.get("collections", []):
+        for coll in collections:
             collection_id = coll["name"]
             output_path = f"{output_base}/{collection_id}"