Login updates for 2FA

- Login updates for 2FA, it now opens a browser

- Adds options for profile ID as well so you dont have to login every time

- Fixed temp file uploads, this gives them a unique file name.

- Updated metadata guidelines to explain overture maps

Author: Rydela

.env.example

@@ -9,6 +9,12 @@ AGOL_PORTAL_URL=https://your-org.maps.arcgis.com
 AGOL_USERNAME=your_username
 AGOL_PASSWORD=your_secure_password_here
 
+# ArcGIS Online 2FA / OAuth (Browser Login)
+# Set these instead of username/password when 2FA is required.
+# AGOL_USE_OAUTH=true
+# AGOL_CLIENT_ID=your_client_id_here
+# AGOL_PROFILE=your_profile_name  # Optional: reuse saved auth across runs
+
 # ArcGIS Online Advanced Settings (Optional)
 # AGOL_TOKEN_EXPIRATION=9999  # Extended timeout for large datasets
 # AGOL_LARGE_DATASET_THRESHOLD=10000  # Feature count threshold

README.md

@@ -30,7 +30,7 @@ The pipeline follows a modular Source -> Transform -> Publish/Export process:
 
 ## Requirements
 - Python 3.11+ (compatible with ArcGIS Python API)
-- ArcGIS Online credentials (optional - only required for `arcgis-upload` and `overture-dump` command)
+- ArcGIS Online credentials or OAuth client ID (required for `arcgis-upload` and `overture-dump` command)
 
 The pipeline can be used for data export without ArcGIS Online credentials. Environment configuration is managed through `.env` files for secure credential storage.
 
@@ -55,6 +55,15 @@ On macOS/Linux:
 - Use the `.env` example to define your AGOL credentials
 - No need to create country-specific config files, you can use the global config with a country argument
 
+**2FA / Browser Login (Recommended)**
+- Set `AGOL_USE_OAUTH=true` and `AGOL_CLIENT_ID=...` (no username/password required).
+- On first run, a browser window opens to complete sign-in and 2FA.
+- You can register an ArcGIS Online app to get a Client ID (Application type: "Other").
+- Optional: set `AGOL_PROFILE=your_profile` to cache the session and avoid repeated prompts in batch runs.
+
+**Username/Password (Non‑2FA or Service Accounts)**
+- Set `AGOL_USERNAME` and `AGOL_PASSWORD` as shown in `.env.example`.
+
 ### 4. Run commands
 The Python CLI has three main commands: uploading to AGOL `arcgis-upload` , downloading as geojson `export`, or download dump for local use as needed `overture-dump`. 
 
@@ -192,4 +201,4 @@ o2agol overture-dump buildings --country afg --force-download
 - Chris Holme's excellent tutorial here: https://github.com/cholmes/duckdb-geoparquet-tutorials/tree/main
 - Georock's post on spatial clipping: https://geo.rocks/post/duckdb_geospatial/
 - Bounding Box extracts from Natural Earth: https://github.com/sandstrom/country-bounding-boxes/tree/master
-- This ArcGIS Pro extension that helped inspire this workflow: https://github.com/COF-RyLopez/ArcGISPro-GeoParquet-Addin
+- This ArcGIS Pro extension that helped inspire this workflow: https://github.com/COF-RyLopez/ArcGISPro-GeoParquet-Addin

src/o2agol/cleanup.py

@@ -133,15 +133,43 @@ def check_temp_size_limits(warning_gb: int = 10, limit_gb: int = 20) -> bool:
     return True
 
 
+def _remove_path_with_retries(path: Path, retries: int = 3, delay_s: float = 1.0) -> bool:
+    """Attempt to remove a file/directory with retries (handles Windows file locks)."""
+    for attempt in range(1, retries + 1):
+        try:
+            if path.is_dir():
+                shutil.rmtree(path)
+            else:
+                path.unlink()
+            return True
+        except OSError as e:
+            if attempt == retries:
+                logging.warning(f"Could not remove {path}: {e}")
+                return False
+            time.sleep(delay_s)
+    return False
+
+
 def cleanup_current_pid() -> None:
     """Clean up temp files for current process."""
     pid_dir = get_project_temp_dir() / f"pid_{os.getpid()}"
-    if pid_dir.exists():
+    if not pid_dir.exists():
+        return
+
+    # Try to remove files first so a single locked file doesn't block all cleanup
+    for item in pid_dir.rglob("*"):
+        if item.is_file():
+            _remove_path_with_retries(item, retries=3, delay_s=1.0)
+
+    # Remove any empty directories, then the pid dir itself
+    for subdir in sorted([p for p in pid_dir.rglob("*") if p.is_dir()], reverse=True):
         try:
-            shutil.rmtree(pid_dir)
-            logging.debug(f"Cleaned up PID temp directory: {pid_dir}")
-        except OSError as e:
-            logging.warning(f"Could not clean PID temp directory {pid_dir}: {e}")
+            subdir.rmdir()
+        except OSError:
+            pass
+
+    if not _remove_path_with_retries(pid_dir, retries=3, delay_s=1.0):
+        logging.warning(f"Could not clean PID temp directory {pid_dir}: file(s) still in use")
 
 
 def cleanup_orphaned_pids() -> int:
@@ -257,4 +285,4 @@ def full_cleanup_check(
         # Then clean up stale files
         cleanup_stale_files(retention_hours)
 
-    return check_temp_size_limits(warning_gb, limit_gb)
+    return check_temp_size_limits(warning_gb, limit_gb)

src/o2agol/cli.py

@@ -10,6 +10,7 @@
 from dotenv import load_dotenv
 
 import geopandas as gpd
+import pandas as pd
 
 # Load environment variables BEFORE importing local modules that use them
 load_dotenv()
@@ -1040,7 +1041,8 @@ def arcgis_upload(
             metadata = format_metadata_from_config(config_dict, query_config, country_config)
 
             #hide any polygons
-            transformed_data = polygons_to_centroids(transformed_data)
+            if isinstance(transformed_data, dict) and query_config.is_multilayer:
+                transformed_data = add_sector_layers(transformed_data)
 
             if isinstance(transformed_data, dict):
                 # Multi-layer service (education, health, markets)
@@ -1245,7 +1247,8 @@ def export_data_command(
 
 
             # create single point from building polygons
-            transformed_data = polygons_to_centroids(transformed_data)
+            if isinstance(transformed_data, dict) and query_config.is_multilayer:
+                transformed_data = add_sector_layers(transformed_data)
 
 ############################################
 ############################################
@@ -1730,6 +1733,9 @@ def format_duration(seconds: float) -> str:
                 if not layer_gdf.empty:
                     normalized_gdf = transformer.normalize(layer_gdf)
                     transformed_data[layer_name] = transformer.add_metadata(normalized_gdf, country_config)
+
+            if isinstance(transformed_data, dict) and query_config.is_multilayer:
+                transformed_data = add_sector_layers(transformed_data)
 
             if not transformed_data:
                 pipeline_logger.info("No valid data after transformation")
@@ -2160,68 +2166,134 @@ def version():
     app()
 
 # This function will check for layers with polygon geometries and convert them to centroids
-def polygons_to_centroids(transformed_data: dict[str, gpd.GeoDataFrame]) -> dict[str, gpd.GeoDataFrame]:
+def polygons_to_centroids(
+    transformed_data: gpd.GeoDataFrame | dict[str, gpd.GeoDataFrame]
+) -> gpd.GeoDataFrame | dict[str, gpd.GeoDataFrame]:
     base_logger = logging.getLogger(__name__)
     pipeline_logger = PipelineLogger(base_logger)
     # create single point from building polygons
     #print("\n############################################\n")
 
-    out: dict[str, gpd.GeoDataFrame] = {k: v.copy() for k, v in transformed_data.items()}
+    def _centroid_layer(name: str, layer: gpd.GeoDataFrame) -> gpd.GeoDataFrame | None:
+        if not hasattr(layer, "geometry") or not hasattr(layer.geometry, "geom_type"):
+            pipeline_logger.info(f"[{name}] No GeoSeries geometry found. Skipping centroid conversion.")
+            return None
 
-    for name,d in transformed_data.items():
-        
-        # What geometry types do we have?
-        gtypes = set(d.geom_type.dropna().unique()) if not d.empty else set()
-        crs_str = d.crs.to_string() if d.crs is not None else "None"
+        gtypes = set(layer.geometry.geom_type.dropna().unique()) if not layer.empty else set()
+        crs_str = layer.crs.to_string() if layer.crs is not None else "None"
         pipeline_logger.info(f"[{name}] geom_types={gtypes} | crs={crs_str}")
-        #print(f"[{name}] geom_types={gtypes} | crs={crs_str}")
 
         if not any(t in {"Polygon", "MultiPolygon"} for t in gtypes):
             pipeline_logger.info(f"[{name}] No (Multi)Polygon geometries to centroid. Skipping.")
-            #print(f"[{name}] No (Multi)Polygon geometries to centroid. Skipping.")
-            continue
+            return None
 
-        if d.crs is None:
+        if layer.crs is None:
             pipeline_logger.info(f"[{name}] CRS is None → set it first (e.g., EPSG:4326).")
-            #print(f"[{name}] CRS is None → set it first (e.g., EPSG:4326).")
-            continue
-        
+            return None
+
         # Compute centroids in a projected CRS, then back to WGS84
-        if d.crs.is_geographic:
+        if layer.crs.is_geographic:
             try:
-                proj_crs = d.estimate_utm_crs()
+                proj_crs = layer.estimate_utm_crs()
                 pipeline_logger.info(f"[{name}] best CRS is: {proj_crs.to_string()}")
-                #print(f"[{name}] best CRS is: {proj_crs.to_string()}")
                 centroids_wgs84 = (
-                    d.to_crs(proj_crs).geometry.centroid
+                    layer.to_crs(proj_crs).geometry.centroid
                     .set_crs(proj_crs)              # ensure CRS on series
                     .to_crs("EPSG:4326")
                 )
             except Exception as e:
                 pipeline_logger.info(f"[{name}] estimate_utm_crs() failed: {e} → fallback (less accurate).")
-                #print(f"[{name}] estimate_utm_crs() failed: {e} → fallback (less accurate).")
-                centroids_wgs84 = gpd.GeoSeries(d.geometry.centroid, crs=d.crs).to_crs("EPSG:4326")
+                centroids_wgs84 = gpd.GeoSeries(layer.geometry.centroid, crs=layer.crs).to_crs("EPSG:4326")
         else:
-            centroids_wgs84 = gpd.GeoSeries(d.geometry.centroid, crs=d.crs).to_crs("EPSG:4326")
+            centroids_wgs84 = gpd.GeoSeries(layer.geometry.centroid, crs=layer.crs).to_crs("EPSG:4326")
 
-        
         # Build a points layer without destroying the polygon layer
-        points_gdf = d.copy()
-        # (Optional) keep original polygon geometry as an attribute column for reference
-        # points_gdf["polygon_geom"] = d.geometry
-
- 
+        points_gdf = layer.copy()
         points_gdf["centroid"] = centroids_wgs84
-        points_gdf.set_geometry("centroid", inplace=True) 
+        points_gdf.set_geometry("centroid", inplace=True)
         points_gdf.drop(columns="geometry", inplace=True)
+        # Normalize geometry column name for downstream publishing
+        try:
+            points_gdf = points_gdf.rename_geometry("geometry")
+        except Exception:
+            points_gdf = points_gdf.set_geometry(points_gdf.geometry.name)
+        return points_gdf
+
+    if isinstance(transformed_data, gpd.GeoDataFrame):
+        centroids = _centroid_layer("features", transformed_data)
+        return centroids if centroids is not None else transformed_data
+
+    out: dict[str, gpd.GeoDataFrame] = {k: v.copy() for k, v in transformed_data.items()}
+
+    for name, layer in transformed_data.items():
+        centroids = _centroid_layer(name, layer)
+        if centroids is None:
+            continue
 
-        # Insert alongside the original polygon layer
-        # print(f"{name}_centroids")
         points_key = f"{name}_centroids"
         if points_key in out:
             pipeline_logger.info(f"[{name}] '{points_key}' already exists, overwriting with new centroids layer.")
-        out[points_key] = points_gdf
+        out[points_key] = centroids
+
+    return out
+
+
+def add_sector_layers(
+    transformed_data: gpd.GeoDataFrame | dict[str, gpd.GeoDataFrame]
+) -> gpd.GeoDataFrame | dict[str, gpd.GeoDataFrame]:
+    """
+    For sectoral queries (education/health/markets), ensure three layers:
+    - places (points)
+    - buildings (polygons)
+    - combined points (places + building centroids)
+    """
+    if not isinstance(transformed_data, dict):
+        return transformed_data
+
+    # Identify primary layers
+    places_key = None
+    buildings_key = None
+    for key in transformed_data.keys():
+        lowered = key.lower()
+        if places_key is None and "place" in lowered:
+            places_key = key
+        if buildings_key is None and "building" in lowered:
+            buildings_key = key
+
+    if not places_key or not buildings_key:
+        return transformed_data
+
+    # Ensure centroid layer exists
+    updated = polygons_to_centroids(transformed_data)
+    if not isinstance(updated, dict):
+        return transformed_data
+
+    centroids_key = f"{buildings_key}_centroids"
+    if centroids_key not in updated:
+        return updated
+
+    places_gdf = updated[places_key]
+    centroids_gdf = updated[centroids_key]
+
+    # Normalize geometry column name for both
+    for gdf in (places_gdf, centroids_gdf):
+        if hasattr(gdf, "geometry"):
+            try:
+                if gdf.geometry.name != "geometry":
+                    gdf.rename_geometry("geometry", inplace=True)
+            except Exception:
+                pass
+
+    combined_key = f"{places_key}_combined"
+    if combined_key not in updated:
+        combined = gpd.GeoDataFrame(
+            pd.concat([places_gdf, centroids_gdf], ignore_index=True, sort=False),
+            geometry="geometry",
+            crs=places_gdf.crs or centroids_gdf.crs
+        )
+        updated[combined_key] = combined
 
-        # Keep the original polygon layer unchanged in 'out[name]'
+    # Remove intermediate centroid layer (keep only combined output)
+    updated.pop(centroids_key, None)
 
-    return out
+    return updated