Preserve custom ext-only features and readable GeoJSON output (0.3.1)

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change log
 
+### 0.3.1
+- Preserve custom `ext:*` features across all geometries: ext-only points keep numeric IDs (no `p` prefix), ext-only lines/polygons are retained, and custom attributes are emitted in the appropriate GeoJSON file.
+- Add schema-safe handling for ext-only geometries during construction to avoid missing-ref crashes.
+- Emit GeoJSON with indentation for easier inspection.
+
 ### 0.3.0
 - Update converters to emit OSW 0.3 schema id and support new vegetation features (trees, tree rows, woods).
 - Extend OSW normalizers to keep `leaf_cycle` and `leaf_type` where allowed for points, lines, and polygons.

README.md

@@ -75,7 +75,14 @@ To install the GDAL library (Geospatial Data Abstraction Library) on your system
    1. It takes the `zip` file which contains edges.geojson, points.geojson, nodes.geojson, zones.geojson, polygons.geojson and lines.geojson files, and output directory path(optional) as input
    2. Process the geojson files
    3. Convert those files into xml file at provided output directory path   
-
+
+## Custom attributes (OSW 0.3)
+- Custom features that contain only `ext:*` attributes are preserved and written to their matching GeoJSON:
+  - Point geometries → `points.geojson` with numeric `_id` (no `p` prefix) and `ext:osm_id`.
+  - LineString geometries → `lines.geojson` with `_id`, `_u_id`, `_v_id`, plus `ext:*`.
+  - Polygon geometries → `polygons.geojson` with `_id` and `ext:*`.
+- Outputs are formatted with indentation to simplify inspection.
+
 
 ## Starting a new project with template  
 
@@ -233,4 +240,4 @@ test_normalize_sidewalk (test_serializer.test_osw_normalizer.TestOSWWayNormalize
 Ran 73 tests in 79.494s
 
 OK
-```
+```

src/osm_osw_reformatter/serializer/osm/osm_graph.py

@@ -121,13 +121,13 @@ def node(self, n) -> None:
         if not self.point_filter(n.tags):
             return
 
-        d = {}
-
         tags = dict(n.tags)
 
-        d2 = {**d, **OSWPointNormalizer(tags).normalize()}
+        normalizer = OSWPointNormalizer(tags)
+        normalized = normalizer.normalize()
 
-        self.G.add_node("p" + str(n.id), lon=n.location.lon, lat=n.location.lat, **d2)
+        node_id = n.id if normalizer.is_custom() else "p" + str(n.id)
+        self.G.add_node(node_id, lon=n.location.lon, lat=n.location.lat, **normalized)
 
 
 class OSMLineParser(osmium.SimpleHandler):
@@ -329,9 +329,11 @@ def node(self, n):
             return
 
         if self.point_filter(tags):
-            normalized = OSWPointNormalizer(tags).normalize()
+            normalizer = OSWPointNormalizer(tags)
+            normalized = normalizer.normalize()
             if normalized:
-                self.G.add_node("p" + str(n.id), lon=n.location.lon, lat=n.location.lat, **normalized)
+                node_id = n.id if normalizer.is_custom() else "p" + str(n.id)
+                self.G.add_node(node_id, lon=n.location.lon, lat=n.location.lat, **normalized)
 
 class OSMGraph:
     def __init__(self, G: nx.MultiDiGraph = None) -> None:
@@ -515,33 +517,44 @@ def construct_geometries(self, progressbar: Optional[callable] = None) -> None:
 
         for n, d in self.G.nodes(data=True):
             if OSWZoneNormalizer.osw_zone_filter(d):
+                ndref = d.get("ndref")
+                indref = d.get("indref", [])
+                if not ndref:
+                    continue
                 coords = []
-                for ref in d["ndref"]:
+                for ref in ndref:
                     node_d = self.G._node[int(ref)]
                     coords.append((node_d["lon"], node_d["lat"]))
 
-                geometry = Polygon(coords, d["indref"])
+                geometry = Polygon(coords, indref)
                 d["geometry"] = geometry
 
                 d["_w_id"] = d.pop("ndref")
-                del d["indref"]
+                d.pop("indref", None)
 
                 if progressbar:
                     progressbar.update(1)
             elif OSWPolygonNormalizer.osw_polygon_filter(d):
-                geometry = Polygon(d["ndref"], d["indref"])
+                ndref = d.get("ndref")
+                indref = d.get("indref", [])
+                if not ndref:
+                    continue
+                geometry = Polygon(ndref, indref)
                 d["geometry"] = geometry
 
-                del d["ndref"]
-                del d["indref"]
+                d.pop("ndref", None)
+                d.pop("indref", None)
 
                 if progressbar:
                     progressbar.update(1)
             elif OSWLineNormalizer.osw_line_filter(d):
-                geometry = LineString(d["ndref"])
+                ndref = d.get("ndref")
+                if not ndref:
+                    continue
+                geometry = LineString(ndref)
                 d["geometry"] = geometry
                 d["length"] = round(self.geod.geometry_length(geometry), 1)
-                del d["ndref"]
+                d.pop("ndref", None)
                 if progressbar:
                     progressbar.update(1)
             else:
@@ -689,27 +702,27 @@ def to_geojson(self, *args) -> None:
 
         if len(edge_features) > 0:
             with open(edges_path, 'w') as f:
-                json.dump(edges_fc, f)
+                json.dump(edges_fc, f, indent=2)
 
         if len(node_features) > 0:
             with open(nodes_path, 'w') as f:
-                json.dump(nodes_fc, f)
+                json.dump(nodes_fc, f, indent=2)
 
         if len(point_features) > 0:
             with open(points_path, "w") as f:
-                json.dump(points_fc, f)
+                json.dump(points_fc, f, indent=2)
 
         if len(line_features) > 0:
             with open(lines_path, "w") as f:
-                json.dump(lines_fc, f)
+                json.dump(lines_fc, f, indent=2)
 
         if len(zone_features) > 0:
             with open(zones_path, "w") as f:
-                json.dump(zones_fc, f)
+                json.dump(zones_fc, f, indent=2)
 
         if len(polygon_features) > 0:
             with open(polygons_path, "w") as f:
-                json.dump(polygons_fc, f)
+                json.dump(polygons_fc, f, indent=2)
 
     @classmethod
     def from_geojson(cls, nodes_path, edges_path):

src/osm_osw_reformatter/serializer/osw/osw_normalizer.py

@@ -25,6 +25,30 @@ def _tag_value(tags, key):
     return tags.get(f"ext:{key}", "")
 
 
+def _tags_to_dict(tags):
+    # Accept dict-like and osmium TagList-like objects
+    if hasattr(tags, "items"):
+        try:
+            return dict(tags)
+        except Exception:
+            pass
+
+    try:
+        return {
+            (item.k if hasattr(item, "k") else item[0]): (item.v if hasattr(item, "v") else item[1])
+            for item in tags
+            if (hasattr(item, "k") and hasattr(item, "v")) or (isinstance(item, tuple) and len(item) == 2)
+        }
+    except Exception:
+        return {}
+
+
+def _has_only_ext_tags(tags):
+    if not tags:
+        return False
+    return all(str(k).startswith("ext:") for k in tags.keys())
+
+
 class OSWWayNormalizer:
 
     ROAD_HIGHWAY_VALUES = (
@@ -271,7 +295,8 @@ def filter(self):
             self.is_manhole()) or (
             self.is_bollard()) or (
             self.is_street_lamp()) or (
-            self.is_tree())
+            self.is_tree()) or (
+            self.is_custom())
 
     @staticmethod
     def osw_point_filter(tags):
@@ -298,6 +323,8 @@ def normalize(self):
                     "leaf_type": leaf_type
                 }
             )
+        elif self.is_custom():
+            return self._normalize_point()
         else:
             print(f"Invalid point skipped. Tags: {self.tags}")
             return {}
@@ -332,13 +359,17 @@ def is_street_lamp(self):
 
     def is_tree(self):
         return _tag_value(self.tags, "natural") == "tree"
+
+    def is_custom(self):
+        tag_dict = _tags_to_dict(self.tags)
+        return _has_only_ext_tags(tag_dict)
 
 class OSWLineNormalizer:
     def __init__(self, tags):
         self.tags = tags
 
     def filter(self):
-        return (self.is_fence()) or (self.is_tree_row())
+        return (self.is_fence()) or (self.is_tree_row()) or (self.is_custom())
 
     @staticmethod
     def osw_line_filter(tags):
@@ -355,6 +386,8 @@ def normalize(self):
                     "leaf_type": leaf_type
                 }
             )
+        elif self.is_custom():
+            return self._normalize_line()
         else:
             raise ValueError("This is an invalid line")
 
@@ -370,6 +403,10 @@ def is_fence(self):
 
     def is_tree_row(self):
         return _tag_value(self.tags, "natural") == "tree_row"
+
+    def is_custom(self):
+        tag_dict = _tags_to_dict(self.tags)
+        return _has_only_ext_tags(tag_dict)
 
 class OSWPolygonNormalizer:
     # Will be fetched from schema soon
@@ -479,7 +516,7 @@ def __init__(self, tags):
         self.tags = tags
 
     def filter(self):
-        return self.is_building() or self.is_wood()
+        return self.is_building() or self.is_wood() or self.is_custom()
 
     @staticmethod
     def osw_polygon_filter(tags):
@@ -506,6 +543,8 @@ def normalize(self):
                     "leaf_type": leaf_type
                 }
             )
+        elif self.is_custom():
+            return self._normalize_polygon()
         else:
             raise ValueError("This is an invalid polygon")
 
@@ -522,6 +561,10 @@ def is_building(self):
     def is_wood(self):
         return _tag_value(self.tags, "natural") == "wood"
 
+    def is_custom(self):
+        tag_dict = _tags_to_dict(self.tags)
+        return _has_only_ext_tags(tag_dict)
+
 class OSWZoneNormalizer:
     def __init__(self, tags):
         self.tags = tags

src/osm_osw_reformatter/version.py

@@ -1 +1 @@
-__version__ = '0.3.0'
+__version__ = '0.3.1'