diff --git a/CHANGELOG.md b/CHANGELOG.md index d5fc080..67e1e26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change log +### 0.3.1 +- Preserve custom `ext:*` features across all geometries: ext-only points keep numeric IDs (no `p` prefix), ext-only lines/polygons are retained, and custom attributes are emitted in the appropriate GeoJSON file. +- Add schema-safe handling for ext-only geometries during construction to avoid missing-ref crashes. +- Emit GeoJSON with indentation for easier inspection. + ### 0.3.0 - Update converters to emit OSW 0.3 schema id and support new vegetation features (trees, tree rows, woods). - Extend OSW normalizers to keep `leaf_cycle` and `leaf_type` where allowed for points, lines, and polygons. diff --git a/README.md b/README.md index 18288cb..52f8a64 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,14 @@ To install the GDAL library (Geospatial Data Abstraction Library) on your system 1. It takes the `zip` file which contains edges.geojson, points.geojson, nodes.geojson, zones.geojson, polygons.geojson and lines.geojson files, and output directory path(optional) as input 2. Process the geojson files 3. Convert those files into xml file at provided output directory path - + +## Custom attributes (OSW 0.3) +- Custom features that contain only `ext:*` attributes are preserved and written to their matching GeoJSON: + - Point geometries → `points.geojson` with numeric `_id` (no `p` prefix) and `ext:osm_id`. + - LineString geometries → `lines.geojson` with `_id`, `_u_id`, `_v_id`, plus `ext:*`. + - Polygon geometries → `polygons.geojson` with `_id` and `ext:*`. +- Outputs are formatted with indentation to simplify inspection. + ## Starting a new project with template @@ -233,4 +240,4 @@ test_normalize_sidewalk (test_serializer.test_osw_normalizer.TestOSWWayNormalize Ran 73 tests in 79.494s OK -``` \ No newline at end of file +``` diff --git a/src/osm_osw_reformatter/serializer/osm/osm_graph.py b/src/osm_osw_reformatter/serializer/osm/osm_graph.py index 41e7117..0c495e1 100644 --- a/src/osm_osw_reformatter/serializer/osm/osm_graph.py +++ b/src/osm_osw_reformatter/serializer/osm/osm_graph.py @@ -121,13 +121,13 @@ def node(self, n) -> None: if not self.point_filter(n.tags): return - d = {} - tags = dict(n.tags) - d2 = {**d, **OSWPointNormalizer(tags).normalize()} + normalizer = OSWPointNormalizer(tags) + normalized = normalizer.normalize() - self.G.add_node("p" + str(n.id), lon=n.location.lon, lat=n.location.lat, **d2) + node_id = n.id if normalizer.is_custom() else "p" + str(n.id) + self.G.add_node(node_id, lon=n.location.lon, lat=n.location.lat, **normalized) class OSMLineParser(osmium.SimpleHandler): @@ -329,9 +329,11 @@ def node(self, n): return if self.point_filter(tags): - normalized = OSWPointNormalizer(tags).normalize() + normalizer = OSWPointNormalizer(tags) + normalized = normalizer.normalize() if normalized: - self.G.add_node("p" + str(n.id), lon=n.location.lon, lat=n.location.lat, **normalized) + node_id = n.id if normalizer.is_custom() else "p" + str(n.id) + self.G.add_node(node_id, lon=n.location.lon, lat=n.location.lat, **normalized) class OSMGraph: def __init__(self, G: nx.MultiDiGraph = None) -> None: @@ -515,33 +517,44 @@ def construct_geometries(self, progressbar: Optional[callable] = None) -> None: for n, d in self.G.nodes(data=True): if OSWZoneNormalizer.osw_zone_filter(d): + ndref = d.get("ndref") + indref = d.get("indref", []) + if not ndref: + continue coords = [] - for ref in d["ndref"]: + for ref in ndref: node_d = self.G._node[int(ref)] coords.append((node_d["lon"], node_d["lat"])) - geometry = Polygon(coords, d["indref"]) + geometry = Polygon(coords, indref) d["geometry"] = geometry d["_w_id"] = d.pop("ndref") - del d["indref"] + d.pop("indref", None) if progressbar: progressbar.update(1) elif OSWPolygonNormalizer.osw_polygon_filter(d): - geometry = Polygon(d["ndref"], d["indref"]) + ndref = d.get("ndref") + indref = d.get("indref", []) + if not ndref: + continue + geometry = Polygon(ndref, indref) d["geometry"] = geometry - del d["ndref"] - del d["indref"] + d.pop("ndref", None) + d.pop("indref", None) if progressbar: progressbar.update(1) elif OSWLineNormalizer.osw_line_filter(d): - geometry = LineString(d["ndref"]) + ndref = d.get("ndref") + if not ndref: + continue + geometry = LineString(ndref) d["geometry"] = geometry d["length"] = round(self.geod.geometry_length(geometry), 1) - del d["ndref"] + d.pop("ndref", None) if progressbar: progressbar.update(1) else: @@ -689,27 +702,27 @@ def to_geojson(self, *args) -> None: if len(edge_features) > 0: with open(edges_path, 'w') as f: - json.dump(edges_fc, f) + json.dump(edges_fc, f, indent=2) if len(node_features) > 0: with open(nodes_path, 'w') as f: - json.dump(nodes_fc, f) + json.dump(nodes_fc, f, indent=2) if len(point_features) > 0: with open(points_path, "w") as f: - json.dump(points_fc, f) + json.dump(points_fc, f, indent=2) if len(line_features) > 0: with open(lines_path, "w") as f: - json.dump(lines_fc, f) + json.dump(lines_fc, f, indent=2) if len(zone_features) > 0: with open(zones_path, "w") as f: - json.dump(zones_fc, f) + json.dump(zones_fc, f, indent=2) if len(polygon_features) > 0: with open(polygons_path, "w") as f: - json.dump(polygons_fc, f) + json.dump(polygons_fc, f, indent=2) @classmethod def from_geojson(cls, nodes_path, edges_path): diff --git a/src/osm_osw_reformatter/serializer/osw/osw_normalizer.py b/src/osm_osw_reformatter/serializer/osw/osw_normalizer.py index 751b3bf..b10c81b 100644 --- a/src/osm_osw_reformatter/serializer/osw/osw_normalizer.py +++ b/src/osm_osw_reformatter/serializer/osw/osw_normalizer.py @@ -25,6 +25,30 @@ def _tag_value(tags, key): return tags.get(f"ext:{key}", "") +def _tags_to_dict(tags): + # Accept dict-like and osmium TagList-like objects + if hasattr(tags, "items"): + try: + return dict(tags) + except Exception: + pass + + try: + return { + (item.k if hasattr(item, "k") else item[0]): (item.v if hasattr(item, "v") else item[1]) + for item in tags + if (hasattr(item, "k") and hasattr(item, "v")) or (isinstance(item, tuple) and len(item) == 2) + } + except Exception: + return {} + + +def _has_only_ext_tags(tags): + if not tags: + return False + return all(str(k).startswith("ext:") for k in tags.keys()) + + class OSWWayNormalizer: ROAD_HIGHWAY_VALUES = ( @@ -271,7 +295,8 @@ def filter(self): self.is_manhole()) or ( self.is_bollard()) or ( self.is_street_lamp()) or ( - self.is_tree()) + self.is_tree()) or ( + self.is_custom()) @staticmethod def osw_point_filter(tags): @@ -298,6 +323,8 @@ def normalize(self): "leaf_type": leaf_type } ) + elif self.is_custom(): + return self._normalize_point() else: print(f"Invalid point skipped. Tags: {self.tags}") return {} @@ -332,13 +359,17 @@ def is_street_lamp(self): def is_tree(self): return _tag_value(self.tags, "natural") == "tree" + + def is_custom(self): + tag_dict = _tags_to_dict(self.tags) + return _has_only_ext_tags(tag_dict) class OSWLineNormalizer: def __init__(self, tags): self.tags = tags def filter(self): - return (self.is_fence()) or (self.is_tree_row()) + return (self.is_fence()) or (self.is_tree_row()) or (self.is_custom()) @staticmethod def osw_line_filter(tags): @@ -355,6 +386,8 @@ def normalize(self): "leaf_type": leaf_type } ) + elif self.is_custom(): + return self._normalize_line() else: raise ValueError("This is an invalid line") @@ -370,6 +403,10 @@ def is_fence(self): def is_tree_row(self): return _tag_value(self.tags, "natural") == "tree_row" + + def is_custom(self): + tag_dict = _tags_to_dict(self.tags) + return _has_only_ext_tags(tag_dict) class OSWPolygonNormalizer: # Will be fetched from schema soon @@ -479,7 +516,7 @@ def __init__(self, tags): self.tags = tags def filter(self): - return self.is_building() or self.is_wood() + return self.is_building() or self.is_wood() or self.is_custom() @staticmethod def osw_polygon_filter(tags): @@ -506,6 +543,8 @@ def normalize(self): "leaf_type": leaf_type } ) + elif self.is_custom(): + return self._normalize_polygon() else: raise ValueError("This is an invalid polygon") @@ -522,6 +561,10 @@ def is_building(self): def is_wood(self): return _tag_value(self.tags, "natural") == "wood" + def is_custom(self): + tag_dict = _tags_to_dict(self.tags) + return _has_only_ext_tags(tag_dict) + class OSWZoneNormalizer: def __init__(self, tags): self.tags = tags diff --git a/src/osm_osw_reformatter/version.py b/src/osm_osw_reformatter/version.py index 0404d81..e1424ed 100644 --- a/src/osm_osw_reformatter/version.py +++ b/src/osm_osw_reformatter/version.py @@ -1 +1 @@ -__version__ = '0.3.0' +__version__ = '0.3.1'