Merge pull request #231 from OpenGeoscience/lint-scripts

Run linting on all scripts, fix associated errors

Author: brianhelba

sample_data/new_york_energy/export_networks.py

@@ -1,11 +1,10 @@
-import json
-import shapely
-
 from datetime import datetime
+import json
 from pathlib import Path
 
-from geoinsight.core.models import Network, Region
+import shapely
 
+from geoinsight.core.models import Network, Region
 
 OUTPUT_FOLDER = Path('sample_data/use_cases/new_york_energy/networks')
 
@@ -22,28 +21,28 @@ def perform_export():
             features = []
             for node in network.nodes.all():
                 geom = shapely.geometry.mapping(shapely.wkt.loads(node.location.wkt))
-                features.append(dict(
-                    geometry=geom,
-                    properties=dict(
-                        **node.metadata,
-                        county=zone.name
-                    )
-                ))
+                features.append(
+                    dict(geometry=geom, properties=dict(**node.metadata, county=zone.name))
+                )
             for edge in network.edges.all():
                 geom = shapely.geometry.mapping(shapely.wkt.loads(edge.line_geometry.wkt))
-                from_point = shapely.geometry.mapping(shapely.wkt.loads(edge.from_node.location.wkt))
+                from_point = shapely.geometry.mapping(
+                    shapely.wkt.loads(edge.from_node.location.wkt)
+                )
                 to_point = shapely.geometry.mapping(shapely.wkt.loads(edge.to_node.location.wkt))
-                features.append(dict(
-                    geometry=geom,
-                    properties=dict(
-                        **edge.metadata,
-                        from_point=from_point,
-                        to_point=to_point,
-                        county=zone.name
+                features.append(
+                    dict(
+                        geometry=geom,
+                        properties=dict(
+                            **edge.metadata,
+                            from_point=from_point,
+                            to_point=to_point,
+                            county=zone.name,
+                        ),
                     )
-                ))
+                )
             geodata = dict(
-                type="FeatureCollection",
+                type='FeatureCollection',
                 crs=dict(type='name', properties=dict(name='EPSG:4326')),
                 features=features,
             )

sample_data/new_york_energy/import_networks.py

@@ -1,25 +1,19 @@
+from datetime import datetime
 import json
+from pathlib import Path
 import tempfile
 import zipfile
 
-from datetime import datetime
-from pathlib import Path
+from django.contrib.gis.geos import LineString, Point
 
-from django.contrib.gis.measure import D
-from django.contrib.gis.db.models.functions import Distance
-from django.contrib.gis.geos import Point, LineString
 from geoinsight.core.models import Network, NetworkEdge, NetworkNode, VectorData
 from geoinsight.core.tasks.networks import create_vector_features_from_network
 
-
 TOLERANCE_METERS = 15
 
 
 def get_metadata(feature):
-    return {
-        k: v for k, v in feature.get('properties', {}).items()
-        if k and v
-    }
+    return {k: v for k, v in feature.get('properties', {}).items() if k and v}
 
 
 def create_network(dataset, network_name, geodata):
@@ -32,27 +26,31 @@ def create_network(dataset, network_name, geodata):
         name=network_name,
         category='energy',
         vector_data=vector_data,
-        metadata=dict(name=network_name)
+        metadata=dict(name=network_name),
     )
     features = geodata.get('features')
     nodes = []
     edges = []
     for feature in features:
         geom = feature.get('geometry')
         if geom.get('type') == 'Point':
-            nodes.append(NetworkNode(
-                name=f'{network_name} {len(nodes)}',
-                network=network,
-                location=Point(*geom.get('coordinates')),
-                metadata=get_metadata(feature),
-            ))
+            nodes.append(
+                NetworkNode(
+                    name=f'{network_name} {len(nodes)}',
+                    network=network,
+                    location=Point(*geom.get('coordinates')),
+                    metadata=get_metadata(feature),
+                )
+            )
         elif geom.get('type') == 'LineString':
-            edges.append(NetworkEdge(
-                name=f'{network_name} {len(edges)}',
-                network=network,
-                line_geometry=LineString(*geom.get('coordinates')),
-                metadata=get_metadata(feature),
-            ))
+            edges.append(
+                NetworkEdge(
+                    name=f'{network_name} {len(edges)}',
+                    network=network,
+                    line_geometry=LineString(*geom.get('coordinates')),
+                    metadata=get_metadata(feature),
+                )
+            )
     NetworkNode.objects.bulk_create(nodes, batch_size=1000)
 
     # fill in node relationships on edges now that nodes exist

sample_data/new_york_energy/ingest.py

@@ -1,10 +1,10 @@
 from pathlib import Path
-from .import_networks import perform_import
-from .export_networks import perform_export
-from .nysdp import create_consolidated_network, create_vector_features
 
 from geoinsight.core.tasks.dataset import create_layers_and_frames
 
+from .export_networks import perform_export
+from .import_networks import perform_import
+from .nysdp import create_consolidated_network, create_vector_features
 
 DOWNLOADS_FOLDER = DOWNLOADS_FOLDER = Path('../../sample_data/downloads')
 PULL_LATEST = False

sample_data/new_york_energy/interpret_network.py

@@ -1,23 +1,33 @@
 import json
+
 import geopandas
 import shapely
-
 from webcolors import name_to_hex
 
-
 TOLERANCE = 0.0001
 
 
 def get_properties(feature):
     properties = json.loads(
         feature.drop(['geometry', 'index'], errors='ignore').fillna('').to_json()
     )
-    properties.update(dict(
-        colors=','.join([
-            name_to_hex((properties.get('color',) or 'black').replace(' ', '')),
-            '#ffffff'
-        ])
-    ))
+    properties.update(
+        dict(
+            colors=','.join(
+                [
+                    name_to_hex(
+                        (
+                            properties.get(
+                                'color',
+                            )
+                            or 'black'
+                        ).replace(' ', '')
+                    ),
+                    '#ffffff',
+                ]
+            )
+        )
+    )
     return properties
 
 
@@ -43,7 +53,7 @@ def merge_properties(p1, p2):
 def cut_crossed_lines(gdf):
     # cut lines at any cross points
     separated_features = []
-    for feat_index, feature in gdf.iterrows():
+    for _, feature in gdf.iterrows():
         properties = json.loads(
             feature.drop(['geometry', 'index'], errors='ignore').fillna('').to_json()
         )
@@ -52,7 +62,7 @@ def cut_crossed_lines(gdf):
         separated = []
         if len(crosses) > 0:
             split_points = []
-            for c_id, cross in crosses.iterrows():
+            for _, cross in crosses.iterrows():
                 p = cross.geometry.intersection(curr_geom)
                 if p.geom_type == 'MultiPoint':
                     split_points.append(p.geoms[0])
@@ -61,14 +71,16 @@ def cut_crossed_lines(gdf):
             separated = shapely.ops.split(curr_geom, shapely.MultiPoint(split_points)).geoms
         else:
             separated = [feature.geometry]
-        separated_features.extend([
-            dict(
-                type='Feature',
-                geometry=json.loads(shapely.to_geojson(s)),
-                properties=properties
-            )
-            for s in separated
-        ])
+        separated_features.extend(
+            [
+                dict(
+                    type='Feature',
+                    geometry=json.loads(shapely.to_geojson(s)),
+                    properties=properties,
+                )
+                for s in separated
+            ]
+        )
 
     gdf = geopandas.GeoDataFrame.from_features(separated_features)
     return gdf
@@ -87,50 +99,55 @@ def merge_lines(gdf):
             not_visited = gdf[~gdf.index.isin(visited)]
             touching = not_visited[not_visited.geometry.touches(curr_geom)]
             snapped = touching.snap(curr_geom, TOLERANCE * 2)
-            merge = shapely.line_merge(shapely.union_all([
-                *snapped.geometry, curr_geom
-            ]))
+            merge = shapely.line_merge(shapely.union_all([*snapped.geometry, curr_geom]))
             curr_segment = None
             if merge.geom_type == 'MultiLineString':
                 for segment in merge.geoms:
                     if segment.contains(curr_geom) and not any(
-                        s.touches(segment)
-                        for s in merge.geoms
-                        if s != segment
+                        s.touches(segment) for s in merge.geoms if s != segment
                     ):
                         curr_segment = segment
             elif merge.geom_type == 'LineString':
                 curr_segment = merge
 
             if curr_segment is None:
                 # no valid merge segment, include feature as-is
-                merged_features.append(dict(
-                    type='Feature',
-                    geometry=json.loads(shapely.to_geojson(curr_geom)),
-                    properties=properties
-                ))
+                merged_features.append(
+                    dict(
+                        type='Feature',
+                        geometry=json.loads(shapely.to_geojson(curr_geom)),
+                        properties=properties,
+                    )
+                )
             else:
                 # valid merge segment, mark constituent features as visited
                 visited_indexes = list(snapped[snapped.geometry.within(curr_segment)].index)
                 visited += visited_indexes
                 visited_features = gdf.loc[visited_indexes]
-                for v_index, v_feature in visited_features.iterrows():
-                    properties = merge_properties(properties, json.loads(
-                        v_feature.drop(['geometry', 'index'], errors='ignore').fillna('').to_json()
-                    ))
-                merged_features.append(dict(
-                    type='Feature',
-                    geometry=json.loads(shapely.to_geojson(curr_segment)),
-                    properties=properties
-                ))
+                for _, v_feature in visited_features.iterrows():
+                    properties = merge_properties(
+                        properties,
+                        json.loads(
+                            v_feature.drop(['geometry', 'index'], errors='ignore')
+                            .fillna('')
+                            .to_json()
+                        ),
+                    )
+                merged_features.append(
+                    dict(
+                        type='Feature',
+                        geometry=json.loads(shapely.to_geojson(curr_segment)),
+                        properties=properties,
+                    )
+                )
     gdf = geopandas.GeoDataFrame.from_features(merged_features)
     return cut_crossed_lines(gdf)
 
 
 def find_nodes(gdf):
     nodes = []
 
-    for feat_index, feature in gdf.iterrows():
+    for _, feature in gdf.iterrows():
         properties = get_properties(feature)
         curr_geom = feature.geometry
         points = [shapely.Point(*p) for p in curr_geom.coords]
@@ -144,22 +161,23 @@ def find_nodes(gdf):
                 # omit duplicates within tolerance radius
                 not existing_node_locations.dwithin(endpoint, TOLERANCE).any()
             ):
-                nodes.append(dict(
-                    location=endpoint,
-                    metadata=properties
-                ))
+                nodes.append(dict(location=endpoint, metadata=properties))
     return nodes
 
 
 def find_edges(gdf, nodes):
     edges = []
     existing_node_locations = geopandas.GeoSeries([n['location'] for n in nodes])
-    for feat_index, feature in gdf.iterrows():
+    for _, feature in gdf.iterrows():
         properties = get_properties(feature)
         curr_geom = feature.geometry
         points = [shapely.Point(*p) for p in curr_geom.coords]
-        nearby_nodes = existing_node_locations[existing_node_locations.dwithin(curr_geom, TOLERANCE)]
-        snapped = shapely.snap(shapely.MultiPoint(list(nearby_nodes.geometry)), curr_geom, TOLERANCE)
+        nearby_nodes = existing_node_locations[
+            existing_node_locations.dwithin(curr_geom, TOLERANCE)
+        ]
+        snapped = shapely.snap(
+            shapely.MultiPoint(list(nearby_nodes.geometry)), curr_geom, TOLERANCE
+        )
         separated = shapely.ops.split(shapely.LineString(points), snapped).geoms
         existing_edge_geometries = geopandas.GeoSeries([e['line_geometry'] for e in edges])
         for segment in separated:
@@ -170,16 +188,20 @@ def find_edges(gdf, nodes):
             from_points = nearby_nodes[nearby_nodes.dwithin(endpoints[0], TOLERANCE)]
             to_points = nearby_nodes[nearby_nodes.dwithin(endpoints[1], TOLERANCE)]
             if (
-                len(from_points) > 0 and
-                len(to_points) > 0 and
-                not shapely.snap(existing_edge_geometries, segment, TOLERANCE).covers(segment).any()
+                len(from_points) > 0
+                and len(to_points) > 0
+                and not shapely.snap(existing_edge_geometries, segment, TOLERANCE)
+                .covers(segment)
+                .any()
             ):
-                edges.append(dict(
-                    line_geometry=segment,
-                    from_point=from_points.iloc[0],
-                    to_point=to_points.iloc[0],
-                    metadata=properties,
-                ))
+                edges.append(
+                    dict(
+                        line_geometry=segment,
+                        from_point=from_points.iloc[0],
+                        to_point=to_points.iloc[0],
+                        metadata=properties,
+                    )
+                )
     return edges