Skip to content

Commit 80ecf28

Browse files
GigasziGigaszi
committed
feat: add functions for spatial sampling and tests for it
1 parent 14ff237 commit 80ecf28

File tree

4 files changed

+282
-0
lines changed

4 files changed

+282
-0
lines changed

‎mapswipe_workers/mapswipe_workers/project_types/street/project.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ def create_tasks(self):
114114
task = StreetTask(
115115
projectId=self.projectId,
116116
groupId=group_id,
117+
# TODO: change when db allows point geometries
117118
data=str(self.imageGeometries.pop()),
118119
geometry="",
119120
taskId=self.imageIds.pop(),

‎mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import numpy as np
2+
import pandas as pd
3+
from shapely import wkt
4+
from shapely.geometry import Point
5+
6+
def distance_on_sphere(p1, p2):
7+
"""
8+
p1 and p2 are two lists that have two elements. They are numpy arrays of the long and lat
9+
coordinates of the points in set1 and set2
10+
11+
Calculate the distance between two points on the Earth's surface using the haversine formula.
12+
13+
Args:
14+
p1 (list): Array containing the longitude and latitude coordinates of points FROM which the distance to be calculated in degree
15+
p2 (list): Array containing the longitude and latitude coordinates of points TO which the distance to be calculated in degree
16+
17+
Returns:
18+
numpy.ndarray: Array containing the distances between the two points on the sphere in kilometers.
19+
20+
This function computes the distance between two points on the Earth's surface using the haversine formula,
21+
which takes into account the spherical shape of the Earth. The input arrays `p1` and `p2` should contain
22+
longitude and latitude coordinates in degrees. The function returns an array containing the distances
23+
between corresponding pairs of points.
24+
"""
25+
earth_radius = 6371 # km
26+
27+
p1 = np.radians(np.array(p1))
28+
p2 = np.radians(np.array(p2))
29+
30+
delta_lat = p2[1] - p1[1]
31+
delta_long = p2[0] - p1[0]
32+
33+
a = np.sin(delta_lat / 2) ** 2 + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2
34+
c = 2 * np.arcsin(np.sqrt(a))
35+
36+
distances = earth_radius * c
37+
return distances
38+
39+
"""-----------------------------------Filtering Points------------------------------------------------"""
40+
def filter_points(df, threshold_distance):
41+
"""
42+
Filter points from a DataFrame based on a threshold distance.
43+
44+
Args:
45+
df (pandas.DataFrame): DataFrame containing latitude and longitude columns.
46+
threshold_distance (float): Threshold distance for filtering points in kms.
47+
48+
Returns:
49+
pandas.DataFrame: Filtered DataFrame containing selected points.
50+
float: Total road length calculated from the selected points.
51+
52+
This function filters points from a DataFrame based on the given threshold distance. It calculates
53+
distances between consecutive points and accumulates them until the accumulated distance surpasses
54+
the threshold distance. It then selects those points and constructs a new DataFrame. Additionally,
55+
it manually checks the last point to include it if it satisfies the length condition. The function
56+
returns the filtered DataFrame along with the calculated road length.
57+
"""
58+
road_length = 0
59+
mask = np.zeros(len(df), dtype=bool)
60+
mask[0] = True
61+
lat = np.array([wkt.loads(point).y for point in df['data']])
62+
long = np.array([wkt.loads(point).x for point in df['data']])
63+
64+
df['lat'] = lat
65+
df['long'] = long
66+
67+
68+
distances = distance_on_sphere([long[1:],lat[1:]],
69+
[long[:-1],lat[:-1]])
70+
road_length = np.sum(distances)
71+
72+
#save the last point if the road segment is relavitely small (< 2*road_length)
73+
if threshold_distance <= road_length < 2 * threshold_distance:
74+
mask[-1] = True
75+
76+
accumulated_distance = 0
77+
for i, distance in enumerate(distances):
78+
accumulated_distance += distance
79+
if accumulated_distance >= threshold_distance:
80+
mask[i+1] = True
81+
accumulated_distance = 0 # Reset accumulated distance
82+
83+
to_be_returned_df = df[mask]
84+
# since the last point has to be omitted in the vectorized distance calculation, it is being checked manually
85+
p2 = to_be_returned_df.iloc[0]
86+
distance = distance_on_sphere([float(p2["long"]),float(p2["lat"])],[long[-1],lat[-1]])
87+
88+
#last point will be added if it suffices the length condition
89+
#last point will be added in case there is only one point returned
90+
if distance >= threshold_distance or len(to_be_returned_df) ==1:
91+
to_be_returned_df = pd.concat([to_be_returned_df,pd.DataFrame(df.iloc[-1],columns=to_be_returned_df.columns)],axis=0)
92+
return to_be_returned_df
93+
94+
95+
def calculate_spacing(df, interval_length):
96+
"""
97+
Calculate spacing between points in a GeoDataFrame.
98+
99+
Args:
100+
df (pandas.DataFrame): DataFrame containing points with timestamps.
101+
interval_length (float): Interval length for filtering points in kms.
102+
103+
Returns:
104+
geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points.
105+
float: Total road length calculated from the selected points.
106+
107+
This function calculates the spacing between points in a GeoDataFrame by filtering points
108+
based on the provided interval length. It first sorts the GeoDataFrame by timestamp and
109+
then filters points using the filter_points function. The function returns the filtered
110+
GeoDataFrame along with the total road length.
111+
"""
112+
road_length = 0
113+
if len(df) == 1:
114+
return df
115+
sorted_sub_df = df.sort_values(by=['timestamp'])
116+
filtered_sorted_sub_df = filter_points(sorted_sub_df,interval_length)
117+
return filtered_sorted_sub_df

‎mapswipe_workers/tests/fixtures/mapillary_sequence.csv‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
,data,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id
2+
9,POINT (38.995129466056824 -6.785243670271996),1453463352000.0,102506575322825.0,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ,
3+
12,POINT (38.9906769990921 -6.783315348346505),1453463400000.0,102506575322825.0,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ,
4+
14,POINT (39.00127172470093 -6.787981661065601),1453463294000.0,102506575322825.0,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ,
5+
18,POINT (38.99769365787506 -6.786351652857817),1453463332000.0,102506575322825.0,1014398349364928,,True,288.49790876724,ywMkSP_5PaJzcbIDa5v1aQ,
6+
19,POINT (38.99540305137634 -6.785360860858361),1453463350000.0,102506575322825.0,165035685525369,,True,293.57188365181,ywMkSP_5PaJzcbIDa5v1aQ,
7+
20,POINT (39.00015592575073 -6.787491593818643),1453463300000.0,102506575322825.0,1139870886511452,,True,305.58569254109,ywMkSP_5PaJzcbIDa5v1aQ,
8+
21,POINT (38.9979350566864 -6.7864262285172),1453463330000.0,102506575322825.0,921248855101166,,True,287.76018227153,ywMkSP_5PaJzcbIDa5v1aQ,
9+
22,POINT (38.99890601634979 -6.786804433469129),1453463318000.0,102506575322825.0,233272625257058,,True,295.57372041131,ywMkSP_5PaJzcbIDa5v1aQ,
10+
23,POINT (38.99618625640869 -6.7857071056060505),1453463344000.0,102506575322825.0,762184237820558,,True,293.88722229187,ywMkSP_5PaJzcbIDa5v1aQ,
11+
26,POINT (38.98969531059265 -6.782905179427416),1453463424000.0,102506575322825.0,1099193250486391,,True,292.87773588969,ywMkSP_5PaJzcbIDa5v1aQ,
12+
27,POINT (38.998627066612244 -6.786671262745287),1453463322000.0,102506575322825.0,831521697449190,,True,296.54786608582,ywMkSP_5PaJzcbIDa5v1aQ,
13+
29,POINT (38.992629647254944 -6.784167646282242),1453463368000.0,102506575322825.0,151617226911336,,True,292.93390738544,ywMkSP_5PaJzcbIDa5v1aQ,
14+
30,POINT (39.00003254413605 -6.7873584232849),1453463302000.0,102506575322825.0,1773698542801178,,True,296.75481871036,ywMkSP_5PaJzcbIDa5v1aQ,
15+
32,POINT (39.000563621520996 -6.787811202949342),1453463298000.0,102506575322825.0,164116422292333,,True,305.80313236719,ywMkSP_5PaJzcbIDa5v1aQ,
16+
37,POINT (38.9902800321579 -6.783144888578377),1453463410000.0,102506575322825.0,329957465177767,,True,293.87481737883,ywMkSP_5PaJzcbIDa5v1aQ,
17+
38,POINT (38.99117052555084 -6.783539076700606),1453463388000.0,102506575322825.0,2948533428757015,,True,294.44274652914,ywMkSP_5PaJzcbIDa5v1aQ,
18+
39,POINT (38.99877190589905 -6.78674051152629),1453463320000.0,102506575322825.0,479325670049740,,True,296.24340777377,ywMkSP_5PaJzcbIDa5v1aQ,
19+
41,POINT (38.990601897239685 -6.783294040878786),1453463402000.0,102506575322825.0,501560441022559,,True,289.98678899102,ywMkSP_5PaJzcbIDa5v1aQ,
20+
42,POINT (38.989362716674805 -6.7828785450699485),1453463432000.0,102506575322825.0,494436578424418,,True,249.25945175736,ywMkSP_5PaJzcbIDa5v1aQ,
21+
44,POINT (38.994566202163696 -6.7850039621655895),1453463356000.0,102506575322825.0,2928848347373461,,True,295.93075138027,ywMkSP_5PaJzcbIDa5v1aQ,
22+
45,POINT (38.993815183639526 -6.784657716912335),1453463362000.0,102506575322825.0,167884815220625,,True,290.65289338004,ywMkSP_5PaJzcbIDa5v1aQ,
23+
47,POINT (38.991841077804565 -6.783837381011111),1453463380000.0,102506575322825.0,2783373888590755,,True,292.94668882511,ywMkSP_5PaJzcbIDa5v1aQ,
24+
48,POINT (38.99052679538727 -6.783262079675453),1453463404000.0,102506575322825.0,500930794384261,,True,296.09431621523,ywMkSP_5PaJzcbIDa5v1aQ,
25+
49,POINT (38.9897757768631 -6.782937140654425),1453463422000.0,102506575322825.0,473363863989539,,True,292.22088072734,ywMkSP_5PaJzcbIDa5v1aQ,
26+
50,POINT (38.99429798126221 -6.784870790943785),1453463358000.0,102506575322825.0,792308461667709,,True,295.36479453372,ywMkSP_5PaJzcbIDa5v1aQ,
27+
51,POINT (38.9997535943985 -6.787219925890724),1453463306000.0,102506575322825.0,1169832606865116,,True,296.51874301031,ywMkSP_5PaJzcbIDa5v1aQ,
28+
54,POINT (38.992860317230225 -6.784268856561923),1453463364000.0,102506575322825.0,143904254368287,,True,292.87299883627,ywMkSP_5PaJzcbIDa5v1aQ,
29+
57,POINT (38.98994743824005 -6.783006389972371),1453463418000.0,102506575322825.0,512708183243254,,True,292.58758044439,ywMkSP_5PaJzcbIDa5v1aQ,
30+
58,POINT (38.99670124053955 -6.785941486524692),1453463340000.0,102506575322825.0,168474601828325,,True,294.32908734047,ywMkSP_5PaJzcbIDa5v1aQ,
31+
59,POINT (38.992136120796204 -6.783959898799395),1453463376000.0,102506575322825.0,171815874817246,,True,292.47687051975,ywMkSP_5PaJzcbIDa5v1aQ,
32+
60,POINT (38.99090766906738 -6.783405905073778),1453463394000.0,102506575322825.0,475809606904698,,True,297.28158189053,ywMkSP_5PaJzcbIDa5v1aQ,
33+
61,POINT (38.99251699447632 -6.7841250314212544),1453463370000.0,102506575322825.0,798930200741228,,True,292.63573224675,ywMkSP_5PaJzcbIDa5v1aQ,
34+
62,POINT (38.99019956588745 -6.7831129273651385),1453463412000.0,102506575322825.0,989719805170705,,True,292.67075887406,ywMkSP_5PaJzcbIDa5v1aQ,
35+
63,POINT (38.991336822509766 -6.783613652795566),1453463386000.0,102506575322825.0,887351401825944,,True,294.11111203184,ywMkSP_5PaJzcbIDa5v1aQ,
36+
64,POINT (38.99745762348175 -6.786271750352796),1453463334000.0,102506575322825.0,820185135568044,,True,292.38261450394,ywMkSP_5PaJzcbIDa5v1aQ,
37+
69,POINT (38.99919033050537 -6.7869376041561225),1453463314000.0,102506575322825.0,1401323273568889,,True,296.07037190303,ywMkSP_5PaJzcbIDa5v1aQ,
38+
70,POINT (38.99229168891907 -6.784023821111347),1453463374000.0,102506575322825.0,971311816939999,,True,293.93020293096,ywMkSP_5PaJzcbIDa5v1aQ,
39+
71,POINT (38.9956659078598 -6.785483378259087),1453463348000.0,102506575322825.0,1888667171315269,,True,294.14026651816,ywMkSP_5PaJzcbIDa5v1aQ,
40+
77,POINT (38.99095058441162 -6.783437866267576),1453463392000.0,102506575322825.0,313013763568992,,True,299.51413646126,ywMkSP_5PaJzcbIDa5v1aQ,
41+
78,POINT (38.99240434169769 -6.784077089698172),1453463372000.0,102506575322825.0,300316491494684,,True,293.91812405268,ywMkSP_5PaJzcbIDa5v1aQ,
42+
79,POINT (38.994094133377075 -6.784774907641307),1453463360000.0,102506575322825.0,799986720927690,,True,293.50534124709,ywMkSP_5PaJzcbIDa5v1aQ,
43+
80,POINT (38.998122811317444 -6.786495477333432),1453463328000.0,102506575322825.0,766866250678914,,True,291.66896324881,ywMkSP_5PaJzcbIDa5v1aQ,
44+
82,POINT (38.99715185165405 -6.786143906317193),1453463336000.0,102506575322825.0,4104687166260249,,True,294.20932475917,ywMkSP_5PaJzcbIDa5v1aQ,
45+
84,POINT (38.99081647396088 -6.783368617011661),1453463396000.0,102506575322825.0,311476673697450,,True,292.1387426333,ywMkSP_5PaJzcbIDa5v1aQ,
46+
85,POINT (38.999614119529724 -6.787150677178687),1453463308000.0,102506575322825.0,486545769438431,,True,296.75245798459,ywMkSP_5PaJzcbIDa5v1aQ,
47+
86,POINT (38.99830520153046 -6.786516784659511),1453463326000.0,102506575322825.0,287818266220456,,True,296.90157475174,ywMkSP_5PaJzcbIDa5v1aQ,
48+
89,POINT (38.991073966026306 -6.78349646178404),1453463390000.0,102506575322825.0,1104648166685558,,True,294.64381621501,ywMkSP_5PaJzcbIDa5v1aQ,
49+
95,POINT (38.99045169353485 -6.783224791602194),1453463406000.0,102506575322825.0,1189096484862313,,True,296.52075233925,ywMkSP_5PaJzcbIDa5v1aQ,
50+
96,POINT (38.989604115486145 -6.782867891326546),1453463426000.0,102506575322825.0,149492403718498,,True,291.35167681135,ywMkSP_5PaJzcbIDa5v1aQ,
51+
99,POINT (38.990371227264404 -6.783187503526065),1453463408000.0,102506575322825.0,1891398304374063,,True,293.92928705821,ywMkSP_5PaJzcbIDa5v1aQ,
52+
102,POINT (38.99695873260498 -6.786058676941238),1453463338000.0,102506575322825.0,823059255257426,,True,294.12835748694,ywMkSP_5PaJzcbIDa5v1aQ,
53+
105,POINT (38.99198055267334 -6.7838959764789735),1453463378000.0,102506575322825.0,3696263247264941,,True,293.15037639266,ywMkSP_5PaJzcbIDa5v1aQ,
54+
108,POINT (38.989861607551575 -6.782974428749938),1453463420000.0,102506575322825.0,862036401042035,,True,291.44770311405,ywMkSP_5PaJzcbIDa5v1aQ,
55+
109,POINT (38.98951292037964 -6.782841256967004),1453463428000.0,102506575322825.0,304865891166877,,True,272.6464361279,ywMkSP_5PaJzcbIDa5v1aQ,
56+
111,POINT (38.99274230003357 -6.784220914853137),1453463366000.0,102506575322825.0,794241458144562,,True,293.2506353664,ywMkSP_5PaJzcbIDa5v1aQ,
57+
113,POINT (38.99847149848938 -6.786596687123861),1453463324000.0,102506575322825.0,939866586773160,,True,295.61868532377,ywMkSP_5PaJzcbIDa5v1aQ,
58+
115,POINT (38.99947464466095 -6.787081428456702),1453463310000.0,102506575322825.0,1143620816058871,,True,296.46351643457,ywMkSP_5PaJzcbIDa5v1aQ,
59+
116,POINT (38.99003326892853 -6.783043678062526),1453463416000.0,102506575322825.0,1405161323178994,,True,293.46012249789,ywMkSP_5PaJzcbIDa5v1aQ,
60+
119,POINT (39.00104105472565 -6.78785914430064),1453463296000.0,102506575322825.0,1273957886333326,,True,279.89824953791,ywMkSP_5PaJzcbIDa5v1aQ,
61+
120,POINT (38.98943245410919 -6.782857237582903),1453463430000.0,102506575322825.0,1171907646606759,,True,252.08147683237,ywMkSP_5PaJzcbIDa5v1aQ,
62+
121,POINT (38.99148166179657 -6.783677575153462),1453463384000.0,102506575322825.0,164230852280713,,True,293.89449177018,ywMkSP_5PaJzcbIDa5v1aQ,
63+
122,POINT (38.999335169792175 -6.787012179724755),1453463312000.0,102506575322825.0,510570906624460,,True,297.86598575521,ywMkSP_5PaJzcbIDa5v1aQ,
64+
124,POINT (38.99592339992523 -6.785595241945558),1453463346000.0,102506575322825.0,136548521821574,,True,293.97664288588,ywMkSP_5PaJzcbIDa5v1aQ,
65+
125,POINT (38.990119099617004 -6.783075639280355),1453463414000.0,102506575322825.0,1112069379291585,,True,293.19285658924,ywMkSP_5PaJzcbIDa5v1aQ,
66+
126,POINT (38.99644374847412 -6.785829622918669),1453463342000.0,102506575322825.0,846432512639247,,True,294.78265773594,ywMkSP_5PaJzcbIDa5v1aQ,
67+
128,POINT (38.99165332317352 -6.783752151226963),1453463382000.0,102506575322825.0,133891088754131,,True,294.07658010782,ywMkSP_5PaJzcbIDa5v1aQ,
68+
129,POINT (38.99078965187073 -6.7833579632791015),1453463398000.0,102506575322825.0,211793933759059,,True,294.17362400652,ywMkSP_5PaJzcbIDa5v1aQ,
69+
131,POINT (38.99904549121857 -6.786873682230961),1453463316000.0,102506575322825.0,137437058395340,,True,295.32783278431,ywMkSP_5PaJzcbIDa5v1aQ,
70+
132,POINT (38.99989306926727 -6.787289174592786),1453463304000.0,102506575322825.0,323401522460093,,True,296.4023335068,ywMkSP_5PaJzcbIDa5v1aQ,
71+
133,POINT (38.994882702827454 -6.785147787043741),1453463354000.0,102506575322825.0,1070660863461711,,True,294.24600460049,ywMkSP_5PaJzcbIDa5v1aQ,

‎mapswipe_workers/tests/unittests/test_spatial_sampling.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
import os
2+
3+
import unittest
4+
import numpy as np
5+
import pandas as pd
6+
from shapely import wkt
7+
from shapely.geometry import Point
8+
9+
from mapswipe_workers.utils.spatial_sampling import distance_on_sphere, filter_points, calculate_spacing
10+
11+
12+
class TestDistanceCalculations(unittest.TestCase):
13+
14+
@classmethod
15+
def setUpClass(cls):
16+
with open(
17+
os.path.join(
18+
os.path.dirname(os.path.abspath(__file__)),
19+
"..",
20+
"fixtures",
21+
"mapillary_sequence.csv",
22+
),
23+
"r",
24+
) as file:
25+
cls.fixture_df = pd.read_csv(file)
26+
27+
def test_distance_on_sphere(self):
28+
p1 = Point(-74.006, 40.7128)
29+
p2 = Point(-118.2437, 34.0522)
30+
31+
distance = distance_on_sphere((p1.x, p1.y), (p2.x, p2.y))
32+
expected_distance = 3940 # Approximate known distance in km
33+
34+
self.assertTrue(np.isclose(distance, expected_distance, atol=50))
35+
36+
def test_filter_points(self):
37+
data = {
38+
"data": [
39+
"POINT (-74.006 40.7128)",
40+
"POINT (-75.006 41.7128)",
41+
"POINT (-76.006 42.7128)",
42+
"POINT (-77.006 43.7128)"
43+
]
44+
}
45+
df = pd.DataFrame(data)
46+
47+
threshold_distance = 100
48+
filtered_df = filter_points(df, threshold_distance)
49+
50+
self.assertIsInstance(filtered_df, pd.DataFrame)
51+
self.assertLessEqual(len(filtered_df), len(df))
52+
53+
54+
def test_calculate_spacing(self):
55+
data = {
56+
"data": [
57+
"POINT (-74.006 40.7128)",
58+
"POINT (-75.006 41.7128)",
59+
"POINT (-76.006 42.7128)",
60+
"POINT (-77.006 43.7128)"
61+
],
62+
'timestamp': [1, 2, 3, 4]
63+
}
64+
df = pd.DataFrame(data)
65+
gdf = pd.DataFrame(df)
66+
67+
interval_length = 100
68+
filtered_gdf = calculate_spacing(gdf, interval_length)
69+
70+
self.assertTrue(filtered_gdf['timestamp'].is_monotonic_increasing)
71+
72+
73+
def test_calculate_spacing_with_sequence(self):
74+
threshold_distance = 5
75+
filtered_df = filter_points(self.fixture_df, threshold_distance)
76+
self.assertIsInstance(filtered_df, pd.DataFrame)
77+
self.assertLess(len(filtered_df), len(self.fixture_df))
78+
79+
filtered_df.reset_index(drop=True, inplace=True)
80+
81+
for i in range(len(filtered_df) - 1):
82+
geom1 = wkt.loads(filtered_df.loc[i, 'data'])
83+
geom2 = wkt.loads(filtered_df.loc[i + 1, 'data'])
84+
85+
distance = geom1.distance(geom2)
86+
87+
self.assertLess(distance, threshold_distance)
88+
89+
90+
91+
92+
if __name__ == "__main__":
93+
unittest.main()

0 commit comments

Comments
 (0)