Skip to content

Commit 34898c9

Browse files
tess1otess1o
authored
Added geonames geocoding for immich (#369)
1 parent b530462 commit 34898c9

27 files changed

+1558
-35
lines changed

‎backend/src/main/java/org/github/tess1o/geopulse/geocoding/GeocodingNativeConfig.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
import org.github.tess1o.geopulse.geocoding.client.PhotonRestClient;
88
import org.github.tess1o.geopulse.geocoding.config.GeocodingConfigurationService;
99
import org.github.tess1o.geopulse.geocoding.dto.*;
10+
import org.github.tess1o.geopulse.geocoding.model.GeonamesCityRecord;
11+
import org.github.tess1o.geopulse.geocoding.model.GeonamesCountryRecord;
12+
import org.github.tess1o.geopulse.geocoding.model.GeonamesNormalizedLocation;
1013
import org.github.tess1o.geopulse.geocoding.model.ReconciliationJobProgress;
1114
import org.github.tess1o.geopulse.geocoding.model.ReverseGeocodingLocationEntity;
1215
import org.github.tess1o.geopulse.geocoding.model.common.FormattableGeocodingResult;
@@ -64,6 +67,9 @@
6467
ReverseGeocodingResource.class,
6568
ReverseGeocodingManagementService.class,
6669
ReconciliationJobProgress.class,
70+
GeonamesCityRecord.class,
71+
GeonamesCountryRecord.class,
72+
GeonamesNormalizedLocation.class,
6773

6874
BulkUpdateGeocodingDto.class,
6975
BulkUpdateGeocodingResult.class,

‎backend/src/main/java/org/github/tess1o/geopulse/geocoding/model/GeonamesCityRecord.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package org.github.tess1o.geopulse.geocoding.model;
2+
3+
import java.time.LocalDate;
4+
5+
/**
6+
* Parsed row from GeoNames "cities500" dataset.
7+
*/
8+
public record GeonamesCityRecord(
9+
Long geonameId,
10+
String name,
11+
String asciiName,
12+
String alternateNames,
13+
Double latitude,
14+
Double longitude,
15+
String featureClass,
16+
String featureCode,
17+
String countryCode,
18+
String cc2,
19+
String admin1Code,
20+
String admin2Code,
21+
String admin3Code,
22+
String admin4Code,
23+
Long population,
24+
Integer elevation,
25+
Integer dem,
26+
String timezone,
27+
LocalDate modificationDate
28+
) {
29+
}

‎backend/src/main/java/org/github/tess1o/geopulse/geocoding/model/GeonamesCountryRecord.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package org.github.tess1o.geopulse.geocoding.model;
2+
3+
/**
4+
* Parsed row from GeoNames countryInfo dataset.
5+
*/
6+
public record GeonamesCountryRecord(
7+
String isoAlpha2,
8+
String isoAlpha3,
9+
Integer isoNumeric,
10+
String fipsCode,
11+
String countryName,
12+
String capital,
13+
Double areaSqKm,
14+
Long population,
15+
String continent,
16+
String tld,
17+
String currencyCode,
18+
String currencyName,
19+
String phone,
20+
String postalCodeFormat,
21+
String postalCodeRegex,
22+
String languages,
23+
Long geonameId,
24+
String neighbors,
25+
String equivalentFipsCode
26+
) {
27+
}

‎backend/src/main/java/org/github/tess1o/geopulse/geocoding/model/GeonamesNormalizedLocation.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package org.github.tess1o.geopulse.geocoding.model;
2+
3+
/**
4+
* Normalized city/country resolved from GeoNames by coordinates.
5+
*/
6+
public record GeonamesNormalizedLocation(
7+
Long geonameId,
8+
String city,
9+
String country,
10+
String countryCode,
11+
Double distanceMeters
12+
) {
13+
}

‎backend/src/main/java/org/github/tess1o/geopulse/geocoding/repository/GeonamesCityRepository.java‎

Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
package org.github.tess1o.geopulse.geocoding.repository;
2+
3+
import jakarta.enterprise.context.ApplicationScoped;
4+
import jakarta.inject.Inject;
5+
import jakarta.persistence.EntityManager;
6+
import jakarta.transaction.Transactional;
7+
import org.github.tess1o.geopulse.geocoding.model.GeonamesCityRecord;
8+
import org.github.tess1o.geopulse.geocoding.model.GeonamesNormalizedLocation;
9+
import org.hibernate.Session;
10+
11+
import java.sql.Date;
12+
import java.sql.PreparedStatement;
13+
import java.sql.SQLException;
14+
import java.sql.Types;
15+
import java.util.List;
16+
import java.util.Optional;
17+
18+
@ApplicationScoped
19+
public class GeonamesCityRepository {
20+
21+
private static final String STAGING_TABLE = "geonames_city_import_staging";
22+
23+
private static final String UPSERT_SQL = """
24+
INSERT INTO geonames_city (
25+
geonameid, name, asciiname, alternatenames, latitude, longitude,
26+
feature_class, feature_code, country_code, cc2, admin1_code, admin2_code, admin3_code, admin4_code,
27+
population, elevation, dem, timezone, modification_date
28+
) VALUES (
29+
?, ?, ?, ?, ?, ?,
30+
?, ?, ?, ?, ?, ?, ?, ?,
31+
?, ?, ?, ?, ?
32+
)
33+
ON CONFLICT (geonameid) DO UPDATE SET
34+
name = EXCLUDED.name,
35+
asciiname = EXCLUDED.asciiname,
36+
alternatenames = EXCLUDED.alternatenames,
37+
latitude = EXCLUDED.latitude,
38+
longitude = EXCLUDED.longitude,
39+
feature_class = EXCLUDED.feature_class,
40+
feature_code = EXCLUDED.feature_code,
41+
country_code = EXCLUDED.country_code,
42+
cc2 = EXCLUDED.cc2,
43+
admin1_code = EXCLUDED.admin1_code,
44+
admin2_code = EXCLUDED.admin2_code,
45+
admin3_code = EXCLUDED.admin3_code,
46+
admin4_code = EXCLUDED.admin4_code,
47+
population = EXCLUDED.population,
48+
elevation = EXCLUDED.elevation,
49+
dem = EXCLUDED.dem,
50+
timezone = EXCLUDED.timezone,
51+
modification_date = EXCLUDED.modification_date
52+
""";
53+
54+
private static final String UPSERT_STAGING_SQL = """
55+
INSERT INTO geonames_city_import_staging (
56+
geonameid, name, asciiname, alternatenames, latitude, longitude,
57+
feature_class, feature_code, country_code, cc2, admin1_code, admin2_code, admin3_code, admin4_code,
58+
population, elevation, dem, timezone, modification_date
59+
) VALUES (
60+
?, ?, ?, ?, ?, ?,
61+
?, ?, ?, ?, ?, ?, ?, ?,
62+
?, ?, ?, ?, ?
63+
)
64+
ON CONFLICT (geonameid) DO UPDATE SET
65+
name = EXCLUDED.name,
66+
asciiname = EXCLUDED.asciiname,
67+
alternatenames = EXCLUDED.alternatenames,
68+
latitude = EXCLUDED.latitude,
69+
longitude = EXCLUDED.longitude,
70+
feature_class = EXCLUDED.feature_class,
71+
feature_code = EXCLUDED.feature_code,
72+
country_code = EXCLUDED.country_code,
73+
cc2 = EXCLUDED.cc2,
74+
admin1_code = EXCLUDED.admin1_code,
75+
admin2_code = EXCLUDED.admin2_code,
76+
admin3_code = EXCLUDED.admin3_code,
77+
admin4_code = EXCLUDED.admin4_code,
78+
population = EXCLUDED.population,
79+
elevation = EXCLUDED.elevation,
80+
dem = EXCLUDED.dem,
81+
timezone = EXCLUDED.timezone,
82+
modification_date = EXCLUDED.modification_date
83+
""";
84+
85+
private final EntityManager entityManager;
86+
87+
@Inject
88+
public GeonamesCityRepository(EntityManager entityManager) {
89+
this.entityManager = entityManager;
90+
}
91+
92+
@Transactional
93+
public long countCities() {
94+
Number result = (Number) entityManager.createNativeQuery("SELECT COUNT(*) FROM geonames_city")
95+
.getSingleResult();
96+
return result.longValue();
97+
}
98+
99+
@Transactional
100+
public void truncateAll() {
101+
entityManager.createNativeQuery("TRUNCATE TABLE geonames_city").executeUpdate();
102+
}
103+
104+
@Transactional
105+
public int upsertBatch(List<GeonamesCityRecord> batch) {
106+
if (batch == null || batch.isEmpty()) {
107+
return 0;
108+
}
109+
110+
Session session = entityManager.unwrap(Session.class);
111+
session.doWork(connection -> {
112+
try (PreparedStatement statement = connection.prepareStatement(UPSERT_SQL)) {
113+
for (GeonamesCityRecord record : batch) {
114+
bind(statement, record);
115+
statement.addBatch();
116+
}
117+
statement.executeBatch();
118+
}
119+
});
120+
121+
entityManager.clear();
122+
return batch.size();
123+
}
124+
125+
@Transactional
126+
public void prepareStagingTable() {
127+
entityManager.createNativeQuery("""
128+
CREATE TABLE IF NOT EXISTS geonames_city_import_staging
129+
(LIKE geonames_city INCLUDING ALL)
130+
""").executeUpdate();
131+
entityManager.createNativeQuery("TRUNCATE TABLE " + STAGING_TABLE).executeUpdate();
132+
}
133+
134+
@Transactional
135+
public int upsertBatchToStaging(List<GeonamesCityRecord> batch) {
136+
if (batch == null || batch.isEmpty()) {
137+
return 0;
138+
}
139+
140+
Session session = entityManager.unwrap(Session.class);
141+
session.doWork(connection -> {
142+
try (PreparedStatement statement = connection.prepareStatement(UPSERT_STAGING_SQL)) {
143+
for (GeonamesCityRecord record : batch) {
144+
bind(statement, record);
145+
statement.addBatch();
146+
}
147+
statement.executeBatch();
148+
}
149+
});
150+
151+
entityManager.clear();
152+
return batch.size();
153+
}
154+
155+
@Transactional
156+
public long countStagingCities() {
157+
Number result = (Number) entityManager.createNativeQuery("SELECT COUNT(*) FROM " + STAGING_TABLE)
158+
.getSingleResult();
159+
return result.longValue();
160+
}
161+
162+
@Transactional
163+
public void replaceMainFromStagingAtomic() {
164+
entityManager.createNativeQuery("TRUNCATE TABLE geonames_city").executeUpdate();
165+
entityManager.createNativeQuery("INSERT INTO geonames_city SELECT * FROM " + STAGING_TABLE).executeUpdate();
166+
}
167+
168+
@Transactional
169+
public Optional<GeonamesNormalizedLocation> findNearestNormalizedLocation(
170+
double latitude,
171+
double longitude,
172+
Double maxDistanceMeters
173+
) {
174+
List<Object[]> rows = entityManager.createNativeQuery("""
175+
SELECT
176+
gc.geonameid,
177+
gc.name AS city_name,
178+
COALESCE(gct.country_name, gc.country_code) AS country_name,
179+
gc.country_code,
180+
ST_DistanceSphere(
181+
ST_SetSRID(ST_MakePoint(gc.longitude, gc.latitude), 4326),
182+
ST_SetSRID(ST_MakePoint(:lon, :lat), 4326)
183+
) AS distance_m
184+
FROM geonames_city gc
185+
LEFT JOIN geonames_country gct ON gct.iso_alpha2 = gc.country_code
186+
WHERE gc.latitude IS NOT NULL
187+
AND gc.longitude IS NOT NULL
188+
AND (
189+
:maxDistanceMeters IS NULL
190+
OR ST_DistanceSphere(
191+
ST_SetSRID(ST_MakePoint(gc.longitude, gc.latitude), 4326),
192+
ST_SetSRID(ST_MakePoint(:lon, :lat), 4326)
193+
) <= :maxDistanceMeters
194+
)
195+
ORDER BY distance_m ASC, COALESCE(gc.population, 0) DESC
196+
LIMIT 1
197+
""")
198+
.setParameter("lat", latitude)
199+
.setParameter("lon", longitude)
200+
.setParameter("maxDistanceMeters", maxDistanceMeters)
201+
.getResultList();
202+
203+
if (rows.isEmpty()) {
204+
return Optional.empty();
205+
}
206+
207+
Object[] row = rows.getFirst();
208+
Long geonameId = row[0] != null ? ((Number) row[0]).longValue() : null;
209+
String city = row[1] != null ? row[1].toString() : null;
210+
String country = row[2] != null ? row[2].toString() : null;
211+
String countryCode = row[3] != null ? row[3].toString() : null;
212+
Double distanceMeters = row[4] != null ? ((Number) row[4]).doubleValue() : null;
213+
214+
return Optional.of(new GeonamesNormalizedLocation(
215+
geonameId,
216+
city,
217+
country,
218+
countryCode,
219+
distanceMeters
220+
));
221+
}
222+
223+
private void bind(PreparedStatement statement, GeonamesCityRecord record) throws SQLException {
224+
int index = 1;
225+
statement.setLong(index++, record.geonameId());
226+
statement.setString(index++, record.name());
227+
setNullableString(statement, index++, record.asciiName());
228+
setNullableString(statement, index++, record.alternateNames());
229+
statement.setDouble(index++, record.latitude());
230+
statement.setDouble(index++, record.longitude());
231+
setNullableString(statement, index++, record.featureClass());
232+
setNullableString(statement, index++, record.featureCode());
233+
setNullableString(statement, index++, record.countryCode());
234+
setNullableString(statement, index++, record.cc2());
235+
setNullableString(statement, index++, record.admin1Code());
236+
setNullableString(statement, index++, record.admin2Code());
237+
setNullableString(statement, index++, record.admin3Code());
238+
setNullableString(statement, index++, record.admin4Code());
239+
setNullableLong(statement, index++, record.population());
240+
setNullableInteger(statement, index++, record.elevation());
241+
setNullableInteger(statement, index++, record.dem());
242+
setNullableString(statement, index++, record.timezone());
243+
setNullableDate(statement, index, record.modificationDate() != null ? Date.valueOf(record.modificationDate()) : null);
244+
}
245+
246+
private void setNullableString(PreparedStatement statement, int index, String value) throws SQLException {
247+
if (value == null) {
248+
statement.setNull(index, Types.VARCHAR);
249+
return;
250+
}
251+
statement.setString(index, value);
252+
}
253+
254+
private void setNullableLong(PreparedStatement statement, int index, Long value) throws SQLException {
255+
if (value == null) {
256+
statement.setNull(index, Types.BIGINT);
257+
return;
258+
}
259+
statement.setLong(index, value);
260+
}
261+
262+
private void setNullableInteger(PreparedStatement statement, int index, Integer value) throws SQLException {
263+
if (value == null) {
264+
statement.setNull(index, Types.INTEGER);
265+
return;
266+
}
267+
statement.setInt(index, value);
268+
}
269+
270+
private void setNullableDate(PreparedStatement statement, int index, Date value) throws SQLException {
271+
if (value == null) {
272+
statement.setNull(index, Types.DATE);
273+
return;
274+
}
275+
statement.setDate(index, value);
276+
}
277+
}

0 commit comments

Comments
 (0)