Skip to content

Commit ef28ff9

Browse files
committed
create backend for osm2pgsql
1 parent 2e2a9a1 commit ef28ff9

File tree

10 files changed

+263
-1
lines changed

10 files changed

+263
-1
lines changed

README.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@ Underpass-API aim to be a [Overpass-API](https://github.com/drolbr/Overpass-API)
99
### With docker (recommended)
1010

1111
Follow the instruction of one of the backends:
12-
* [Postgres+PostGIS / Osmosis](backends/postgres_osmosis/README.md), Osmosis schema
1312
* [DuckDB+Spatial / QuackOSM](backends/duckdb_quackosm/README.md), Quackosm schema
13+
* [Postgres+PostGIS / Osmosis](backends/postgres_osmosis/README.md), Osmosis schema
14+
* [Postgres+PostGIS / Osm2pgsql](backends/postgres_osm2pgsql/README.md), Osm2pgsql schema
15+
1416

1517
### Without docker
1618

@@ -19,6 +21,7 @@ It is possible to use Underpass-API without docker with the following instructio
1921
* declare environment variables (add new lines in `~/.bashrc` or `~/.profile` then reload with `source ~/.bashrc`) :
2022
- DuckDB+Spatial / QuackOSM: `export BACKEND="DuckdbQuackosm"` and `export DB="/data/database.parquet"`
2123
- Postgres+PostGIS / Osmosis: `export BACKEND="PostgresOsmosis"` and `export "DATABASE_URL='postgresql://user:pw@host:5432/database"`
24+
- Postgres+PostGIS / Osm2pgsql: `export BACKEND="PostgresOsm2pgsql"` and `export "DATABASE_URL='postgresql://user:pw@host:5432/database"`
2225
* install software : `bundle install`
2326
* start server : `bundle exec rackup`
2427

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Postgres/PostGIS, Osm2pgsql schema
2+
3+
Prepare Docker
4+
```sh
5+
docker compose --profile '*' build
6+
```
7+
8+
## Prepare the data
9+
10+
Create you database using osm2pgsql.
11+
If you do not use the `-s (--slim)` option, you will have to modify view.sql (see comments)
12+
13+
## Run the server
14+
15+
Run the HTTP server
16+
```
17+
docker compose up
18+
```
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
version: "3.3"
2+
3+
services:
4+
osm2pgsql:
5+
profiles: [tools]
6+
build:
7+
context: docker/osm2pgsql
8+
environment:
9+
DATABASE_URL: postgresql://postgres@postgres:5432/postgres
10+
volumes:
11+
- ../../data:/data
12+
depends_on:
13+
- postgres
14+
15+
postgres:
16+
image: postgis/postgis:15-3.4
17+
shm_size: 1g
18+
environment:
19+
POSTGRES_HOST_AUTH_METHOD: trust
20+
volumes:
21+
- ./docker/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
22+
- pgdata:/var/lib/postgresql/data
23+
restart: unless-stopped
24+
25+
api:
26+
extends:
27+
file: ../../docker-compose-base.yaml
28+
service: api
29+
environment:
30+
BACKEND: PostgresOsm2pgsql
31+
DATABASE_URL: postgresql://postgres@postgres:5432/postgres
32+
volumes:
33+
- .:/srv/app/backends/postgres_osm2pgsql
34+
depends_on:
35+
- postgres
36+
37+
volumes:
38+
pgdata:
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM debian:12
2+
3+
RUN apt update -y && apt install -y \
4+
osm2pgsql \
5+
postgresql-client
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
DROP SCHEMA IF EXISTS tiger CASCADE;
2+
DROP EXTENSION IF EXISTS postgis_tiger_geocoder;
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
CREATE EXTENSION IF NOT EXISTS htsore;
2+
3+
-- Same as ->, for code compatibility with json
4+
CREATE OPERATOR ->> (
5+
LEFTARG = hstore,
6+
RIGHTARG = text,
7+
PROCEDURE = fetchval
8+
);
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE EXTENSION IF NOT EXISTS postgis;
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# frozen_string_literal: true
2+
3+
require 'pg'
4+
require 'overpass_parser/sql_dialect/postgres'
5+
6+
class PostgresOsm2pgsql
7+
def initialize
8+
9+
@@con = PG.connect(ENV['DATABASE_URL'])
10+
@@con.query(File.read(File.dirname(__FILE__) + '/view.sql'))
11+
@dialect = OverpassParser::SqlDialect::Postgres.new(postgres_escape_literal: ->(s) { @@con.escape_literal(s) })
12+
end
13+
14+
def exec(query)
15+
request = OverpassParser.parse(query)
16+
sql = request.to_sql(@dialect)
17+
puts sql
18+
result = @@con.exec(sql)
19+
[sql, result.collect { |row| row['j'].gsub('+00:00', 'Z') }]
20+
end
21+
end
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/************** ABOUT TABLES *****************/
2+
/*
3+
- without the --slim option, only tables _point _line and _polygon are created, with filtered data. These tables also contain the geometry.
4+
- with --slim, 3 other tables containing the raw OSM elements are created : _nodes, _ways, _rels
5+
*/
6+
7+
/************** ABOUT TAGS COLUMNS *****************/
8+
/*
9+
- tags in tables _nodes, _ways, _rels are identical to the ones in OSM (they are "raw data" tables)
10+
- but tags in tables _point _line and _polygon are always (slightly) different depending on the hstore options:
11+
# With --hstore any tags without a column will be added to the hstore column.
12+
# With --hstore-all all tags are added to the hstore column unless they appear in the style file with a delete flag
13+
*/
14+
15+
/************** ABOUT METADATA *****************/
16+
/* adding metadata to the DB requires -x option */
17+
/* without it, all the CAST(tags->...) will return NULL */
18+
19+
20+
/************** NODES *****************/
21+
/* read data only in table planet_osm_point since planet_osm_nodes may be absent if you used flat nodes mode or did not use --slim */
22+
CREATE OR REPLACE TEMP VIEW node AS
23+
SELECT
24+
osm_id AS id,
25+
CAST(tags->'osm_version' AS integer) AS version,
26+
CAST(tags->'osm_timestamp' AS timestamp without time zone) AS created,
27+
CAST(tags->'osm_changeset' AS bigint) AS changeset,
28+
CAST(tags->'osm_uid' AS integer) AS uid,
29+
CAST(tags->'osm_user' AS text) AS user,
30+
to_jsonb(tags) AS tags,
31+
NULL::bigint[] AS nodes,
32+
NULL::jsonb AS members,
33+
ST_Transform(way,4326) AS geom,
34+
'n' AS osm_type
35+
FROM planet_osm_point;
36+
37+
/************** WAYS *****************/
38+
/* ways are in tables _line and _polygon (excluding id<0 which are relations) */
39+
CREATE OR REPLACE TEMP VIEW way AS
40+
SELECT
41+
l.osm_id AS id,
42+
CAST(l.tags->'osm_version' AS integer) AS version,
43+
CAST(l.tags->'osm_timestamp' AS timestamp without time zone) AS created,
44+
CAST(l.tags->'osm_changeset' AS bigint) AS changeset,
45+
CAST(l.tags->'osm_uid' AS integer) AS uid,
46+
CAST(l.tags->'osm_user' AS text) AS user,
47+
to_jsonb(l.tags) AS tags,
48+
w.nodes AS nodes, /* replace by `NULL::bigint[] AS nodes` if did not use --slim */
49+
NULL::jsonb AS members,
50+
ST_Transform(l.way,4326) AS geom,
51+
'w' AS osm_type
52+
FROM planet_osm_line AS l
53+
LEFT JOIN planet_osm_ways AS w ON osm_id = id /* remove if you did not use --slim */
54+
UNION ALL
55+
SELECT
56+
p.osm_id AS id,
57+
CAST(p.tags->'osm_version' AS integer) AS version,
58+
CAST(p.tags->'osm_timestamp' AS timestamp without time zone) AS created,
59+
CAST(p.tags->'osm_changeset' AS bigint) AS changeset,
60+
CAST(p.tags->'osm_uid' AS integer) AS uid,
61+
CAST(p.tags->'osm_user' AS text) AS user,
62+
to_jsonb(p.tags) as tags,
63+
w.nodes AS nodes, /* replace by `NULL::bigint[] AS nodes` if did not use --slim */
64+
NULL::jsonb AS members,
65+
ST_Transform(p.way,4326) AS geom,
66+
'w' AS osm_type
67+
FROM planet_osm_polygon AS p
68+
LEFT JOIN planet_osm_ways AS w ON osm_id = id /* remove if you did not use --slim */
69+
WHERE p.osm_id > 0
70+
;
71+
72+
/************** RELATIONS *****************/
73+
/* complete version if you used --slim */
74+
CREATE OR REPLACE TEMP VIEW relation AS
75+
SELECT
76+
r.id AS id,
77+
CAST(r.tags->>'osm_version' AS integer) AS version,
78+
CAST(r.tags->>'osm_timestamp' AS timestamp without time zone) AS created,
79+
CAST(r.tags->>'osm_changeset' AS bigint) AS changeset,
80+
CAST(r.tags->>'osm_uid' AS integer) AS uid,
81+
CAST(r.tags->>'osm_user' AS text) AS user,
82+
r.tags as tags,
83+
NULL::bigint[] AS nodes,
84+
r.members AS members,
85+
ST_Transform(p.way,4326) AS geom,
86+
'r' AS osm_type
87+
FROM planet_osm_rels AS r
88+
LEFT JOIN planet_osm_polygon AS p ON id = -osm_id;
89+
90+
/* simple version if you did not used --slim */
91+
/* returns only some relations : multipolygon, boundary, routes */
92+
/*
93+
CREATE OR REPLACE TEMP VIEW relation AS
94+
SELECT
95+
-osm_id as id, /* relation ids are stored as negative values in planet_osm_polygon */
96+
CAST(tags->'osm_version' AS integer) AS version,
97+
CAST(tags->'osm_timestamp' AS timestamp without time zone) AS created,
98+
CAST(tags->'osm_changeset' AS bigint) AS changeset,
99+
CAST(tags->'osm_uid' AS integer) AS uid,
100+
CAST(tags->'osm_user' AS text) AS user,
101+
to_jsonb(tags) as tags,
102+
NULL::bigint[] AS nodes,
103+
NULL::jsonb AS members,
104+
ST_Transform(way,4326) AS geom,
105+
'r' AS osm_type
106+
FROM planet_osm_polygon;
107+
*/
108+
109+
/************** NWR *****************/
110+
CREATE OR REPLACE TEMP VIEW nwr AS
111+
SELECT * FROM node
112+
UNION ALL
113+
SELECT * FROM way
114+
UNION ALL
115+
SELECT * FROM relation
116+
;
117+
118+
/************** AREA *****************/
119+
/* complete version if you used --slim */
120+
CREATE OR REPLACE TEMP VIEW area AS
121+
SELECT
122+
CASE
123+
WHEN p.osm_id<0 THEN 3600000000-p.osm_id /* transform the negative values used for relations in _polygon table to the id format used for areas in overpass */
124+
ELSE p.osm_id
125+
END AS id,
126+
CAST(p.tags->'osm_version' AS integer) AS version,
127+
CAST(p.tags->'osm_timestamp' AS timestamp without time zone) AS created,
128+
CAST(p.tags->'osm_changeset' AS bigint) AS changeset,
129+
CAST(p.tags->'osm_uid' AS integer) AS uid,
130+
CAST(p.tags->'osm_user' AS text) AS user,
131+
to_jsonb(p.tags) as tags,
132+
w.nodes AS nodes, /* For ways only : get nodes from table planet_osm_ways */
133+
r.members AS members, /* For relations only : get members from table planet_osm_rels */
134+
ST_Transform(p.way,4326) AS geom,
135+
'a' AS osm_type
136+
FROM planet_osm_polygon AS p
137+
LEFT JOIN planet_osm_ways AS w ON osm_id=w.id
138+
LEFT JOIN planet_osm_rels AS r ON osm_id=-r.id
139+
/* I don't know if JOIN to get nodes and members are useful for areas. Maybe geom is enough? */
140+
;
141+
142+
/* simple version if you did not used --slim */
143+
/*
144+
CREATE OR REPLACE TEMP VIEW area AS
145+
SELECT
146+
CASE
147+
WHEN p.osm_id<0 THEN 3600000000-p.osm_id /* transform the negative values used here for relations to the id format used for areas in overpass */
148+
ELSE p.osm_id
149+
END AS id,
150+
CAST(p.tags->'osm_version' AS integer) AS version,
151+
CAST(p.tags->'osm_timestamp' AS timestamp without time zone) AS created,
152+
CAST(p.tags->'osm_changeset' AS bigint) AS changeset,
153+
CAST(p.tags->'osm_uid' AS integer) AS uid,
154+
CAST(p.tags->'osm_user' AS text) AS user,
155+
to_jsonb(p.tags) as tags,
156+
NULL::bigint[] AS nodes,
157+
NULL::jsonb AS members,
158+
ST_Transform(p.way,4326) AS geom,
159+
'a' AS osm_type
160+
FROM planet_osm_polygon AS p
161+
;
162+
*/

config.ru

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ when 'DuckdbQuackosm'
99
require_relative 'backends/duckdb_quackosm/duckdb_quackosm'
1010
when 'PostgresOsmosis'
1111
require_relative 'backends/postgres_osmosis/postgres_osmosis'
12+
when 'PostgresOsm2pgsql'
13+
require_relative 'backends/postgres_osm2pgsql/postgres_osm2pgsql'
1214
end
1315

1416
class App < Hanami::API
@@ -19,6 +21,8 @@ class App < Hanami::API
1921
DuckdbQuackosm.new
2022
when 'PostgresOsmosis'
2123
PostgresOsmosis.new
24+
when 'PostgresOsm2pgsql'
25+
PostgresOsm2pgsql.new
2226
end
2327
end
2428

0 commit comments

Comments
 (0)