rewrite flex.lua and view.sql to solve index issue with negative ids

Author: etienneJr

backends/postgres_osm2pgsql/README.md

@@ -7,20 +7,56 @@ docker compose --profile '*' build
 
 ## Prepare the data
 
-Create you database with osm2pgsql, using the script `geometries-alone.lua` in this folder.
+### 1. View for a new, dedicated, updatable and complete DB
 
-Example command :
+This view is configured to be a true alternative to overpass, in the sense that there is no filtering on the elements included in the DB. They are all included in the DB with all their tags, whereas osm2pgsql is usually configured not to include tags or elements that are useless for generating tiles.
 
+Warning : this wiew will not work on an existing osm2pgsql database (see below). It is specific for a database created with the lua script above and the `-s (--slim)` option.
+
+Create you database with osm2pgsql, using the script `geometries-alone.lua` available in this folder.
+
+Example command:
+
+```
+osm2pgsql -U user -d database -c -s --flat-nodes FILE --middle-with-nodes -x -O flex -S geometries-alone.lua extract.osm.pbf
+```
+
+Explanation of the command:
+- `-c -s -x`: create an updatable table including metadata
+- `--flat-nodes FILE --middle-with-nodes`: use the file `FILE` to store nodes (to reduce the size of the DB), but nodes with tags are also stored in the database (so that filtering by tag will be possible on nodes)
+- `-O flex -S geometries-alone.lua`: use flex output mode with specific `.lua` file
+
+One finished, add index for tags:
 ```
-osm2pgsql -U user -d database -c -s --flat-nodes DB-flat-nodes --middle-with-nodes -x -O flex -S geometries-alone.lua extract.osm.pbf
+CREATE INDEX nodes_tags_idx ON planet_osm_nodes USING GIN (tags);
+CREATE INDEX ways_tags_idx ON planet_osm_ways USING GIN (tags);
+CREATE INDEX rels_tags_idx ON planet_osm_rels USING GIN (tags);
 ```
 
-Warning : this backend will not work on a existing osm2pgsql database. it is specific for a database created with the lua script above and the `-s (--slim)` option.
+Use `osm2pgsql-replication` to update the DB.
+
+Sizing
+- 57GB for France (29GB for middle tables (metadata+tags) + 2GB for tag index + 26GB for output tables (geometries))
 
 If your database was created "outside" docker, you will have to modify `docker-compose.yaml` to:
   - delete services `osm2pgsql` and `postgress`
   - in service api : delete reference `depends on: -postgres` and set your `DATABASE_URL: postgres://user:pw@host:5432/database`
 
+Explanation of the `.lua` script and principle of the DB structure:
+- `-s` option creates `middle` tables which include all raw OSM elements (`planet_osm_nodes`, `planet_osm_ways`, `planet_osm_rels`) with all their tags. This view uses these 3 tables to get tags (no need to duplicate them in `output` tables), but we need to manually index them at the beginning.
+- `-x` option add columns for metadata in these tables, and table `planet_osm_users` for usernames.
+- so only the geometries are missing. The `.lua` script of the `flex output` creates 3 additionnal `output` tables: `nodes_geom`, `ways_geom` and `rels_geom` with the id and the geometry.
+  - I had to use tables by element type instead of geometry type (as usual for osm2pgsql) since the negative id used for relations in area type table is problematic for the join with the `planet_osm_rels` table which uses normal positive id (join is slow since indexing is not working).
+- For a list of expected areas (as usual in osm2pgsql), the lua script creates polygon-like geometries (instead of line-like) and adds the area size in a 3rd column (it will be usefull to get areas).
+
+### 2. View for an existing DB created for cartocss
+
+not written yet
+
+### 3. View for a simple static DB (without synchronisation)
+
+not written yet
+
 ## Run the server
 
 Run the HTTP server

backends/postgres_osm2pgsql/geometries-alone.lua

@@ -1,51 +1,34 @@
 -- This is a very simple Lua config for the Flex output
 -- which only stores the geometries (not even the tags)
--- for use with Underpass-API to mimics Overpass
+-- for use with Underpass-API to mimic Overpass
 
 local tables = {}
 
-tables.points = osm2pgsql.define_table({
-    name = 'points',
-    ids = { type = 'node', id_column = 'id' },
+tables.nodes_geom = osm2pgsql.define_table({
+    name = 'nodes_geom',
+    ids = { type = 'node', id_column = 'id', create_index='unique'  },
     columns = {
-        -- { column = 'tags', type = 'jsonb' },
         { column = 'geom', type = 'point', projection = 4326, not_null = true  }
 }})
 
-tables.lines = osm2pgsql.define_table({
-    name = 'lines',
-    ids = { type = 'way', id_column = 'id' },
+tables.ways_geom = osm2pgsql.define_table({
+    name = 'ways_geom',
+    ids = { type = 'way', id_column = 'id', create_index='unique' },
     columns = {
-        -- { column = 'tags', type = 'jsonb' },
-        { column = 'geom', type = 'multilinestring', projection = 4326, not_null = true }
+        { column = 'geom', type = 'geometry', projection = 4326, not_null = true },
+        { column = 'area', type = 'real' }
 }})
 
-tables.polygons = osm2pgsql.define_table({
-    name = 'polygons',
-    ids = { type = 'area', id_column = 'id' },
+tables.rels_geom = osm2pgsql.define_table({
+    name = 'rels_geom',
+    ids = { type = 'relation', id_column = 'id', create_index='unique' },
     columns = {
-        -- { column = 'tags', type = 'jsonb' },
         { column = 'geom', type = 'geometry', projection = 4326, not_null = true },
-        -- In this column we'll put the true area calculated on the spheroid
         { column = 'area', type = 'real' }
 }})
 
 
--- Helper function to remove some of the tags we usually are not interested in.
--- Returns true if there are no tags left.
-
--- modifié : retourne vrai si aucun tag
-local function clean_tags(tags)
-    -- tags.odbl = nil
-    -- tags.created_by = nil
-    -- tags.source = nil
-    -- tags['source:ref'] = nil
-
-    return next(tags) == nil
-end
-
--- Helper function that looks at the tags and decides if this is possibly
--- an area.
+-- Helper function that looks at the tags and decides if this is possibly an area
 local function has_area_tags(tags)
     if tags.area == 'yes' or tags.area == 'true' or tags.area == '1' then
         return true
@@ -82,65 +65,49 @@ local function has_area_tags(tags)
         or tags['area:highway']
 end
 
+-- Store geometry of nodes (so that they can be indexed)
 function osm2pgsql.process_node(object)
-    if clean_tags(object.tags) then
-        return
-    end
-
-    local geom = object:as_point()
-
-    tables.points:insert({
-        -- tags = object.tags,
-        geom = geom -- the point will be automatically be projected to 3857
+    tables.nodes_geom:insert({
+        geom = object:as_point()
     })
-
 end
 
+
 function osm2pgsql.process_way(object)
-    if clean_tags(object.tags) then
-        return
-    end
 
     -- A closed way that also has the right tags for an area is a polygon.
     if object.is_closed and has_area_tags(object.tags) then
         -- Creating the polygon geometry takes time, so we do it once here
         -- and later store it in the table and use it to calculate the area.
         local geom = object:as_polygon()
-        tables.polygons:insert({
+        tables.ways_geom:insert({  
             geom = geom,
             area = geom:spherical_area()  -- calculate "real" area in spheroid
         })
     else
-        -- modif : on enregistre la géométrie directement en multilinestring
-        -- en mergeant les lignes le plus possible
-        tables.lines:insert({
-            geom = object:as_multilinestring():line_merge()
+        -- Store way as line
+        tables.ways_geom:insert({
+            geom = object:as_linestring()
         })
     end
 end
 
 function osm2pgsql.process_relation(object)
-    if clean_tags(object.tags) then
-        return
-    end
 
     local relation_type = object:grab_tag('type')
 
-    -- Store route relations as multilinestrings
-    if relation_type == 'route' or relation_type == 'associatedStreet' or relation_type == 'public_transport' or relation_type == 'waterway' then
-        tables.lines:insert({
-            geom = object:as_multilinestring():line_merge()
-        })
-        return
-    end
-
-    -- Store multipolygon and boundary relations as polygons
+    -- Store multipolygon and boundary relations as multipolygons, with their area
     if relation_type == 'multipolygon' or relation_type == 'boundary' then
         local geom = object:as_multipolygon()
-        tables.polygons:insert({
+        tables.rels_geom:insert({
             geom = geom,
             area = geom:spherical_area()
         })
+    else
+    -- Store other relations as geometryCollection
+        tables.rels_geom:insert({
+            geom = object:as_geometrycollection()
+        })
     end
 end
 

backends/postgres_osm2pgsql/view.sql

@@ -1,41 +1,18 @@
-/************** ABOUT TABLES *****************/
+/************** WARNING *****************/
 /*
-- without the --slim option, only "output" tables are created (PREFIX_point PREFIX_line and PREFIX_polygon in the case of the deprecated pgsql output) and they contain elements filtered based on their tags (using list and parameters in .style file). These tables also contain the geometry.
-- with --slim, 3 other "middle" tables containing ALL the raw OSM elements are created : _nodes, _ways, _rels
+This view is specific for a new database created with flex output geometries-alone.lua
 */
 
-/************** ABOUT GEOMETRY *****************/
-/*
-Underpass as Overpass uses SRID 4326 whereas osm2pgsql uses by default SRID 3857
-To make Underpass works on your osm2pgsl database, you have to CREATE it with option --proj=4326 (for the case of deprecated pgsql output) or define that the tables will use SRID 4326 (for the case of the modern flex output)
-If you want to deploy Underpass on an existing osm2pgsl database, you will have to modify both this backend and overpass_parser-rb, so that the parser transforms bbox and areas and others in SRID 3857 to allow comparison with the geom in your DB
-*/
 
-/************** ABOUT TAGS COLUMNS *****************/
+/************** ABOUT TABLES *****************/
 /*
-- tags in "middle" tables _nodes, _ways, _rels are identical to the ones in OSM (they are "raw data" tables)
-- but tags in "output" tables are always (slightly) different depending on the hstore options:
-  # With --hstore any tags without a column will be added to the hstore column.
-  # With --hstore-all all tags are added to the hstore column unless they appear in the style file with a delete flag
+The flex output lua script creates:
+- the 3 normal "middle" tables planet_osm_nodes, _ ways and _rels which include (all ans synchronised) OSM elements with (all) their tags and metadata. (tags need to be indexed after DB creation). And also _users table
+- 3 additionnal "output" tables to store the geometries using SRID 4326 (to be compatible with overpass). 1 table per element type : nodes_geom, ways_geom and rels_geom
+- the views join middle table (tags) with the corresponding output table (geometry) and also with users table (username)
 */
 
-/************** ABOUT METADATA *****************/
-/* adding metadata to the DB requires -x option */
-/* without it, all the CAST(tags->...) will return NULL */
-
-
-/******** UNION LINES AND POLYGONS **********/
-CREATE OR REPLACE TEMP VIEW lines_and_polygons AS
-SELECT id, geom, NULL::real as area FROM lines
-UNION ALL
-SELECT * FROM polygons
-;
-
-
 /************** NODES *****************/
-/* requires database creation with either :
-    - option --slim without --flat-nodes
-    - options --slim --flat-nodes FILE --middle-with-nodes */
 CREATE OR REPLACE TEMP VIEW node AS
 SELECT
   n.id AS id,
@@ -44,7 +21,7 @@ SELECT
   n.changeset_id AS changeset,
   n.user_id AS uid,
   u.name AS user,
-  /* if you did not use --extra-attributes, replace above lines by
+  /* if you did not use -x to get metadata, replace above lines by
   NULL::integer AS version,
   NULL::timestamp without time zone AS created,
   NULL::bigint AS changeset,
@@ -54,15 +31,14 @@ SELECT
   n.tags AS tags,
   NULL::bigint[] AS nodes,
   NULL::jsonb AS members,
-  p.geom AS geom,
+  g.geom AS geom,
   NULL::real AS area,
   'n' AS osm_type
 FROM planet_osm_nodes as n
 LEFT JOIN planet_osm_users AS u ON n.user_id = u.id /* also remove this line */
-LEFT JOIN points AS p ON n.id = p.id
+LEFT JOIN nodes_geom AS g ON n.id = g.id
 ;
 
-
 /************** WAYS *****************/
 CREATE OR REPLACE TEMP VIEW way AS
 SELECT
@@ -72,7 +48,7 @@ SELECT
   w.changeset_id AS changeset,
   w.user_id AS uid,
   u.name AS user,
-  /* if you did not use --extra-attributes, replace above lines by
+  /* if you did not use -x to get metadata, replace above lines by
   NULL::integer AS version,
   NULL::timestamp without time zone AS created,
   NULL::bigint AS changeset,
@@ -82,17 +58,15 @@ SELECT
   w.tags as tags,
   w.nodes AS nodes,
   NULL::jsonb AS members,
-  lp.geom AS geom,
-  lp.area AS area,
+  g.geom AS geom,
+  g.area AS area,
   'w' AS osm_type
 FROM planet_osm_ways AS w
 LEFT JOIN planet_osm_users AS u ON w.user_id = u.id /* also remove this line */
-LEFT JOIN lines_and_polygons AS lp ON w.id = lp.id
+LEFT JOIN ways_geom AS g ON w.id = g.id
 ;
 
-
 /************** RELATIONS *****************/
-/* complete version (based on table _rels) if you used --slim */
 CREATE OR REPLACE TEMP VIEW relation AS
 SELECT
   r.id AS id,
@@ -101,7 +75,7 @@ SELECT
   r.changeset_id AS changeset,
   r.user_id AS uid,
   u.name AS user,
-  /* if you did not use --extra-attributes, replace above lines by
+  /* if you did not use -x to get metadata, replace above lines by
   NULL::integer AS version,
   NULL::timestamp without time zone AS created,
   NULL::bigint AS changeset,
@@ -111,15 +85,14 @@ SELECT
   r.tags as tags,
   NULL::bigint[] AS nodes,
   r.members AS members,
-  lp.geom AS geom,
-  lp.area AS area,
+  g.geom AS geom,
+  g.area AS area,
   'r' AS osm_type
 FROM planet_osm_rels AS r
 LEFT JOIN planet_osm_users AS u ON r.user_id = u.id /* also remove this line */
-LEFT JOIN lines_and_polygons AS lp ON r.id = -lp.id
+LEFT JOIN rels_geom AS g ON r.id = g.id
 ;
 
-
 /************** NWR *****************/
 CREATE OR REPLACE TEMP VIEW nwr AS
 SELECT * FROM node
@@ -129,11 +102,11 @@ UNION ALL
 SELECT * FROM relation
 ;
 
-/************** AREA *****************/
+/************** AREAS *****************/
 CREATE OR REPLACE TEMP VIEW area AS
 SELECT
   CASE
-    WHEN osm_type='r' THEN 3600000000+id /* transform if of relations to be consistent with overpass */
+    WHEN osm_type='r' THEN 3600000000+id /* transform id of relations to be consistent with overpass */
     ELSE id
   END AS id,
   version,
@@ -147,10 +120,6 @@ SELECT
   geom,
   area,
   REPLACE(osm_type, 'w', 'a') AS osm_type
-  /*CASE
-    WHEN osm_type='r' THEN 'a' /* transform r in a for underpass ? */
-    ELSE osm_type
-  END AS osm_type,*/
 FROM nwr
 WHERE area IS NOT NULL
 ;