Merge pull request #655 from geoadmin/master

update from Master

Author: ltclm

.github/workflows/create-milestone.yml

@@ -8,4 +8,4 @@ jobs:
     uses: geoadmin/.github/.github/workflows/create-milestone.yml@master
     secrets: inherit
     with:
-      ci_status_check_name: ''
+      ci_status_check_name: 'AWS CodeBuild eu-central-1 (service-search-sphinx-pr)'

Dockerfile

@@ -26,9 +26,10 @@ FROM sphinxsearch_base as sphinxsearch_geodata
     # add geodata user, same uid/gid as the EFS owner is needed here
 RUN groupadd -r geodata -g 2500 && \
     useradd -u 2500 -r -g geodata -s /sbin/nologin --create-home geodata && \
-    # create mountpoint folders with geodata ownership
+    # create mountpoint for Amazon EFS CSI driver
+    install -o geodata -g geodata -d /var/local/ && \
+    # create mountpoint folder for infra-vhost/k8s ebs/ssd volume
     install -o geodata -g geodata -d /var/lib/sphinxsearch/data/index/ && \
-    install -o geodata -p geodata -d /var/lib/sphinxsearch/data/index_efs/ && \
     # change ownerships to geodata which will run the service or the maintenance scripts
     # and mount the efs folder
     chown -R geodata:geodata /var/run/sphinxsearch/ && \

Makefile

@@ -73,6 +73,12 @@ endif
 
 PPID := $(shell echo $$PPID)
 
+# Docker resosource throttling
+# we allow 70% of the currently available memory to be used by sphinx indexer per job
+PERCENTAGE := 70
+free_mem := $(shell free -m | awk '/Mem:/{print $$4}')
+export DOCKER_FREE_MEM := $(shell echo $$(( ${free_mem} * ${PERCENTAGE} / 100 ))m)
+
 # Maintenance / Index Commands
 # EFS Index will be mounted as bind mount
 # DOCKER_EXEC will always check if a newer image exists on ecr -> develop.latest support
@@ -153,6 +159,7 @@ help:
 	@echo
 	@echo "- CPUS:                     ${YELLOW}${CPUS}${RESET}"
 	@echo "- DB_ACCESS:                ${YELLOW}${DB_ACCESS}${RESET}"
+	@echo "- DOCKER_FREE_MEM:          ${YELLOW}${DOCKER_FREE_MEM}${RESET}"
 
 
 # Build targets. Calling setup is all that is needed for the local files to be installed as needed.

README.md

@@ -57,7 +57,7 @@ This command is executed/triggered by the database deploy script on `geodatasync
 
 ### service mode
 #### local sphinxsearch server
-you can run a local sphinxsearch server on port 913 with the following make targets:
+You can run a local sphinxsearch server on port 9312 with the following make targets:
 ```bash
 make dockerrun
 make dockerrundebug
@@ -83,4 +83,4 @@ The swisssearch index (zipcodes) has to be computed after a wordforms update.
 ```bash
 cd test
 python test.py -h localhost -p 9312 -i swisssearch "birgmattenweg 5"
-```
+```

conf/bafu.conf.part

@@ -1173,24 +1173,6 @@ source src_ch_bafu_gewaesser_uferbestockung : def_searchable_features
         from wasser.gewaesser_uferbestockung
 }
 
-source src_bafu_lebensraumkarte_schweiz : def_searchable_features
-{
-    sql_db = bafu_${DBSTAGING}
-    sql_query = \
-        SELECT bgdi_id as id \
-            , polyid as label \
-            , 'feature' as origin \
-            , remove_accents(concat_ws(' ', polyid)) as  detail \
-            , 'ch.bafu.lebensraumkarte-schweiz' as layer \
-            , quadindex(the_geom) as geom_quadindex \
-            , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
-            , st_x(st_transform(st_centroid(the_geom),4326)) as lon \
-            , box2d(st_transform(the_geom, 21781)) as geom_st_box2d \
-            , box2d(st_transform(the_geom, 2056)) as geom_st_box2d_lv95 \
-            , bgdi_id::text as feature_id  \
-        from lebensraumkarte.lebensraumkarte_schweiz
-}
-
 source src_bafu_tranquillity_gebiete : def_searchable_features
 {
     sql_db = bafu_${DBSTAGING}
@@ -1640,12 +1622,6 @@ index ch_bafu_gewaesser_uferbestockung : ch_bafu_hydrologie_wassertemperaturmess
     path = /var/lib/sphinxsearch/data/index/ch_bafu_gewaesser_uferbestockung
 }
 
-index ch_bafu_lebensraumkarte_schweiz : ch_bafu_hydrologie_wassertemperaturmessstationen
-{
-    source = src_bafu_lebensraumkarte_schweiz
-    path = /var/lib/sphinxsearch/data/index/ch_bafu_lebensraumkarte_schweiz
-}
-
 index ch_bafu_tranquillity_gebiete : ch_bafu_hydrologie_wassertemperaturmessstationen
 {
     source = src_bafu_tranquillity_gebiete

conf/dritte.conf.part

@@ -1,23 +1,5 @@
 ## SOURCES
 
-source src_ch_pronatura_waldreservate : def_searchable_features
-{
-    sql_db = dritte_${DBSTAGING}
-    sql_query = \
-        SELECT bgdi_id as id \
-            , name::text as label \
-            , 'feature' as origin \
-            , remove_accents(concat_ws(' ', objnummer, name, mcpfe)) as detail \
-            , 'ch.pronatura.waldreservate' as layer \
-            , quadindex(the_geom) as geom_quadindex \
-            , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
-            , st_x(st_transform(st_centroid(the_geom),4326)) as lon \
-            , box2d(st_transform(the_geom, 21781)) as geom_st_box2d \
-            , box2d(st_transform(the_geom, 2056)) as geom_st_box2d_lv95 \
-            , bgdi_id::text as feature_id \
-        from pronatura.waldreservate
-}
-
 source src_ch_sem_sachplan_asyl_anhoerung : def_searchable_features
 {
     sql_db = dritte_${DBSTAGING}
@@ -56,25 +38,18 @@ source src_ch_sem_sachplan_asyl_kraft : def_searchable_features
 
 
 ## INDICES
-index ch_pronatura_waldreservate
+index ch_sem_sachplan_asyl_anhoerung
 {
     type = plain
     dict=crc
-    source = src_ch_pronatura_waldreservate
-    path = /var/lib/sphinxsearch/data/index/ch_pronatura_waldreservate
-    min_infix_len = 2
-    expand_keywords = 1
-}
-
-
-index ch_sem_sachplan_asyl_anhoerung : ch_pronatura_waldreservate
-{
     source = src_ch_sem_sachplan_asyl_anhoerung
     path = /var/lib/sphinxsearch/data/index/ch_sem_sachplan_asyl_anhoerung
+    min_infix_len = 2
+    expand_keywords = 1
 }
 
 
-index ch_sem_sachplan_asyl_kraft : ch_pronatura_waldreservate
+index ch_sem_sachplan_asyl_kraft : ch_sem_sachplan_asyl_anhoerung
 {
     source = src_ch_sem_sachplan_asyl_kraft
     path = /var/lib/sphinxsearch/data/index/ch_sem_sachplan_asyl_kraft

conf/edi.conf.part

@@ -19,25 +19,6 @@ source src_ch_bfs_generalisierte_grenzen_agglomerationen_g1 : def_searchable_fea
 }
 
 
-source src_ch_bfs_generalisierte_grenzen_agglomerationen_g2 : def_searchable_features
-{
-    sql_db = edi_${DBSTAGING}
-    sql_query = \
-        SELECT bgdi_id::bigint as id \
-            , gmd_name as label \
-            , 'feature' as origin \
-            , remove_accents(concat_ws(' ',gmd_name,gmd_nr::text,aname,acode::text)) as detail \
-            , 'ch.bfs.generalisierte-grenzen_agglomerationen_g2' as layer \
-            , quadindex(the_geom) as geom_quadindex \
-            , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
-            , st_x(st_transform(st_centroid(the_geom),4326)) as lon \
-            , box2d(st_transform(the_geom, 21781)) as geom_st_box2d \
-            , box2d(st_transform(the_geom, 2056)) as geom_st_box2d_lv95 \
-            , bgdi_id::text as feature_id \
-        from bfs.gen_grenzen_agglo_g2
-}
-
-
 source src_ch_bfs_landschaftswandel : def_searchable_features
 {
     sql_db = edi_${DBSTAGING}
@@ -168,12 +149,6 @@ index ch_bfs_generalisierte_grenzen_agglomerationen_g1
     expand_keywords = 1
 }
 
-index ch_bfs_generalisierte_grenzen_agglomerationen_g2 : ch_bfs_generalisierte_grenzen_agglomerationen_g1
-{
-    source = src_ch_bfs_generalisierte_grenzen_agglomerationen_g2
-    path = /var/lib/sphinxsearch/data/index/ch_bfs_generalisierte_grenzen_agglomerationen_g2
-}
-
 index ch_bfs_landschaftswandel : ch_bfs_generalisierte_grenzen_agglomerationen_g1
 {
     source = src_ch_bfs_landschaftswandel

conf/evd.conf.part

@@ -396,6 +396,24 @@ source src_ch_blw_ursprungsbezeichnungen_spirituosen : def_searchable_features
         from blw.ursprungsbezeichnungen_spirituosen
 }
 
+source src_ch_blw_landwirtschaftliche_nutzungsflaechen : def_searchable_features
+{
+    sql_db = evd_${DBSTAGING}
+    sql_query = \
+        SELECT bgdi_id::bigint as id \
+            , t_id as label \
+            , 'feature' as origin \
+            , remove_accents(concat_ws(' ', bur_nr, nutzungsidentifikator)) as detail \
+            , 'ch.blw.landwirtschaftliche-nutzungsflaechen' as layer \
+            , quadindex(the_geom) as geom_quadindex \
+            , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
+            , st_x(st_transform(st_centroid(the_geom),4326)) as lon \
+            , box2d(st_transform(the_geom, 21781)) as geom_st_box2d \
+            , box2d(st_transform(the_geom, 2056)) as geom_st_box2d_lv95 \
+            , bgdi_id::text as feature_id \
+        from blw.landwirtschaftliche_nutzungsflaechen
+}
+
 ## INDICES
 
 index ch_blw_klimaeignung_typ
@@ -533,3 +551,9 @@ index ch_blw_ursprungsbezeichnungen_spirituosen : ch_blw_klimaeignung_typ
     source = src_ch_blw_ursprungsbezeichnungen_spirituosen
     path = /var/lib/sphinxsearch/data/index/ch_blw_ursprungsbezeichnungen_spirituosen
 }
+
+index ch_blw_landwirtschaftliche_nutzungsflaechen : ch_blw_klimaeignung_typ
+{
+    source = src_ch_blw_landwirtschaftliche_nutzungsflaechen
+    path = /var/lib/sphinxsearch/data/index/ch_blw_landwirtschaftliche_nutzungsflaechen
+}

conf/lubis.conf.part

@@ -26,7 +26,7 @@ source src_ch_swisstopo_lubis_luftbilder_schraegaufnahmen : def_searchable_featu
         SELECT row_number() OVER(ORDER BY ebkey asc) as id \
             , concat_ws(' ',flightdate::date, ebkey::text)  as label \
             , 'feature' as origin \
-            , concat_ws(' ',flightdate::date, ebkey::text)  as detail \
+            , concat_ws(' ',flightdate::date, ebkey::text, ebkey_old)  as detail \
             , 'ch.swisstopo.lubis-luftbilder_schraegaufnahmen' as layer \
             , bgdi_quadindex as geom_quadindex \
             , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
@@ -43,9 +43,9 @@ source src_ch_swisstopo_lubis_luftbilder_farbe : def_searchable_features_with_ye
     sql_db = lubis_${DBSTAGING}
     sql_query = \
         SELECT row_number() OVER(ORDER BY ebkey asc) as id \
-            , concat(flugdatum, ' ', bildnummer, ' (', concat_ws(', ', ort, ebkey), ')' ) as label \
+            , concat(flugdatum, ' ', feature_id, ' (', concat_ws(', ', ebkey), ')' ) as label \
             , 'feature' as origin \
-            , remove_accents(concat_ws(' ', ebkey, ort)) as detail \
+            , remove_accents(concat_ws(' ', ebkey, ebkey_old)) as detail \
             , 'ch.swisstopo.lubis-luftbilder_farbe' as layer \
             , bgdi_quadindex as geom_quadindex \
             , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
@@ -62,9 +62,9 @@ source src_ch_swisstopo_lubis_luftbilder_schwarzweiss: def_searchable_features_w
     sql_db = lubis_${DBSTAGING}
     sql_query = \
         SELECT row_number() OVER(ORDER BY ebkey asc) as id \
-            , concat(flugdatum, ' ', bildnummer, ' (', concat_ws(', ', ort, ebkey), ')' ) as label \
+            , concat(flugdatum, ' ', feature_id, ' (', concat_ws(', ', ebkey), ')' ) as label \
             , 'feature' as origin \
-            , remove_accents(concat_ws(' ', ebkey, ort, ebkey_old)) as detail \
+            , remove_accents(concat_ws(' ', ebkey, ebkey_old)) as detail \
             , 'ch.swisstopo.lubis-luftbilder_schwarzweiss' as layer \
             , bgdi_quadindex as geom_quadindex \
             , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
@@ -81,9 +81,9 @@ source src_ch_swisstopo_lubis_luftbilder_infrarot: def_searchable_features_with_
     sql_db = lubis_${DBSTAGING}
     sql_query = \
         SELECT row_number() OVER(ORDER BY ebkey asc) as id \
-            , concat(flugdatum, ' ', bildnummer, ' (', concat_ws(', ', ort, ebkey), ')' ) as label \
+            , concat(flugdatum, ' ', feature_id, ' (', concat_ws(', ', ebkey, ebkey_old), ')' ) as label \
             , 'feature' as origin \
-            , remove_accents(concat_ws(' ', ebkey, ort)) as detail \
+            , remove_accents(ebkey) as detail \
             , 'ch.swisstopo.lubis-luftbilder_infrarot' as layer \
             , bgdi_quadindex as geom_quadindex \
             , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
@@ -140,7 +140,7 @@ source src_ch_swisstopo_lubis_terrestrische_aufnahmen : def_searchable_features_
         SELECT row_number() OVER(ORDER BY inventory_number asc) as id \
             , lpad(bilder.inventory_number::text, 14, '0'::text) as label \
             , 'feature' as origin \
-            , remove_accents(concat_ws(' ', inventory_number, image_number, station, operate_name, year)) as detail \
+            , remove_accents(concat_ws(' ', inventory_number, station, operate_name, year, inventarnummer_old)) as detail \
             , 'ch.swisstopo.lubis-terrestrische_aufnahmen' as layer \
             , bgdi_quadindex as geom_quadindex \
             , st_y(st_transform(st_centroid(the_geom),4326)) as lat \
@@ -149,7 +149,7 @@ source src_ch_swisstopo_lubis_terrestrische_aufnahmen : def_searchable_features_
             , box2d(st_transform(the_geom_hilight, 2056)) as geom_st_box2d_lv95 \
             , year \
             , lpad(bilder.inventory_number::text, 14, '0'::text)::text as feature_id \
-        FROM swisstopo_terrestrial.bilder
+        FROM swisstopo_terrestrial.view_bilder bilder
 }
 
 ## INDICES

conf/search.conf.part

@@ -288,25 +288,25 @@ source src_zipcode : src_swisssearch
     sql_db=stopo_${DBSTAGING}
     sql_query = \
         SELECT \
-        gid as id \
+        os_uuid::int as id \
         , os_uuid::text as feature_id \
         , plz::text as detail \
         , concat('<b>', plz, ' - ', trim(langtext), '</b>') as label \
         , NULL as objectclass \
         , 'zipcode' as origin \
         , quadindex(the_geom) as geom_quadindex \
         , box2d(st_transform(the_geom, 21781)) as geom_st_box2d \
-        , box2d(st_transform(the_geom, 2056)) as geom_st_box2d_lv95 \
+        , box2d(the_geom) as geom_st_box2d_lv95 \
         , 1::integer as rank \
-        , st_y(st_transform(st_setsrid(st_point(label_x,label_y),2056),21781)) as x \
-        , st_x(st_transform(st_setsrid(st_point(label_x,label_y),2056),21781)) as y \
-        , st_y(st_transform(st_setsrid(st_point(label_x,label_y),2056),2056)) as x_lv95 \
-        , st_x(st_transform(st_setsrid(st_point(label_x,label_y),2056),2056)) as y_lv95 \
-        , st_y(st_transform(st_setsrid(st_point(label_x,label_y),2056),4326)) as lat \
-        , st_x(st_transform(st_setsrid(st_point(label_x,label_y),2056),4326)) as lon \
+        , st_y(st_transform(ST_PointOnSurface(the_geom),21781)) as x \
+        , st_x(st_transform(ST_PointOnSurface(the_geom),21781)) as y \
+        , st_y(ST_PointOnSurface(the_geom)) as x_lv95 \
+        , st_x(ST_PointOnSurface(the_geom)) as y_lv95 \
+        , st_y(st_transform(ST_PointOnSurface(the_geom),4326)) as lat \
+        , st_x(st_transform(ST_PointOnSurface(the_geom),4326)) as lon \
         , -1 as zoomlevel \
         , 1 as num \
-        FROM vd.gabmo_plz p
+        FROM vd.amtovz_ortschaften p
 }
 
 source src_haltestellen : src_swisssearch
@@ -405,6 +405,8 @@ index district_metaphone : zipcode
     source = src_district
     path = /var/lib/sphinxsearch/data/index/district_metaphone
     preopen = 0
+    wordforms = /dev/null
+    expand_keywords = 1
 }
 
 index kantone_metaphone : zipcode
@@ -413,6 +415,8 @@ index kantone_metaphone : zipcode
     source = src_kantone
     path = /var/lib/sphinxsearch/data/index/kantone_metaphone
     preopen = 0
+    wordforms = /dev/null
+    expand_keywords = 1
 }
 
 index gg25_metaphone : zipcode
@@ -421,6 +425,8 @@ index gg25_metaphone : zipcode
     source = src_gg25
     path = /var/lib/sphinxsearch/data/index/gg25_metaphone
     preopen = 0
+    wordforms = /dev/null
+    expand_keywords = 1
 }
 
 index swissnames3d_metaphone : zipcode
@@ -429,6 +435,8 @@ index swissnames3d_metaphone : zipcode
     source = src_swissnames3d
     path = /var/lib/sphinxsearch/data/index/swissnames3d_metaphone
     preopen = 0
+    wordforms = /dev/null
+    expand_keywords = 1
 }
 
 index haltestellen_metaphone : zipcode
@@ -437,6 +445,8 @@ index haltestellen_metaphone : zipcode
     source = src_haltestellen
     path = /var/lib/sphinxsearch/data/index/haltestellen_metaphone
     preopen = 0
+    wordforms = /dev/null
+    expand_keywords = 1
 }
 
 index address_metaphone: zipcode
@@ -445,6 +455,8 @@ index address_metaphone: zipcode
     source = src_address
     path = /var/lib/sphinxsearch/data/index/address_metaphone
     preopen = 0
+    wordforms = /dev/null
+    expand_keywords = 1
 }
 
 # only create on demand