Merge pull request #530 from hotosm/feature/vector-tiles

Feature : PMtiles , MVT tiles

Author: kshitijrajsharma

api/serializers.py

@@ -5,6 +5,7 @@
 Used by the View classes api/views.py to serialize API responses as JSON or HTML.
 See DEFAULT_RENDERER_CLASSES setting in core.settings.contrib for the enabled renderers.
 """
+
 # -*- coding: utf-8 -*-
 import logging
 
@@ -113,7 +114,7 @@ class Meta:
             "mbtiles_maxzoom",
             "pinned",
             "unfiltered",
-            "preserve_geom",
+            "userinfo",
         )
         extra_kwargs = {
             "the_geom": {"write_only": True},

jobs/migrations/0001_initial.py

@@ -96,7 +96,7 @@ class Migration(migrations.Migration):
                 ("expire_old_runs", models.BooleanField(default=True)),
                 ("pinned", models.BooleanField(default=False)),
                 ("unfiltered", models.BooleanField(default=False)),
-                ("preserve_geom", models.BooleanField(default=False)),
+                ("userinfo", models.BooleanField(default=False)),
                 (
                     "user",
                     models.ForeignKey(

jobs/models.py

@@ -95,6 +95,8 @@ def validate_export_formats(value):
             "shp",
             "geojson",
             "fgb",
+            "mvt",
+            "pmtiles",
             "csv",
             "sql",
             "geopackage",
@@ -203,7 +205,7 @@ class Job(models.Model):
     expire_old_runs = models.BooleanField(default=True)
     pinned = models.BooleanField(default=False)
     unfiltered = models.BooleanField(default=False)
-    preserve_geom = models.BooleanField(default=False)
+    userinfo = models.BooleanField(default=False)
 
     class Meta:  # pragma: no cover
         managed = True

tasks/task_runners.py

@@ -254,7 +254,7 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False)
                     total_bytes += file.size()
                 task.filesize_bytes = total_bytes
             LOG.debug(total_bytes)
-            
+
         task.save()
 
     is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists()
@@ -271,6 +271,8 @@ def finish_task(name, created_files=None, response_back=None, planet_file=False)
         "kml",
         "shp",
         "fgb",
+        "mvt",
+        "pmtiles",
         "csv",
         "sql",
         "mbtiles",
@@ -673,17 +675,16 @@ def add_metadata(z, theme):
         mapping_filter = mapping
         if job.unfiltered:
             mapping_filter = None
+        userinfo = job.userinfo
 
         if "geojson" in export_formats:
-            preserved_geom = geom
-            if job.preserve_geom:
-                preserved_geom = load_geometry(job.the_geom.json)
             geojson = Galaxy(
                 settings.RAW_DATA_API_URL,
-                preserved_geom,
+                geom,
                 mapping=mapping_filter,
                 file_name=valid_name,
                 access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                userinfo=userinfo,
             )
             start_task("geojson")
 
@@ -694,6 +695,7 @@ def add_metadata(z, theme):
                 mapping=mapping_filter,
                 file_name=valid_name,
                 access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                userinfo=userinfo,
             )
             start_task("fgb")
 
@@ -714,6 +716,7 @@ def add_metadata(z, theme):
                 mapping=mapping_filter,
                 file_name=valid_name,
                 access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                userinfo=userinfo,
             )
             start_task("sql")
 
@@ -724,6 +727,7 @@ def add_metadata(z, theme):
                 mapping=mapping_filter,
                 file_name=valid_name,
                 access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                userinfo=userinfo,
             )
             # geopackage = tabular.Geopackage(join(stage_dir,valid_name),mapping)
             # tabular_outputs.append(geopackage)
@@ -736,6 +740,7 @@ def add_metadata(z, theme):
                 mapping=mapping_filter,
                 file_name=valid_name,
                 access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                userinfo=userinfo,
             )
             start_task("shp")
 
@@ -746,6 +751,7 @@ def add_metadata(z, theme):
                 mapping=mapping_filter,
                 file_name=valid_name,
                 access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                userinfo=userinfo,
             )
             # kml = tabular.Kml(join(stage_dir,valid_name),mapping)
             # tabular_outputs.append(kml)
@@ -910,6 +916,7 @@ def add_metadata(z, theme):
                     mapping=mapping_filter,
                     file_name=valid_name,
                     access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                    userinfo=userinfo,
                 )
                 start_task("mbtiles")
                 LOG.debug(
@@ -934,6 +941,68 @@ def add_metadata(z, theme):
                 stop_task("mbtiles")
                 raise ex
 
+        if "pmtiles" in export_formats:
+            try:
+                pmtiles = Galaxy(
+                    settings.RAW_DATA_API_URL,
+                    geom,
+                    mapping=mapping_filter,
+                    file_name=valid_name,
+                    access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                    userinfo=userinfo,
+                )
+                start_task("pmtiles")
+                LOG.debug(
+                    "Raw Data API fetch started for pmtiles run: {0}".format(run_uid)
+                )
+                all_feature_filter_json = join(
+                    os.getcwd(), "tasks/tests/fixtures/all_features_filters.json"
+                )
+                response_back = pmtiles.fetch(
+                    "pmtiles",
+                    all_feature_filter_json=all_feature_filter_json,
+                    min_zoom=job.mbtiles_minzoom,
+                    max_zoom=job.mbtiles_maxzoom,
+                )
+                write_file_size(response_back)
+                LOG.debug(
+                    "Raw Data API fetch ended for mbtiles run: {0}".format(run_uid)
+                )
+                finish_task("pmtiles", response_back=response_back)
+
+            except Exception as ex:
+                stop_task("pmtiles")
+                raise ex
+
+        if "mvt" in export_formats:
+            try:
+                mvt = Galaxy(
+                    settings.RAW_DATA_API_URL,
+                    geom,
+                    mapping=mapping_filter,
+                    file_name=valid_name,
+                    access_token=settings.RAW_DATA_ACCESS_TOKEN,
+                    userinfo=userinfo,
+                )
+                start_task("mvt")
+                LOG.debug("Raw Data API fetch started for mvt run: {0}".format(run_uid))
+                all_feature_filter_json = join(
+                    os.getcwd(), "tasks/tests/fixtures/all_features_filters.json"
+                )
+                response_back = mvt.fetch(
+                    "mvt",
+                    all_feature_filter_json=all_feature_filter_json,
+                    min_zoom=job.mbtiles_minzoom,
+                    max_zoom=job.mbtiles_maxzoom,
+                )
+                write_file_size(response_back)
+                LOG.debug("Raw Data API fetch ended for mvt run: {0}".format(run_uid))
+                finish_task("mvt", response_back=response_back)
+
+            except Exception as ex:
+                stop_task("mvt")
+                raise ex
+
         if use_only_galaxy == False:
             LOG.debug("Source start for run: {0}".format(run_uid))
             source_path = source.path()

ui/app/actions/exports.js

@@ -64,7 +64,7 @@ export const cloneExport = e => (dispatch, getState) => {
           name: e.name,
           published: e.published,
           unfiltered: e.unfiltered,
-          preserve_geom: e.preserve_geom,
+          userinfo: e.userinfo,
           the_geom: rsp.data.the_geom,
           aoi: {
             description: "Cloned Area",

ui/app/components/Summary.js

@@ -15,9 +15,9 @@ const messages = defineMessages({
   //   id: "export.bundle_for_posm.description",
   //   defaultMessage: "Bundle for POSM"
   // },
-  preserveGeometry: {
-    id: "export.preserve_geom.description",
-    defaultMessage: "Preserve Geometry - Avoid simplify ( Only supports for geojson )"
+  userinfo: {
+    id: "export.userinfo.description",
+    defaultMessage: "Include user info on exports"
   },
   publishedDescription: {
     id: "export.published.description",
@@ -84,12 +84,12 @@ export default injectIntl(
           component={renderCheckbox}
           type="checkbox"
         /> */}
-        {/* <Field
-          name="preserve_geom"
-          description={formatMessage(messages.preserveGeometry)}
+        <Field
+          name="userinfo"
+          description={formatMessage(messages.userinfo)}
           component={renderCheckbox}
           type="checkbox"
-        /> */}
+        /> 
         <Button
           bsStyle="danger"
           disabled={submitting}

ui/app/components/help/ExportFormats.js

@@ -101,7 +101,7 @@ export default () =>
               </ul>
             </div>
             <div>
-              <h2 id="gpkg">GeoJSON  .geojson</h2>
+              <h2 id="geojson">GeoJSON  .geojson</h2>
               <p>
               GeoJSON is an open standard geospatial data interchange format that represents simple geographic features and their nonspatial attributes. Based on JavaScript Object Notation (JSON), GeoJSON is a format for encoding a variety of geographic data structures. It uses a geographic coordinate reference system, World Geodetic System 1984, and units of decimal degrees.
               </p>
@@ -142,7 +142,7 @@ export default () =>
               </ul>
             </div>
             <div>
-              <h2 id="gpkg">FlatGeobuf .fgb</h2>
+              <h2 id="fgb">FlatGeobuf .fgb</h2>
               <p>
               FlatGeobuf is a binary file format for storing geospatial vector data in a compact and efficient manner. It uses a hierarchical structure to organize features into layers, and stores attribute data in a separate file.
               </p>
@@ -183,7 +183,7 @@ export default () =>
               </ul>
             </div>
             <div>
-              <h2 id="gpkg">CSV .csv</h2>
+              <h2 id="csv">CSV .csv</h2>
               <p>
               CSV is a file format for storing tabular data in plain text format. Each row of data represents a record, and each column represents a field of that record. CSV files are widely used because they are simple and easy to create and manipulate, making them a popular choice for data exchange.              </p>
               <h4>
@@ -203,7 +203,7 @@ export default () =>
                 </ul>
             </div>
             <div>
-              <h2 id="gpkg">SQL .sql</h2>
+              <h2 id="sql">SQL .sql</h2>
               <p>
               SQL files are plain text files that contain SQL commands to create, modify or interact with a relational database. They can be used to define database schemas, constraints, and indexes, as well as to insert, update, and query data.
               </p>

ui/app/components/utils.js

@@ -63,6 +63,16 @@ export const AVAILABLE_EXPORT_FORMATS = {
       MBTiles <code>.mbtiles</code>
     </span>
   ),
+  pmtiles: (
+    <span key="pmtiles">
+      Pmtiles <code>.pmtiles</code>
+    </span>
+  ),
+  mvt: (
+    <span key="mvt">
+      Mapbox Vector Tiles <code>.mvt</code>
+    </span>
+  ),
   garmin_img: (
     <span key="garmin_img">
       Garmin <code>.img</code>
@@ -99,6 +109,8 @@ export const REQUIRES_FEATURE_SELECTION = {
   shp: true,
   geojson:true,
   fgb:true,
+  mvt:true,
+  pmtiles:true,
   sql:true,
   csv:true,
   geopackage: true,
@@ -110,12 +122,14 @@ export const REQUIRES_FEATURE_SELECTION = {
 };
 
 export const REQUIRES_TILE_SOURCE = {
-  mbtiles: true
+  mbtiles: true,
+  pmtiles:true,
+  mvt:true,
 };
 
 export const OMIT_FROM_FORMAT_OPTIONS = {
   osm_xml: true,
-  bundle: true
+  bundle: true 
 };
 
 export const getRootUrl = () => {

utils/aoi_utils.py

@@ -11,22 +11,22 @@
 #   be buffered efficiently, they must be simplified first.
 #   so first simplify them to 0.01 degrees.
 
+
 def force2d(geom):
     # force geom to be 2d: https://groups.google.com/forum/#!topic/django-users/7c1NZ76UwRU
     wkt = wkt_w(dim=2).write(geom).decode()
     return GEOSGeometry(wkt)
 
-def simplify_geom(geom,force_buffer=False, preserve_geom=False):
-    if preserve_geom is False:
-        if geom.num_coords > 10000:
-            geom = geom.simplify(0.01)
-        if geom.num_coords > 500:
-            geom = geom.buffer(0.02)
-        param = 0.01
-        while geom.num_coords > 500:
-            geom = geom.simplify(param, preserve_topology=True)
-            param = param * 2
+
+def simplify_geom(geom, force_buffer=False):
+    if geom.num_coords > 10000:
+        geom = geom.simplify(0.01)
+    if geom.num_coords > 500:
+        geom = geom.buffer(0.02)
+    param = 0.01
+    while geom.num_coords > 500:
+        geom = geom.simplify(param, preserve_topology=True)
+        param = param * 2
     if force_buffer:
-            geom = geom.buffer(0.02)
+        geom = geom.buffer(0.02)
     return geom
-