Support for non-UTM coordinate systems

app/admin.py

@@ -44,7 +44,28 @@ def has_add_permission(self, request):
     list_display = ('id', 'name', 'project', 'processing_node', 'created_at', 'status', 'last_error')
     list_filter = ('status', 'project',)
     search_fields = ('id', 'name', 'project__name')
-
+    exclude = ('orthophoto_extent', 'dsm_extent', 'dtm_extent', 'crop', )
+    readonly_fields = ('orthophoto_extent_wkt', 'dsm_extent_wkt', 'dtm_extent_wkt', 'crop_wkt', )
+
+    def orthophoto_extent_wkt(self, obj):
+        if obj.orthophoto_extent:
+            return obj.orthophoto_extent.wkt
+        return None
+
+    def dsm_extent_wkt(self, obj):
+        if obj.dsm_extent:
+            return obj.dsm_extent.wkt
+        return None
+
+    def dtm_extent_wkt(self, obj):
+        if obj.dtm_extent:
+            return obj.dtm_extent.wkt
+        return None
+
+    def crop_wkt(self, obj):
+        if obj.crop:
+            return obj.crop.wkt
+        return None
 
 admin.site.register(Task, TaskAdmin)
 

app/geoutils.py

@@ -1,6 +1,9 @@
 import rasterio.warp
 import numpy as np
 from rasterio.crs import CRS
+from rasterio.warp import transform_bounds
+from osgeo import osr
+osr.DontUseExceptions()
 
 # GEOS has some weird bug where
 # we can't simply call geom.tranform(srid)
@@ -79,3 +82,38 @@ def geom_transform(geom, epsg):
         return list(zip(tx, ty))
     else:
         raise ValueError("Cannot transform complex geometries to WKT")
+
+
+def epsg_from_wkt(wkt):
+    srs = osr.SpatialReference()
+    if srs.ImportFromWkt(wkt) != 0:
+        return None
+
+    epsg = srs.GetAuthorityCode(None)
+    if epsg is not None:
+        return None
+
+    # Try to get the 2D component
+    if srs.IsCompound():
+        if srs.DemoteTo2D() != 0:
+            return None
+
+    epsg = srs.GetAuthorityCode(None)
+    if epsg is not None:
+        return epsg
+
+
+def get_raster_bounds_wkt(raster_path, target_srs="EPSG:4326"):
+    with rasterio.open(raster_path) as src:
+        if src.crs is None:
+            return None
+
+        left, bottom, right, top = src.bounds
+        w, s, e, n = transform_bounds(
+            src.crs, 
+            target_srs, 
+            left, bottom, right, top
+        )
+
+        wkt = f"POLYGON(({w} {s}, {w} {n}, {e} {n}, {e} {s}, {w} {s}))"
+        return wkt

app/models/task.py

@@ -22,8 +22,6 @@
 import requests
 from PIL import Image
 Image.MAX_IMAGE_PIXELS = 4096000000
-from django.contrib.gis.gdal import GDALRaster
-from django.contrib.gis.gdal import OGRGeometry
 from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.postgres import fields
 from django.core.files.uploadedfile import InMemoryUploadedFile
@@ -41,7 +39,7 @@
 from app.pointcloud_utils import is_pointcloud_georeferenced
 from app.testwatch import testWatch
 from app.security import path_traversal_check
-from app.geoutils import geom_transform
+from app.geoutils import geom_transform, epsg_from_wkt, get_raster_bounds_wkt
 from nodeodm import status_codes
 from nodeodm.models import ProcessingNode
 from pyodm.exceptions import NodeResponseError, NodeConnectionError, NodeServerError, OdmError
@@ -1012,32 +1010,16 @@ def extract_assets_and_complete(self):
                 except IOError as e:
                     logger.warning("Cannot create Cloud Optimized GeoTIFF for %s (%s). This will result in degraded visualization performance." % (raster_path, str(e)))
 
-                # Read extent and SRID
-                raster = GDALRaster(raster_path)
-                extent = OGRGeometry.from_bbox(raster.extent)
-
-                # Make sure PostGIS supports it
-                with connection.cursor() as cursor:
-                    cursor.execute("SELECT SRID FROM spatial_ref_sys WHERE SRID = %s", [raster.srid])
-                    if cursor.rowcount == 0:
-                        raise NodeServerError(gettext("Unsupported SRS %(code)s. Please make sure you picked a supported SRS.") % {'code': str(raster.srid)})
-
-                # It will be implicitly transformed into the SRID of the model’s field
-                # self.field = GEOSGeometry(...)
-                setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
-
-                logger.info("Populated extent field with {} for {}".format(raster_path, self))
+                # Read extent
+                extent_wkt = get_raster_bounds_wkt(raster_path)
+                if extent_wkt is not None:
+                    extent = GEOSGeometry(extent_wkt, srid=4326)
+                    setattr(self, field, extent)
+                    logger.info("Populated extent field with {} for {}".format(raster_path, self))
+                else:
+                    logger.warning("Cannot populate extent field with {} for {}, not georeferenced".format(raster_path, self))
 
         self.check_ept()
-
-        # Flushes the changes to the *_extent fields
-        # and immediately reads them back into Python
-        # This is required because GEOS screws up the X/Y conversion
-        # from the raster CRS to 4326, whereas PostGIS seems to do it correctly :/
-        self.status = status_codes.RUNNING # avoid telling clients that task is completed prematurely
-        self.save()
-        self.refresh_from_db()
-
         self.update_available_assets_field()
         self.update_epsg_field()
         self.update_orthophoto_bands_field()
@@ -1225,8 +1207,18 @@ def update_epsg_field(self, commit=False):
                 try:
                     with rasterio.open(asset_path) as f:
                         if f.crs is not None:
-                            epsg = f.crs.to_epsg()
-                            break # We assume all assets are in the same CRS
+                            code = f.crs.to_epsg()
+                            if code is not None:
+                                epsg = code
+                                break # We assume all assets are in the same CRS
+                            else:
+                                # Try to get code from WKT
+                                wkt = f.crs.to_wkt()
+                                if wkt is not None:
+                                    code = epsg_from_wkt(wkt)
+                                    if code is not None:
+                                        epsg = code
+                                        break
                 except Exception as e:
                     logger.warning(e)
 

app/static/app/js/classes/Gcp.js

@@ -1,58 +1,53 @@
+import { _, interpolate } from './gettext';
+
 class Gcp{
   constructor(text){
-    this.text = text;
-  }
-
-  // Scale the image location of GPCs
-  // according to the values specified in the map
-  // @param imagesRatioMap {Object} object in which keys are image names and values are scaling ratios
-  //    example: {'DJI_0018.jpg': 0.5, 'DJI_0019.JPG': 0.25}
-  // @return {Gcp} a new GCP object
-  resize(imagesRatioMap, muteWarnings = false){
-    // Make sure dict is all lower case and values are floats
-    let ratioMap = {};
-    for (let k in imagesRatioMap) ratioMap[k.toLowerCase()] = parseFloat(imagesRatioMap[k]);
+    this.crs = "";
+    this.errors = [];
+    // this.entries = [];
 
-    const lines = this.text.split(/\r?\n/);
-    let output = "";
+    const lines = text.split(/\r?\n/);
 
     if (lines.length > 0){
-      output += lines[0] + '\n'; // coordinate system description
+      this.crs = lines[0];
+
+      // Check header
+      let c = this.crs.toUpperCase();
+      console.log(c);
+      if (!c.startsWith("WGS84") && !c.startsWith("+PROJ") && !c.startsWith("EPSG:")){
+        this.errors.push(interpolate(_("Invalid CRS: %(line)s"), { line: this.crs } ));
+      }
 
       for (let i = 1; i < lines.length; i++){
         let line = lines[i].trim();
         if (line !== "" && line[0] !== "#"){
           let parts = line.split(/\s+/);
           if (parts.length >= 6){
             let [x, y, z, px, py, imagename, ...extracols] = parts;
-            let ratio = ratioMap[imagename.toLowerCase()];
 
             px = parseFloat(px);
             py = parseFloat(py);
-
-            if (ratio !== undefined){
-              px *= ratio;
-              py *= ratio;
-            }else{
-              if (!muteWarnings) console.warn(`${imagename} not found in ratio map. Are you missing some images?`);
+            x = parseFloat(x);
+            y = parseFloat(y);
+            z = parseFloat(y);
+            if (isNaN(px) || isNaN(py) || isNaN(x) || isNaN(y)){
+              this.errors.push(interpolate(_("Invalid line %(num)s: %(line)s"), { num: i + 1, line }));
+              continue;
             }
-
-            let extra = extracols.length > 0 ? ' ' + extracols.join(' ') : '';
-            output += `${x} ${y} ${z} ${px.toFixed(8)} ${py.toFixed(8)} ${imagename}${extra}\n`;
           }else{
-            if (!muteWarnings) console.warn(`Invalid GCP format at line ${i}: ${line}`);
-            output += line + '\n';
+            this.errors.push(interpolate(_("Invalid line %(num)s: %(line)s"), { num: i + 1, line }));
           }
         }
       }
-    }
 
-    return new Gcp(output);
+    }else{
+      this.errors.push(_("Empty GCP file"));
+    }
   }
 
-  toString(){
-    return this.text;
+  valid(){
+    return this.errors.length === 0;
   }
 }
 
-module.exports = Gcp;
+export default Gcp;

app/static/app/js/components/EditTaskForm.jsx

@@ -28,7 +28,8 @@ class EditTaskForm extends React.Component {
       inReview: PropTypes.bool,
       task: PropTypes.object,
       suggestedTaskName: PropTypes.oneOfType([PropTypes.string, PropTypes.func]),
-      getCropPolygon: PropTypes.func
+      getCropPolygon: PropTypes.func,
+      getGcpFile: PropTypes.func
   };
 
   constructor(props){
@@ -368,6 +369,16 @@ class EditTaskForm extends React.Component {
       }
     }
 
+    // If a processing node supports "crs" as an option
+    // and a GCP file is provided, and the user hasn't specified
+    // a preference, default to "gcp" (set the CRS to use the GCP's CRS)
+    if (this.props.getGcpFile){
+      if (this.props.getGcpFile() && optionNames['crs']){
+        let crsOpt = optsCopy.find(opt => opt.name === 'crs');
+        if (!crsOpt) optsCopy.push({name: 'crs', value: 'gcp'});
+      }
+    }
+
     return optsCopy.filter(opt => optionNames[opt.name]);
   }
 

app/static/app/js/components/NewTaskPanel.jsx

@@ -8,6 +8,7 @@ import MapPreview from './MapPreview';
 import update from 'immutability-helper';
 import PluginsAPI from '../classes/plugins/API';
 import statusCodes from '../classes/StatusCodes';
+import Gcp from '../classes/Gcp';
 import { _, interpolate } from '../classes/gettext';
 
 class NewTaskPanel extends React.Component {
@@ -45,6 +46,7 @@ class NewTaskPanel extends React.Component {
       loading: false,
       showMapPreview: false,
       dismissImageCountWarning: false,
+      showMalformedGcpErrors: false,
     };
 
     this.save = this.save.bind(this);
@@ -172,6 +174,32 @@ class NewTaskPanel extends React.Component {
     return this.mapPreview.getCropPolygon();
   };
 
+  getGcpFile = () => {
+    if (!this.props.getFiles) return null;
+
+    const files = this.props.getFiles();
+    for (let i = 0; i < files.length; i++){
+      const f = files[i];
+      if (f.type.indexOf("text") === 0 && ["geo.txt", "image_groups.txt"].indexOf(f.name.toLowerCase()) === -1){
+        if (!f._gcp){
+          const reader = new FileReader();
+          reader.onload = (e) => {
+            if (e.target.result){
+              const gcp = new Gcp(e.target.result);
+              if (!gcp.valid()){
+                this.setState({showMalformedGcpErrors: true});
+              }
+              f._gcp = gcp;
+            }
+          };
+          reader.readAsText(f);
+        }
+
+        return f;
+      }
+    }
+  }
+
   handlePolygonChange = () => {
     if (this.taskForm) this.taskForm.forceUpdate();
   }
@@ -201,18 +229,34 @@ class NewTaskPanel extends React.Component {
     let filesCountOk = true;
     if (this.taskForm && !this.taskForm.checkFilesCount(this.props.filesCount)) filesCountOk = false;
 
+    let fileCountInfo = interpolate(_("%(count)s files selected."), { count: this.props.filesCount });
+    let gcp = this.getGcpFile();
+    if (gcp){
+      fileCountInfo = interpolate(_("%(count)s files and GCP file (%(name)s) selected."), { count: this.props.filesCount - 1, name: gcp.name });
+    }
+
     return (
       <div className="new-task-panel theme-background-highlight">
         <div className="form-horizontal">
           <div className={this.state.inReview ? "disabled" : ""}>
-            <p>{interpolate(_("%(count)s files selected. Please check these additional options:"), { count: this.props.filesCount})}</p>
+            <p>{fileCountInfo} {_("Please check these additional options:")}</p>
             {this.props.filesCount === 999 && !this.state.dismissImageCountWarning ? 
             <div className="alert alert-warning alert-dismissible alert-images">
               <button type="button" className="close" title={_("Close")} onClick={() => this.setState({dismissImageCountWarning: true})}><span aria-hidden="true">&times;</span></button>
               <i className="fa fa-hand-point-right"></i> {_("Did you forget any images? When images exceed 1000, they are often stored inside multiple folders on the SD card.")}
             </div>
             : ""}
 
+            {gcp && gcp._gcp && this.state.showMalformedGcpErrors ? 
+            <div className="alert alert-warning alert-dismissible alert-images">
+              <button type="button" className="close" title={_("Close")} onClick={() => this.setState({showMalformedGcpErrors: false})}><span aria-hidden="true">&times;</span></button>
+              <div dangerouslySetInnerHTML={{__html: interpolate(_("Whoops! It looks like your GCP file is not formatted properly: %(errors)s See %(link)s for information on the GCP file format"), {
+                link: `<a href="${window.__gcpDocsLink}" target="_blank">${_("GCP File")}</a>`,
+                errors: "<ul>" + gcp._gcp.errors.map(err => `<li>${err}</li>`) + "</ul>"
+              })}}></div>
+            </div> : ""}
+
+
             {!filesCountOk ? 
             <div className="alert alert-warning">
               {interpolate(_("Number of files selected exceeds the maximum of %(count)s allowed on this processing node."), { count: this.taskForm.selectedNodeMaxImages() })}
@@ -236,6 +280,7 @@ class NewTaskPanel extends React.Component {
               inReview={this.state.inReview}
               suggestedTaskName={this.handleSuggestedTaskName}
               getCropPolygon={this.getCropPolygon}
+              getGcpFile={this.getGcpFile}
               ref={(domNode) => { if (domNode) this.taskForm = domNode; }}
             />
 

app/templates/app/base.html

@@ -123,6 +123,9 @@
 <script>
 {% task_options_docs_link as to_link %}
 window.__taskOptionsDocsLink = "{{ to_link|safe }}";
+{% gcp_docs_link as gcp_link %}
+window.__gcpDocsLink = "{{ gcp_link|safe }}";
+
 
 $(function(){
     $(window).bind("load resize", function() {

app/templates/app/dashboard.html

@@ -35,7 +35,7 @@ <h3>{% trans 'Welcome!' %} ☺</h3>
 					<li>{% trans 'You need at least 5 images, but 16-32 is typically the minimum.' %}</li>
 					<li>{% trans 'Images must overlap by 65% or more. Aim for 70-72%' %}</li>
 					<li>{% trans 'For great 3D, images must overlap by 83%' %}</li>
-					<li>{% gcp_docs_link as gcp_link %}{% blocktrans with link_start=gcp_link|safe link_end='</a>' %}A {{link_start}}GCP File{{link_end}} is optional, but can increase georeferencing accuracy{% endblocktrans %}</li>
+					<li>{% gcp_docs_link as gcp_link %}{% blocktrans with link_start='<a href="'|add:gcp_link|add:'">'|safe link_end='</a>' %}A {{link_start}}GCP File{{link_end}} is optional, but can increase georeferencing accuracy{% endblocktrans %}</li>
 				</ul>
 			</p>
 		{% endif %}

app/templatetags/settings.py

@@ -15,7 +15,7 @@ def task_options_docs_link():
 
 @register.simple_tag
 def gcp_docs_link():
-    return '<a href="%s" target="_blank">' % settings.GCP_DOCS_LINK
+    return settings.GCP_DOCS_LINK
 
 @register.simple_tag
 def reset_password_link():