diff --git a/app/admin.py b/app/admin.py index 59ce7d40d..397040cc8 100644 --- a/app/admin.py +++ b/app/admin.py @@ -44,7 +44,28 @@ def has_add_permission(self, request): list_display = ('id', 'name', 'project', 'processing_node', 'created_at', 'status', 'last_error') list_filter = ('status', 'project',) search_fields = ('id', 'name', 'project__name') - + exclude = ('orthophoto_extent', 'dsm_extent', 'dtm_extent', 'crop', ) + readonly_fields = ('orthophoto_extent_wkt', 'dsm_extent_wkt', 'dtm_extent_wkt', 'crop_wkt', ) + + def orthophoto_extent_wkt(self, obj): + if obj.orthophoto_extent: + return obj.orthophoto_extent.wkt + return None + + def dsm_extent_wkt(self, obj): + if obj.dsm_extent: + return obj.dsm_extent.wkt + return None + + def dtm_extent_wkt(self, obj): + if obj.dtm_extent: + return obj.dtm_extent.wkt + return None + + def crop_wkt(self, obj): + if obj.crop: + return obj.crop.wkt + return None admin.site.register(Task, TaskAdmin) diff --git a/app/geoutils.py b/app/geoutils.py index 816a8f2bd..218247a3e 100644 --- a/app/geoutils.py +++ b/app/geoutils.py @@ -1,6 +1,9 @@ import rasterio.warp import numpy as np from rasterio.crs import CRS +from rasterio.warp import transform_bounds +from osgeo import osr +osr.DontUseExceptions() # GEOS has some weird bug where # we can't simply call geom.tranform(srid) @@ -79,3 +82,38 @@ def geom_transform(geom, epsg): return list(zip(tx, ty)) else: raise ValueError("Cannot transform complex geometries to WKT") + + +def epsg_from_wkt(wkt): + srs = osr.SpatialReference() + if srs.ImportFromWkt(wkt) != 0: + return None + + epsg = srs.GetAuthorityCode(None) + if epsg is not None: + return None + + # Try to get the 2D component + if srs.IsCompound(): + if srs.DemoteTo2D() != 0: + return None + + epsg = srs.GetAuthorityCode(None) + if epsg is not None: + return epsg + + +def get_raster_bounds_wkt(raster_path, target_srs="EPSG:4326"): + with rasterio.open(raster_path) as src: + if src.crs is None: + return None + + left, bottom, right, top = src.bounds + w, s, e, n = transform_bounds( + src.crs, + target_srs, + left, bottom, right, top + ) + + wkt = f"POLYGON(({w} {s}, {w} {n}, {e} {n}, {e} {s}, {w} {s}))" + return wkt \ No newline at end of file diff --git a/app/models/task.py b/app/models/task.py index a6e9519d6..40a89805e 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -22,8 +22,6 @@ import requests from PIL import Image Image.MAX_IMAGE_PIXELS = 4096000000 -from django.contrib.gis.gdal import GDALRaster -from django.contrib.gis.gdal import OGRGeometry from django.contrib.gis.geos import GEOSGeometry from django.contrib.postgres import fields from django.core.files.uploadedfile import InMemoryUploadedFile @@ -41,7 +39,7 @@ from app.pointcloud_utils import is_pointcloud_georeferenced from app.testwatch import testWatch from app.security import path_traversal_check -from app.geoutils import geom_transform +from app.geoutils import geom_transform, epsg_from_wkt, get_raster_bounds_wkt from nodeodm import status_codes from nodeodm.models import ProcessingNode from pyodm.exceptions import NodeResponseError, NodeConnectionError, NodeServerError, OdmError @@ -1012,32 +1010,16 @@ def extract_assets_and_complete(self): except IOError as e: logger.warning("Cannot create Cloud Optimized GeoTIFF for %s (%s). This will result in degraded visualization performance." % (raster_path, str(e))) - # Read extent and SRID - raster = GDALRaster(raster_path) - extent = OGRGeometry.from_bbox(raster.extent) - - # Make sure PostGIS supports it - with connection.cursor() as cursor: - cursor.execute("SELECT SRID FROM spatial_ref_sys WHERE SRID = %s", [raster.srid]) - if cursor.rowcount == 0: - raise NodeServerError(gettext("Unsupported SRS %(code)s. Please make sure you picked a supported SRS.") % {'code': str(raster.srid)}) - - # It will be implicitly transformed into the SRID of the model’s field - # self.field = GEOSGeometry(...) - setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) - - logger.info("Populated extent field with {} for {}".format(raster_path, self)) + # Read extent + extent_wkt = get_raster_bounds_wkt(raster_path) + if extent_wkt is not None: + extent = GEOSGeometry(extent_wkt, srid=4326) + setattr(self, field, extent) + logger.info("Populated extent field with {} for {}".format(raster_path, self)) + else: + logger.warning("Cannot populate extent field with {} for {}, not georeferenced".format(raster_path, self)) self.check_ept() - - # Flushes the changes to the *_extent fields - # and immediately reads them back into Python - # This is required because GEOS screws up the X/Y conversion - # from the raster CRS to 4326, whereas PostGIS seems to do it correctly :/ - self.status = status_codes.RUNNING # avoid telling clients that task is completed prematurely - self.save() - self.refresh_from_db() - self.update_available_assets_field() self.update_epsg_field() self.update_orthophoto_bands_field() @@ -1225,8 +1207,18 @@ def update_epsg_field(self, commit=False): try: with rasterio.open(asset_path) as f: if f.crs is not None: - epsg = f.crs.to_epsg() - break # We assume all assets are in the same CRS + code = f.crs.to_epsg() + if code is not None: + epsg = code + break # We assume all assets are in the same CRS + else: + # Try to get code from WKT + wkt = f.crs.to_wkt() + if wkt is not None: + code = epsg_from_wkt(wkt) + if code is not None: + epsg = code + break except Exception as e: logger.warning(e) diff --git a/app/static/app/js/classes/Gcp.js b/app/static/app/js/classes/Gcp.js index c377d30b8..4280e9056 100644 --- a/app/static/app/js/classes/Gcp.js +++ b/app/static/app/js/classes/Gcp.js @@ -1,23 +1,22 @@ +import { _, interpolate } from './gettext'; + class Gcp{ constructor(text){ - this.text = text; - } - - // Scale the image location of GPCs - // according to the values specified in the map - // @param imagesRatioMap {Object} object in which keys are image names and values are scaling ratios - // example: {'DJI_0018.jpg': 0.5, 'DJI_0019.JPG': 0.25} - // @return {Gcp} a new GCP object - resize(imagesRatioMap, muteWarnings = false){ - // Make sure dict is all lower case and values are floats - let ratioMap = {}; - for (let k in imagesRatioMap) ratioMap[k.toLowerCase()] = parseFloat(imagesRatioMap[k]); + this.crs = ""; + this.errors = []; + // this.entries = []; - const lines = this.text.split(/\r?\n/); - let output = ""; + const lines = text.split(/\r?\n/); if (lines.length > 0){ - output += lines[0] + '\n'; // coordinate system description + this.crs = lines[0]; + + // Check header + let c = this.crs.toUpperCase(); + console.log(c); + if (!c.startsWith("WGS84") && !c.startsWith("+PROJ") && !c.startsWith("EPSG:")){ + this.errors.push(interpolate(_("Invalid CRS: %(line)s"), { line: this.crs } )); + } for (let i = 1; i < lines.length; i++){ let line = lines[i].trim(); @@ -25,34 +24,30 @@ class Gcp{ let parts = line.split(/\s+/); if (parts.length >= 6){ let [x, y, z, px, py, imagename, ...extracols] = parts; - let ratio = ratioMap[imagename.toLowerCase()]; px = parseFloat(px); py = parseFloat(py); - - if (ratio !== undefined){ - px *= ratio; - py *= ratio; - }else{ - if (!muteWarnings) console.warn(`${imagename} not found in ratio map. Are you missing some images?`); + x = parseFloat(x); + y = parseFloat(y); + z = parseFloat(y); + if (isNaN(px) || isNaN(py) || isNaN(x) || isNaN(y)){ + this.errors.push(interpolate(_("Invalid line %(num)s: %(line)s"), { num: i + 1, line })); + continue; } - - let extra = extracols.length > 0 ? ' ' + extracols.join(' ') : ''; - output += `${x} ${y} ${z} ${px.toFixed(8)} ${py.toFixed(8)} ${imagename}${extra}\n`; }else{ - if (!muteWarnings) console.warn(`Invalid GCP format at line ${i}: ${line}`); - output += line + '\n'; + this.errors.push(interpolate(_("Invalid line %(num)s: %(line)s"), { num: i + 1, line })); } } } - } - return new Gcp(output); + }else{ + this.errors.push(_("Empty GCP file")); + } } - toString(){ - return this.text; + valid(){ + return this.errors.length === 0; } } -module.exports = Gcp; +export default Gcp; diff --git a/app/static/app/js/components/EditTaskForm.jsx b/app/static/app/js/components/EditTaskForm.jsx index 4d76fa7a5..3426b7f1b 100644 --- a/app/static/app/js/components/EditTaskForm.jsx +++ b/app/static/app/js/components/EditTaskForm.jsx @@ -28,7 +28,8 @@ class EditTaskForm extends React.Component { inReview: PropTypes.bool, task: PropTypes.object, suggestedTaskName: PropTypes.oneOfType([PropTypes.string, PropTypes.func]), - getCropPolygon: PropTypes.func + getCropPolygon: PropTypes.func, + getGcpFile: PropTypes.func }; constructor(props){ @@ -368,6 +369,16 @@ class EditTaskForm extends React.Component { } } + // If a processing node supports "crs" as an option + // and a GCP file is provided, and the user hasn't specified + // a preference, default to "gcp" (set the CRS to use the GCP's CRS) + if (this.props.getGcpFile){ + if (this.props.getGcpFile() && optionNames['crs']){ + let crsOpt = optsCopy.find(opt => opt.name === 'crs'); + if (!crsOpt) optsCopy.push({name: 'crs', value: 'gcp'}); + } + } + return optsCopy.filter(opt => optionNames[opt.name]); } diff --git a/app/static/app/js/components/NewTaskPanel.jsx b/app/static/app/js/components/NewTaskPanel.jsx index 8ba6b4c02..3a44b9694 100644 --- a/app/static/app/js/components/NewTaskPanel.jsx +++ b/app/static/app/js/components/NewTaskPanel.jsx @@ -8,6 +8,7 @@ import MapPreview from './MapPreview'; import update from 'immutability-helper'; import PluginsAPI from '../classes/plugins/API'; import statusCodes from '../classes/StatusCodes'; +import Gcp from '../classes/Gcp'; import { _, interpolate } from '../classes/gettext'; class NewTaskPanel extends React.Component { @@ -45,6 +46,7 @@ class NewTaskPanel extends React.Component { loading: false, showMapPreview: false, dismissImageCountWarning: false, + showMalformedGcpErrors: false, }; this.save = this.save.bind(this); @@ -172,6 +174,32 @@ class NewTaskPanel extends React.Component { return this.mapPreview.getCropPolygon(); }; + getGcpFile = () => { + if (!this.props.getFiles) return null; + + const files = this.props.getFiles(); + for (let i = 0; i < files.length; i++){ + const f = files[i]; + if (f.type.indexOf("text") === 0 && ["geo.txt", "image_groups.txt"].indexOf(f.name.toLowerCase()) === -1){ + if (!f._gcp){ + const reader = new FileReader(); + reader.onload = (e) => { + if (e.target.result){ + const gcp = new Gcp(e.target.result); + if (!gcp.valid()){ + this.setState({showMalformedGcpErrors: true}); + } + f._gcp = gcp; + } + }; + reader.readAsText(f); + } + + return f; + } + } + } + handlePolygonChange = () => { if (this.taskForm) this.taskForm.forceUpdate(); } @@ -201,11 +229,17 @@ class NewTaskPanel extends React.Component { let filesCountOk = true; if (this.taskForm && !this.taskForm.checkFilesCount(this.props.filesCount)) filesCountOk = false; + let fileCountInfo = interpolate(_("%(count)s files selected."), { count: this.props.filesCount }); + let gcp = this.getGcpFile(); + if (gcp){ + fileCountInfo = interpolate(_("%(count)s files and GCP file (%(name)s) selected."), { count: this.props.filesCount - 1, name: gcp.name }); + } + return (
-

{interpolate(_("%(count)s files selected. Please check these additional options:"), { count: this.props.filesCount})}

+

{fileCountInfo} {_("Please check these additional options:")}

{this.props.filesCount === 999 && !this.state.dismissImageCountWarning ?
@@ -213,6 +247,16 @@ class NewTaskPanel extends React.Component {
: ""} + {gcp && gcp._gcp && this.state.showMalformedGcpErrors ? +
+ +
${_("GCP File")}`, + errors: "
    " + gcp._gcp.errors.map(err => `
  • ${err}
    • `) + "
" + })}}>
+
: ""} + + {!filesCountOk ?
{interpolate(_("Number of files selected exceeds the maximum of %(count)s allowed on this processing node."), { count: this.taskForm.selectedNodeMaxImages() })} @@ -236,6 +280,7 @@ class NewTaskPanel extends React.Component { inReview={this.state.inReview} suggestedTaskName={this.handleSuggestedTaskName} getCropPolygon={this.getCropPolygon} + getGcpFile={this.getGcpFile} ref={(domNode) => { if (domNode) this.taskForm = domNode; }} /> diff --git a/app/templates/app/base.html b/app/templates/app/base.html index e7a895fa7..d18c47961 100644 --- a/app/templates/app/base.html +++ b/app/templates/app/base.html @@ -123,6 +123,9 @@