Skip to content

Improved DroneDB plugin #1801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HeDo88TH wants to merge 5 commits into
base: master
Choose a base branch
from
+723 −195
Open

Improved DroneDB plugin #1801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7acc921
Improved dronedb plugin
HeDo88TH Dec 11, 2025
dceb8cf
Added tag format validation
HeDo88TH Dec 30, 2025
34c1f2d
Improved timeout and memory management
HeDo88TH Dec 30, 2025
eda0de4
Improved code reliability
HeDo88TH Dec 30, 2025
c3d71d5
Fixed comment
HeDo88TH Dec 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 64 additions & 44 deletions coreplugins/dronedb/api_views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import importlib
import json
from posixpath import join
import re
import time
import requests
import os
Expand All @@ -24,20 +25,23 @@

VALID_IMAGE_EXTENSIONS = ['.tiff', '.tif', '.png', '.jpeg', '.jpg']

# Regex pattern for valid tag format: "org" or "org/dataset" with lowercase alphanumeric and hyphens
TAG_PATTERN = re.compile(r'^[a-z0-9][a-z0-9\-]*(/[a-z0-9][a-z0-9\-]*)?$')

def is_valid(file):
_, file_extension = path.splitext(file)
return file_extension.lower() in VALID_IMAGE_EXTENSIONS or file == 'gcp_list.txt' or file == 'geo.txt'

def get_settings(request):
ds = get_current_plugin().get_user_data_store(request.user)

registry_url = ds.get_string('registry_url') or DEFAULT_HUB_URL
username = ds.get_string('username') or None
password = ds.get_string('password') or None
token = ds.get_string('token') or None

return registry_url, username, password, token


def update_token(request, token):
ds = get_current_plugin().get_user_data_store(request.user)
Expand Down Expand Up @@ -70,7 +74,7 @@ def post(self, request):

ddb = DroneDB(hub_url, username, password)

return Response({'success': ddb.login()}, status=status.HTTP_200_OK)
return Response({'success': ddb.login()}, status=status.HTTP_200_OK)

except(Exception) as e:
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
Expand Down Expand Up @@ -137,36 +141,36 @@ def post(self, request):

try:

result, org, ds, folder, count, size = verify_url(url, username, password).values()
result, org, ds, folder, count, size = verify_url(url, username, password).values()

if (not result):
return Response({'error': 'Invalid url.'}, status=status.HTTP_400_BAD_REQUEST)
return Response({'error': 'Invalid url.'}, status=status.HTTP_400_BAD_REQUEST)

return Response({'count': count, 'success': result, 'ds' : ds, 'org': org, 'folder': folder or None, 'size': size}
return Response({'count': count, 'success': result, 'ds' : ds, 'org': org, 'folder': folder or None, 'size': size}
if org else {'success': False}, status=status.HTTP_200_OK)

except Exception as e:
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)
return Response({'error': str(e)}, status=status.HTTP_400_BAD_REQUEST)

class InfoTaskView(TaskView):
def get(self, request):

registry_url, username, _, _ = get_settings(request)

return Response({ 'hubUrl': registry_url, 'username': username }, status=status.HTTP_200_OK)


class ImportDatasetTaskView(TaskView):
def post(self, request, project_pk=None, pk=None):

task = self.get_and_check_task(request, pk)

# Read form data
ddb_url = request.data.get('ddb_url', None)

if ddb_url == None:
return Response({'error': 'DroneDB url must be set.'}, status=status.HTTP_400_BAD_REQUEST)

registry_url, orgSlug, dsSlug, folder = parse_url(ddb_url).values()

_, username, password, token = get_settings(request)
Expand All @@ -175,27 +179,27 @@ def post(self, request, project_pk=None, pk=None):
# Get the files from the folder
rawfiles = ddb.get_files_list(orgSlug, dsSlug, folder)
files = [file for file in rawfiles if is_valid(file['path'])]
# Verify that the folder url is valid

# Verify that the folder url is valid
if len(files) == 0:
return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST)

# Update the task with the new information
task.console += "Importing {} images...\n".format(len(files))
task.images_count = len(files)
task.pending_action = pending_actions.IMPORT
task.save()

# Associate the folder url with the project and task
combined_id = "{}_{}".format(project_pk, pk)

datastore = get_current_plugin().get_global_data_store()
datastore.set_json(combined_id, {
"ddbUrl": ddb_url,
"token": ddb.token,
"ddbUrl": ddb_url,
"token": ddb.token,
"ddbWebUrl": "{}/r/{}/{}/{}".format(to_web_protocol(registry_url), orgSlug, dsSlug, folder.rstrip('/'))
})

#ddb.refresh_token()

# Start importing the files in the background
Expand All @@ -211,7 +215,7 @@ def import_files(task_id, carrier):
from app.security import path_traversal_check

files = carrier['files']

headers = {}

if carrier['token'] != None:
Expand All @@ -225,15 +229,15 @@ def download_file(task, file):
with open(path, 'wb') as fd:
for chunk in download_stream.iter_content(4096):
fd.write(chunk)

logger.info("Will import {} files".format(len(files)))
task = models.Task.objects.get(pk=task_id)
task.create_task_directories()
task.save()

try:
downloaded_total = 0
for file in files:
for file in files:
download_file(task, file)
task.check_if_canceled()
models.Task.objects.filter(pk=task.id).update(upload_progress=(float(downloaded_total) / float(len(files))))
Expand Down Expand Up @@ -286,7 +290,7 @@ def ddb_cleanup(sender, task_id, **kwargs):
logger.info("Info task {0} ({1})".format(str(task_id), status_key))

datastore.del_key(status_key)


class StatusTaskView(TaskView):
def get(self, request, pk):
Expand All @@ -295,7 +299,7 @@ def get(self, request, pk):

# Associate the folder url with the project and task
status_key = get_status_key(pk)

datastore = get_current_plugin().get_global_data_store()

task_info = datastore.get_json(status_key, {
Expand All @@ -311,7 +315,7 @@ def get(self, request, pk):
return Response(task_info, status=status.HTTP_200_OK)

DRONEDB_ASSETS = [
'orthophoto.tif',
'orthophoto.tif',
'orthophoto.png',
'georeferenced_model.laz',
'dtm.tif',
Expand All @@ -321,15 +325,31 @@ def get(self, request, pk):
'ground_control_points.geojson'
]

class ShareTaskView(TaskView):
class ShareTaskView(TaskView):
def post(self, request, pk):

from app.plugins import logger

task = self.get_and_check_task(request, pk)

# Get optional tag and datasetName from request
tag = request.data.get('tag', None)
dataset_name = request.data.get('datasetName', None) or task.name

# Validate tag format if provided
if tag is not None and tag.strip():
tag = tag.strip().lower()
if not TAG_PATTERN.match(tag):
return Response({
'error': 'Invalid tag format. Must be "org" or "org/dataset" with lowercase alphanumeric characters and hyphens only.'
}, status=status.HTTP_400_BAD_REQUEST)

# Sanitize dataset_name (remove potentially dangerous characters)
if dataset_name:
dataset_name = dataset_name.strip()[:255] # Limit length

status_key = get_status_key(pk)

datastore = get_current_plugin().get_global_data_store()

data = {
Expand All @@ -348,33 +368,33 @@ def post(self, request, pk):

available_assets = [task.get_asset_file_or_stream(f) for f in list(set(task.available_assets) & set(DRONEDB_ASSETS))]

if 'textured_model.zip' in task.available_assets:
if 'textured_model.zip' in task.available_assets:
texture_files = [join(task.assets_path('odm_texturing'), f) for f in listdir(task.assets_path('odm_texturing')) if isfile(join(task.assets_path('odm_texturing'), f))]
available_assets.extend(texture_files)

assets_path = task.assets_path()

files = [{'path': f, 'name': f[len(assets_path)+1:], 'size': os.path.getsize(f)} for f in available_assets]

share_to_ddb.delay(pk, settings, files)
share_to_ddb.delay(pk, settings, files, tag, dataset_name)

return Response(data, status=status.HTTP_200_OK)
return Response(data, status=status.HTTP_200_OK)


@task
def share_to_ddb(pk, settings, files):
def share_to_ddb(pk, settings, files, tag=None, dataset_name=None):

from app.plugins import logger
status_key = get_status_key(pk)

status_key = get_status_key(pk)
datastore = get_current_plugin().get_global_data_store()

registry_url, username, password, token = settings

ddb = DroneDB(registry_url, username, password, token)

# Init share (to check)
share_token = ddb.share_init()
# Init share with optional tag and dataset name
share_token = ddb.share_init(tag=tag, dataset_name=dataset_name)

status = datastore.get_json(status_key)

Expand All @@ -394,9 +414,9 @@ def share_to_ddb(pk, settings, files):

while attempt < 3:
try:

attempt += 1

up = ddb.share_upload(share_token, file['path'], file['name'])

logger.info("Uploaded " + file['name'] + " to Dronedb (hash: " + up['hash'] + ")")
Expand All @@ -405,7 +425,7 @@ def share_to_ddb(pk, settings, files):
status['uploadedSize'] += file['size']

datastore.set_json(status_key, status)

break

except Exception as e:
Expand All @@ -423,7 +443,7 @@ def share_to_ddb(pk, settings, files):


res = ddb.share_commit(share_token)

status['status'] = 3 # Done
status['shareUrl'] = registry_url + res['url']

Expand Down
Loading