upload via rclone is now possible for every rclone remote (only tested for nextcloud)

Author: gg46ixav

README.md

@@ -171,7 +171,7 @@ Usage: databusclient deploy [OPTIONS] [DISTRIBUTIONS]...
 
  - Metadata-based deployment
 
- - Upload & deploy via Nextcloud
+ - Upload & deploy via Rclone
 
 Arguments:
   DISTRIBUTIONS...  Depending on mode:
@@ -192,10 +192,8 @@ Options:
   --license TEXT      License (see dalicc.net)  [required]
   --apikey TEXT       API key  [required]
   --metadata PATH     Path to metadata JSON file (for metadata mode)
-  --webdav-url TEXT   WebDAV URL (e.g.,
-                      https://cloud.example.com/remote.php/webdav)
-  --remote TEXT       rclone remote name (e.g., 'nextcloud')
-  --path TEXT         Remote path on Nextcloud (e.g., 'datasets/mydataset')
+  --remote TEXT       rclone remote name (e.g., 'my-nextcloud')
+  --path TEXT         Remote path on Rclone Remote (e.g., 'datasets/mydataset')
   --help              Show this message and exit.
   
 ```
@@ -251,20 +249,19 @@ Metadata file structure (file_format and compression are optional):
 ```
 
 
-##### Mode 3: Upload & Deploy via Nextcloud
+##### Mode 3: Upload & Deploy via Rclone
 
-Upload local files or folders to a WebDAV/Nextcloud instance and automatically deploy to DBpedia Databus. 
+Upload local files or folders to a Rclone remote and automatically deploy to DBpedia Databus. 
 Rclone is required.
 
 ```bash
 databusclient deploy \
-  --webdav-url https://cloud.example.com/remote.php/webdav \
-  --remote nextcloud \
+  --remote my-nextcloud \
   --path datasets/mydataset \
   --version-id https://databus.org/user/dataset/version/1.0 \
   --title "Test Dataset" \
   --abstract "Short abstract of dataset" \
-  --description "This dataset was uploaded for testing the Nextcloud → Databus pipeline." \
+  --description "This dataset was uploaded for testing the Rclone → Databus pipeline." \
   --license https://dalicc.net/licenselibrary/Apache-2.0 \
   --apikey "API-KEY" \
   ./localfile1.ttl \

databusclient/cli.py

@@ -29,28 +29,27 @@ def app():
 
 @click.option("--metadata", "metadata_file", type=click.Path(exists=True),
               help="Path to metadata JSON file (for metadata mode)")
-@click.option("--webdav-url", "webdav_url", help="WebDAV URL (e.g., https://cloud.example.com/remote.php/webdav)")
-@click.option("--remote", help="rclone remote name (e.g., 'nextcloud')")
-@click.option("--path", help="Remote path on Nextcloud (e.g., 'datasets/mydataset')")
+@click.option("--remote", help="rclone remote name (e.g., 'my-nextcloud')")
+@click.option("--path", help="Remote path on Rclone Remote (e.g., 'datasets/mydataset')")
 
 @click.argument("distributions", nargs=-1)
 def deploy(version_id, title, abstract, description, license_url, apikey,
-           metadata_file, webdav_url, remote, path, distributions: List[str]):
+           metadata_file, remote, path, distributions: List[str]):
     """
     Flexible deploy to Databus command supporting three modes:\n
     - Classic deploy (distributions as arguments)\n
     - Metadata-based deploy (--metadata <file>)\n
-    - Upload & deploy via Nextcloud (--webdav-url, --remote, --path)
+    - Upload & deploy via Rclone (--remote, --path)
     """
 
     # Sanity checks for conflicting options
-    if metadata_file and any([distributions, webdav_url, remote, path]):
-        raise click.UsageError("Invalid combination: when using --metadata, do not provide --webdav-url, --remote, --path, or distributions.")
-    if any([webdav_url, remote, path]) and not all([webdav_url, remote, path]):
-        raise click.UsageError("Invalid combination: when using WebDAV/Nextcloud mode, please provide --webdav-url, --remote, and --path together.")
+    if metadata_file and any([distributions, remote, path]):
+        raise click.UsageError("Invalid combination: when using --metadata, do not provide --remote, --path, or distributions.")
+    if any([remote, path]) and not all([remote, path]):
+        raise click.UsageError("Invalid combination: when using Rclone mode, please provide --remote, and --path together.")
 
     # === Mode 1: Classic Deploy ===
-    if distributions and not (metadata_file or webdav_url or remote or path):
+    if distributions and not (metadata_file or remote or path):
         click.echo("[MODE] Classic deploy with distributions")
         click.echo(f"Deploying dataset version: {version_id}")
 
@@ -66,27 +65,27 @@ def deploy(version_id, title, abstract, description, license_url, apikey,
         client.deploy_from_metadata(metadata, version_id, title, abstract, description, license_url, apikey)
         return
 
-    # === Mode 3: Upload & Deploy (Nextcloud) ===
-    if webdav_url and remote and path:
+    # === Mode 3: Upload & Deploy (Rclone) ===
+    if remote and path:
         if not distributions:
-            raise click.UsageError("Please provide files to upload when using WebDAV/Nextcloud mode.")
+            raise click.UsageError("Please provide files to upload when using Rclone mode.")
 
         #Check that all given paths exist and are files or directories.#
         invalid = [f for f in distributions if not os.path.exists(f)]
         if invalid:
             raise click.UsageError(f"The following input files or folders do not exist: {', '.join(invalid)}")
 
-        click.echo("[MODE] Upload & Deploy to DBpedia Databus via Nextcloud")
+        click.echo("[MODE] Upload & Deploy to DBpedia Databus via Rclone")
         click.echo(f"→ Uploading to: {remote}:{path}")
-        metadata = upload.upload_to_nextcloud(distributions, remote, path, webdav_url)
+        metadata = upload.upload_with_rclone(distributions, remote, path)
         client.deploy_from_metadata(metadata, version_id, title, abstract, description, license_url, apikey)
         return
 
     raise click.UsageError(
         "No valid input provided. Please use one of the following modes:\n"
         "  - Classic deploy: pass distributions as arguments\n"
         "  - Metadata deploy: use --metadata <file>\n"
-        "  - Upload & deploy: use --webdav-url, --remote, --path, and file arguments"
+        "  - Upload & deploy: use --remote, --path, and file arguments"
     )
 
 

databusclient/rclone_wrapper/upload.py

@@ -1,8 +1,11 @@
 import hashlib
 import os
-import subprocess
 import posixpath
-from urllib.parse import urljoin, quote
+import re
+import subprocess
+from typing import List, Dict, Union
+
+from rclone_python import rclone
 
 
 def compute_sha256_and_length(filepath):
@@ -17,6 +20,7 @@ def compute_sha256_and_length(filepath):
             total_length += len(chunk)
     return sha256.hexdigest(), total_length
 
+
 def get_all_files(path):
     if os.path.isfile(path):
         return [path]
@@ -26,7 +30,23 @@ def get_all_files(path):
             files.append(os.path.join(root, name))
     return files
 
-def upload_to_nextcloud(source_paths: list[str], remote_name: str, remote_path: str, webdav_url: str):
+
+def upload_with_rclone(source_paths: list[str], remote_name: str, remote_path: str) -> List[Dict[str, Union[str, int]]]:
+    """
+       Upload files or directories to any rclone remote using rclone-python.
+
+       Args:
+           source_paths: List of local paths to upload.
+           remote_name: Name of the rclone remote (e.g., 'mydrive').
+           remote_path: Remote path inside the remote (e.g., 'backup/').
+
+       Returns:
+           List of dictionaries with keys: filename, checksum, size, url.
+       """
+
+    if not rclone.is_installed():
+        raise RuntimeError("rclone not found. Please install rclone.")
+
     result = []
     for path in source_paths:
         if not os.path.exists(path):
@@ -37,46 +57,62 @@ def upload_to_nextcloud(source_paths: list[str], remote_name: str, remote_path:
         basename = os.path.basename(abs_path)
         files = get_all_files(abs_path)
 
-        tmp_results = []
-
         for file in files:
             checksum,size  = compute_sha256_and_length(file)
 
             if os.path.isdir(path):
                 rel_file = os.path.relpath(file, abs_path)
-                # Normalize to POSIX for WebDAV/URLs
+                # Normalize to POSIX
                 rel_file = rel_file.replace(os.sep, "/")
-                remote_webdav_path = posixpath.join(remote_path, basename, rel_file)
+                dest_subpath = f"{remote_path.rstrip('/')}/{basename}/{rel_file}"
             else:
-                remote_webdav_path = posixpath.join(remote_path, os.path.basename(file))
+                dest_subpath = f"{remote_path.rstrip('/')}/{os.path.basename(file)}"
 
-            # Preserve scheme/host and percent-encode path segments
-            url = urljoin(webdav_url.rstrip("/") + "/", quote(remote_webdav_path.lstrip("/"), safe="/"))
+            destination = f"{remote_name}:{dest_subpath}"
 
-            filename = os.path.basename(file)
-            tmp_results.append({
-                "filename": filename,
-                "checksum": checksum,
-                "size": size,
-                "url": url,
-            })
+            # Upload File
+            try:
+                if os.path.isdir(path):
+                    rclone.copy(abs_path, f"{remote_name}:{remote_path.rstrip('/')}/{basename}", args=["--progress"])
+                else:
+                    rclone.copyto(abs_path, destination, args=["--progress"])
 
-        dest_subpath = posixpath.join(remote_path.lstrip("/"), basename)
-        if os.path.isdir(path):
-            destination = f"{remote_name}:{dest_subpath}"
-            command = ["rclone", "copy", abs_path, destination, "--progress"]
-        else:
-            destination = f"{remote_name}:{dest_subpath}"
-            command = ["rclone", "copyto", abs_path, destination, "--progress"]
-
-        print(f"Upload: {path} → {destination}")
-        try:
-            subprocess.run(command, check=True)
-            result.extend(tmp_results)
-            print("✅ Uploaded successfully.\n")
-        except subprocess.CalledProcessError as e:
-            print(f"❌ Error uploading {path}: {e}\n")
-        except FileNotFoundError:
-            print("❌ rclone not found on PATH. Install rclone and retry.")
-
-    return result
+                # Get URL
+                try:
+                    url = rclone.link(destination).strip()
+                    if not url:
+                        url = None
+                except Exception:
+                    url = None
+
+                if not url:
+                    try:
+                        proc = subprocess.run(["rclone", "config", "show", remote_name],
+                                              capture_output=True, text=True, check=True)
+                        out = proc.stdout
+                        m = re.search(r"(?m)^\s*url\s*=\s*(.+)$", out)
+                        if m:
+                            remote_url = m.group(1).strip()
+                            url = posixpath.join(remote_url, dest_subpath)
+                        else:
+                            url = None
+                    except Exception as e:
+                        print(f"Cannot resolve remote URL: {e}")
+
+                if url:
+                    result.append({
+                        "filename": os.path.basename(file),
+                        "checksum": checksum,
+                        "size": size,
+                        "url": url,
+                    })
+
+                    print(f"✅ Uploaded {file} → {destination}")
+                    print(f"   Cloud URL: {url}\n")
+                else:
+                    print("   No cloud URL available for this remote.\n")
+
+            except Exception as e:
+                print(f"❌ Error uploading {file}: {e}\n")
+
+    return result

poetry.lock

@@ -13,6 +13,7 @@ requests = "^2.28.1"
 tqdm = "^4.42.1"
 SPARQLWrapper = "^2.0.0"
 rdflib = "^7.2.1"
+rclone-python = "^0.1.23"
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.6.0"