added deploy script with uploading to given rclone remote

Author: gg46ixav

README.md

@@ -5,6 +5,27 @@
 python3 -m pip install databusclient
 ```
 
+## Upload to Nextcloud and Deploy to Databus
+Please add databus API_KEY to .env file
+
+The script uploads all given files and all files in the given folders to the given remote.
+Then registers them on the databus.
+### Example Call
+```bash
+cd databusclient 
+
+python deploy.py \
+--remote scads-nextcloud \
+--path test \
+--version-id https://databus.dbpedia.org/gg46ixav/test_group/test_artifact/2023-07-03 \
+--title "Test Dataset" \
+--abstract "This is a short abstract of the test dataset." \
+--description "This dataset was uploaded for testing the Nextcloud → Databus deployment pipeline." \
+--license https://dalicc.net/licenselibrary/Apache-2.0 \
+/home/theo/Work/SCADS.AI/Projects/CSVTest/newtestoutputfolder \
+/home/theo/Work/SCADS.AI/Projects/CSVTest/output.csv.bz2
+
+```
 ## CLI Usage
 ```bash
 databusclient --help

databusclient/client.py

@@ -342,7 +342,7 @@ def append_to_dataset_graph_if_existent(add_key: str, add_value: str):
     graphs.append(dataset_graph)
 
     dataset = {
-        "@context": "https://downloads.dbpedia.org/databus/context.jsonld",
+        "@context": "https://databus.dbpedia.org/res/context.jsonld",
         "@graph": graphs,
     }
     return dataset

databusclient/deploy.py

@@ -0,0 +1,92 @@
+import os
+import sys
+import argparse
+
+from databusclient import create_distribution, create_dataset, deploy
+from dotenv import load_dotenv
+
+from nextcloudclient.upload import upload_to_nextcloud
+
+
+def deploy_to_databus(
+    metadata,
+    version_id,
+    title,
+    abstract,
+    description,
+    license_url
+):
+
+    load_dotenv()
+    api_key = os.getenv("API_KEY")
+    if not api_key:
+        raise ValueError("API_KEY not found in .env")
+
+    distributions = []
+    counter=0
+    for filename, checksum, size, url in metadata:
+
+        parts = filename.split(".")
+        if len(parts) == 1:
+            file_format = "none"
+            compression = "none"
+        elif len(parts) == 2:
+            file_format = parts[-1]
+            compression = "none"
+        else:
+            file_format = parts[-2]
+            compression = parts[-1]
+
+        distributions.append(
+            create_distribution(
+                url=url,
+                cvs={"count":f"{counter}"},
+                file_format=file_format,
+                compression=compression,
+                sha256_length_tuple=(checksum, size)
+            )
+        )
+        counter+=1
+
+    dataset = create_dataset(
+        version_id=version_id,
+        title=title,
+        abstract=abstract,
+        description=description,
+        license_url=license_url,
+        distributions=distributions
+    )
+
+    deploy(dataset, api_key)
+    metadata_string = ",\n".join([entry[-1] for entry in metadata])
+
+    print(f"Successfully deployed\n{metadata_string}\nto databus {version_id}")
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Upload files to Nextcloud and deploy to DBpedia Databus.")
+
+    parser.add_argument("files", nargs="+", help="Path(s) to file(s) or folder(s) to upload")
+    parser.add_argument("--remote", required=True, help="rclone remote name (e.g., 'nextcloud')")
+    parser.add_argument("--path", required=True, help="Remote path on Nextcloud (e.g., 'datasets/mydataset')")
+    parser.add_argument("--version-id", required=True, help="Databus version URI")
+    parser.add_argument("--title", required=True, help="Title of the dataset")
+    parser.add_argument("--abstract", required=True, help="Short abstract of the dataset")
+    parser.add_argument("--description", required=True, help="Detailed description of the dataset")
+    parser.add_argument("--license", required=True, help="License URL (e.g., https://dalicc.net/licenselibrary/Apache-2.0)")
+
+    return parser.parse_args()
+
+if __name__ == '__main__':
+
+    args = parse_args()
+
+    metadata = upload_to_nextcloud(args.files, args.remote, args.path)
+
+    deploy_to_databus(
+        metadata,
+        version_id=args.version_id,
+        title=args.title,
+        abstract=args.abstract,
+        description=args.description,
+        license_url=args.license
+    )

nextcloudclient/upload.py

@@ -0,0 +1,69 @@
+import hashlib
+import os
+import subprocess
+import posixpath
+
+BASE_URL = "https://cloud.scadsai.uni-leipzig.de"
+
+def compute_sha256_and_length(filepath):
+    sha256 = hashlib.sha256()
+    total_length = 0
+    with open(filepath, 'rb') as f:
+        while True:
+            chunk = f.read(4096)
+            if not chunk:
+                break
+            sha256.update(chunk)
+            total_length += len(chunk)
+    return sha256.hexdigest(), total_length
+
+def get_all_files(path):
+    if os.path.isfile(path):
+        return [path]
+    files = []
+    for root, _, filenames in os.walk(path):
+        for name in filenames:
+            files.append(os.path.join(root, name))
+    return files
+
+def upload_to_nextcloud(source_paths: str, remote_name: str, remote_path: str):
+    result = []
+    for path in source_paths:
+        if not os.path.exists(path):
+            print(f"Path not found: {path}")
+            continue
+
+        abs_path = os.path.abspath(path)
+        basename = os.path.basename(abs_path)
+        files = get_all_files(abs_path)
+
+        for file in files:
+            checksum,size  = compute_sha256_and_length(file)
+
+            if os.path.isdir(path):
+                rel_file = os.path.relpath(file, abs_path)
+                remote_webdav_path = posixpath.join(remote_path, basename, rel_file)
+            else:
+                remote_webdav_path = posixpath.join(remote_path, os.path.basename(file))
+
+            url = f"{BASE_URL}/remote.php/webdav/{remote_webdav_path}"
+
+            filename = file.split("/")[-1]
+            result.append((filename, checksum, size, url))
+
+        if os.path.isdir(path):
+            destination = f"{remote_name}:{remote_path}/{basename}"
+            command = ["rclone", "copy", abs_path, destination, "--progress"]
+        else:
+            destination = f"{remote_name}:{remote_path}/{basename}"
+            command = ["rclone", "copyto", abs_path, destination, "--progress"]
+
+        print(f"Upload: {path} → {destination}")
+        try:
+            subprocess.run(command, check=True)
+            print("✅ Uploaded successfully.\n")
+        except subprocess.CalledProcessError as e:
+            print(f"❌ Error uploading {path}: {e}\n")
+
+
+    return result