|
1 | | -# TODO find suitable bucket |
| 1 | +import os |
| 2 | + |
| 3 | +import s3fs |
| 4 | + |
| 5 | +from mobie.metadata import add_remote_project_metadata |
| 6 | +from tqdm import tqdm |
| 7 | + |
| 8 | +# Using incucyte s3 as a temporary measure. |
| 9 | +MOBIE_FOLDER = "/mnt/lustre-emmy-hdd/projects/nim00007/data/moser/lightsheet/mobie" |
| 10 | +SERVICE_ENDPOINT = "https://s3.gwdg.de/" |
| 11 | +BUCKET_NAME = "incucyte-general/lightsheet" |
| 12 | + |
| 13 | +# For MoBIE: |
| 14 | +# https://s3.gwdg.de/incucyte-general/lightsheet |
| 15 | + |
| 16 | + |
| 17 | +def read_s3_credentials(credential_file): |
| 18 | + key, secret = None, None |
| 19 | + with open(credential_file) as f: |
| 20 | + for line in f: |
| 21 | + if line.startswith("aws_access_key_id"): |
| 22 | + key = line.rstrip("\n").strip().split(" ")[-1] |
| 23 | + if line.startswith("aws_secret_access_key"): |
| 24 | + secret = line.rstrip("\n").strip().split(" ")[-1] |
| 25 | + if key is None or secret is None: |
| 26 | + raise ValueError(f"Invalid credential file {credential_file}") |
| 27 | + return key, secret |
| 28 | + |
| 29 | + |
| 30 | +def create_s3_target(url, anon=False, credential_file=None): |
| 31 | + client_kwargs = {"endpoint_url": url} |
| 32 | + if credential_file is not None: |
| 33 | + key, secret = read_s3_credentials(credential_file) |
| 34 | + fs = s3fs.S3FileSystem(key=key, secret=secret, client_kwargs=client_kwargs) |
| 35 | + else: |
| 36 | + fs = s3fs.S3FileSystem(anon=anon, client_kwargs=client_kwargs) |
| 37 | + return fs |
| 38 | + |
| 39 | + |
| 40 | +def remote_metadata(): |
| 41 | + add_remote_project_metadata(MOBIE_FOLDER, BUCKET_NAME, SERVICE_ENDPOINT) |
| 42 | + |
| 43 | + |
| 44 | +def upload_data(): |
| 45 | + target = create_s3_target( |
| 46 | + SERVICE_ENDPOINT, |
| 47 | + credential_file="./credentials.incucyte" |
| 48 | + ) |
| 49 | + to_upload = [] |
| 50 | + for root, dirs, files in os.walk(MOBIE_FOLDER): |
| 51 | + dirs.sort() |
| 52 | + for ff in files: |
| 53 | + if ff.endswith(".xml"): |
| 54 | + to_upload.append(os.path.join(root, ff)) |
| 55 | + |
| 56 | + print("Uploading", len(to_upload), "files to") |
| 57 | + |
| 58 | + for path in tqdm(to_upload): |
| 59 | + rel_path = os.path.relpath(path, MOBIE_FOLDER) |
| 60 | + target.put( |
| 61 | + path, os.path.join(BUCKET_NAME, rel_path) |
| 62 | + ) |
| 63 | + |
| 64 | + |
| 65 | +# FIXME: access via s3 is not working due to permission issues. |
| 66 | +# Maybe this is not working due to bdv fileformat?! |
| 67 | +# Make an issue in MoBIE. |
| 68 | +def main(): |
| 69 | + # remote_metadata() |
| 70 | + upload_data() |
| 71 | + |
| 72 | + |
| 73 | +if __name__ == "__main__": |
| 74 | + main() |
0 commit comments