|
| 1 | +# Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. |
| 2 | +# |
| 3 | +# Uploads all files from a local directory to an object storage bucket |
| 4 | +# using multiple processes so that the uploads are done in parallel. |
| 5 | +# |
| 6 | +# Assumptions: Object storage bucket already exists. See object_crud.py for |
| 7 | +# an example of creating a bucket. |
| 8 | +# Loads configuration from default profile in the default config |
| 9 | +# file |
| 10 | + |
| 11 | +import oraclebmc |
| 12 | +import os |
| 13 | +import argparse |
| 14 | +from multiprocessing import Process |
| 15 | +from glob import glob |
| 16 | + |
| 17 | + |
| 18 | +def upload_to_object_storage(config, namespace, bucket, path): |
| 19 | + """ |
| 20 | + upload_to_object_storage will upload a file to an object storage bucket. |
| 21 | + This function is intended to be run as a separate process. The client is |
| 22 | + created with each invocation so that the separate processes do |
| 23 | + not have a reference to the same client. |
| 24 | +
|
| 25 | + :param config: a configuration dictionary used to create ObjectStorageClient |
| 26 | + :param namespace: Namespace where the bucket resides |
| 27 | + :param bucket: Name of the bucket in which the object will be stored |
| 28 | + :param path: path to file to upload to object storage |
| 29 | + :rtype: None |
| 30 | + """ |
| 31 | + with open(path, "rb") as in_file: |
| 32 | + name = os.path.basename(path) |
| 33 | + ostorage = oraclebmc.object_storage.ObjectStorageClient(config) |
| 34 | + ostorage.put_object(namespace, |
| 35 | + bucket, |
| 36 | + name, |
| 37 | + in_file) |
| 38 | + print("Finished uploading {}".format(name)) |
| 39 | + |
| 40 | + |
| 41 | +if __name__ == "__main__": |
| 42 | + config = oraclebmc.config.from_file() |
| 43 | + object_storage = oraclebmc.object_storage.ObjectStorageClient(config) |
| 44 | + namespace = object_storage.get_namespace().data |
| 45 | + |
| 46 | + description = "\n".join(["This is an example to show how multiple files can be uploaded to in", |
| 47 | + "parallel. The example uses multiple processes.", |
| 48 | + "", |
| 49 | + "All the files in 'directory' will be uploaded to the object storage bucket", |
| 50 | + "specified by 'bucket_name' The default profile will is used.", |
| 51 | + "", |
| 52 | + "The bucket must already exist. See object_crud.py for a bucket creation", |
| 53 | + "example."]) |
| 54 | + |
| 55 | + parser = argparse.ArgumentParser(description=description, |
| 56 | + formatter_class=argparse.RawDescriptionHelpFormatter) |
| 57 | + parser.add_argument(dest='bucket_name', |
| 58 | + help="Name of object storage bucket") |
| 59 | + parser.add_argument(dest='directory', |
| 60 | + help="Path to local directory containing files to upload. Do not include trailing path delimiter.") |
| 61 | + args = parser.parse_args() |
| 62 | + |
| 63 | + bucket_name = args.bucket_name |
| 64 | + directory = args.directory |
| 65 | + if not os.path.isdir(directory): |
| 66 | + parser.usage() |
| 67 | + else: |
| 68 | + dir = directory + os.path.sep + "*" |
| 69 | + |
| 70 | + proc_list = [] |
| 71 | + for file_path in glob(dir): |
| 72 | + print("Starting upload for {}".format(file_path)) |
| 73 | + p = Process(target=upload_to_object_storage, args=(config, |
| 74 | + namespace, |
| 75 | + args.bucket_name, |
| 76 | + file_path)) |
| 77 | + p.start() |
| 78 | + proc_list.append(p) |
| 79 | + |
| 80 | + for job in proc_list: |
| 81 | + job.join() |
0 commit comments