support cos scheme and multiple config.

Author: bassel-zeidan

python/ibmos2spark/osconfig.py

@@ -19,6 +19,8 @@
 
 import warnings
 
+DEFAULT_SERVICE_NAME = "service"
+
 def swifturl2d(name, container_name, object_name):
   return 'swift2d://{}.{}/{}'.format(container_name, name, object_name)
 
@@ -216,7 +218,13 @@ def __init__(self, sparkcontext, credentials, cos_id='', bucket_name=''):
                 raise ValueError("Invalid input: credentials.{} is required!".format(key))
 
         # setup config
-        prefix = "fs.s3d.service"
+        prefix = "fs.cos"
+
+        if (cos_id):
+            prefix = "{}.{}".format(prefix, cos_id)
+        else:
+            prefix = prefix + "." + DEFAULT_SERVICE_NAME
+
         hconf = sparkcontext._jsc.hadoopConfiguration()
         hconf.set(prefix + ".endpoint", credentials['endpoint'])
         hconf.set(prefix + ".access.key", credentials['access_key'])
@@ -227,11 +235,18 @@ def get_os_id():
 
     def url(self, object_name, bucket_name=''):
         bucket_name_var = ''
+        service_name = DEFAULT_SERVICE_NAME
+
+        # determine the bucket to use
         if (bucket_name):
             bucket_name_var = bucket_name
         elif (self.bucket_name):
             bucket_name_var = self.bucket_name
         else:
             raise ValueError("Invalid input: bucket_name is required!")
 
-        return "s3d://{}.service/{}".format(bucket_name_var, object_name)
+        # use service name that we set up hadoop config for
+        if (self.cos_id):
+            service_name = self.cos_id
+
+        return "cos://{}.{}/{}".format(bucket_name_var, service_name, object_name)