@@ -174,18 +174,11 @@ def url(self, container_name, object_name):
174
174
175
175
class CloudObjectStorage (object ):
176
176
177
- def __init__ (self , sparkcontext , credentials , bucket_name = '' ):
177
+ def __init__ (self , sparkcontext , credentials , cos_id = '' , bucket_name = '' ):
178
178
179
179
'''
180
180
sparkcontext: a SparkContext object.
181
181
182
- bucket_name (projectId in DSX) [optional]: string that identifies the bucket you want
183
- to access files from in the COS service instance.
184
- In DSX, bucket_name is the same as projectId. One bucket is
185
- associated with one project.
186
- If this value is not specified, you need to pass it when
187
- you use the url function.
188
-
189
182
credentials: a dictionary with the following required keys:
190
183
* endpoint
191
184
* access_key
@@ -195,8 +188,24 @@ def __init__(self, sparkcontext, credentials, bucket_name=''):
195
188
in DSX - Notebooks by clicking on the datasources palette then
196
189
choose the datasource you want to access then hit insert credentials.
197
190
191
+ cos_id [optional]: this parameter is the cloud object storage unique id. It is useful
192
+ to keep in the class instance for further checks after the initialization. However,
193
+ it is not mandatory for the class instance to work. This value can be retrieved by
194
+ calling the get_os_id function.
195
+
196
+ bucket_name (projectId in DSX) [optional]: string that identifies the defult
197
+ bucket nameyou want to access files from in the COS service instance.
198
+ In DSX, bucket_name is the same as projectId. One bucket is
199
+ associated with one project.
200
+ If this value is not specified, you need to pass it when
201
+ you use the url function.
202
+
203
+ Warning: creating a new instance of this class would overwrite the existing
204
+ spark hadoop configs if set before if used with the same spark context instance.
205
+
198
206
'''
199
207
self .bucket_name = bucket_name
208
+ self .cos_id = cos_id
200
209
201
210
# check if all required values are availble
202
211
credential_key_list = ["endpoint" , "access_key" , "secret_key" ]
@@ -213,6 +222,9 @@ def __init__(self, sparkcontext, credentials, bucket_name=''):
213
222
hconf .set (prefix + ".access.key" , credentials ['access_key' ])
214
223
hconf .set (prefix + ".secret.key" , credentials ['secret_key' ])
215
224
225
+ def get_os_id ():
226
+ return self .cos_id
227
+
216
228
def url (self , object_name , bucket_name = '' ):
217
229
bucket_name_var = ''
218
230
if (bucket_name ):
0 commit comments