🐛 Fix storage for IBM COS (api_key) non-read ops. (#1447)

Co-authored-by: German Telmo Eizaguirre Suarez <germantelmo.eizaguirre@urv.cat>

Author: kikemolina3

config/config_template.yaml

@@ -141,6 +141,7 @@
 #ibm_cos:
     #storage_bucket: <BUCKET_NAME>
     #region: <REGION>
+    #service_instance_id: <SERVICE_INSTANCE_ID>
     #endpoint: <REGION_ENDPOINT>
     #private_endpoint: <PRIVATE_REGION_ENDPOINT>
     #api_key: <API_KEY>

docs/source/storage_config/ibm_cos.md

@@ -127,12 +127,13 @@ Alternative to using region, you can configure the public and private endpoints
 
 ### IBM Cloud Object Storage:
 
-|Group|Key|Default|Mandatory|Additional info|
-|---|---|---|---|---|
-|ibm_cos | region | |yes | Region of your bucket. One of: `eu-gb`, `eu-de`, `us-south`, `us-east`, `br-sao`, `ca-tor`, `jp-tok`, `jp-osa`, `au-syd`. Lithops will use the region set under the `ibm` section if it is not set here|
-|ibm_cos | api_key | |yes | API Key to your COS account. Not needed if using IAM API Key|
-|ibm_cos | storage_bucket | | yes | The name of a bucket that exists in you account. This will be used by Lithops for intermediate data. You must provide HMAC Credentials if you want the bucket to be automatically created |
-|ibm_cos | access_key_id | |no | HMAC Credentials. **Mandatory** if no api_key. Not needed if using IAM API Key|
-|ibm_cos | secret_access_key | |no | HMAC Credentials. **Mandatory** if no api_key. Not needed if using IAM API Key|
-|ibm_cos | endpoint | |no | Endpoint to your COS account. **Mandatory** if no region. Make sure to use the full path with 'https://' as prefix |
-|ibm_cos | private_endpoint | |no | Private endpoint to your COS account. **Mandatory** if no region. Make sure to use the full path with 'https://' or http:// as prefix |
+| Group   | Key                 |Default|Mandatory| Additional info                                                                                                                                                                                         |
+|---------|---------------------|---|---|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ibm_cos | region              | |yes | Region of your bucket. One of: `eu-gb`, `eu-de`, `us-south`, `us-east`, `br-sao`, `ca-tor`, `jp-tok`, `jp-osa`, `au-syd`. Lithops will use the region set under the `ibm` section if it is not set here |
+| ibm_cos | api_key             | |yes | API Key to your COS account. Not needed if using IAM API Key                                                                                                                                            |
+| ibm_cos | storage_bucket      | | yes | The name of a bucket that exists in you account. This will be used by Lithops for intermediate data. You must provide HMAC Credentials if you want the bucket to be automatically created               |
+| ibm_cos | service_instance_id | |no | The service instance (CRN format) of your COS instance. **Mandatory** if no HMAC Credentials provided.                                                                                                  |                                                                                                 |                                                                                                   |                                                                                                                                  |                                                                                                                                        |
+| ibm_cos | access_key_id       | |no | HMAC Credentials. **Mandatory** if no api_key. Not needed if using IAM API Key                                                                                                                          |
+| ibm_cos | secret_access_key   | |no | HMAC Credentials. **Mandatory** if no api_key. Not needed if using IAM API Key                                                                                                                          |
+| ibm_cos | endpoint            | |no | Endpoint to your COS account. **Mandatory** if no region. Make sure to use the full path with 'https://' as prefix                                                                                      |
+| ibm_cos | private_endpoint    | |no | Private endpoint to your COS account. **Mandatory** if no region. Make sure to use the full path with 'https://' or http:// as prefix                                                                   |

lithops/storage/backends/ibm_cos/config.py

@@ -65,7 +65,8 @@ def load_config(config_data):
         if 'private_endpoint' not in config_data['ibm_cos']:
             raise Exception('You must provide the private_endpoint to access to IBM COS')
         elif 'direct' not in config_data['ibm_cos']['private_endpoint']:
-            raise Exception('The private_endpoint you provided to access to IBM COS is not valid. You must use the "direct" endpoint')
+            raise Exception('The private_endpoint you provided to access to IBM COS is not valid. You must use the '
+                            '"direct" endpoint')
         if not config_data['ibm_cos']['private_endpoint'].startswith('http'):
             raise Exception('IBM COS Private Endpoint must start with http:// or https://')
 
@@ -78,3 +79,8 @@ def load_config(config_data):
     if 'region' not in config_data['ibm_cos']:
         endpoint = config_data['ibm_cos']['endpoint']
         config_data['ibm_cos']['region'] = endpoint.split('//')[1].split('.')[1]
+
+    if ('service_instance_id' not in config_data['ibm_cos']
+            and {'access_key_id', 'secret_access_key'}.isdisjoint(config_data['ibm_cos'])):
+        raise Exception('You must provide the service_instance_id or the (access_key_id,secret_access_key) to '
+                        'access to IBM COS')

lithops/storage/backends/ibm_cos/ibm_cos.py

@@ -103,7 +103,8 @@ def __init__(self, ibm_cos_config):
                 aws_secret_access_key="",
                 aws_session_token=token,
                 config=client_config,
-                endpoint_url=service_endpoint
+                endpoint_url=service_endpoint,
+                ibm_service_instance_id=self.config.get('service_instance_id')
             )
 
         msg = STORAGE_CLI_MSG.format('IBM COS')