Releasing version 3.20.3

Releasing version 3.20.3

Author: HamadaGabrIbrahim

CHANGELOG.rst

@@ -6,9 +6,15 @@ All notable changes to this project will be documented in this file.
 
 The format is based on `Keep a Changelog <http://keepachangelog.com/>`__.
 
-3.20.2 - 2022-11-15
+3.20.3 - 2022-11-22
 --------------------
+Fixed
+~~~~~
 
+* Upgraded the cryptography version to (>=3.2.1,<39.0.0) to fix the `OpenSSL Security bug <https://www.openssl.org/blog/blog/2022/11/01/email-address-overflows/>`_
+
+3.20.2 - 2022-11-15
+--------------------
 Added
 ~~~~~
 

THIRD_PARTY_LICENSES.txt

@@ -8,13 +8,13 @@ certifi
 cffi>=1.9.1
 click==7.1.2
 coverage==4.5.2
-cryptography>=3.2.1,<=37.0.2
+cryptography>=3.2.1,<39.0.0
 httpsig-cffi==15.0.0
 Jinja2==3.0.3
 jmespath==0.10.0
 ndg-httpsclient==0.4.2
 mock==2.0.0
-oci==2.88.1
+oci==2.88.2
 packaging==20.2
 pluggy==0.13.0
 py==1.10.0

requirements.txt

@@ -304,13 +304,12 @@ def install_cli(install_dir, tmp_dir, version, optional_features, dependency_dir
     # Check if we should install a local full-install bundle.
     oci_cli_whl_files = glob.glob(match_wheel)
     if os.path.exists('./' + dependency_dir) and len(oci_cli_whl_files) > 0:
+        dependency_dir = DEFAULT_DEPENDENCY_DIR + '/python{}{}.html'.format(sys.version_info.major, sys.version_info.minor)
         print_status("Installing {} from local resources.".format(oci_cli_whl_files[0]))
         cmd = [path_to_pip, 'install', '--cache-dir', tmp_dir, oci_cli_whl_files[0], '--upgrade', '--find-links', dependency_dir]
 
     elif OFFLINE_INSTALL:
-        # # Since cffi is a 4th party library which cryptography uses, it needs to be installed first in the offline installation
-        # cmd = [path_to_pip, 'install', 'cffi', '--find-links', dependency_dir, '--no-index']
-        # exec_command(cmd, env=env)
+        dependency_dir = DEFAULT_DEPENDENCY_DIR + '/python{}{}.html'.format(sys.version_info.major, sys.version_info.minor)
         cmd = [path_to_pip, 'install', cli_package_name, '--find-links', dependency_dir, '--no-index', '--ignore-requires-python']
     else:
         cmd = [path_to_pip, 'install', '--cache-dir', tmp_dir, cli_package_name, '--upgrade']
@@ -626,9 +625,7 @@ def main():
     global OFFLINE_INSTALL
     OFFLINE_INSTALL = args.offline_install
     dependency_dir = args.dependency_dir
-    if dependency_dir is None and OFFLINE_INSTALL:
-        dependency_dir = DEFAULT_DEPENDENCY_DIR + '/python{}{}'.format(sys.version_info.major, sys.version_info.minor)
-    elif dependency_dir is None:
+    if dependency_dir is None:
         dependency_dir = DEFAULT_DEPENDENCY_DIR
     global DRY_RUN
     DRY_RUN = args.dry_run

scripts/install/install.py

@@ -0,0 +1,10 @@
+# coding: utf-8
+# Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
+
+import os
+import shutil
+
+
+def remove_dir_at_path(path):
+    if os.path.exists(path):
+        shutil.rmtree(path)

services/object_storage/tests/common/util.py

@@ -932,7 +932,8 @@ def create_empty_folders_local(path, no_of_folders_to_create):
 
 def cleanup_files_from_local(file_set):
     for file in file_set:
-        os.remove(file)
+        if os.path.exists(file):
+            os.remove(file)
 
 
 def create_new_objects_remote(client, bucket_name, no_of_files_to_create, with_content=False, extension='txt',

services/object_storage/tests/integ/test_object_storage_sync_download.py

@@ -16,6 +16,7 @@
 from test_object_storage_sync_download import parse_dry_run_result, \
     generate_random_string, create_new_objects_remote, create_new_files_local, cleanup_files_from_local
 from tests import util
+import services.object_storage.tests.common.util as object_storage_util
 
 OBJECTS_TO_CREATE_IN_REMOTE_FOR_SYNC = 20
 
@@ -30,6 +31,10 @@
     '': []
 }
 
+new_path = 'new_path'
+new_files_set = 'new_files_set'
+new_dir = 'new_dir'
+
 
 @pytest.fixture(params=[False])
 def debug(request):
@@ -86,6 +91,14 @@ def teardown(debug):
                            sync_upload_bucket_name, '--force'], debug=debug)
 
 
+@pytest.fixture()
+def cleanup_new_content_set():
+    data = {new_files_set: {}, new_dir: ''}
+    yield data
+    cleanup_files_from_local(data[new_files_set])
+    object_storage_util.remove_dir_at_path(data[new_dir])
+
+
 @util.skip_while_rerecording
 def test_sync_src_dry_run(object_storage_client, debug):
     """
@@ -162,7 +175,7 @@ def test_sync_src_updated_objects(object_storage_client):
 
 
 @util.skip_while_rerecording
-def test_sync_src_new_objects(object_storage_client):
+def test_sync_src_new_objects(object_storage_client, cleanup_new_content_set):
     """
     1. Upload the files to remote and validate the output and file contents
     2. Create a new file in local
@@ -176,6 +189,7 @@ def test_sync_src_new_objects(object_storage_client):
     new_object_name = 'a/new_object'
     new_file_contents = {new_object_name: bulk_operation.generate_random_string(bulk_operation.CONTENT_STRING_LENGTH)}
     new_file_path = os.path.join(sync_upload_test_dir, new_object_name)
+    cleanup_new_content_set[new_files_set] = {new_file_path}
     with open(new_file_path, 'w') as fh:
         fh.write(new_file_contents[new_object_name])
 
@@ -425,7 +439,7 @@ def test_sync_src_with_delete_paging(object_storage_client, debug):
 
 
 @util.skip_while_rerecording
-def test_sync_src_with_delete_and_include(object_storage_client):
+def test_sync_src_with_delete_and_include(object_storage_client, cleanup_new_content_set):
     """
     1. Create new set of objects in local with .pdf and .doc extensions.
     2. Perform a dry run with --delete to include only *.pdf and validate that only .pdf files are transferred.
@@ -439,6 +453,8 @@ def test_sync_src_with_delete_and_include(object_storage_client):
 
     l_file_set_1 = create_new_files_local(sync_upload_test_dir, 3, extension='pdf')
     l_file_set_2 = create_new_files_local(sync_upload_test_dir, 5, extension='doc')
+    cleanup_new_content_set[new_files_set] = l_file_set_1.union(l_file_set_2)
+    cleanup_new_content_set[new_dir] = os.path.join(sync_upload_test_dir, 'dir')
 
     result = bulk_operation.invoke(['os', 'object', 'sync', '--namespace', util.NAMESPACE, '--bucket-name',
                                     sync_upload_bucket_name, '--src-dir', sync_upload_test_dir, '--delete',
@@ -479,11 +495,9 @@ def test_sync_src_with_delete_and_include(object_storage_client):
     assert parsed_result['skipped-objects'] == []
     assert set(parsed_result['deleted-objects']) == r_obj_set_1
 
-    cleanup_files_from_local(l_file_set_1.union(l_file_set_2))
-
 
 @util.skip_while_rerecording
-def test_sync_src_with_delete_and_exclude(object_storage_client, debug):
+def test_sync_src_with_delete_and_exclude(object_storage_client, debug, cleanup_new_content_set):
     """
     1. Create new set of objects in local with .pdf and .doc extensions.
     2. Perform a dry run with --delete to exclude only *.pdf and validate that all other files are transferred.
@@ -497,6 +511,8 @@ def test_sync_src_with_delete_and_exclude(object_storage_client, debug):
 
     l_file_set_1 = create_new_files_local(sync_upload_test_dir, 3, extension='pdf')
     l_file_set_2 = create_new_files_local(sync_upload_test_dir, 5, extension='doc')
+    cleanup_new_content_set[new_files_set] = l_file_set_1.union(l_file_set_2)
+    cleanup_new_content_set[new_dir] = os.path.join(sync_upload_test_dir, 'dir')
 
     result = bulk_operation.invoke(['os', 'object', 'sync', '--namespace', util.NAMESPACE, '--bucket-name',
                                     sync_upload_bucket_name, '--src-dir', sync_upload_test_dir, '--delete',
@@ -537,8 +553,6 @@ def test_sync_src_with_delete_and_exclude(object_storage_client, debug):
     assert parsed_result['skipped-objects'] == []
     assert set(parsed_result['deleted-objects']) == r_obj_set_2
 
-    cleanup_files_from_local(l_file_set_1.union(l_file_set_2))
-
 
 @util.skip_while_rerecording
 def test_sync_src_with_delete_and_prefix(object_storage_client, debug):
@@ -601,7 +615,7 @@ def test_sync_src_with_delete_and_prefix(object_storage_client, debug):
 
 
 @util.skip_while_rerecording
-def test_sync_src_with_delete_include_and_prefix(object_storage_client, debug):
+def test_sync_src_with_delete_include_and_prefix(object_storage_client, debug, cleanup_new_content_set):
     """
     Assert that scope of file transfer and delete is only limited to --include pattern within the bucket prefix
 
@@ -621,6 +635,8 @@ def test_sync_src_with_delete_include_and_prefix(object_storage_client, debug):
 
     l_file_set_1 = create_new_files_local(sync_upload_test_dir, 6, extension='pdf')
     l_file_set_2 = create_new_files_local(sync_upload_test_dir, 4, extension='doc')
+    cleanup_new_content_set[new_files_set] = l_file_set_1.union(l_file_set_2)
+    cleanup_new_content_set[new_dir] = os.path.join(sync_upload_test_dir, 'dir')
 
     result = bulk_operation.invoke(['os', 'object', 'sync', '--namespace', util.NAMESPACE, '--bucket-name',
                                     sync_upload_bucket_name, '--src-dir', sync_upload_test_dir, '--prefix',
@@ -663,11 +679,10 @@ def test_sync_src_with_delete_include_and_prefix(object_storage_client, debug):
     assert set(parsed_result['uploaded-objects']) == set([(os.path.join(_prefix, f[len(sync_upload_test_dir):].replace(os.sep, '/').strip('/'))) for f in l_file_set_1.union(l_file_set_2)])
     assert parsed_result['skipped-objects'] == []
     assert set(parsed_result['deleted-objects']) == r_obj_set_1
-    cleanup_files_from_local(l_file_set_1.union(l_file_set_2))
 
 
 @util.skip_while_rerecording
-def test_sync_src_with_delete_exclude_and_prefix(object_storage_client, debug):
+def test_sync_src_with_delete_exclude_and_prefix(object_storage_client, debug, cleanup_new_content_set):
     """
     Assert that scope of file transfer and delete is only limited to --exclude pattern within the bucket prefix
 
@@ -686,6 +701,8 @@ def test_sync_src_with_delete_exclude_and_prefix(object_storage_client, debug):
     _prefix = 'upload_prefix_exclude_delete/'
     l_file_set_1 = create_new_files_local(sync_upload_test_dir, 6, extension='pdf')
     l_file_set_2 = create_new_files_local(sync_upload_test_dir, 4, extension='doc')
+    cleanup_new_content_set[new_files_set] = l_file_set_1.union(l_file_set_2)
+    cleanup_new_content_set[new_dir] = os.path.join(sync_upload_test_dir, 'dir')
 
     result = bulk_operation.invoke(['os', 'object', 'sync', '--namespace', util.NAMESPACE, '--bucket-name',
                                     sync_upload_bucket_name, '--src-dir', sync_upload_test_dir, '--prefix',
@@ -728,7 +745,6 @@ def test_sync_src_with_delete_exclude_and_prefix(object_storage_client, debug):
     assert set(parsed_result['uploaded-objects']) == set([(os.path.join(_prefix, o).replace(os.sep, '/')) for o in sync_local_object_content.keys()])
     assert parsed_result['skipped-objects'] == []
     assert set(parsed_result['deleted-objects']) == r_obj_set_2
-    cleanup_files_from_local(l_file_set_1.union(l_file_set_2))
 
 
 @util.skip_while_rerecording

services/object_storage/tests/integ/test_object_storage_sync_upload.py

@@ -469,7 +469,7 @@ def test_preauth_request_create(self):
         assert "Error: Missing option(s) --bucket-name, --name, --access-type, --time-expires." in result.output
 
         result = util.invoke_command(['os', 'preauth-request', 'create', '--bucket-name', 'b001', '--name', 'par', '--access-type', 'new-access', '--time-expires', '2017-09-15T20:30:00.123456Z'])
-        assert "Error: Invalid value for '--access-type': invalid choice: new-access. (choose from ObjectRead, ObjectWrite, ObjectReadWrite, AnyObjectWrite, AnyObjectRead, AnyObjectReadWrite)" in result.output
+        assert "Error: Invalid value for '--access-type': invalid choice: new-access." in result.output
 
     def test_preauth_request_delete(self):
         result = util.invoke_command(['os', 'preauth-request', 'delete'])

services/object_storage/tests/unit/test_objectstorage_extended.py

@@ -96,6 +96,71 @@ def modify_image_workload_name(image_workload_name):
     return image_workload_name
 
 
+def get_object_storage_helper(ctx):
+    # Override the region for r1 for os public endpoint
+    if 'config' in ctx.obj:
+        if 'region' in ctx.obj['config'] and ctx.obj['config']['region']:  # region present in ~/.oci/config
+            if ctx.obj['config']['region'].strip().split(".")[0] == "r1":
+                region = ctx.obj['config']['region']
+                ctx.obj['region'] = "r1.oracleiaas.com"
+                object_cli = cli_util.build_client('object_storage', 'object_storage', ctx)
+                ctx.obj['region'] = region
+                return object_cli
+    return cli_util.build_client('object_storage', 'object_storage', ctx)
+
+
+def validate_get_image(ctx, **kwargs):
+    if 'image_id' in kwargs and not kwargs['image_id']:
+        raise click.UsageError('Parameter image-id cannot be whitespace or empty string')
+    image_id = kwargs['image_id']
+    try:
+        compute_image_obj = get_compute_image_helper(ctx, image_id)
+        if compute_image_obj is None or compute_image_obj.data is None:
+            raise click.UsageError("Image not authorized or not found")
+    except Exception as e:
+        raise click.UsageError("Image not authorized or not found")
+    return compute_image_obj
+
+
+def validate_bucket(compartment_id, ctx, **kwargs):
+    if kwargs['type'].lower() == "bucket":
+        if not ('bucket_name' in kwargs and kwargs['bucket_name']):
+            raise click.UsageError('Parameter bucket-name cannot be whitespace or empty string')
+        try:
+            object_storage_obj = get_object_storage_helper(ctx)
+            kwargs_os = {
+                'compartment_id': compartment_id}
+            namespace = object_storage_obj.get_namespace(**kwargs_os).data
+            result = object_storage_obj.get_bucket(
+                namespace_name=namespace,
+                bucket_name=kwargs['bucket_name']
+            )
+            if result is None or result.data is None:
+                raise click.UsageError("Bucket not authorized or not found")
+        except Exception as e:
+            raise click.UsageError("Bucket not authorized or not found")
+    return result
+
+
+def prepare_bucket_workload_data(result_bucket, **kwargs):
+    workload_data = [{
+        "workloadType": "BUCKET", "id": result_bucket.data.name, "name": kwargs['bucket_name'],
+        "compartmentId": result_bucket.data.compartment_id,
+        'prefix': kwargs['prefix'], 'rangeStart': kwargs['range_start'], 'rangeEnd': kwargs['range_end']
+    }]
+    return workload_data
+
+
+def prepare_image_workload_data(compute_image_obj, image_id):
+    compute_image_obj_name = modify_image_workload_name(compute_image_obj.data.display_name)
+    workload_data = [
+        {'workloadType': "IMAGE", 'id': image_id, 'name': compute_image_obj_name,
+         'size': compute_image_obj.data.size_in_mbs, 'compartmentId': compute_image_obj.data.compartment_id,
+         }
+    ]
+    return workload_data
+
+
 def create_master_key_policy_rover_resource(resource_name, ctx, **kwargs):
     confirm_prompt_policy = "You are providing your own master key ID, please create a policy to allow " \
                             "Roving Edge Infrastructure to create DEKs and use the master key ID to encrypt " \
@@ -201,66 +266,3 @@ def remove_additional_params_after_policy(**kwargs):
     if 'policy_compartment_id' in kwargs:
         kwargs.pop('policy_compartment_id')
     return kwargs
-
-
-def get_object_storage_helper(ctx):
-    # Override the region for r1 for os public endpoint
-    if 'config' in ctx.obj:
-        if 'region' in ctx.obj['config'] and ctx.obj['config']['region']:  # region present in ~/.oci/config
-            if ctx.obj['config']['region'].strip().split(".")[0] == "r1":
-                region = ctx.obj['config']['region']
-                ctx.obj['region'] = "r1.oracleiaas.com"
-                object_cli = cli_util.build_client('object_storage', 'object_storage', ctx)
-                ctx.obj['region'] = region
-                return object_cli
-    return cli_util.build_client('object_storage', 'object_storage', ctx)
-
-
-def validate_get_image(ctx, **kwargs):
-    if 'image_id' in kwargs and not kwargs['image_id']:
-        raise click.UsageError('Parameter image-id cannot be whitespace or empty string')
-    image_id = kwargs['image_id']
-    try:
-        compute_image_obj = get_compute_image_helper(ctx, image_id)
-        if compute_image_obj is None or compute_image_obj.data is None:
-            raise click.UsageError("Image not authorized or not found")
-    except Exception as e:
-        raise click.UsageError("Image not authorized or not found")
-    return compute_image_obj
-
-
-def validate_bucket(ctx, **kwargs):
-    if kwargs['type'].lower() == "bucket":
-        if not ('bucket_name' in kwargs and kwargs['bucket_name']):
-            raise click.UsageError('Parameter bucket-name cannot be whitespace or empty string')
-        try:
-            object_storage_obj = get_object_storage_helper(ctx)
-            namespace = object_storage_obj.get_namespace().data
-            result = object_storage_obj.get_bucket(
-                namespace_name=namespace,
-                bucket_name=kwargs['bucket_name']
-            )
-            if result is None or result.data is None:
-                raise click.UsageError("Bucket not authorized or not found")
-        except Exception as e:
-            raise click.UsageError("Bucket not authorized or not found")
-    return result
-
-
-def prepare_bucket_workload_data(result_bucket, **kwargs):
-    workload_data = [{
-        "workloadType": "BUCKET", "id": result_bucket.data.name, "name": kwargs['bucket_name'],
-        "compartmentId": result_bucket.data.compartment_id,
-        'prefix': kwargs['prefix'], 'rangeStart': kwargs['range_start'], 'rangeEnd': kwargs['range_end']
-    }]
-    return workload_data
-
-
-def prepare_image_workload_data(compute_image_obj, image_id):
-    compute_image_obj_name = modify_image_workload_name(compute_image_obj.data.display_name)
-    workload_data = [
-        {'workloadType': "IMAGE", 'id': image_id, 'name': compute_image_obj_name,
-         'size': compute_image_obj.data.size_in_mbs, 'compartmentId': compute_image_obj.data.compartment_id,
-         }
-    ]
-    return workload_data

services/rover/src/oci_cli_rover/rover_utils.py

@@ -174,7 +174,7 @@ def add_workload_common(ctx, **kwargs):
     result = get_rover_cluster_helper(ctx, kwargs['cluster_id'])
 
     if kwargs['type'].lower() == "bucket":
-        result_bucket = validate_bucket(ctx, **kwargs)
+        result_bucket = validate_bucket(result.data.compartment_id, ctx, **kwargs)
 
         workload_id = result_bucket.data.name
         workload_data = prepare_bucket_workload_data(result_bucket, **kwargs)