diff --git a/examples/custom_crypto.py b/examples/custom_crypto.py index 0aca143d..e7c5d40b 100644 --- a/examples/custom_crypto.py +++ b/examples/custom_crypto.py @@ -5,6 +5,7 @@ import oss2 from oss2.crypto import BaseCryptoProvider from oss2.utils import b64encode_as_string, b64decode_from_string, to_bytes +from oss2.headers import * from Crypto.Cipher import PKCS1_OAEP from Crypto.PublicKey import RSA @@ -36,7 +37,7 @@ def get_key(): def get_start(): return 'fake_start' - def __init__(self, key=None, start=None): + def __init__(self, key=None, start=None, count=None): pass def encrypt(self, raw): @@ -74,22 +75,44 @@ def __init__(self, cipher=FakeCrypto): self.private_key = self.public_key - def build_header(self, headers=None): + def build_header(self, headers=None, multipart_context=None): if not isinstance(headers, CaseInsensitiveDict): headers = CaseInsensitiveDict(headers) if 'content-md5' in headers: - headers['x-oss-meta-unencrypted-content-md5'] = headers['content-md5'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] del headers['content-md5'] if 'content-length' in headers: - headers['x-oss-meta-unencrypted-content-length'] = headers['content-length'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] del headers['content-length'] - headers['x-oss-meta-oss-crypto-key'] = b64encode_as_string(self.public_key.encrypt(self.plain_key)) - headers['x-oss-meta-oss-crypto-start'] = b64encode_as_string(self.public_key.encrypt(to_bytes(str(self.plain_start)))) - headers['x-oss-meta-oss-cek-alg'] = self.cipher.ALGORITHM - headers['x-oss-meta-oss-wrap-alg'] = 'custom' + headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY] = b64encode_as_string(self.public_key.encrypt(self.plain_key)) + headers[OSS_CLIENT_SIDE_ENCRYPTION_START] = b64encode_as_string(self.public_key.encrypt(to_bytes(str(self.plain_start)))) + headers[OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG] = self.cipher.ALGORITHM + headers[OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG] = 'custom' + + # multipart file build header + if multipart_context: + headers[OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE] = str(multipart_context.data_size) + headers[OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE] = str(multipart_context.part_size) + + self.plain_key = None + self.plain_start = None + + return headers + + def build_header_for_upload_part(self, headers=None): + if not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + if 'content-md5' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] + del headers['content-md5'] + + if 'content-length' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] + del headers['content-length'] self.plain_key = None self.plain_start = None @@ -110,6 +133,12 @@ def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x): except: return None + def decrypt_from_str(self, key, value, conv=lambda x:x): + try: + return conv(self.private_key.decrypt(b64decode_from_string(value))) + except: + return None + # 首先初始化AccessKeyId、AccessKeySecret、Endpoint等信息。 @@ -162,4 +191,52 @@ def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x): with open(filename, 'rb') as fileobj: assert fileobj.read() == content -os.remove(filename) \ No newline at end of file +os.remove(filename) + +""" +分片上传 +""" +# 初始化上传分片 +part_a = b'a' * 1024 * 100 +part_b = b'b' * 1024 * 100 +part_c = b'c' * 1024 * 100 +multi_content = [part_a, part_b, part_c] + +parts = [] +data_size = 100 * 1024 * 3 +part_size = 100 * 1024 +multi_key = "test_crypto_multipart" + +res = bucket.init_multipart_upload(multi_key, data_size, part_size) +upload_id = res.upload_id +crypto_multipart_context = res.crypto_multipart_context; + +# 分片上传 +for i in range(3): + result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) + parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) + +## 分片上传时，若意外中断丢失crypto_multipart_context, 利用list_parts找回。 +#for i in range(2): +# result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) +# parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) +# +#res = bucket.list_parts(multi_key, upload_id) +#crypto_multipart_context_new = res.crypto_multipart_context +# +#result = bucket.upload_part(multi_key, upload_id, 3, multi_content[2], crypto_multipart_context_new) +#parts.append(oss2.models.PartInfo(3, result.etag, size = part_size, part_crc = result.crc)) + +# 完成上传 +result = bucket.complete_multipart_upload(multi_key, upload_id, parts) + +# 下载全部文件 +result = bucket.get_object(multi_key) + +# 验证一下 +content_got = b'' +for chunk in result: + content_got += chunk +assert content_got[0:102400] == part_a +assert content_got[102400:204800] == part_b +assert content_got[204800:307200] == part_c diff --git a/examples/object_crypto.py b/examples/object_crypto.py index 5576abd7..14d5c266 100644 --- a/examples/object_crypto.py +++ b/examples/object_crypto.py @@ -5,7 +5,7 @@ import oss2 from oss2.crypto import LocalRsaProvider, AliKMSProvider -# 以下代码展示了客户端文件加密上传下载的用法，如下载文件、上传文件等，注意在客户端加密的条件下，oss暂不支持文件分片上传下载操作。 +# 以下代码展示了客户端文件加密上传下载的用法，如下载文件、上传文件等。 # 首先初始化AccessKeyId、AccessKeySecret、Endpoint等信息。 @@ -30,11 +30,9 @@ content = b'a' * 1024 * 1024 filename = 'download.txt' - -# 创建Bucket对象，可以进行客户端数据加密(用户端RSA)，此模式下只提供对象整体上传下载操作 +# 创建Bucket对象，可以进行客户端数据加密(用户端RSA) bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, crypto_provider=LocalRsaProvider()) -key1 = 'motto-copy.txt' # 上传文件 bucket.put_object(key, content, headers={'content-length': str(1024 * 1024)}) @@ -62,12 +60,67 @@ os.remove(filename) +# 下载部分文件 +result = bucket.get_object(key, byte_range=(32,1024)) -# 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS)，此模式下只提供对象整体上传下载操作 -bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, - crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk, '1234')) +#验证一下 +content_got = b'' +for chunk in result: + content_got +=chunk +assert content_got == content[32:1025] + + +""" +分片上传 +""" +# 初始化上传分片 +part_a = b'a' * 1024 * 100 +part_b = b'b' * 1024 * 100 +part_c = b'c' * 1024 * 100 +multi_content = [part_a, part_b, part_c] + +parts = [] +data_size = 100 * 1024 * 3 +part_size = 100 * 1024 +multi_key = "test_crypto_multipart" + +res = bucket.init_multipart_upload(multi_key, data_size, part_size) +upload_id = res.upload_id +crypto_multipart_context = res.crypto_multipart_context; + +# 分片上传 +for i in range(3): + result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) + parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) + +## 分片上传时，若意外中断丢失crypto_multipart_context, 利用list_parts找回。 +#for i in range(2): +# result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) +# parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) +# +#res = bucket.list_parts(multi_key, upload_id) +#crypto_multipart_context_new = res.crypto_multipart_context +# +#result = bucket.upload_part(multi_key, upload_id, 3, multi_content[2], crypto_multipart_context_new) +#parts.append(oss2.models.PartInfo(3, result.etag, size = part_size, part_crc = result.crc)) + +# 完成上传 +result = bucket.complete_multipart_upload(multi_key, upload_id, parts) + +# 下载全部文件 +result = bucket.get_object(multi_key) -key1 = 'motto-copy.txt' +# 验证一下 +content_got = b'' +for chunk in result: + content_got += chunk +assert content_got[0:102400] == part_a +assert content_got[102400:204800] == part_b +assert content_got[204800:307200] == part_c + +# 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS) +bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, + crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk)) # 上传文件 bucket.put_object(key, content, headers={'content-length': str(1024 * 1024)}) @@ -93,4 +146,61 @@ with open(filename, 'rb') as fileobj: assert fileobj.read() == content -os.remove(filename) \ No newline at end of file +os.remove(filename) + +# 下载部分文件 +result = bucket.get_object(key, byte_range=(32,1024)) + +#验证一下 +content_got = b'' +for chunk in result: + content_got +=chunk +assert content_got == content[32:1025] + +""" +分片上传 +""" +# 初始化上传分片 +part_a = b'a' * 1024 * 100 +part_b = b'b' * 1024 * 100 +part_c = b'c' * 1024 * 100 +multi_content = [part_a, part_b, part_c] + +parts = [] +data_size = 100 * 1024 * 3 +part_size = 100 * 1024 +multi_key = "test_crypto_multipart" + +res = bucket.init_multipart_upload(multi_key, data_size, part_size) +upload_id = res.upload_id +crypto_multipart_context = res.crypto_multipart_context; + +# 分片上传 +for i in range(3): + result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) + parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) + +## 分片上传时，若意外中断丢失crypto_multipart_context, 利用list_parts找回。 +#for i in range(2): +# result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) +# parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) +# +#res = bucket.list_parts(multi_key, upload_id) +#crypto_multipart_context_new = res.crypto_multipart_context +# +#result = bucket.upload_part(multi_key, upload_id, 3, multi_content[2], crypto_multipart_context_new) +#parts.append(oss2.models.PartInfo(3, result.etag, size = part_size, part_crc = result.crc)) + +# 完成上传 +result = bucket.complete_multipart_upload(multi_key, upload_id, parts) + +# 下载全部文件 +result = bucket.get_object(multi_key) + +# 验证一下 +content_got = b'' +for chunk in result: + content_got += chunk +assert content_got[0:102400] == part_a +assert content_got[102400:204800] == part_b +assert content_got[204800:307200] == part_c diff --git a/oss2/api.py b/oss2/api.py index 8daf5677..160f1e2c 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -177,6 +177,8 @@ def progress_callback(bytes_consumed, total_bytes): from .crypto import BaseCryptoProvider from .headers import * +from .utils import calc_aes_ctr_offset_by_data_offset, is_valid_crypto_part_size, determine_crypto_part_size + import time import shutil import base64 @@ -621,6 +623,8 @@ def get_object(self, key, resp = self.__do_object('GET', key, headers=headers, params=params) logger.debug("Get object done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) + if models._hget(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY): + raise ClientError('Could not use normal bucket to decrypt an encrypted object') return GetObjectResult(resp, progress_callback, self.enable_crc) def select_object(self, key, sql, @@ -1173,7 +1177,7 @@ def upload_part_copy(self, source_bucket_name, source_key, byte_range, return PutObjectResult(resp) def list_parts(self, key, upload_id, - marker='', max_parts=1000): + marker='', max_parts=1000, headers=None): """列举已经上传的分片。支持分页。 :param str key: 文件名 @@ -1188,7 +1192,8 @@ def list_parts(self, key, upload_id, resp = self.__do_object('GET', key, params={'uploadId': upload_id, 'part-number-marker': marker, - 'max-parts': str(max_parts)}) + 'max-parts': str(max_parts)}, + headers=headers) logger.debug("List parts done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) return self._parse_result(resp, xml_utils.parse_list_parts, ListPartsResult) @@ -1599,7 +1604,7 @@ def __convert_data(self, klass, converter, data): return data -class CryptoBucket(): +class CryptoBucket(_Base): """用于加密Bucket和Object操作的类，诸如上传、下载Object等。创建、删除bucket的操作需使用Bucket类接口。 用法（假设Bucket属于杭州区域） :: @@ -1641,6 +1646,11 @@ def __init__(self, auth, endpoint, bucket_name, crypto_provider, if not isinstance(crypto_provider, BaseCryptoProvider): raise ClientError('Crypto bucket must provide a valid crypto_provider') + logger.debug("Init oss crypto bucket, endpoint: {0}, isCname: {1}, connect_timeout: {2}, app_name: {3}, enabled_crc: " + "{4}".format(endpoint, is_cname, connect_timeout, app_name, enable_crc)) + super(CryptoBucket, self).__init__(auth, endpoint, is_cname, session, connect_timeout, + app_name, enable_crc) + self.crypto_provider = crypto_provider self.bucket_name = bucket_name.strip() self.enable_crc = enable_crc @@ -1703,6 +1713,7 @@ def put_object_from_file(self, key, filename, return self.put_object(key, f, headers=headers, progress_callback=progress_callback) def get_object(self, key, + byte_range=None, headers=None, progress_callback=None, params=None): @@ -1715,6 +1726,7 @@ def get_object(self, key, 'hello world' :param key: 文件名 + :param byte_range: 指定下载范围。参见 :ref:`byte_range` :param headers: HTTP头部 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict @@ -1730,12 +1742,23 @@ def get_object(self, key, """ headers = http.CaseInsensitiveDict(headers) - if 'range' in headers: - raise ClientError('Crypto bucket do not support range get') + if byte_range and (not utils.is_multiple_sizeof_encrypt_block(byte_range[0])): + raise ClientError('Crypto bucket get range start must align to encrypt block') + + range_string = _make_range_string(byte_range) + if range_string: + headers['range'] = range_string + + params = {} if params is None else params - encrypted_result = self.bucket.get_object(key, headers=headers, params=params, progress_callback=None) + logger.debug("Start to get object, bucket: {0}， key: {1}, range: {2}, headers: {3}, params: {4}".format( + self.bucket_name, to_string(key), range_string, headers, params)) + resp = self.__do_object('GET', key, headers=headers, params=params) + logger.debug("Get object done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) - return GetObjectResult(encrypted_result.resp, progress_callback, self.enable_crc, + if models._hget(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY) is None: + raise ClientError('Could not use crypto bucket to decrypt an unencrypted object') + return GetObjectResult(resp, progress_callback, self.enable_crc, crypto_provider=self.crypto_provider) def get_object_to_file(self, key, filename, @@ -1768,6 +1791,150 @@ def get_object_to_file(self, key, filename, return result + def init_multipart_upload(self, key, data_size, part_size = None, headers=None): + """客户端加密初始化分片上传。 + + :param str key: 待上传的文件名 + :param int data_size : 待上传文件总大小 + :param int part_size : 后续分片上传时除最后一个分片之外的其他分片大小 + + :param headers: HTTP头部 + :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict + + :return: :class:`InitMultipartUploadResult ` + 返回值中的 `crypto_multipart_context` 记录了加密Meta信息，在upload_part时需要一并传入 + """ + if part_size is not None: + res = is_valid_crypto_part_size(part_size, data_size) + if not res: + raise ClientError("Crypto bucket get an invalid part_size") + else: + part_size = determine_crypto_part_size(data_size) + + logger.info("Start to init multipart upload by crypto bucket, data_size: {0}, part_size: {1}".format(data_size, part_size)) + + crypto_key = self.crypto_provider.get_key() + crypto_start = self.crypto_provider.get_start() + + part_number = int((data_size - 1) / part_size + 1) + context = CryptoMultipartContext(crypto_key, crypto_start, data_size, part_size) + + headers = self.crypto_provider.build_header(headers, context) + + resp = self.bucket.init_multipart_upload(key, headers) + resp.crypto_multipart_context = context; + + logger.info("Init multipart upload by crypto bucket done, upload_id = {0}.".format(resp.upload_id)) + + return resp + + def upload_part(self, key, upload_id, part_number, data, crypto_multipart_context, progress_callback=None, headers=None): + """客户端加密上传一个分片。 + + :param str key: 待上传文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 + :param str upload_id: 分片上传ID + :param int part_number: 分片号，最小值是1. + :param data: 待上传数据。 + :param crypto_multipart_context: 加密Meta信息，在`init_multipart_upload` 时获得 + :param progress_callback: 用户指定进度回调函数。可以用来实现进度条等功能。参考 :ref:`progress_callback` 。 + + :param headers: 用户指定的HTTP头部。可以指定Content-MD5头部等 + :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict + + :return: :class:`PutObjectResult ` + """ + logger.info("Start upload part by crypto bucket, upload_id = {0}, part_number = {1}".format(upload_id, part_number)) + + headers = http.CaseInsensitiveDict(headers) + headers[FLAG_CLIENT_SIDE_ENCRYPTION_MULTIPART_FILE] = "true" + headers = self.crypto_provider.build_header_for_upload_part(headers) + + crypto_key = crypto_multipart_context.crypto_key + start = crypto_multipart_context.crypto_start + offset = crypto_multipart_context.part_size * (part_number - 1) + count_offset = utils.calc_aes_ctr_offset_by_data_offset(offset) + + data = self.crypto_provider.make_encrypt_adapter(data, crypto_key, start, count_offset=count_offset) + if self.enable_crc: + data = utils.make_crc_adapter(data) + + resp = self.bucket.upload_part(key, upload_id, part_number, data, progress_callback, headers) + + logger.info("Upload part {0} by crypto bucket done.".format(part_number)) + + return resp + + + def complete_multipart_upload(self, key, upload_id, parts, headers=None): + """客户端加密完成分片上传，创建文件。 + 当所有分片均已上传成功，才可以调用此函数 + + :param str key: 待上传的文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 + :param str upload_id: 分片上传ID + + :param parts: PartInfo列表。PartInfo中的part_number和etag是必填项。其中的etag可以从 :func:`upload_part` 的返回值中得到。 + :type parts: list of `PartInfo ` + + :param headers: HTTP头部 + :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict + + :return: :class:`PutObjectResult ` + """ + logger.info("Start complete multipart upload by crypto bucket, upload_id = {0}".format(upload_id)) + + headers = http.CaseInsensitiveDict(headers) + headers[FLAG_CLIENT_SIDE_ENCRYPTION_MULTIPART_FILE] = "true" + + res = self.bucket.complete_multipart_upload(key, upload_id, parts, headers) + + logger.info("Complete multipart upload by crypto bucket done, upload_id = {0}.".format(upload_id)) + + return res + + def abort_multipart_upload(self, key, upload_id): + """取消分片上传。 + + :param str key: 待上传的文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 + :param str upload_id: 分片上传ID + + :return: :class:`RequestResult ` + """ + logger.info("Start abort multipart upload by crypto bucket, upload_id = {0}".format(upload_id)) + + res = self.bucket.abort_multipart_upload(key, upload_id) + + logger.info("Abort multipart upload by crypto bucket done, upload_id = {0}.".format(upload_id)) + + return res + + def list_parts(self, key, upload_id, + marker='', max_parts=1000): + """列举已经上传的分片。支持分页。 + + :param str key: 文件名 + :param str upload_id: 分片上传ID + :param str marker: 分页符 + :param int max_parts: 一次最多罗列多少分片 + + :return: :class:`ListPartsResult ` + """ + logger.info("Start list parts by crypto bucket, upload_id = {0}".format(upload_id)) + + headers = http.CaseInsensitiveDict() + headers[FLAG_CLIENT_SIDE_ENCRYPTION_MULTIPART_FILE] = "true" + + res = self.bucket.list_parts(key, upload_id, marker = marker, max_parts = max_parts, headers=headers) + + crypto_key = self.crypto_provider.decrypt_from_str(OSS_CLIENT_SIDE_ENCRYPTION_KEY, res.crypto_key) + crypto_start = int(self.crypto_provider.decrypt_from_str(OSS_CLIENT_SIDE_ENCRYPTION_START, res.crypto_start)) + context = CryptoMultipartContext(crypto_key, crypto_start, res.client_encryption_data_size, res.client_encryption_part_size) + res.crypto_multipart_context = context + + logger.info("List parts by crypto bucket done, upload_id = {0}".format(upload_id)) + return res + + def __do_object(self, method, key, **kwargs): + return self._do(method, self.bucket_name, key, **kwargs) def _normalize_endpoint(endpoint): if not endpoint.startswith('http://') and not endpoint.startswith('https://'): diff --git a/oss2/crypto.py b/oss2/crypto.py index 66faa1c3..5f8f776c 100644 --- a/oss2/crypto.py +++ b/oss2/crypto.py @@ -13,6 +13,7 @@ from . import utils from .compat import to_string, to_bytes, to_unicode from .exceptions import OssError, ClientError, OpenApiFormatError, OpenApiServerError +from .headers import * from Crypto.Cipher import PKCS1_OAEP from Crypto.PublicKey import RSA @@ -35,12 +36,14 @@ def __init__(self, cipher): self.plain_start = None self.cipher = cipher - def make_encrypt_adapter(self, stream, key, start): - return utils.make_cipher_adapter(stream, partial(self.cipher.encrypt, self.cipher(key, start))) + def make_encrypt_adapter(self, stream, key, count_start, count_offset=0): + return utils.make_cipher_adapter(stream, partial(self.cipher.encrypt, self.cipher(key, count_start, count_offset))) - def make_decrypt_adapter(self, stream, key, start): - return utils.make_cipher_adapter(stream, partial(self.cipher.decrypt, self.cipher(key, start))) + def make_decrypt_adapter(self, stream, key, count_start, count_offset=0): + return utils.make_cipher_adapter(stream, partial(self.cipher.decrypt, self.cipher(key, count_start, count_offset))) + def check_plain_key_valid(self, plain_key, plain_key_hmac): + pass _LOCAL_RSA_TMP_DIR = '.oss-local-rsa' @@ -88,22 +91,45 @@ def __init__(self, dir=None, key='', passphrase=None, cipher=utils.AESCipher): except (ValueError, TypeError, IndexError) as e: raise ClientError(str(e)) - def build_header(self, headers=None): + def build_header(self, headers=None, multipart_context=None): if not isinstance(headers, CaseInsensitiveDict): headers = CaseInsensitiveDict(headers) if 'content-md5' in headers: - headers['x-oss-meta-unencrypted-content-md5'] = headers['content-md5'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] del headers['content-md5'] if 'content-length' in headers: - headers['x-oss-meta-unencrypted-content-length'] = headers['content-length'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] del headers['content-length'] - headers['x-oss-meta-oss-crypto-key'] = b64encode_as_string(self.__encrypt_obj.encrypt(self.plain_key)) - headers['x-oss-meta-oss-crypto-start'] = b64encode_as_string(self.__encrypt_obj.encrypt(to_bytes(str(self.plain_start)))) - headers['x-oss-meta-oss-cek-alg'] = self.cipher.ALGORITHM - headers['x-oss-meta-oss-wrap-alg'] = 'rsa' + headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY] = b64encode_as_string(self.__encrypt_obj.encrypt(self.plain_key)) + headers[OSS_CLIENT_SIDE_ENCRYPTION_START] = b64encode_as_string(self.__encrypt_obj.encrypt(to_bytes(str(self.plain_start)))) + headers[OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG] = self.cipher.ALGORITHM + headers[OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG] = 'rsa' + + headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC] = b64encode_as_string(str(hash(self.plain_key))) + # multipart file build header + if multipart_context: + headers[OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE] = str(multipart_context.data_size) + headers[OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE] = str(multipart_context.part_size) + + self.plain_key = None + self.plain_start = None + + return headers + + def build_header_for_upload_part(self, headers=None): + if not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + if 'content-md5' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] + del headers['content-md5'] + + if 'content-length' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] + del headers['content-length'] self.plain_key = None self.plain_start = None @@ -120,10 +146,25 @@ def get_start(self): def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x): try: - return conv(self.__decrypt_obj.decrypt(utils.b64decode_from_string(headers[key]))) + if key.lower() == OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC.lower(): + return conv(utils.b64decode_from_string(headers[key])) + else: + return conv(self.__decrypt_obj.decrypt(utils.b64decode_from_string(headers[key]))) except: return None + def decrypt_from_str(self, key, value, conv=lambda x:x): + try: + if key.lower() == OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC.lower(): + return conv(utils.b64decode_from_string(value)) + else: + return conv(self.__decrypt_obj.decrypt(utils.b64decode_from_string(value))) + except: + return None + + def check_plain_key_valid(self, plain_key, plain_key_hmac): + if str(hash(plain_key)) != plain_key_hmac: + raise ClientError("The decrypted key is inconsistent, make sure use right RSA key pair") class AliKMSProvider(BaseCryptoProvider): """使用aliyun kms服务加密数据密钥。kms的详细说明参见 @@ -152,27 +193,50 @@ def __init__(self, access_key_id, access_key_secret, region, cmkey, sts_token = self.encrypted_key = None - def build_header(self, headers=None): + def build_header(self, headers=None, multipart_context=None): if not isinstance(headers, CaseInsensitiveDict): headers = CaseInsensitiveDict(headers) + if 'content-md5' in headers: - headers['x-oss-meta-unencrypted-content-md5'] = headers['content-md5'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] del headers['content-md5'] if 'content-length' in headers: - headers['x-oss-meta-unencrypted-content-length'] = headers['content-length'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] del headers['content-length'] - headers['x-oss-meta-oss-crypto-key'] = self.encrypted_key - headers['x-oss-meta-oss-crypto-start'] = self.__encrypt_data(to_bytes(str(self.plain_start))) - headers['x-oss-meta-oss-cek-alg'] = self.cipher.ALGORITHM - headers['x-oss-meta-oss-wrap-alg'] = 'kms' + headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY] = self.encrypted_key + headers[OSS_CLIENT_SIDE_ENCRYPTION_START] = self.__encrypt_data(to_bytes(str(self.plain_start))) + headers[OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG] = self.cipher.ALGORITHM + headers[OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG] = 'kms' + + # multipart file build header + if multipart_context: + headers[OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE] = str(multipart_context.data_size) + headers[OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE] = str(multipart_context.part_size) self.encrypted_key = None self.plain_start = None return headers + def build_header_for_upload_part(self, headers=None): + if not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + if 'content-md5' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] + del headers['content-md5'] + + if 'content-length' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] + del headers['content-length'] + + self.plain_key = None + self.plain_start = None + + return headers + def get_key(self): plain_key, self.encrypted_key = self.__generate_data_key() return plain_key @@ -241,11 +305,22 @@ def __do(self, req): def decrypt_oss_meta_data(self, headers, key, conv=lambda x: x): try: - if key.lower() == 'x-oss-meta-oss-crypto-key'.lower(): + if key.lower() == OSS_CLIENT_SIDE_ENCRYPTION_KEY.lower(): return conv(b64decode_from_string(self.__decrypt_data(headers[key]))) else: return conv(self.__decrypt_data(headers[key])) except OssError as e: raise e except: - return None \ No newline at end of file + return None + + def decrypt_from_str(self, key, value, conv=lambda x:x): + try: + if key.lower() == OSS_CLIENT_SIDE_ENCRYPTION_KEY.lower(): + return conv(b64decode_from_string(self.__decrypt_data(value))) + else: + return conv(self.__decrypt_data(value)) + except OssError as e: + raise e + except: + return None diff --git a/oss2/exceptions.py b/oss2/exceptions.py index 67518f01..dedcb6f8 100644 --- a/oss2/exceptions.py +++ b/oss2/exceptions.py @@ -146,6 +146,17 @@ class InvalidObjectName(ServerError): status = 400 code = 'InvalidObjectName' +class NotImplemented(ServerError): + status = 400 + code = 'NotImplemented' + +class UnexpectedClientEncryptionPartsList(ServerError): + status = 400 + code = 'UnexpectedClientEncryptionPartsList' + +class DuplicateClientEncryptionMetaSettings(ServerError): + status = 400 + code = 'DuplicateClientEncryptionMetaSettings' class NoSuchBucket(NotFound): status = 404 diff --git a/oss2/headers.py b/oss2/headers.py index 5ddf76ca..86413cc4 100644 --- a/oss2/headers.py +++ b/oss2/headers.py @@ -30,6 +30,25 @@ OSS_SERVER_SIDE_ENCRYPTION = "x-oss-server-side-encryption" OSS_SERVER_SIDE_ENCRYPTION_KEY_ID = "x-oss-server-side-encryption-key-id" +OSS_CLIENT_SIDE_ENCRYPTION_KEY = "x-oss-client-side-encryption-key" +OSS_CLIENT_SIDE_ENCRYPTION_START = "x-oss-client-side-encryption-start" +OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG = "x-oss-client-side-encryption-cek-alg" +OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG = "x-oss-client-side-encryption-wrap-alg" +OSS_CLIENT_SIDE_ENCRYTPION_MATDESC = "x-oss-client-side-encryption-matdesc" +OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH = "x-oss-client-side-encryption-unencrypted-content-length" +OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5 = "x-oss-client-side-encryption-unencrypted-content-md5" +OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC = "x-oss-client-side-encryption-key-hmac" +OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE = "x-oss-client-side-encryption-data-size" +OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE = "x-oss-client-side-encryption-part-size" +FLAG_CLIENT_SIDE_ENCRYPTION_MULTIPART_FILE = "flag-client-side-encryption-multipart-file" + +DEPRECATED_CLIENT_SIDE_ENCRYPTION_KEY = "x-oss-meta-oss-crypto-key" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_START = "x-oss-meta-oss-crypto-start" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_CEK_ALG = "x-oss-meta-oss-cek-alg" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_WRAP_ALG = "x-oss-meta-oss-wrap-alg" +DEPRECATED_CLIENT_SIDE_ENCRYTPION_MATDESC = "x-oss-meta-oss-crypto-matdesc" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH = "x-oss-meta-oss-crypto-unencrypted-content-length" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5 = "x-oss-meta-oss-crypto-unencrypted-content-md5" class RequestHeader(dict): def __init__(self, *arg, **kw): diff --git a/oss2/models.py b/oss2/models.py index 2f61dde2..393fab74 100644 --- a/oss2/models.py +++ b/oss2/models.py @@ -7,7 +7,8 @@ 该模块包含Python SDK API接口所需要的输入参数以及返回值类型。 """ -from .utils import http_to_unixtime, make_progress_adapter, make_crc_adapter +from .utils import http_to_unixtime, make_progress_adapter, make_crc_adapter, \ + calc_aes_ctr_offset_by_data_offset, is_multiple_sizeof_encrypt_block from .exceptions import ClientError, InconsistentError from .compat import urlunquote, to_string from .select_response import SelectResponseAdapter @@ -33,6 +34,14 @@ def __init__(self, part_number, etag, size=None, last_modified=None, part_crc=No self.last_modified = last_modified self.part_crc = part_crc +class CryptoMultipartContext(object): + """表示客户端加密文件通过Multipart接口上传的meta信息 + """ + def __init__(self, crypto_key, crypto_start, data_size, part_size): + self.crypto_key = crypto_key + self.crypto_start = crypto_start + self.data_size = data_size + self.part_size = part_size def _hget(headers, key, converter=lambda x: x): if key in headers: @@ -127,8 +136,11 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi self.__crc_enabled = crc_enabled self.__crypto_provider = crypto_provider - if _hget(resp.headers, 'x-oss-meta-oss-crypto-key') and _hget(resp.headers, 'Content-Range'): - raise ClientError('Could not get an encrypted object using byte-range parameter') + content_range = _hget(resp.headers, 'Content-Range') + if _hget(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY) and content_range: + byte_range = self._parse_range_str(content_range) + if not is_multiple_sizeof_encrypt_block(byte_range[0]): + raise ClientError('Could not get an encrypted object using byte-range parameter') if progress_callback: self.stream = make_progress_adapter(self.resp, progress_callback, self.content_length) @@ -139,14 +151,36 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi self.stream = make_crc_adapter(self.stream) if self.__crypto_provider: - key = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, 'x-oss-meta-oss-crypto-key') - start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, 'x-oss-meta-oss-crypto-start') - cek_alg = _hget(resp.headers, 'x-oss-meta-oss-cek-alg') - if key and start and cek_alg: - self.stream = self.__crypto_provider.make_decrypt_adapter(self.stream, key, start) + key = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY) + count_start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_START) + + # if content range , adjust the decrypt adapter + count_offset = 0; + if content_range: + byte_range = self._parse_range_str(content_range) + count_offset = calc_aes_ctr_offset_by_data_offset(byte_range[0]) + + cek_alg = _hget(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG) + + # check the key wrap algorthm is correct if rsa + if cek_alg == "rsa": + key_hmac = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC) + self.__crypto_provider.check_plain_key_valid(key, to_string(key_hmac)) + + if key and count_start and cek_alg: + self.stream = self.__crypto_provider.make_decrypt_adapter(self.stream, key, count_start, count_offset) else: - raise InconsistentError('all metadata keys are required for decryption (x-oss-meta-oss-crypto-key, \ - x-oss-meta-oss-crypto-start, x-oss-meta-oss-cek-alg)', self.request_id) + err_msg = 'all metadata keys are required for decryption (' \ + + OSS_CLIENT_SIDE_ENCRYPTION_KEY + ', ' \ + + OSS_CLIENT_SIDE_ENCRYPTION_START + ', ' \ + + OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG + ')' + raise InconsistentError(err_msg, self.request_id) + + def _parse_range_str(self, content_range): + # :param str content_range: sample 'bytes 0-128/1024' + range_data = (content_range.split(' ',2)[1]).split('/',2)[0] + range_start, range_end = range_data.split('-',2) + return (int(range_start), int(range_end)) def read(self, amt=None): return self.stream.read(amt) @@ -216,6 +250,8 @@ def __init__(self, resp): #: 新生成的Upload ID self.upload_id = None + # 客户端加密Bucket关于Multipart文件的context + self.crypto_multipart_context = None class ListObjectsResult(RequestResult): def __init__(self, resp): @@ -234,6 +270,7 @@ def __init__(self, resp): self.prefix_list = [] + class SimplifiedObjectInfo(object): def __init__(self, key, last_modified, etag, type, size, storage_class): #: 文件名，或公共前缀名。 @@ -359,6 +396,23 @@ def __init__(self, resp): # 罗列出的Part信息，类型为 `PartInfo` 列表。 self.parts = [] + # 是否是客户端加密 + self.is_client_encryption = False + + # 客户端加密文件密钥 + self.crypto_key = None + + # 客户端加密文件初始向量 + self.crypto_start = None + + # 客户端加密Multipart文件总大小 + self.client_encryption_data_size = 0 + + # 客户端加密Multipart文件块大小 + self.client_encryption_part_size = 0 + + # 客户端加密Bucket关于Multipart文件的context + self.crypto_multipart_context = None BUCKET_ACL_PRIVATE = 'private' BUCKET_ACL_PUBLIC_READ = 'public-read' diff --git a/oss2/resumable.py b/oss2/resumable.py index 4a574417..89e5fc2d 100644 --- a/oss2/resumable.py +++ b/oss2/resumable.py @@ -19,6 +19,7 @@ from .compat import json, stringify, to_unicode, to_string from .task_queue import TaskQueue from .headers import * +from .utils import _MAX_PART_COUNT, _MIN_PART_SIZE import functools import threading @@ -29,10 +30,6 @@ logger = logging.getLogger(__name__) -_MAX_PART_COUNT = 10000 -_MIN_PART_SIZE = 100 * 1024 - - def resumable_upload(bucket, key, filename, store=None, headers=None, diff --git a/oss2/utils.py b/oss2/utils.py index 8c80d702..4e044ef2 100644 --- a/oss2/utils.py +++ b/oss2/utils.py @@ -588,11 +588,57 @@ def random_counter(begin=1, end=10): # aes 256, key always is 32 bytes _AES_256_KEY_SIZE = 32 -_AES_CTR_COUNTER_BITS_LEN = 8 * 16 +_AES_CTR_COUNTER_LEN = 16 +_AES_CTR_COUNTER_BITS_LEN = 8 * _AES_CTR_COUNTER_LEN _AES_GCM = 'AES/GCM/NoPadding' +_MAX_PART_COUNT = 10000 +_MIN_PART_SIZE = 100 * 1024 + +def is_multiple_sizeof_encrypt_block(data_offset): + if data_offset is None: + data_offset = 0 + return (data_offset % _AES_CTR_COUNTER_LEN == 0) + +def calc_aes_ctr_offset_by_data_offset(data_offset): + if not is_multiple_sizeof_encrypt_block(data_offset): + raise ClientError('data_offset is not align to encrypt block') + return data_offset / _AES_CTR_COUNTER_LEN + +def is_valid_crypto_part_size(part_size, data_size): + if not is_multiple_sizeof_encrypt_block(part_size) or part_size < _MIN_PART_SIZE: + return False + part_num = (data_size - 1) / part_size + 1 + if part_num > _MAX_PART_COUNT: + return False + return True + +def determine_crypto_part_size(data_size, excepted_part_size = None): + if excepted_part_size: + # excepted_part_size is valid + if is_valid_crypto_part_size(excepted_part_size, data_size): + return excepted_part_size + # excepted_part_size is enough big but not algin + elif excepted_part_size > data_size/_MAX_PART_COUNT: + part_size = int(excepted_part_size/_AES_CTR_COUNTER_LEN + 1) * _AES_CTR_COUNTER_LEN + return part_size + + # if excepted_part_size is None or is too small, calculate a correct part_size + if data_size % _MAX_PART_COUNT == 0: + part_size = data_size / _MAX_PART_COUNT + else: + part_size = int(data_size / (_MAX_PART_COUNT - 1)) + + if part_size < _MIN_PART_SIZE: + part_size = _MIN_PART_SIZE + elif not is_multiple_sizeof_encrypt_block(part_size): + part_size = int(part_size / _AES_CTR_COUNTER_LEN + 1) * _AES_CTR_COUNTER_LEN + + return part_size + + class AESCipher: """AES256 加密实现。 :param str key: 对称加密数据密钥 @@ -613,15 +659,16 @@ def get_key(): def get_start(): return random_counter() - def __init__(self, key=None, start=None): + def __init__(self, key=None, count_start=None, count_offset=0): self.key = key + self.count_offset = int(count_offset) if not self.key: self.key = random_aes256_key() - if not start: - self.start = random_counter() + if not count_start: + self.count_start = random_counter() else: - self.start = int(start) - ctr = Counter.new(_AES_CTR_COUNTER_BITS_LEN, initial_value=self.start) + self.count_start = int(count_start) + ctr = Counter.new(_AES_CTR_COUNTER_BITS_LEN, initial_value=(self.count_start + self.count_offset)) self.__cipher = AES.new(self.key, AES.MODE_CTR, counter=ctr) def encrypt(self, raw): diff --git a/oss2/xml_utils.py b/oss2/xml_utils.py index 292e0d8f..f6e4e05f 100644 --- a/oss2/xml_utils.py +++ b/oss2/xml_utils.py @@ -179,6 +179,20 @@ def parse_list_parts(result, body): result.is_truncated = _find_bool(root, 'IsTruncated') result.next_marker = _find_tag(root, 'NextPartNumberMarker') + + try: + result.is_client_encryption = _find_bool(root, 'IsClientEncryption') + result.crypto_key = _find_tag(root, 'ClientEncryptionKey') + result.crypto_start = _find_tag(root, 'ClientEncryptionStart') + result.client_encryption_data_size = _find_int(root, 'ClientEncryptionDataSize') + result.client_encryption_part_size = _find_int(root, 'ClientEncryptionPartSize') + except RuntimeError as e: + result.is_client_encryption = False + result.crypto_key = None + result.crypto_start = None + result.client_encryption_data_size = 0 + result.client_encryption_part_size = 0 + for part_node in root.findall('Part'): result.parts.append(PartInfo( _find_int(part_node, 'PartNumber'), diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 3bf840d0..52501f32 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -6,6 +6,7 @@ from common import * +from oss2.compat import is_py2, is_py33 class TestMultipart(OssTestCase): def do_multipart_internal(self, do_md5): @@ -97,6 +98,481 @@ def test_upload_part_copy(self): self.assertEqual(len(content_got), len(content)) self.assertEqual(content_got, content) + def do_crypto_multipart_internal(self, do_md5, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(100 * 1024) + content = [content_1, content_2, content_3] + + parts = [] + data_size = 1024 * 300 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + self.assertEqual(crypto_multipart_context.data_size, 1024 * 300) + self.assertEqual(crypto_multipart_context.part_size, 1024 * 100) + + for i in range(3): + if do_md5: + headers = {'Content-Md5': oss2.utils.content_md5(content[i])} + else: + headers = None + upload_result = bucket.upload_part(key, upload_id, i+1, content[i], crypto_multipart_context, headers=headers) + parts.append(oss2.models.PartInfo(i+1, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + complete_result = bucket.complete_multipart_upload(key, upload_id, parts) + self.assertTrue(complete_result.status == 200) + + get_result_range_1 = bucket.get_object(key, byte_range=(0, 102399)) + self.assertTrue(get_result_range_1.status == 206) + content_got_1 = get_result_range_1.read() + self.assertEqual(content_1, content_got_1) + + get_result_range_2 = bucket.get_object(key, byte_range=(102400, 204799)) + self.assertTrue(get_result_range_2.status == 206) + content_got_2 = get_result_range_2.read() + self.assertEqual(content_2, content_got_2) + + get_result_range_3 = bucket.get_object(key, byte_range=(204800, 307199)) + self.assertTrue(get_result_range_3.status == 206) + content_got_3 = get_result_range_3.read() + self.assertEqual(content_3, content_got_3) + + get_result = bucket.get_object(key) + self.assertTrue(get_result.status == 200) + content_got = get_result.read() + self.assertEqual(content_1, content_got[0:102400]) + self.assertEqual(content_2, content_got[102400:204800]) + self.assertEqual(content_3, content_got[204800:307200]) + + def do_crypto_abort_multipart(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content = random_bytes(100 * 1024) + + data_size = 1024 * 100 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + upload_result = bucket.upload_part(key, upload_id, 1, content, crypto_multipart_context) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_list_parts(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content = random_bytes(100 * 1024) + + data_size = 1024 * 300 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + upload_result = bucket.upload_part(key, upload_id, 1, content, crypto_multipart_context) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + list_result = bucket.list_parts(key, upload_id) + self.assertTrue(list_result.status == 200) + crypto_multipart_context_new = list_result.crypto_multipart_context + + self.assertEqual(crypto_multipart_context_new.crypto_key, crypto_multipart_context.crypto_key) + self.assertEqual(crypto_multipart_context_new.crypto_start, crypto_multipart_context.crypto_start) + self.assertEqual(crypto_multipart_context_new.data_size, crypto_multipart_context.data_size) + self.assertEqual(crypto_multipart_context_new.part_size, crypto_multipart_context.part_size) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_init_multipart_invalid_parameter(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content = random_bytes(100 * 1024) + + data_size = 1024 * 100 + part_size = 1 + + #init multipart with invalid part_size + self.assertRaises(oss2.exceptions.ClientError, bucket.init_multipart_upload, key, data_size, part_size=part_size) + + #init multipart without part_size + init_result = bucket.init_multipart_upload(key, data_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + part_size = crypto_multipart_context.part_size; + self.assertEqual(part_size, 100*1024) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_upload_invalid_part_content(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(50 * 1024) + content = [content_1, content_2, content_3] + content_invalid = random_bytes(100 * 1024 - 1) + + parts = [] + data_size = 1024 * 250 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + self.assertRaises(oss2.exceptions.InvalidArgument, bucket.upload_part, key, upload_id, 1, content_invalid, crypto_multipart_context) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_upload_invalid_last_part_content(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(50 * 1024) + content = [content_1, content_2, content_3] + content_invalid = random_bytes(100 * 1024 - 1) + + parts = [] + data_size = 1024 * 250 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + for i in range(2): + upload_result = bucket.upload_part(key, upload_id, i+1, content[i], crypto_multipart_context) + parts.append(oss2.models.PartInfo(i+1, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + self.assertRaises(oss2.exceptions.InvalidArgument, bucket.upload_part, key, upload_id, 3, content_invalid, crypto_multipart_context) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_upload_invalid_part_number(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(50 * 1024) + content = [content_1, content_2, content_3] + + parts = [] + data_size = 1024 * 250 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + self.assertRaises(oss2.exceptions.InvalidArgument, bucket.upload_part, key, upload_id, 4, content_1, crypto_multipart_context) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_complete_multipart_miss_parts(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(50 * 1024) + content = [content_1, content_2, content_3] + + parts = [] + data_size = 1024 * 250 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + for i in range(2): + upload_result = bucket.upload_part(key, upload_id, i+1, content[i], crypto_multipart_context) + parts.append(oss2.models.PartInfo(i+1, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + self.assertRaises(oss2.exceptions.UnexpectedClientEncryptionPartsList, bucket.complete_multipart_upload, key, upload_id, parts) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_resume_upload_after_loss_context(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(100 * 1024) + content = [content_1, content_2, content_3] + + parts = [] + data_size = 1024 * 300 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + upload_result = bucket.upload_part(key, upload_id, 1, content[0], crypto_multipart_context) + parts.append(oss2.models.PartInfo(1, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + list_result = bucket.list_parts(key, upload_id) + self.assertTrue(list_result.status == 200) + crypto_multipart_context_new_1 = list_result.crypto_multipart_context + + upload_result = bucket.upload_part(key, upload_id, 2, content[1], crypto_multipart_context_new_1) + parts.append(oss2.models.PartInfo(2, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + list_result = bucket.list_parts(key, upload_id) + self.assertTrue(list_result.status == 200) + crypto_multipart_context_new_2 = list_result.crypto_multipart_context + + upload_result = bucket.upload_part(key, upload_id, 3, content[2], crypto_multipart_context_new_2) + parts.append(oss2.models.PartInfo(3, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + complete_result = bucket.complete_multipart_upload(key, upload_id, parts) + self.assertTrue(complete_result.status == 200) + + get_result_range_1 = bucket.get_object(key, byte_range=(0, 102399)) + self.assertTrue(get_result_range_1.status == 206) + content_got_1 = get_result_range_1.read() + self.assertEqual(content_1, content_got_1) + + get_result_range_2 = bucket.get_object(key, byte_range=(102400, 204799)) + self.assertTrue(get_result_range_2.status == 206) + content_got_2 = get_result_range_2.read() + self.assertEqual(content_2, content_got_2) + + get_result_range_3 = bucket.get_object(key, byte_range=(204800, 307199)) + self.assertTrue(get_result_range_3.status == 206) + content_got_3 = get_result_range_3.read() + self.assertEqual(content_3, content_got_3) + + get_result = bucket.get_object(key) + self.assertTrue(get_result.status == 200) + content_got = get_result.read() + self.assertEqual(content_1, content_got[0:102400]) + self.assertEqual(content_2, content_got[102400:204800]) + self.assertEqual(content_3, content_got[204800:307200]) + + def do_upload_part_copy_from_crypto_source(self, bucket, crypto_bucket, is_kms=False): + if is_py33 and is_kms: + return + + src_object = self.random_key() + dst_object = self.random_key() + + content = random_bytes(200 * 1024) + + # 上传源文件 + crypto_bucket.put_object(src_object, content) + + # part copy到目标文件 + parts = [] + upload_id = bucket.init_multipart_upload(dst_object).upload_id + + self.assertRaises(oss2.exceptions.NotImplemented, bucket.upload_part_copy, self.bucket.bucket_name, + src_object, (0, 100 * 1024 - 1), dst_object, upload_id, 1) + + abort_result = bucket.abort_multipart_upload(dst_object, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_multipart_concurrent(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key1 = self.random_key() + key1_content_1 = random_bytes(100 * 1024) + key1_content_2 = random_bytes(100 * 1024) + key1_content_3 = random_bytes(100 * 1024) + key1_content = [key1_content_1, key1_content_2, key1_content_3] + + key1_parts = [] + key1_data_size = 1024 * 300 + key1_part_size = 1024 * 100 + + key1_init_result = bucket.init_multipart_upload(key1, key1_data_size, key1_part_size) + self.assertTrue(key1_init_result.status == 200) + key1_upload_id = key1_init_result.upload_id + key1_crypto_multipart_context = key1_init_result.crypto_multipart_context + + self.assertEqual(key1_crypto_multipart_context.data_size, 1024 * 300) + self.assertEqual(key1_crypto_multipart_context.part_size, 1024 * 100) + + key2 = self.random_key() + key2_content_1 = random_bytes(200 * 1024) + key2_content_2 = random_bytes(200 * 1024) + key2_content_3 = random_bytes(100 * 1024) + key2_content = [key2_content_1, key2_content_2, key2_content_3] + + key2_parts = [] + key2_data_size = 1024 * 500 + key2_part_size = 1024 * 200 + + key2_init_result = bucket.init_multipart_upload(key2, key2_data_size, key2_part_size) + self.assertTrue(key2_init_result.status == 200) + key2_upload_id = key2_init_result.upload_id + key2_crypto_multipart_context = key2_init_result.crypto_multipart_context + + self.assertEqual(key2_crypto_multipart_context.data_size, 1024 * 500) + self.assertEqual(key2_crypto_multipart_context.part_size, 1024 * 200) + + for i in range(3): + key1_upload_result = bucket.upload_part(key1, key1_upload_id, i+1, key1_content[i], key1_crypto_multipart_context) + key1_parts.append(oss2.models.PartInfo(i+1, key1_upload_result.etag, size = key1_part_size, part_crc = key1_upload_result.crc)) + self.assertTrue(key1_upload_result.status == 200) + self.assertTrue(key1_upload_result.crc is not None) + + key2_upload_result = bucket.upload_part(key2, key2_upload_id, i+1, key2_content[i], key2_crypto_multipart_context) + key2_parts.append(oss2.models.PartInfo(i+1, key2_upload_result.etag, size = key2_part_size, part_crc = key2_upload_result.crc)) + self.assertTrue(key2_upload_result.status == 200) + self.assertTrue(key2_upload_result.crc is not None) + + key1_complete_result = bucket.complete_multipart_upload(key1, key1_upload_id, key1_parts) + self.assertTrue(key1_complete_result.status == 200) + + key1_get_result = bucket.get_object(key1) + self.assertTrue(key1_get_result.status == 200) + key1_content_got = key1_get_result.read() + self.assertEqual(key1_content_1, key1_content_got[0:102400]) + self.assertEqual(key1_content_2, key1_content_got[102400:204800]) + self.assertEqual(key1_content_3, key1_content_got[204800:307200]) + + key2_complete_result = bucket.complete_multipart_upload(key2, key2_upload_id, key2_parts) + self.assertTrue(key2_complete_result.status == 200) + + key2_get_result = bucket.get_object(key2) + self.assertTrue(key2_get_result.status == 200) + key2_content_got = key2_get_result.read() + self.assertEqual(key2_content_1, key2_content_got[0:204800]) + self.assertEqual(key2_content_2, key2_content_got[204800:409600]) + self.assertEqual(key2_content_3, key2_content_got[409600:512000]) + + def test_rsa_crypto_multipart(self): + self.do_crypto_multipart_internal(False, self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_upload_part_content_md5_good(self): + self.do_crypto_multipart_internal(True, self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_abort_multipart(self): + self.do_crypto_abort_multipart(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_list_parts(self): + self.do_crypto_list_parts(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_init_multipart_invalid_parameter(self): + self.do_crypto_init_multipart_invalid_parameter(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_upload_invalid_part_content(self): + self.do_crypto_upload_invalid_part_content(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_upload_invalid_last_part_content(self): + self.do_crypto_upload_invalid_last_part_content(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_upload_invalid_part_number(self): + self.do_crypto_upload_invalid_part_number(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_complete_multipart_miss_parts(self): + self.do_crypto_complete_multipart_miss_parts(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_resume_upload_after_loss_context(self): + self.do_crypto_resume_upload_after_loss_context(self.rsa_crypto_bucket, is_kms=False) + + def test_upload_part_copy_from_rsa_crypto_source(self): + self.do_upload_part_copy_from_crypto_source(self.bucket, self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_multipart_concurrent(self): + self.do_crypto_multipart_concurrent(self.rsa_crypto_bucket, is_kms=False) + + def test_kms_crypto_multipart(self): + self.do_crypto_multipart_internal(False, self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_upload_part_content_md5_good(self): + self.do_crypto_multipart_internal(True, self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_abort_multipart(self): + self.do_crypto_abort_multipart(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_list_parts(self): + self.do_crypto_list_parts(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_init_multipart_invalid_parameter(self): + self.do_crypto_init_multipart_invalid_parameter(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_upload_invalid_part_content(self): + self.do_crypto_upload_invalid_part_content(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_upload_invalid_last_part_content(self): + self.do_crypto_upload_invalid_last_part_content(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_upload_invalid_part_number(self): + self.do_crypto_upload_invalid_part_number(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_complete_multipart_miss_parts(self): + self.do_crypto_complete_multipart_miss_parts(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_resume_upload_after_loss_context(self): + self.do_crypto_resume_upload_after_loss_context(self.kms_crypto_bucket, is_kms=True) + + def test_upload_part_copy_from_kms_crypto_source(self): + self.do_upload_part_copy_from_crypto_source(self.bucket, self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_multipart_concurrent(self): + self.do_crypto_multipart_concurrent(self.rsa_crypto_bucket, is_kms=True) if __name__ == '__main__': unittest.main() diff --git a/tests/test_object.py b/tests/test_object.py index 31d22417..76f5da62 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -84,6 +84,121 @@ def assert_result(result): self.assertTrue(get_result.server_crc is not None) self.assertTrue(get_result.client_crc == get_result.server_crc) + def test_rsa_crypto_range_get(self): + key = self.random_key() + content = random_bytes(1024) + + self.rsa_crypto_bucket.put_object(key, content) + + get_result = self.rsa_crypto_bucket.get_object(key, byte_range=(None, None)) + self.assertEqual(get_result.read(), content[:]) + + get_result = self.rsa_crypto_bucket.get_object(key, byte_range=(32, None)) + self.assertEqual(get_result.read(), content[32:]) + + get_result = self.rsa_crypto_bucket.get_object(key, byte_range=(None, 32)) + self.assertEqual(get_result.read(), content[-32:]) + + get_result = self.rsa_crypto_bucket.get_object(key, byte_range=(32, 103)) + self.assertEqual(get_result.read(), content[32:103+1]) + + self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(31, None)) + self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(None, 31)) + + def test_rsa_crypto_object_decrypt_by_normal_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.rsa_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + self.assertRaises(ClientError, self.bucket.get_object, key) + + def test_get_unencrypt_object_decrypt_by_rsa_crypto_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + self.assertRaises(ClientError, self.rsa_crypto_bucket.get_object, key) + + def test_copy_rsa_crypto_object_by_normal_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))} + result = self.rsa_crypto_bucket.put_object(key, content, headers=headers) + self.assertTrue(result.status == 200) + + copy_key = key + "_copy"; + result = self.bucket.copy_object(self.bucket.bucket_name, key, copy_key) + self.assertTrue(result.status == 200) + + def assert_result(result): + self.assertEqual(result.content_length, len(content)) + self.assertEqual(result.content_type, 'application/javascript') + self.assertEqual(result.object_type, 'Normal') + self.assertTrue(result.etag) + + get_result = self.rsa_crypto_bucket.get_object(copy_key) + self.assertEqual(get_result.read(), content) + assert_result(get_result) + self.assertTrue(get_result.client_crc is not None) + self.assertTrue(get_result.server_crc is not None) + self.assertTrue(get_result.client_crc == get_result.server_crc) + + def test_replace_rsa_crypto_object_by_normal_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.rsa_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + replace_key = key + "_replace"; + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content)), + 'x-oss-metadata-directive':'REPLACE'} + result = self.bucket.copy_object(self.bucket.bucket_name, key, replace_key, headers=headers) + self.assertTrue(result.status == 200) + + def assert_result(result): + self.assertEqual(result.content_length, len(content)) + self.assertEqual(result.content_type, 'application/javascript') + self.assertEqual(result.object_type, 'Normal') + self.assertTrue(result.etag) + + get_result = self.rsa_crypto_bucket.get_object(replace_key) + self.assertEqual(get_result.read(), content) + assert_result(get_result) + self.assertTrue(get_result.client_crc is not None) + self.assertTrue(get_result.server_crc is not None) + self.assertTrue(get_result.client_crc == get_result.server_crc) + + def test_update_crypto_meta_rsa_crypto_object_by_normal_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.rsa_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content)), + 'x-oss-client-side-encryption-key':'aaaa'} + self.assertRaises(oss2.exceptions.DuplicateClientEncryptionMetaSettings, self.bucket.copy_object, + self.bucket.bucket_name, key, key, headers=headers) + def test_kms_crypto_object(self): if is_py33: return @@ -116,6 +231,141 @@ def assert_result(result): self.assertTrue(get_result.server_crc is not None) self.assertTrue(get_result.client_crc == get_result.server_crc) + def test_kms_crypto_range_get(self): + if is_py33: + return + + key = self.random_key() + content = random_bytes(1024) + + self.kms_crypto_bucket.put_object(key, content, headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))}) + + get_result = self.kms_crypto_bucket.get_object(key, byte_range=(None, None)) + self.assertEqual(get_result.read(), content[:]) + + get_result = self.kms_crypto_bucket.get_object(key, byte_range=(32, None)) + self.assertEqual(get_result.read(), content[32:]) + + get_result = self.kms_crypto_bucket.get_object(key, byte_range=(None, 32)) + self.assertEqual(get_result.read(), content[-32:]) + + get_result = self.kms_crypto_bucket.get_object(key, byte_range=(32, 103)) + self.assertEqual(get_result.read(), content[32:103+1]) + + self.assertRaises(oss2.exceptions.ClientError, self.kms_crypto_bucket.get_object, key, byte_range=(31, None)) + self.assertRaises(oss2.exceptions.ClientError, self.kms_crypto_bucket.get_object, key, byte_range=(None, 31)) + + def test_kms_crypto_object_decrypt_by_normal_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.kms_crypto_bucket.put_object(key, content, headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))}) + self.assertTrue(result.status == 200) + + self.assertRaises(ClientError, self.bucket.get_object, key) + + def test_get_unencrypt_object_decrypt_by_kms_crypto_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + self.assertRaises(ClientError, self.kms_crypto_bucket.get_object, key) + + def test_copy_kms_crypto_object_by_normal_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))} + result = self.kms_crypto_bucket.put_object(key, content, headers=headers) + self.assertTrue(result.status == 200) + + copy_key = key + "_copy"; + result = self.bucket.copy_object(self.bucket.bucket_name, key, copy_key) + self.assertTrue(result.status == 200) + + def assert_result(result): + self.assertEqual(result.content_length, len(content)) + self.assertEqual(result.content_type, 'application/javascript') + self.assertEqual(result.object_type, 'Normal') + self.assertTrue(result.etag) + + get_result = self.kms_crypto_bucket.get_object(copy_key) + self.assertEqual(get_result.read(), content) + assert_result(get_result) + self.assertTrue(get_result.client_crc is not None) + self.assertTrue(get_result.server_crc is not None) + self.assertTrue(get_result.client_crc == get_result.server_crc) + + def test_replace_kms_crypto_object_by_normal_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.kms_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + replace_key = key + "_replace"; + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content)), + 'x-oss-metadata-directive':'REPLACE'} + result = self.bucket.copy_object(self.bucket.bucket_name, key, replace_key, headers=headers) + self.assertTrue(result.status == 200) + + def assert_result(result): + self.assertEqual(result.content_length, len(content)) + self.assertEqual(result.content_type, 'application/javascript') + self.assertEqual(result.object_type, 'Normal') + self.assertTrue(result.etag) + + get_result = self.kms_crypto_bucket.get_object(replace_key) + self.assertEqual(get_result.read(), content) + assert_result(get_result) + self.assertTrue(get_result.client_crc is not None) + self.assertTrue(get_result.server_crc is not None) + self.assertTrue(get_result.client_crc == get_result.server_crc) + + def test_update_crypto_meta_kms_crypto_object_by_normal_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.kms_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content)), + 'x-oss-client-side-encryption-key':'aaaa'} + self.assertRaises(oss2.exceptions.DuplicateClientEncryptionMetaSettings, self.bucket.copy_object, + self.bucket.bucket_name, key, key, headers=headers) + def test_restore_object(self): auth = oss2.Auth(OSS_ID, OSS_SECRET) bucket = oss2.Bucket(auth, OSS_ENDPOINT, random_string(63).lower()) diff --git a/unittests/test_models.py b/unittests/test_models.py new file mode 100644 index 00000000..25cea838 --- /dev/null +++ b/unittests/test_models.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import unittest +from oss2.models import * +from unittests.common import * + + +class TestModels(unittest.TestCase): + def test_parse_range_str(self): + resp = do4body('', 0, body='') + get_obj_result = GetObjectResult(resp) + + content_range = 'bytes 0-128/1024' + range_data = get_obj_result._parse_range_str(content_range) + self.assertEqual(range_data[0], 0) + self.assertEqual(range_data[1], 128) diff --git a/unittests/test_utils.py b/unittests/test_utils.py new file mode 100644 index 00000000..e92b0efc --- /dev/null +++ b/unittests/test_utils.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +import unittest +from oss2.utils import * + + +class TestUtils(unittest.TestCase): + def test_is_multiple_sizeof_encrypt_block(self): + byte_range_start = 1024 + is_multiple = is_multiple_sizeof_encrypt_block(byte_range_start) + self.assertTrue(is_multiple) + + byte_range_start = 1025 + is_multiple = is_multiple_sizeof_encrypt_block(byte_range_start) + self.assertFalse(is_multiple) + + def test_calc_aes_ctr_offset_by_data_offset(self): + byte_range_start = 1024 + cout_offset = calc_aes_ctr_offset_by_data_offset(byte_range_start) + self.assertEqual(cout_offset, 1024 / 16) + + def test_is_valid_crypto_part_size(self): + self.assertFalse(is_valid_crypto_part_size(1, 1024*1024*100)) + self.assertFalse(is_valid_crypto_part_size(1024 * 100 + 1, 1024*1024*100)) + self.assertFalse(is_valid_crypto_part_size(1024 * 100, 1024*1024*1024)) + self.assertTrue(is_valid_crypto_part_size(1024 * 100, 1024*1024*100)) + + def test_determine_crypto_part_size(self): + self.assertEqual(determine_crypto_part_size(1024*100*100000), 1024*1000) + self.assertEqual(determine_crypto_part_size(1024*100*100000 - 1), 1024112) + self.assertEqual(determine_crypto_part_size(1024*100*99), 1024*100) + + self.assertEqual(determine_crypto_part_size(1024*100*1000, 1024*100), 1024*100) + self.assertEqual(determine_crypto_part_size(1024*100*1000, 1024*100-1), 1024*100) + self.assertEqual(determine_crypto_part_size(1024*100*10000, 1024), 1024*100) + +if __name__ == '__main__': + unittest.main()