Skip to content

Commit 0be14bc

Browse files
dt3310321dt3310321
authored
Merge pull request #48 from dt3310321/s3
S3
2 parents f8d36b5 + 37e3789 commit 0be14bc

File tree

3 files changed

+106
-40
lines changed

3 files changed

+106
-40
lines changed

‎qcloud_cos/cos_auth.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,13 @@ def __call__(self, r):
5454
path = self._path
5555
uri_params = self._params
5656
headers = filter_headers(r.headers)
57+
uri_params = dict([(k.lower(), v.lower()) for k, v in uri_params.items()])
5758
# reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+)
5859
headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode
5960
format_str = "{method}\n{host}\n{params}\n{headers}\n".format(
6061
method=r.method.lower(),
6162
host=path,
62-
params=urllib.urlencode(sorted(uri_params.items())),
63+
params=urllib.urlencode(sorted(uri_params.items())).replace('+', '%20'),
6364
headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items())))
6465
)
6566
logger.debug("format str: " + format_str)

‎qcloud_cos/cos_client.py‎

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ def send_request(self, method, url, timeout=30, **kwargs):
153153
timeout = self._conf._timeout
154154
if self._conf._token is not None:
155155
kwargs['headers']['x-cos-security-token'] = self._conf._token
156-
kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.1'
156+
kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.2'
157157
try:
158158
for j in range(self._retry):
159159
if method == 'POST':
@@ -391,7 +391,7 @@ def delete_objects(self, Bucket, Delete={}, **kwargs):
391391
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
392392
headers=headers)
393393
data = xml_to_dict(rt.text)
394-
data = format_dict(data, ['Deleted', 'Error'])
394+
format_dict(data, ['Deleted', 'Error'])
395395
return data
396396

397397
def head_object(self, Bucket, Key, **kwargs):
@@ -723,9 +723,9 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, Part
723723
headers=headers,
724724
params=params)
725725
data = xml_to_dict(rt.text)
726-
data = format_dict(data, ['Part'])
726+
format_dict(data, ['Part'])
727727
if decodeflag:
728-
data = decode_result(data, ['Key'], [])
728+
decode_result(data, ['Key'], [])
729729
return data
730730

731731
def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs):
@@ -938,9 +938,9 @@ def list_objects(self, Bucket, Prefix="", Delimiter="", Marker="", MaxKeys=1000,
938938
headers=headers,
939939
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
940940
data = xml_to_dict(rt.text)
941-
data = format_dict(data, ['Contents', 'CommonPrefixes'])
941+
format_dict(data, ['Contents', 'CommonPrefixes'])
942942
if decodeflag:
943-
data = decode_result(
943+
decode_result(
944944
data,
945945
[
946946
'Prefix',
@@ -1006,9 +1006,9 @@ def list_objects_versions(self, Bucket, Prefix="", Delimiter="", KeyMarker="", V
10061006
headers=headers,
10071007
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
10081008
data = xml_to_dict(rt.text)
1009-
data = format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes'])
1009+
format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes'])
10101010
if decodeflag:
1011-
data = decode_result(
1011+
decode_result(
10121012
data,
10131013
[
10141014
'Prefix',
@@ -1078,9 +1078,9 @@ def list_multipart_uploads(self, Bucket, Prefix="", Delimiter="", KeyMarker="",
10781078
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
10791079

10801080
data = xml_to_dict(rt.text)
1081-
data = format_dict(data, ['Upload', 'CommonPrefixes'])
1081+
format_dict(data, ['Upload', 'CommonPrefixes'])
10821082
if decodeflag:
1083-
data = decode_result(
1083+
decode_result(
10841084
data,
10851085
[
10861086
'Prefix',
@@ -1349,7 +1349,17 @@ def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs):
13491349
LifecycleConfiguration=lifecycle_config
13501350
)
13511351
"""
1352-
lst = ['<Rule>', '<Tag>', '</Tag>', '</Rule>'] # 类型为list的标签
1352+
# 类型为list的标签
1353+
lst = [
1354+
'<Rule>',
1355+
'<Tag>',
1356+
'<Transition>',
1357+
'<NoncurrentVersionTransition>',
1358+
'</NoncurrentVersionTransition>',
1359+
'</Transition>',
1360+
'</Tag>',
1361+
'</Rule>'
1362+
]
13531363
xml_config = format_xml(data=LifecycleConfiguration, root='LifecycleConfiguration', lst=lst)
13541364
headers = mapped(kwargs)
13551365
headers['Content-MD5'] = get_md5(xml_config)
@@ -1393,7 +1403,12 @@ def get_bucket_lifecycle(self, Bucket, **kwargs):
13931403
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
13941404
headers=headers)
13951405
data = xml_to_dict(rt.text)
1396-
data = format_dict(data, ['Rule'])
1406+
format_dict(data, ['Rule'])
1407+
if 'Rule' in data.keys():
1408+
for rule in data['Rule']:
1409+
format_dict(rule, ['Transition', 'NoncurrentVersionTransition'])
1410+
if 'Filter' in rule.keys():
1411+
format_dict(rule['Filter'], ['Tag'])
13971412
return data
13981413

13991414
def delete_bucket_lifecycle(self, Bucket, **kwargs):
@@ -1596,7 +1611,7 @@ def get_bucket_replication(self, Bucket, **kwargs):
15961611
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
15971612
headers=headers)
15981613
data = xml_to_dict(rt.text)
1599-
data = format_dict(data, ['Rule'])
1614+
format_dict(data, ['Rule'])
16001615
return data
16011616

16021617
def delete_bucket_replication(self, Bucket, **kwargs):
@@ -1814,11 +1829,12 @@ def _check_all_upload_parts(self, bucket, key, uploadid, local_path, parts_num,
18141829
UploadId=uploadid,
18151830
PartNumberMarker=part_number_marker
18161831
)
1817-
parts_info.extend(response['Part'])
1832+
if 'Part' in response:
1833+
parts_info.extend(response['Part'])
18181834
if response['IsTruncated'] == 'false':
18191835
list_over_status = True
18201836
else:
1821-
part_number_marker = int(response['NextMarker'])
1837+
part_number_marker = int(response['NextPartNumberMarker'])
18221838
for part in parts_info:
18231839
part_num = int(part['PartNumber'])
18241840
# 如果分块数量大于本地计算出的最大数量,校验失败

‎ut/test.py‎

Lines changed: 73 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@
1212

1313
SECRET_ID = os.environ["SECRET_ID"]
1414
SECRET_KEY = os.environ["SECRET_KEY"]
15-
test_bucket = "test01-1252448703"
15+
TRAVIS_FLAG = os.environ["TRAVIS_FLAG"]
16+
test_bucket = 'cos-python-v5-test-' + str(sys.version_info[0]) + '-' + str(sys.version_info[1]) + '-' + '1252448703'
1617
test_object = "test.txt"
1718
special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
1819
conf = CosConfig(
@@ -23,6 +24,19 @@
2324
client = CosS3Client(conf)
2425

2526

27+
def _create_test_bucket(test_bucket):
28+
try:
29+
response = client.create_bucket(
30+
Bucket=test_bucket,
31+
)
32+
except Exception as e:
33+
if e.get_error_code() == 'BucketAlreadyOwnedByYou':
34+
print('BucketAlreadyOwnedByYou')
35+
else:
36+
raise e
37+
return None
38+
39+
2640
def get_raw_md5(data):
2741
m2 = hashlib.md5(data)
2842
etag = '"' + str(m2.hexdigest()) + '"'
@@ -37,22 +51,24 @@ def gen_file(path, size):
3751

3852

3953
def print_error_msg(e):
40-
print e.get_origin_msg()
41-
print e.get_digest_msg()
42-
print e.get_status_code()
43-
print e.get_error_code()
44-
print e.get_error_msg()
45-
print e.get_resource_location()
46-
print e.get_trace_id()
47-
print e.get_request_id()
54+
print (e.get_origin_msg())
55+
print (e.get_digest_msg())
56+
print (e.get_status_code())
57+
print (e.get_error_code())
58+
print (e.get_error_msg())
59+
print (e.get_resource_location())
60+
print (e.get_trace_id())
61+
print (e.get_request_id())
4862

4963

5064
def setUp():
51-
print "start test..."
65+
print ("start test...")
66+
print ("start create bucket " + test_bucket)
67+
_create_test_bucket(test_bucket)
5268

5369

5470
def tearDown():
55-
print "function teardown"
71+
print ("function teardown")
5672

5773

5874
def test_put_get_delete_object_10MB():
@@ -146,6 +162,11 @@ def test_put_object_non_exist_bucket():
146162

147163
def test_put_object_acl():
148164
"""设置object acl"""
165+
response = client.put_object(
166+
Bucket=test_bucket,
167+
Key=test_object,
168+
Body='test acl'
169+
)
149170
response = client.put_object_acl(
150171
Bucket=test_bucket,
151172
Key=test_object,
@@ -160,6 +181,10 @@ def test_get_object_acl():
160181
Key=test_object
161182
)
162183
assert response
184+
response = client.delete_object(
185+
Bucket=test_bucket,
186+
Key=test_object
187+
)
163188

164189

165190
def test_copy_object_diff_bucket():
@@ -352,7 +377,7 @@ def test_get_bucket_acl_normal():
352377
def test_list_objects():
353378
"""列出bucket下的objects"""
354379
response = client.list_objects(
355-
Bucket=test_bucket,
380+
Bucket='test01-1252448703',
356381
MaxKeys=100,
357382
Prefix='中文',
358383
Delimiter='/'
@@ -376,7 +401,7 @@ def test_get_presigned_url():
376401
Key='中文.txt'
377402
)
378403
assert url
379-
print url
404+
print (url)
380405

381406

382407
def test_get_bucket_location():
@@ -430,10 +455,27 @@ def test_put_get_delete_lifecycle():
430455
lifecycle_config = {
431456
'Rule': [
432457
{
433-
'Expiration': {'Date': get_date(2030, 5, 1)},
434-
'ID': '123',
435-
'Filter': {'Prefix': ''},
436458
'Status': 'Enabled',
459+
'Filter': {
460+
# 作用于带标签键 datalevel 和值 backup 的标签的对象
461+
'Tag': [
462+
{
463+
'Key': 'datalevel',
464+
'Value': 'backup'
465+
}
466+
]
467+
},
468+
'Transation': [
469+
{
470+
# 30天后转换为Standard_IA
471+
'Days': 30,
472+
'StorageClass': 'Standard_IA'
473+
}
474+
],
475+
'Expiration': {
476+
# 3650天后过期删除
477+
'Days': 3650
478+
}
437479
}
438480
]
439481
}
@@ -479,7 +521,7 @@ def test_put_get_delete_replication():
479521
{
480522
'ID': '123',
481523
'Status': 'Enabled',
482-
'Prefix': 'replication',
524+
'Prefix': '中文',
483525
'Destination': {
484526
'Bucket': 'qcs:id/0:cos:cn-south:appid/1252448703:replicationsouth'
485527
}
@@ -499,7 +541,7 @@ def test_put_get_delete_replication():
499541
Bucket=test_bucket
500542
)
501543
assert response
502-
# delete lifecycle
544+
# delete replication
503545
response = client.delete_bucket_replication(
504546
Bucket=test_bucket
505547
)
@@ -547,7 +589,10 @@ def test_list_multipart_uploads():
547589
def test_upload_file_multithreading():
548590
"""根据文件大小自动选择分块大小,多线程并发上传提高上传速度"""
549591
file_name = "thread_1GB"
550-
gen_file(file_name, 5) # set 5MB beacuse travis too slow
592+
file_size = 1024
593+
if TRAVIS_FLAG == 'true':
594+
file_size = 5 # set 5MB beacuse travis too slow
595+
gen_file(file_name, file_size)
551596
st = time.time() # 记录开始时间
552597
response = client.upload_file(
553598
Bucket=test_bucket,
@@ -560,7 +605,7 @@ def test_upload_file_multithreading():
560605
ed = time.time() # 记录结束时间
561606
if os.path.exists(file_name):
562607
os.remove(file_name)
563-
print ed - st
608+
print (ed - st)
564609

565610

566611
def test_copy_file_automatically():
@@ -608,7 +653,8 @@ def test_use_get_auth():
608653
Key='test.txt',
609654
Params={'acl': '', 'unsed': '123'}
610655
)
611-
response = requests.get('http://test01-1252448703.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123', headers={'Authorization': auth})
656+
url = 'http://' + test_bucket + '.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123'
657+
response = requests.get(url, headers={'Authorization': auth})
612658
assert response.status_code == 200
613659

614660

@@ -646,16 +692,15 @@ def test_put_get_bucket_logging():
646692
response = logging_client.get_bucket_logging(
647693
Bucket=logging_bucket
648694
)
649-
print response
695+
print (response)
650696
assert response['LoggingEnabled']['TargetBucket'] == logging_bucket
651697
assert response['LoggingEnabled']['TargetPrefix'] == 'test'
652698

653699

654700
def test_put_object_enable_md5():
655701
"""上传文件,SDK计算content-md5头部"""
656-
file_size = 10
657702
file_name = 'test_object_sdk_caculate_md5.file'
658-
gen_file(file_name, 10)
703+
gen_file(file_name, 1)
659704
with open(file_name, 'rb') as f:
660705
etag = get_raw_md5(f.read())
661706
with open(file_name, 'rb') as fp:
@@ -686,6 +731,10 @@ def test_put_object_from_local_file():
686731
Key=file_name
687732
)
688733
assert put_response['ETag'] == etag
734+
response = client.delete_object(
735+
Bucket=test_bucket,
736+
Key=file_name
737+
)
689738
if os.path.exists(file_name):
690739
os.remove(file_name)
691740

0 commit comments

Comments
 (0)