|
| 1 | +from qcloud_cos import CosConfig |
| 2 | +from qcloud_cos import CosS3Client |
| 3 | +import sys |
| 4 | +import logging |
| 5 | +import crcmod |
| 6 | +import os |
| 7 | +import subprocess |
| 8 | + |
| 9 | +logging.basicConfig(level=logging.ERROR, stream=sys.stdout) |
| 10 | + |
| 11 | +cos_client = None |
| 12 | +run_dir = "" |
| 13 | + |
| 14 | + |
| 15 | +def init_cos_client(region, bucket_name, mnt_point): |
| 16 | + global cos_client |
| 17 | + global run_dir |
| 18 | + secret_id = '' |
| 19 | + secret_key = '' |
| 20 | + with open('/etc/passwd-cosfs') as fl: |
| 21 | + for line in fl.readlines(): |
| 22 | + if bucket_name in line: |
| 23 | + parts = line.split(":") |
| 24 | + if len(parts) < 3: |
| 25 | + raise ValueError("unexpected secret info in /etc/passwd-cosfs") |
| 26 | + secret_id = parts[1].strip() |
| 27 | + secret_key = parts[2].strip() |
| 28 | + if len(region) == 0 or len(secret_id) == 0 or len(secret_key) == 0: |
| 29 | + raise ValueError("Unexpected region or secret_id or secret_key") |
| 30 | + print("init cos client") |
| 31 | + config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=None, Scheme='http') |
| 32 | + cos_client = CosS3Client(config) |
| 33 | + run_dir = os.path.dirname(os.path.normpath(mnt_point)) |
| 34 | + print("run_dir:" + run_dir) |
| 35 | + |
| 36 | + |
| 37 | +def get_cos_file_crc64(bucket, key): |
| 38 | + response = cos_client.get_object(Bucket=bucket, Key=key) |
| 39 | + c64 = crcmod.mkCrcFun(0x142F0E1EBA9EA3693, initCrc=0, xorOut=0xffffffffffffffff, rev=True) |
| 40 | + body = response['Body'].read(1024*1024*100) |
| 41 | + return str(c64(body)) |
| 42 | + |
| 43 | + |
| 44 | +def get_local_file_crc64(file_path): |
| 45 | + result = subprocess.run([os.path.join(run_dir, "coscli-linux"), '-c', |
| 46 | + os.path.join(run_dir, 'test/cos.yaml'), 'hash', file_path], |
| 47 | + stdout=subprocess.PIPE) |
| 48 | + return str(result.stdout).split(':')[-1].strip("'n\\ ") |
| 49 | + |
| 50 | + |
| 51 | +def verify_file_checksum_length_with_local(bucket, key, local_file_path): |
| 52 | + object_attr = cos_client.head_object(Bucket=bucket, Key=key) |
| 53 | + test_crc64 = crcmod.Crc(0x142F0E1EBA9EA3693, initCrc=0, xorOut=0xffffffffffffffff, rev=True) |
| 54 | + # c64 = crcmod.mkCrcFun(0x142F0E1EBA9EA3693, initCrc=0, xorOut=0xffffffffffffffff, rev=True) |
| 55 | + cos_len = int(object_attr['Content-Length']) |
| 56 | + cos_crc64 = object_attr['x-cos-hash-crc64ecma'] |
| 57 | + local_len = os.stat(local_file_path).st_size |
| 58 | + # read_bytes = 0 |
| 59 | + # buffer_size = 128 * 1024 |
| 60 | + # with open(local_file_path, 'rb') as fl: |
| 61 | + # byte_array = fl.read(buffer_size) |
| 62 | + # read_bytes = read_bytes + len(byte_array) |
| 63 | + # while len(byte_array) > 0: |
| 64 | + # test_crc64.update(byte_array) |
| 65 | + # byte_array = fl.read(buffer_size) |
| 66 | + # read_bytes = read_bytes + len(byte_array) |
| 67 | + # if read_bytes % (100 * 1024 * 1024) == 0: |
| 68 | + # print('read bytes ' + str(read_bytes)) |
| 69 | + local_crc64 = get_local_file_crc64(local_file_path) |
| 70 | + # local_crc64 = str(int(test_crc64.hexdigest(), 16)) |
| 71 | + # local_crc64 = str(c64(open(local_file_path, 'rb').read())) |
| 72 | + if local_len > 0 and cos_crc64 == '': |
| 73 | + cos_crc64 = get_cos_file_crc64(bucket, key) |
| 74 | + print('key' + key + " cos crc64:" + cos_crc64) |
| 75 | + if cos_len != local_len or (cos_len != 0 and cos_crc64 != local_crc64): |
| 76 | + print(object_attr) |
| 77 | + raise ValueError('file diff, local file:' + local_file_path + ', len:' + str(local_len) + |
| 78 | + ', crc64:' + local_crc64 + '; cos file:' + key + ', len:' + str(cos_len) + |
| 79 | + ', crc64:' + cos_crc64) |
| 80 | + |
| 81 | + |
| 82 | +def verify_file_checksum_length(bucket, key, local_len, local_crc64): |
| 83 | + object_attr = cos_client.head_object(Bucket=bucket, Key=key) |
| 84 | + cos_len = int(object_attr['Content-Length']) |
| 85 | + cos_crc64 = object_attr['x-cos-hash-crc64ecma'] |
| 86 | + if local_len > 0 and cos_crc64 == '': |
| 87 | + cos_crc64 = get_cos_file_crc64(bucket, key) |
| 88 | + print('key' + key + " cos crc64:" + cos_crc64) |
| 89 | + if cos_len != local_len or (cos_len != 0 and cos_crc64 != local_crc64): |
| 90 | + raise ValueError('file diff, local len:' + str(local_len) + |
| 91 | + ', crc64:' + local_crc64 + '; cos file:' + key + ', len:' + str(cos_len) + |
| 92 | + ', crc64:' + cos_crc64) |
0 commit comments