support ftruncate file

Author: yongqingliu

src/fdcache.cpp

@@ -1413,6 +1413,7 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
     }
 
   }else{
+    S3FS_PRN_INFO3("[path=%s][fd=%d] already being upload", path.c_str(), fd);
     // upload rest data
     if(0 < mp_size){
       if(0 != (result = NoCacheMultipartPost(fd, mp_start, mp_size))){
@@ -1591,6 +1592,42 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
   return wsize;
 }
 
+int FdEntity::GetRefCount() {
+  AutoLock auto_lock(&fdent_lock);
+  return refcnt;
+}
+
+// this function only truncate file size in local, not flush to cos
+// the refcnt must bigger than 1, in order to flush the truncate result later
+// the flush trigger after user call close
+int FdEntity::Ftruncate(ssize_t size) {
+    AutoLock auto_lock(&fdent_lock);
+        // if size is equal, do nothing
+    if (static_cast<size_t>(size) == pagelist.Size()) {
+      return 0;
+    }
+    if (refcnt <= 1) {
+        S3FS_PRN_ERR("failed to truncate file %s, refcnt(%d)", path.c_str(), refcnt);
+        return -EIO;
+    }
+    // file being upload, this is due to space not enough, we not support truncate in this situation
+    if (!upload_id.empty()) {
+      S3FS_PRN_ERR("failed to truncate file %s, because not enough space.", path.c_str());
+      return -ENOSPC;
+    }
+    // truncate temporary file size
+    if(-1 == ftruncate(fd, size) || -1 == fsync(fd)){
+      S3FS_PRN_ERR("failed to truncate temporary file %s by errno(%d).",  path.c_str(), errno);
+      return -errno;
+    }
+    if(!is_modify){
+      is_modify = true;
+    }
+    // resize pagelist
+    pagelist.Resize(static_cast<size_t>(size), false);
+    return 0;
+}
+
 
 // [NOTE]
 // Returns true if merged to orgmeta.
@@ -1649,6 +1686,7 @@ int FdEntity::UploadPendingMeta()
 
     return result;
 }
+
 //------------------------------------------------
 // FdManager symbol
 //------------------------------------------------

src/fdcache.h

@@ -178,6 +178,8 @@ class FdEntity
 
     ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false);
     ssize_t Write(const char* bytes, off_t start, size_t size);
+    int GetRefCount();
+    int Ftruncate(ssize_t size);
 };
 typedef std::map<std::string, class FdEntity*> fdent_map_t;   // key=path, value=FdEntity*
 

src/s3fs.cpp

@@ -2093,7 +2093,16 @@ static int s3fs_truncate(const char* path, off_t size)
       FdManager::get()->Close(ent);
       return result;
     }
-
+     // the file already opened for write, for exmaple open->ftruncate
+    // in this case we truncate in local disk, the flush will be delay to nearest flush
+    if (ent->GetRefCount() > 1) {
+      S3FS_PRN_DBG("[path=%s] already opened for writing, truncate it in local", path);
+      result = ent->Ftruncate(size);
+      FdManager::get()->Close(ent);
+      return result;
+    }
+    // the file not writing for others, we can safe flush to cos
+    S3FS_PRN_DBG("[path=%s] not being written, ready flush to cos", path);
   }else{
     // Not found -> Make tmpfile(with size)
 

test/cos.yaml

@@ -0,0 +1,11 @@
+cos:
+  base:
+    secretid: xxxxx
+    secretkey: yyyyy
+    sessiontoken: zzzz
+    protocol: https
+  buckets:
+  - name: ""
+    alias: ""
+    region: ""
+    endpoint: ""

test/cos_utils.py

@@ -2,17 +2,19 @@
 from qcloud_cos import CosS3Client
 import sys
 import logging
-import hashlib
 import crcmod
 import os
+import subprocess
 
 logging.basicConfig(level=logging.ERROR, stream=sys.stdout)
 
 cos_client = None
+run_dir = ""
 
 
-def init_cos_client(region, bucket_name):
+def init_cos_client(region, bucket_name, mnt_point):
     global cos_client
+    global run_dir
     secret_id = ''
     secret_key = ''
     with open('/etc/passwd-cosfs') as fl:
@@ -21,12 +23,15 @@ def init_cos_client(region, bucket_name):
                 parts = line.split(":")
                 if len(parts) < 3:
                     raise ValueError("unexpected secret info in /etc/passwd-cosfs")
-                secret_id = parts[1]
-                secret_key = parts[2]
+                secret_id = parts[1].strip()
+                secret_key = parts[2].strip()
     if len(region) == 0 or len(secret_id) == 0 or len(secret_key) == 0:
         raise ValueError("Unexpected region or secret_id or secret_key")
+    print("init cos client")
     config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=None, Scheme='http')
     cos_client = CosS3Client(config)
+    run_dir = os.path.dirname(os.path.normpath(mnt_point))
+    print("run_dir:" + run_dir)
 
 
 def get_cos_file_crc64(bucket, key):
@@ -36,13 +41,34 @@ def get_cos_file_crc64(bucket, key):
     return str(c64(body))
 
 
+def get_local_file_crc64(file_path):
+    result = subprocess.run([os.path.join(run_dir, "coscli-linux"), '-c',
+                             os.path.join(run_dir, 'test/cos.yaml'), 'hash', file_path],
+                            stdout=subprocess.PIPE)
+    return str(result.stdout).split(':')[-1].strip("'n\\ ")
+
+
 def verify_file_checksum_length_with_local(bucket, key, local_file_path):
     object_attr = cos_client.head_object(Bucket=bucket, Key=key)
-    c64 = crcmod.mkCrcFun(0x142F0E1EBA9EA3693, initCrc=0, xorOut=0xffffffffffffffff, rev=True)
+    test_crc64 = crcmod.Crc(0x142F0E1EBA9EA3693, initCrc=0, xorOut=0xffffffffffffffff, rev=True)
+    # c64 = crcmod.mkCrcFun(0x142F0E1EBA9EA3693, initCrc=0, xorOut=0xffffffffffffffff, rev=True)
     cos_len = int(object_attr['Content-Length'])
     cos_crc64 = object_attr['x-cos-hash-crc64ecma']
     local_len = os.stat(local_file_path).st_size
-    local_crc64 =str(c64(open(local_file_path, 'rb').read()))
+    # read_bytes = 0
+    # buffer_size = 128 * 1024
+    # with open(local_file_path, 'rb') as fl:
+    #     byte_array = fl.read(buffer_size)
+    #     read_bytes = read_bytes + len(byte_array)
+    #     while len(byte_array) > 0:
+    #         test_crc64.update(byte_array)
+    #         byte_array = fl.read(buffer_size)
+    #         read_bytes = read_bytes + len(byte_array)
+    #         if read_bytes % (100 * 1024 * 1024) == 0:
+    #             print('read bytes ' + str(read_bytes))
+    local_crc64 = get_local_file_crc64(local_file_path)
+    # local_crc64 = str(int(test_crc64.hexdigest(), 16))
+    # local_crc64 = str(c64(open(local_file_path, 'rb').read()))
     if local_len > 0 and cos_crc64 == '':
         cos_crc64 = get_cos_file_crc64(bucket, key)
         print('key' + key + " cos crc64:" + cos_crc64)
@@ -63,4 +89,4 @@ def verify_file_checksum_length(bucket, key, local_len, local_crc64):
     if cos_len != local_len or (cos_len != 0 and cos_crc64 != local_crc64):
         raise ValueError('file diff, local len:' + str(local_len) +
                          ', crc64:' + local_crc64 + '; cos file:' + key + ', len:' + str(cos_len) +
-                         ', crc64:' + cos_crc64)
+                         ', crc64:' + cos_crc64)

test/integration-test-main.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #Usage:
 #1. Mount bucket: cosfs test-12500000 mnt_dir -ourl=http://cos.ap-shanghai.myqcloud.com -odbglevel=dbg -ocurldbg -oallow_other 
-#2. Run the script with mount point parameter: ./integration-test-main.sh mnt_dir
+#2. Run the script with mount point parameter:bash test/integration-test-main.sh /root/cosfs-yongqing/mnt/ ap-guangzhou test-125000000
 #3. If all test cases pass, "All tests complete" will be output, otherwise the corresponding error message will be output.
 
 set -o xtrace
@@ -21,6 +21,8 @@ BIG_FILE=big-file-s3fs.txt
 BIG_FILE_LENGTH=$((25 * 1024 * 1024))
 CUR_DIR=`pwd`
 TEST_BUCKET_MOUNT_POINT_1=$1
+TEST_BUCKET_REGION=$2
+TEST_BUCKET_NAME=$3
 
 function mk_test_file {
     if [ $# == 0 ]; then
@@ -120,7 +122,7 @@ function test_truncate_file {
 function test_truncate_empty_file {
     echo "Testing truncate empty file ..."
     # Write an empty test file
-    touch ${TEST_TEXT_FILE}
+    echo "data" > ${TEST_TEXT_FILE}
 
     # Truncate the file to 1024 length
     t_size=1024
@@ -133,6 +135,30 @@ function test_truncate_empty_file {
         echo "error: expected ${TEST_TEXT_FILE} to be $t_size length, got $size"
         exit 1
     fi
+    first_chars=`head -c 4 ${TEST_TEXT_FILE}`
+    if [ "$first_chars" != "data" ]
+    then
+        echo "error: expected first 4 byte is 'data' got $first_chars"
+        exit 1
+    fi
+     # Truncate the file to 1024 length
+    t_size=3
+    truncate ${TEST_TEXT_FILE} -s $t_size
+
+    # Verify file is zero length
+    size=$(stat -c %s ${TEST_TEXT_FILE})
+    if [ $t_size -ne $size ]
+    then
+        echo "error: expected ${TEST_TEXT_FILE} to be $t_size length, got $size"
+        exit 1
+    fi
+    first_chars=`head -c 4 ${TEST_TEXT_FILE}`
+    if [ "$first_chars" != "dat" ]
+    then
+        echo "error: expected first 4 byte is 'dat' got $first_chars"
+        exit 1
+    fi
+
     rm_test_file
 }
 
@@ -522,10 +548,16 @@ function test_file_size_in_stat_cache {
     python3 $CUR_DIR/test/stat_cache_test.py $TEST_BUCKET_MOUNT_POINT_1
 }
 
+function test_integration_test {
+   echo "Testing integration test..."
+   python3 $CUR_DIR/test/integration-test.py $TEST_BUCKET_MOUNT_POINT_1 $TEST_BUCKET_REGION $TEST_BUCKET_NAME
+}
+
 function run_all_tests {
+    test_integration_test
     test_append_file
-    # test_truncate_file
-    # test_truncate_empty_file
+    test_truncate_file
+    test_truncate_empty_file
     test_mv_file
     test_mv_bigfile
     test_mv_directory
@@ -548,21 +580,24 @@ function run_all_tests {
     test_mtime_file
     test_file_size_in_stat_cache
 }
-
+pip install -U cos-python-sdk-v5
+if [ ! -f coscli-linux ]; then
+  wget "https://cosbrowser.cloud.tencent.com/software/coscli/coscli-linux"
+  chmod a+x coscli-linux
+fi
 # Mount the bucket
 if [ "$TEST_BUCKET_MOUNT_POINT_1" == "" ]; then
     echo "Mountpoint missing"
     exit 1
 fi
 cd $TEST_BUCKET_MOUNT_POINT_1
+rm -rf *
 
 if [ -e $TEST_TEXT_FILE ]
 then
   rm -f $TEST_TEXT_FILE
 fi
 
-rm -rf *
-pip install -U cos-python-sdk-v5
 SECONDS=0
 run_all_tests
 duration=$SECONDS