Merge branch 'v1.0.21' into 'master' (merge request !5)

v1.0.21
Merge branch 'v1.0.21' into 'v1.0.21' (merge request !4)

enable content md5 by default

Author: yongqingliu

src/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "files.associations": {
+        "string": "cpp",
+        "iosfwd": "cpp"
+    }
+}

src/curl.cpp

@@ -294,7 +294,7 @@ storage_class_t  S3fsCurl::storage_class       = STANDARD;
 sseckeylist_t    S3fsCurl::sseckeys;
 std::string      S3fsCurl::ssekmsid            = "";
 sse_type_t       S3fsCurl::ssetype             = SSE_DISABLE;
-bool             S3fsCurl::is_content_md5      = false;
+bool             S3fsCurl::is_content_md5      = true;
 bool             S3fsCurl::is_verbose          = false;
 string           S3fsCurl::COSAccessKeyId;
 string           S3fsCurl::COSSecretAccessKey;
@@ -2974,8 +2974,16 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string&
 
   string strMD5;
   if(S3fsCurl::is_content_md5){
-	  strMD5 = md5base64;
-	  requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", strMD5.c_str());
+     unsigned char *md5raw = s3fs_md5_fd(partdata.fd, partdata.startpos, partdata.size);
+      if(md5raw == NULL){
+          S3FS_PRN_ERR("Could not make md5 for file(part %d)", part_num);
+          return -EIO;
+      }
+      partdata.etag = s3fs_hex_lower(md5raw, get_md5_digest_length());
+      char* md5base64p = s3fs_base64(md5raw, get_md5_digest_length());
+      requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", md5base64p);
+      delete[] md5base64p;
+      delete[] md5raw;
   }
 
   if(!S3fsCurl::IsPublicBucket()){

src/fdcache.cpp

@@ -200,6 +200,49 @@ bool CacheFileStat::Open(void)
   return true;
 }
 
+bool CacheFileStat::RenameCacheFileStat(const char* oldpath, const char* newpath)
+{
+    if(!oldpath || '\0' == oldpath[0] || !newpath || '\0' == newpath[0]){
+        return false;
+    }
+
+    // stat path
+    std::string old_filestat;
+    std::string new_filestat;
+    if(!CacheFileStat::MakeCacheFileStatPath(oldpath, old_filestat, false) || !CacheFileStat::MakeCacheFileStatPath(newpath, new_filestat, false)){
+        return false;
+    }
+
+    // check new stat path
+    struct stat st;
+    if(0 == stat(new_filestat.c_str(), &st)){
+        // new stat path is existed, then unlink it.
+        if(-1 == unlink(new_filestat.c_str())){
+            S3FS_PRN_ERR("failed to unlink new cache file stat path(%s) by errno(%d).", new_filestat.c_str(), errno);
+            return false;
+        }
+    }
+
+    // check old stat path
+    if(0 != stat(old_filestat.c_str(), &st)){
+        // old stat path is not existed, then nothing to do any more.
+        return true;
+    }
+
+    // create parent dir
+    CacheFileStat::MakeCacheFileStatPath(newpath, new_filestat, true);
+    // link and unlink
+    if(-1 == link(old_filestat.c_str(), new_filestat.c_str())){
+        S3FS_PRN_ERR("failed to link old cache file stat path(%s) to new cache file stat path(%s) by errno(%d).", old_filestat.c_str(), new_filestat.c_str(), errno);
+        return false;
+    }
+    if(-1 == unlink(old_filestat.c_str())){
+        S3FS_PRN_ERR("failed to unlink old cache file stat path(%s) by errno(%d).", old_filestat.c_str(), errno);
+        return false;
+    }
+   return true;
+}
+
 bool CacheFileStat::Release(void)
 {
   if(-1 == fd){
@@ -410,6 +453,11 @@ bool PageList::FindUnloadedPage(off_t start, off_t& resstart, size_t& ressize) c
 
 size_t PageList::GetTotalUnloadedPageSize(off_t start, size_t size) const
 {
+  if(0 == size){
+    if(start < Size()){
+      size = Size() - start;
+    }
+  }
   size_t restsize = 0;
   off_t  next     = static_cast<off_t>(start + size);
   for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
@@ -620,7 +668,7 @@ int FdEntity::FillFile(int fd, unsigned char byte, size_t size, off_t start)
 FdEntity::FdEntity(const char* tpath, const char* cpath)
         : is_lock_init(false), refcnt(0), path(SAFESTRPTR(tpath)), cachepath(SAFESTRPTR(cpath)),
           fd(-1), pfile(NULL), is_modify(false), size_orgmeta(0), upload_id(""), mp_start(0), mp_size(0),
-          is_meta_pending(false)
+          is_meta_pending(false), is_no_disk_space_flushed(false)
 {
   try{
     pthread_mutexattr_t attr;
@@ -843,6 +891,55 @@ int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time)
   return 0;
 }
 
+//
+// Rename file path.
+//
+// This method sets the FdManager::fent map registration key to fentmapkey.
+//
+// [NOTE]
+// This method changes the file path of FdEntity.
+// Old file is deleted after linking to the new file path, and this works
+// without problem because the file descriptor is not affected even if the
+// cache file is open.
+// The mirror file descriptor is also the same. The mirror file path does
+// not need to be changed and will remain as it is.
+//
+bool FdEntity::RenamePath(const std::string& newpath, std::string& fentmapkey) {
+  AutoLock auto_lock(&fdent_lock);
+  if(!cachepath.empty()){
+        // has cache path
+
+        // make new cache path
+        std::string newcachepath;
+        if(!FdManager::MakeCachePath(newpath.c_str(), newcachepath, true)){
+          S3FS_PRN_ERR("failed to make cache path for object(%s).", newpath.c_str());
+          return false;
+        }
+
+        // rename cache file
+        if(-1 == rename(cachepath.c_str(), newcachepath.c_str())){
+          S3FS_PRN_ERR("failed to rename old cache path(%s) to new cache path(%s) by errno(%d).", cachepath.c_str(), newcachepath.c_str(), errno);
+          return false;
+        }
+
+        // link and unlink cache file stat
+        if(!CacheFileStat::RenameCacheFileStat(path.c_str(), newpath.c_str())){
+          S3FS_PRN_ERR("failed to rename cache file stat(%s to %s).", path.c_str(), newpath.c_str());
+          return false;
+        }
+        fentmapkey = newpath;
+        cachepath  = newcachepath;
+    }else{
+        // does not have cache path
+        fentmapkey.erase();
+        FdManager::MakeRandomTempPath(newpath.c_str(), fentmapkey);
+    }
+    // set new path
+    path = newpath;
+
+    return true;
+}
+
 // [NOTE]
 // This method is called from olny nocopapi functions.
 // So we do not check disk space for this option mode, if there is no enough
@@ -1069,7 +1166,12 @@ int FdEntity::Load(off_t start, size_t size)
       }else{
         // single request
         S3fsCurl s3fscurl;
-        result = s3fscurl.GetObjectRequest(path.c_str(), fd, (*iter)->offset, need_load_size);
+        if(0 < need_load_size){
+          S3fsCurl s3fscurl;
+          result = s3fscurl.GetObjectRequest(path.c_str(), fd, (*iter)->offset, need_load_size);
+        }else{
+          result = 0;
+        }
       }
       if(0 != result){
         break;
@@ -1398,6 +1500,10 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
         backup = S3fsCurl::SetReadwriteTimeout(120);
       }
       result = S3fsCurl::ParallelMultipartUploadRequest(tpath ? tpath : path.c_str(), orgmeta, fd);
+      if (result != 0) {
+        S3FS_PRN_ERR("failed to flush object(errno=%d)", result);
+        return result;
+      }
       if(0 != backup){
         S3fsCurl::SetReadwriteTimeout(backup);
       }
@@ -1414,8 +1520,19 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
 
   }else{
     S3FS_PRN_INFO3("[path=%s][fd=%d] already being upload", path.c_str(), fd);
+    if (is_no_disk_space_flushed) {
+        // 磁盘空间不够，进入的顺序写模式只能往cos上传一次
+        S3FS_PRN_ERR("not enough disk space for writting file, allocate more space[path=%s].", path.c_str());
+        return -EIO;
+    }
+    is_no_disk_space_flushed = true;
     // upload rest data
     if(0 < mp_size){
+      // 磁盘空间不够，进入的顺序写模式要求为新写入的文件，或者长度比原有的更长
+      if (mp_start + mp_size < pagelist.Size()) {
+          S3FS_PRN_ERR("not enough disk space for writting file[path=%s].", path.c_str());
+          return -EIO;
+      }
       if(0 != (result = NoCacheMultipartPost(fd, mp_start, mp_size))){
         S3FS_PRN_ERR("failed to multipart post(start=%zd, size=%zu) for file(%d).", mp_start, mp_size, fd);
         return result;
@@ -1548,6 +1665,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
     }
   }else{
     // alreay start miltipart uploading
+
   }
 
   // Writing
@@ -1564,6 +1682,11 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
 
   // check multipart uploading
   if(0 < upload_id.length()){
+    // sequence write mode check offset
+    if (start != mp_start) {
+        S3FS_PRN_ERR("failed to write file [path=%s], make sure you have enough disk space.");
+        return -EIO;
+    }
     mp_size += static_cast<size_t>(wsize);
     if(static_cast<size_t>(S3fsCurl::GetMultipartSize()) <= mp_size){
       // over one multipart size
@@ -2051,18 +2174,42 @@ FdEntity* FdManager::ExistOpen(const char* path, int existfd, bool ignore_existf
   return ent;
 }
 
-void FdManager::Rename(const std::string &from, const std::string &to)
+bool FdManager::Rename(const std::string &from, const std::string &to)
 {
   AutoLock auto_lock(&FdManager::fd_manager_lock);
   fdent_map_t::iterator iter = fent.find(from);
+  if(fent.end() == iter && !FdManager::IsCacheDir()){
+    // If the cache directory is not specified, s3fs opens a temporary file
+    // when the file is opened.
+    // Then if it could not find a entity in map for the file, s3fs should
+    // search a entity in all which opened the temporary file.
+    //
+    for(iter = fent.begin(); iter != fent.end(); ++iter){
+      if((*iter).second && (*iter).second->IsOpen() && 0 == strcmp((*iter).second->GetPath(), from.c_str())){
+        break;              // found opened fd in mapping
+      }
+    }
+  }
   if(fent.end() != iter){
     // found
     S3FS_PRN_DBG("[from=%s][to=%s]", from.c_str(), to.c_str());
     FdEntity* ent = (*iter).second;
     fent.erase(iter);
-    ent->SetPath(to);
-    fent[to] = ent;
+    // rename path and caches in fd entity
+    string fentmapkey;
+    if(!ent->RenamePath(to, fentmapkey)){
+      S3FS_PRN_ERR("Failed to rename FdEntity obejct for %s to %s", from.c_str(), to.c_str());
+      return false;
+    }
+    if (fentmapkey.size() <= 0) {
+      S3FS_PRN_ERR("invalid rename object status from %s to %s", from.c_str(), to.c_str());
+      return false;
+    }
+
+    // set new fd entity to map
+    fent[fentmapkey] = ent;
   }
+  return true;
 }
 
 bool FdManager::Close(FdEntity* ent)

src/fdcache.h

@@ -38,6 +38,7 @@ class CacheFileStat
   public:
     static bool DeleteCacheFileStat(const char* path);
     static bool CheckCacheFileStatTopDir(void);
+    static bool RenameCacheFileStat(const char* oldpath, const char* newpath);
 
     explicit CacheFileStat(const char* tpath = NULL);
     ~CacheFileStat();
@@ -98,7 +99,6 @@ class PageList
     bool FindUnloadedPage(off_t start, off_t& resstart, size_t& ressize) const;
     size_t GetTotalUnloadedPageSize(off_t start = 0, size_t size = 0) const;    // size=0 is checking to end of list
     int GetUnloadedPages(fdpage_list_t& unloaded_list, off_t start = 0, size_t size = 0) const;  // size=0 is checking to end of list
-
     bool Serialize(CacheFileStat& file, bool is_output);
     void Dump(void);
 };
@@ -127,6 +127,7 @@ class FdEntity
     off_t           mp_start;       // start position for no cached multipart(write method only)
     size_t          mp_size;        // size for no cached multipart(write method only)
     bool            is_meta_pending;
+    bool            is_no_disk_space_flushed;
   private:
     static size_t max_prefetch_bytes;
   private:
@@ -152,7 +153,7 @@ class FdEntity
     int Dup(void);
 
     const char* GetPath(void) const { return path.c_str(); }
-    void SetPath(const std::string &newpath) { path = newpath; }
+    bool RenamePath(const std::string& newpath, std::string& fentmapkey);
     int GetFd(void) const { return fd; }
 
     bool GetStats(struct stat& st);
@@ -225,7 +226,7 @@ class FdManager
     FdEntity* GetFdEntity(const char* path, int existfd = -1);
     FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true);
     FdEntity* ExistOpen(const char* path, int existfd = -1, bool ignore_existfd = false);
-    void Rename(const std::string &from, const std::string &to);
+    bool Rename(const std::string &from, const std::string &to);
     bool Close(FdEntity* ent);
     bool ChangeEntityToTempPath(FdEntity* ent, const char* path);
     static bool SetTmpDir(const char* dir);

src/openssl_auth.cpp

@@ -271,6 +271,45 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
   return result;
 }
 
+unsigned char* s3fs_md5_fd(int fd, off_t start, off_t size)
+{
+    MD5_CTX md5ctx;
+    off_t   bytes;
+    unsigned char* result;
+
+    if(-1 == size){
+        struct stat st;
+        if(-1 == fstat(fd, &st)){
+            return NULL;
+        }
+        size = st.st_size;
+    }
+
+    MD5_Init(&md5ctx);
+
+    for(off_t total = 0; total < size; total += bytes){
+        const off_t len = 512;
+        char buf[len];
+        bytes = len < (size - total) ? len : (size - total);
+        bytes = pread(fd, buf, bytes, start + total);
+        if(0 == bytes){
+            // end of file
+            break;
+        }else if(-1 == bytes){
+            // error
+            S3FS_PRN_ERR("file read error(%d)", errno);
+            return NULL;
+        }
+        MD5_Update(&md5ctx, buf, bytes);
+    }
+
+    result = new unsigned char[get_md5_digest_length()];
+    MD5_Final(result, &md5ctx);
+
+    return result;
+}
+
+
 //-------------------------------------------------------------------
 // Utility Function for SHA256
 //-------------------------------------------------------------------