diff --git a/src/cache.cpp b/src/cache.cpp index b898cb4..04693eb 100644 --- a/src/cache.cpp +++ b/src/cache.cpp @@ -165,7 +165,7 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove S3FS_PRN_DBG("stat cache not hit by ETag[path=%s][time=%jd][hit count=%lu][ETag(%s)!=(%s)]", strpath.c_str(), (intmax_t)(ent->cache_date), ent->hit_count, petag ? petag : "null", ent->meta["ETag"].c_str()); }else{ - // hit + // hit S3FS_PRN_DBG("stat cache hit [path=%s][time=%jd][hit count=%lu]", strpath.c_str(), (intmax_t)(ent->cache_date), ent->hit_count); if(pst!= NULL){ @@ -307,7 +307,7 @@ bool StatCache::IncSize(const std::string& key, ssize_t sz) stat_cache_entry* entry = iter->second; entry->stbuf.st_size += sz; S3FS_PRN_INFO3( - "Update file size in stat cache. [path=%s][size=%ld][delta=%ld]", + "Update file size in stat cache. [path=%s][size=%ld][delta=%ld]", key.c_str(), entry->stbuf.st_size, sz); } diff --git a/src/curl.cpp b/src/curl.cpp index 99b33a5..263ece7 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -608,7 +608,7 @@ string S3fsCurl::LookupMimeType(string name) } // neither the last extension nor the second-to-last extension - // matched a mimeType, return the default mime type + // matched a mimeType, return the default mime type return result; } @@ -617,7 +617,7 @@ bool S3fsCurl::LocateBundle(void) // See if environment variable CURL_CA_BUNDLE is set // if so, check it, if it is a good path, then set the // curl_ca_bundle variable to it - char *CURL_CA_BUNDLE; + char *CURL_CA_BUNDLE; if(0 == S3fsCurl::curl_ca_bundle.size()){ CURL_CA_BUNDLE = getenv("CURL_CA_BUNDLE"); @@ -629,7 +629,7 @@ bool S3fsCurl::LocateBundle(void) return false; } BF.close(); - S3fsCurl::curl_ca_bundle.assign(CURL_CA_BUNDLE); + S3fsCurl::curl_ca_bundle.assign(CURL_CA_BUNDLE); return true; } } @@ -650,10 +650,10 @@ bool S3fsCurl::LocateBundle(void) // dnl /usr/local/share/certs/ca-root.crt FreeBSD // dnl /etc/ssl/cert.pem OpenBSD // dnl /etc/ssl/certs/ (ca path) SUSE - ifstream BF("/etc/pki/tls/certs/ca-bundle.crt"); + ifstream BF("/etc/pki/tls/certs/ca-bundle.crt"); if(BF.good()){ BF.close(); - S3fsCurl::curl_ca_bundle.assign("/etc/pki/tls/certs/ca-bundle.crt"); + S3fsCurl::curl_ca_bundle.assign("/etc/pki/tls/certs/ca-bundle.crt"); }else{ S3FS_PRN_ERR("%s: /etc/pki/tls/certs/ca-bundle.crt is not readable", program_name.c_str()); return false; @@ -926,7 +926,7 @@ bool S3fsCurl::SetSseKmsid(const char* kmsid) } // [NOTE] -// Because SSE is set by some options and environment, +// Because SSE is set by some options and environment, // this function check the integrity of the SSE data finally. bool S3fsCurl::FinalCheckSse(void) { @@ -955,7 +955,7 @@ bool S3fsCurl::FinalCheckSse(void) } return true; } - + bool S3fsCurl::LoadEnvSseCKeys(void) { char* envkeys = getenv("OSSSSECKEYS"); @@ -1093,7 +1093,7 @@ bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl) // if(NULL == strstr(s3fscurl->headdata->str(), upper(s3fscurl->partdata.etag).c_str())){ // return false; // } - S3FS_PRN_ERR("headdata is : %s", s3fscurl->headdata->str()); + S3FS_PRN_INFO("headdata is : %s", s3fscurl->headdata->str()); string header_str(s3fscurl->headdata->str(), s3fscurl->headdata->size()); int pos = header_str.find("ETag: \""); if (pos != std::string::npos) { @@ -1103,7 +1103,7 @@ bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl) } else { s3fscurl->partdata.etag = header_str.substr(pos + 7, 32); // ETag get md5 value } - S3FS_PRN_ERR("partdata.etag : %s", s3fscurl->partdata.etag.c_str()); + S3FS_PRN_INFO("partdata.etag : %s", s3fscurl->partdata.etag.c_str()); } s3fscurl->partdata.etaglist->at(s3fscurl->partdata.etagpos).assign(s3fscurl->partdata.etag); s3fscurl->partdata.uploaded = true; @@ -1111,6 +1111,97 @@ bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl) return true; } +int S3fsCurl::ParallelMultipartUploadWithoutPreRequest(const char* tpath, headers_t& meta, int fd, + off_t offset, size_t size, string upload_id, + etaglist_t* list) +{ + S3FS_PRN_DBG("tpath=%s, fd=%d, offset=%lu, size=%lu, upload_id=%s", tpath, fd, offset, size, upload_id.c_str()); + int result; + struct stat st; + int fd2; + off_t remaining_bytes; + off_t cur_pos = offset; + S3fsCurl s3fscurl(true); + + S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd); + + if (NULL == list) { + S3FS_PRN_ERR("Invalid parameter, list could not be null."); + return -1; + } + + // duplicate fd + if(-1 == (fd2 = dup(fd)) || 0 != lseek(fd2, 0, SEEK_SET)){ + S3FS_PRN_ERR("Could not duplicate file descriptor(errno=%d)", errno); + if(-1 != fd2){ + close(fd2); + } + return -errno; + } + if(-1 == fstat(fd2, &st)){ + S3FS_PRN_ERR("Invalid file descriptor(errno=%d)", errno); + close(fd2); + return -errno; + } + + // cycle through open fd, pulling off 10MB chunks at a time + for(remaining_bytes = size; 0 < remaining_bytes; ){ + S3fsMultiCurl curlmulti; + int para_cnt; + off_t chunk; + + // Initialize S3fsMultiCurl + curlmulti.SetSuccessCallback(S3fsCurl::UploadMultipartPostCallback); + curlmulti.SetRetryCallback(S3fsCurl::UploadMultipartPostRetryCallback); + + // Loop for setup parallel upload(multipart) request. + for(para_cnt = 0; para_cnt < S3fsCurl::max_parallel_cnt && 0 < remaining_bytes; para_cnt++, remaining_bytes -= chunk){ + // chunk size + // if remaining_bytes less than 2*multipart_size, upload all remaining bytes, + // in order to avoid sending part that less than S3fsCurl::multipart_size + chunk = remaining_bytes > 2 * S3fsCurl::multipart_size ? S3fsCurl::multipart_size : remaining_bytes; + + // s3fscurl sub object + S3fsCurl* s3fscurl_para = new S3fsCurl(true); + s3fscurl_para->partdata.fd = fd2; + s3fscurl_para->partdata.startpos = cur_pos; + cur_pos += chunk; + s3fscurl_para->partdata.size = chunk; + s3fscurl_para->b_partdata_startpos = s3fscurl_para->partdata.startpos; + s3fscurl_para->b_partdata_size = s3fscurl_para->partdata.size; + s3fscurl_para->partdata.add_etag_list(list); + + // initiate upload part for parallel + if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, list->size(), upload_id))){ + S3FS_PRN_ERR("failed uploading part setup(%d)", result); + close(fd2); + delete s3fscurl_para; + return result; + } + + // set into parallel object + if(!curlmulti.SetS3fsCurlObject(s3fscurl_para)){ + S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath); + close(fd2); + delete s3fscurl_para; + return -1; + } + } + + // Multi request + if(0 != (result = curlmulti.Request())){ + S3FS_PRN_ERR("error occuered in multi request(errno=%d).", result); + break; + } + + // reinit for loop. + curlmulti.Clear(); + } + close(fd2); + + return 0; +} + S3fsCurl* S3fsCurl::UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl) { if(!s3fscurl){ @@ -1436,7 +1527,7 @@ int S3fsCurl::CurlDebugFunc(CURL* hcurl, curl_infotype type, char* data, size_t //------------------------------------------------------------------- // Methods for S3fsCurl //------------------------------------------------------------------- -S3fsCurl::S3fsCurl(bool ahbe) : +S3fsCurl::S3fsCurl(bool ahbe) : hCurl(NULL), path(""), base_path(""), saved_path(""), url(""), requestHeaders(NULL), bodydata(NULL), headdata(NULL), LastResponseCode(-1), postdata(NULL), postdata_remaining(0), is_use_ahbe(ahbe), retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0), @@ -1679,7 +1770,7 @@ bool S3fsCurl::RemakeHandle(void) case REQTYPE_CHKBUCKET: curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); - // XXX + // XXX //curl_easy_setopt(hCurl, CURLOPT_FAILONERROR, true); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); @@ -1780,8 +1871,8 @@ int S3fsCurl::RequestPerform(void) for(int retrycnt = S3fsCurl::retries; 0 < retrycnt; retrycnt--){ // Requests // XXX - //curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&responseHeaders); - //curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback); + //curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&responseHeaders); + //curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback); CURLcode curlCode = curl_easy_perform(hCurl); // Check result @@ -1799,7 +1890,7 @@ int S3fsCurl::RequestPerform(void) if(500 <= LastResponseCode){ S3FS_PRN_INFO3("HTTP response code %ld", LastResponseCode); sleep(4); - break; + break; } // Service response codes which are >= 400 && < 500 @@ -1829,38 +1920,38 @@ int S3fsCurl::RequestPerform(void) case CURLE_WRITE_ERROR: S3FS_PRN_ERR("### CURLE_WRITE_ERROR"); sleep(2); - break; + break; case CURLE_OPERATION_TIMEDOUT: S3FS_PRN_ERR("### CURLE_OPERATION_TIMEDOUT"); sleep(2); - break; + break; case CURLE_COULDNT_RESOLVE_HOST: S3FS_PRN_ERR("### CURLE_COULDNT_RESOLVE_HOST"); sleep(2); - break; + break; case CURLE_COULDNT_CONNECT: S3FS_PRN_ERR("### CURLE_COULDNT_CONNECT"); sleep(4); - break; + break; case CURLE_GOT_NOTHING: S3FS_PRN_ERR("### CURLE_GOT_NOTHING"); sleep(4); - break; + break; case CURLE_ABORTED_BY_CALLBACK: S3FS_PRN_ERR("### CURLE_ABORTED_BY_CALLBACK"); sleep(4); S3fsCurl::curl_times[hCurl] = time(0); - break; + break; case CURLE_PARTIAL_FILE: S3FS_PRN_ERR("### CURLE_PARTIAL_FILE"); sleep(4); - break; + break; case CURLE_SEND_ERROR: S3FS_PRN_ERR("### CURLE_SEND_ERROR"); @@ -1920,7 +2011,7 @@ int S3fsCurl::RequestPerform(void) } S3FS_PRN_INFO3("HTTP response code =%ld", LastResponseCode); - // Let's try to retrieve the + // Let's try to retrieve the if(404 == LastResponseCode){ return -ENOENT; } @@ -1981,14 +2072,14 @@ string S3fsCurl::CalcSignature(string method, string strMD5, string content_type FormatString += get_canonical_headers(requestHeaders); // \n has been append S3FS_PRN_INFO("Format string is : %s", FormatString.c_str()); - + const unsigned char* sdata = reinterpret_cast(FormatString.data()); int sdata_len = FormatString.size(); unsigned char* md = NULL; unsigned int md_len = 0; string format_string_sha1 = s3fs_sha1_hex(sdata, sdata_len, &md, &md_len); - S3FS_PRN_ERR("format string sha1 : %s", format_string_sha1.c_str()); + S3FS_PRN_INFO("format string sha1 : %s", format_string_sha1.c_str()); string StringToSign; StringToSign += string("sha1\n"); StringToSign += q_key_time + "\n"; @@ -2296,7 +2387,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) } else if(STANDARD_IA == GetStorageClass()){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-cos-storage-class", "STANDARD_IA"); } - + string date = get_date_rfc850(); requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); @@ -2614,14 +2705,12 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string { S3FS_PRN_INFO("[tpath=%s]", SAFESTRPTR(tpath)); - S3FS_PRN_ERR("PreMultipartPostRequest"); if(!tpath){ return -1; } if(!CreateCurlHandle(true)){ return -1; } - S3FS_PRN_ERR("PreMultipartPostRequest1"); string resource; string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); @@ -2686,7 +2775,6 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string return result; } - S3FS_PRN_ERR("PreMultipartPostRequest3"); // Parse XML body for UploadId if(!S3fsCurl::GetUploadId(upload_id)){ delete bodydata; @@ -2719,7 +2807,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id, postContent += " " + str(cnt + 1) + "\n"; postContent += " \"" + parts[cnt] + "\"\n"; postContent += "\n"; - } + } postContent += "\n"; // set postdata @@ -2962,7 +3050,7 @@ int S3fsCurl::UploadMultipartPostRequest(const char* tpath, int part_num, string // request if(0 == (result = RequestPerform())){ // check etag - // cos's no check etag + // cos's no check etag // if(NULL != strstr(headdata->str(), upper(partdata.etag).c_str())){ // get etag from response header S3FS_PRN_ERR("headdata is : %s", headdata->str()); @@ -3268,12 +3356,12 @@ int S3fsCurl::MultipartRenameRequest(const char* from, const char* to, headers_t } //------------------------------------------------------------------- -// Class S3fsMultiCurl +// Class S3fsMultiCurl //------------------------------------------------------------------- #define MAX_MULTI_HEADREQ 20 // default: max request count in readdir curl_multi. //------------------------------------------------------------------- -// Class method for S3fsMultiCurl +// Class method for S3fsMultiCurl //------------------------------------------------------------------- int S3fsMultiCurl::max_multireq = MAX_MULTI_HEADREQ; @@ -3285,7 +3373,7 @@ int S3fsMultiCurl::SetMaxMultiRequest(int max) } //------------------------------------------------------------------- -// method for S3fsMultiCurl +// method for S3fsMultiCurl //------------------------------------------------------------------- S3fsMultiCurl::S3fsMultiCurl() : hMulti(NULL), SuccessCallback(NULL), RetryCallback(NULL) { @@ -3334,14 +3422,14 @@ S3fsMultiSuccessCallback S3fsMultiCurl::SetSuccessCallback(S3fsMultiSuccessCallb SuccessCallback = function; return old; } - + S3fsMultiRetryCallback S3fsMultiCurl::SetRetryCallback(S3fsMultiRetryCallback function) { S3fsMultiRetryCallback old = RetryCallback; RetryCallback = function; return old; } - + bool S3fsMultiCurl::SetS3fsCurlObject(S3fsCurl* s3fscurl) { if(hMulti){ diff --git a/src/curl.h b/src/curl.h index 683b46d..d222634 100644 --- a/src/curl.h +++ b/src/curl.h @@ -28,12 +28,12 @@ //---------------------------------------------- // class BodyData //---------------------------------------------- -// memory class for curl write memory callback +// memory class for curl write memory callback // class BodyData { private: - char* text; + char* text; size_t lastpos; size_t bufsize; @@ -180,7 +180,7 @@ enum sse_type_t { // class S3fsCurl { - friend class S3fsMultiCurl; + friend class S3fsMultiCurl; private: enum REQTYPE { @@ -315,6 +315,9 @@ class S3fsCurl // class methods static bool InitS3fsCurl(const char* MimeFile = NULL); static bool DestroyS3fsCurl(void); + static int ParallelMultipartUploadWithoutPreRequest(const char* tpath, headers_t& meta, int fd, + off_t offset, size_t size, std::string upload_id, + etaglist_t* list); static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd); static int ParallelGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size); static bool CheckRAMCredentialUpdate(void); diff --git a/src/fdcache.cpp b/src/fdcache.cpp index cf35182..23d4bbb 100644 --- a/src/fdcache.cpp +++ b/src/fdcache.cpp @@ -112,6 +112,7 @@ bool CacheFileStat::DeleteCacheFileStat(const char* path) S3FS_PRN_ERR("failed to create cache stat file path(%s)", path); return false; } + if(0 != unlink(sfile_path.c_str())){ if(ENOENT == errno){ S3FS_PRN_DBG("failed to delete file(%s): errno=%d", path, errno); @@ -882,7 +883,7 @@ bool FdEntity::GetStats(struct stat& st) } AutoLock auto_lock(&fdent_lock); - memset(&st, 0, sizeof(struct stat)); + memset(&st, 0, sizeof(struct stat)); if(-1 == fstat(fd, &st)){ S3FS_PRN_ERR("fstat failed. errno(%d)", errno); return false; @@ -1339,16 +1340,16 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync) /* * Make decision to do multi upload (or not) based upon file size - * + * * According to the OSS spec: * - 1 to 10,000 parts are allowed * - minimum size of parts is 5MB (expect for the last part) - * + * * For our application, we will define minimum part size to be 10MB (10 * 2^20 Bytes) - * minimum file size will be 64 GB - 2 ** 36 - * + * minimum file size will be 64 GB - 2 ** 36 + * * Initially uploads will be done serially - * + * * If file is > 20MB, then multipart will kick in */ if(pagelist.Size() > static_cast(MAX_MULTIPART_CNT * S3fsCurl::GetMultipartSize())){ @@ -1484,29 +1485,40 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) ssize_t wsize; if(0 == upload_id.length()){ - // check disk space - size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size; - if(FdManager::IsSafeDiskSpace(NULL, restsize)){ - // enough disk space - - // Load unitialized area which starts from 0 to (start + size) before writing. - if(0 < start && 0 != (result = Load(0, static_cast(start)))){ - S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result); - return static_cast(result); - } - }else{ - // no enough disk space + // upload file directly + if (direct_upload) { if(0 != (result = NoCachePreMultipartPost())){ S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result); return static_cast(result); } - // start multipart uploading - if(0 != (result = NoCacheLoadAndPost(0, start))){ - S3FS_PRN_ERR("failed to load uninitialized area and multipart uploading it(errno=%d)", result); - return static_cast(result); - } + mp_start = start; mp_size = 0; + } else { + // check disk space + size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size; + if(FdManager::IsSafeDiskSpace(NULL, restsize)){ + // enough disk space + + // Load unitialized area which starts from 0 to (start + size) before writing. + if(0 < start && 0 != (result = Load(0, static_cast(start)))){ + S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result); + return static_cast(result); + } + }else{ + // no enough disk space + if(0 != (result = NoCachePreMultipartPost())){ + S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result); + return static_cast(result); + } + // start multipart uploading + if(0 != (result = NoCacheLoadAndPost(0, start))){ + S3FS_PRN_ERR("failed to load uninitialized area and multipart uploading it(errno=%d)", result); + return static_cast(result); + } + mp_start = start; + mp_size = 0; + } } }else{ // alreay start miltipart uploading @@ -1525,14 +1537,29 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) } // check multipart uploading - if(0 < upload_id.length()){ + if(0 < upload_id.length()) { mp_size += static_cast(wsize); - if(static_cast(S3fsCurl::GetMultipartSize()) <= mp_size){ + size_t part_size = static_cast(S3fsCurl::GetMultipartSize()); + bool need_trunc = false; + if (direct_upload) { + if (direct_upload_part_num * part_size <= mp_size) { + // over direct_upload_part_num multipart size + if (0 != (result = S3fsCurl::ParallelMultipartUploadWithoutPreRequest( + path.c_str(), orgmeta, fd, mp_start, mp_size, upload_id, &etaglist))) { + return result; + } + need_trunc = true; + } + } else if (part_size <= mp_size) { // over one multipart size if(0 != (result = NoCacheMultipartPost(fd, mp_start, mp_size))){ S3FS_PRN_ERR("failed to multipart post(start=%zd, size=%zu) for file(%d).", mp_start, mp_size, fd); return result; } + need_trunc = true; + } + + if (need_trunc) { // [NOTE] // truncate file to zero and set length to part offset + size // after this, file length is (offset + size), but file does not use any disk space. @@ -1549,7 +1576,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) // Update the stat cache. bool success = StatCache::getStatCacheData()->IncSize(path, wsize); if (!success) { - S3FS_PRN_ERR("failed to update file size in stat cache(path=%s, size=%ld).", path.c_str(), wsize); + S3FS_PRN_ERR("failed to update file size in stat cache(path=%s, size=%ld).", path.c_str(), wsize); } return wsize; } diff --git a/src/fdcache.h b/src/fdcache.h index 1b06974..01d2da9 100644 --- a/src/fdcache.h +++ b/src/fdcache.h @@ -21,6 +21,9 @@ #include #include "curl.h" +extern bool direct_upload; +extern int direct_upload_part_num; + //------------------------------------------------ // CacheFileStat //------------------------------------------------ diff --git a/src/s3fs.cpp b/src/s3fs.cpp index f9d94d5..5841a75 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -88,6 +88,8 @@ bool nomultipart = false; bool pathrequeststyle = false; bool is_specified_endpoint = false; bool noxattr = false; +bool direct_upload = false; +int direct_upload_part_num = 10; int s3fs_init_deferred_exit_status = 0; std::string program_name; std::string service_path = "/"; @@ -144,7 +146,7 @@ static int readdir_multi_head(const char* path, S3ObjList& head, void* buf, fuse static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, bool check_content_only = false); static int directory_empty(const char* path); static bool is_truncated(xmlDocPtr doc);; -static int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, +static int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head); static int append_objects_from_xml(const char* path, xmlDocPtr doc, S3ObjList& head); static bool GetXmlNsUrl(xmlDocPtr doc, string& nsurl); @@ -1293,7 +1295,7 @@ static int rename_directory(const char* from, const char* to) string newpath; // should be from name(not used) string nowcache; // now cache path(not used) int DirType; - bool normdir; + bool normdir; MVNODE* mn_head = NULL; MVNODE* mn_tail = NULL; MVNODE* mn_cur; @@ -1306,7 +1308,7 @@ static int rename_directory(const char* from, const char* to) // // Initiate and Add base directory into MVNODE struct. // - strto += "/"; + strto += "/"; if(0 == chk_dir_object_type(from, newpath, strfrom, nowcache, NULL, &DirType) && DIRTYPE_UNKNOWN != DirType){ if(DIRTYPE_NOOBJ != DirType){ normdir = false; @@ -1328,7 +1330,7 @@ static int rename_directory(const char* from, const char* to) // (CommonPrefixes is empty, but all object is listed in Key.) if(0 != (result = list_bucket(basepath.c_str(), head, NULL))){ S3FS_PRN_ERR("list_bucket returns error."); - return result; + return result; } head.GetNameList(headlist); // get name without "/". S3ObjList::MakeHierarchizedList(headlist, false); // add hierarchized dir. @@ -1362,7 +1364,7 @@ static int rename_directory(const char* from, const char* to) is_dir = false; normdir = false; } - + // push this one onto the stack if(NULL == add_mvnode(&mn_head, &mn_tail, from_name.c_str(), to_name.c_str(), is_dir, normdir)){ return -ENOMEM; @@ -1566,7 +1568,7 @@ static int s3fs_chmod_nocopy(const char* path, mode_t mode) if(S_ISDIR(stbuf.st_mode)){ // Should rebuild all directory object // Need to remove old dir("dir" etc) and make new dir("dir/") - + // At first, remove directory old object if(IS_RMTYPEDIR(nDirType)){ S3fsCurl s3fscurl; @@ -2040,7 +2042,7 @@ static int s3fs_open(const char* path, struct fuse_file_info* fi) if(NULL == (ent = FdManager::get()->Open(path, &meta, static_cast(st.st_size), st.st_mtime, false, true))){ return -EIO; } - + if (needs_flush){ if(0 != (result = ent->RowFlush(path, true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result); @@ -2399,7 +2401,7 @@ static int s3fs_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, bool check_content_only) { - int result; + int result; string s3_realpath; string query_delimiter;; string query_prefix;; @@ -2495,7 +2497,7 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, const char* c_strErrorObjectName = "FILE or SUBDIR in DIR"; -static int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, +static int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head) { xmlXPathObjectPtr contents_xp; @@ -3303,7 +3305,7 @@ static int s3fs_removexattr(const char* path, const char* name) return 0; } - + static void* s3fs_init(struct fuse_conn_info* conn) { // check bucket @@ -3604,7 +3606,7 @@ static int s3fs_utility_mode(void) // // If calling with wrong region, s3fs gets following error body as 400 erro code. -// "AuthorizationHeaderMalformedThe authorization header is +// "AuthorizationHeaderMalformedThe authorization header is // malformed; the region 'us-east-1' is wrong; expecting 'ap-northeast-1' // ap-northeast-1...... // " @@ -3777,7 +3779,7 @@ static int check_for_cos_format(void) // // check_passwd_file_perms -// +// // expect that global passwd_file variable contains // a non-empty value and is readable by the current user // @@ -3796,19 +3798,19 @@ static int check_passwd_file_perms(void) return EXIT_FAILURE; } - // return error if any file has others permissions + // return error if any file has others permissions if( (info.st_mode & S_IROTH) || - (info.st_mode & S_IWOTH) || + (info.st_mode & S_IWOTH) || (info.st_mode & S_IXOTH)) { S3FS_PRN_EXIT("credentials file %s should not have others permissions.", passwd_file.c_str()); return EXIT_FAILURE; } - // Any local file should not have any group permissions - // /etc/passwd-cosfs can have group permissions + // Any local file should not have any group permissions + // /etc/passwd-cosfs can have group permissions if(passwd_file != "/etc/passwd-cosfs"){ if( (info.st_mode & S_IRGRP) || - (info.st_mode & S_IWGRP) || + (info.st_mode & S_IWGRP) || (info.st_mode & S_IXGRP)) { S3FS_PRN_EXIT("credentials file %s should not have group permissions.", passwd_file.c_str()); return EXIT_FAILURE; @@ -3831,10 +3833,10 @@ static int check_passwd_file_perms(void) // read_passwd_file // // Support for per bucket credentials -// +// // Format for the credentials file: // [bucket:]AccessKeyId:SecretAccessKey -// +// // Lines beginning with # are considered comments // and ignored, as are empty lines // @@ -3939,7 +3941,7 @@ static int read_passwd_file(void) // // get_access_keys // -// called only when were are not mounting a +// called only when were are not mounting a // public bucket // // Here is the order precedence for getting the @@ -4030,7 +4032,7 @@ static int get_access_keys(void) } // 5 - from the system default location - passwd_file.assign("/etc/passwd-cosfs"); + passwd_file.assign("/etc/passwd-cosfs"); ifstream PF(passwd_file.c_str()); if(PF.good()){ PF.close(); @@ -4071,10 +4073,10 @@ static int set_moutpoint_attribute(struct stat& mpst) } // This is repeatedly called by the fuse option parser -// if the key is equal to FUSE_OPT_KEY_OPT, it's an option passed in prefixed by +// if the key is equal to FUSE_OPT_KEY_OPT, it's an option passed in prefixed by // '-' or '--' e.g.: -f -d -ousecache=/tmp // -// if the key is equal to FUSE_OPT_KEY_NONOPT, it's either the bucket name +// if the key is equal to FUSE_OPT_KEY_NONOPT, it's either the bucket name // or the mountpoint. The bucket name will always come before the mountpoint static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_args* outargs) { @@ -4532,6 +4534,30 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar pathrequeststyle = true; return 0; } + + if (0 == STR2NCMP(arg, "direct_upload=")) { + const char *p = strchr(arg, '=') + sizeof(char); + const char *temp = p; + while (*temp) { + if (!isdigit(*(temp++))) { + printf("Illegal input direct_upload number.\n"); + exit(EXIT_FAILURE); + } + } + direct_upload_part_num = atoi(p); + if (direct_upload_part_num <= 0) { + printf("direct_upload_part_num must be bigger than 0.\n"); + exit(EXIT_FAILURE); + } + direct_upload = true; + return 0; + } + + if (0 == STR2NCMP(arg, "direct_upload")) { + direct_upload = true; + return 0; + } + // // debug option for s3fs // @@ -4594,7 +4620,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar // s3fs_init calls this function to exit cleanly from the fuse event loop. // // -// // There's no way to pass an exit status to the high-level event loop API, so +// // There's no way to pass an exit status to the high-level event loop API, so // // this function stores the exit value in a global for main() static void s3fs_exit_fuseloop(int exit_status) { S3FS_PRN_ERR("Exiting FUSE event loop due to errors\n"); @@ -4609,7 +4635,7 @@ int main(int argc, char* argv[]) { int ch; int fuse_res; - int option_index = 0; + int option_index = 0; struct fuse_operations s3fs_oper; static const struct option long_opts[] = { @@ -4630,7 +4656,7 @@ int main(int argc, char* argv[]) xmlInitParser(); LIBXML_TEST_VERSION - // get progam name - emulate basename + // get progam name - emulate basename size_t found = string::npos; program_name.assign(argv[0]); found = program_name.find_last_of("/"); @@ -4736,7 +4762,7 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } // More error checking on the access key pair can be done - // like checking for appropriate lengths and characters + // like checking for appropriate lengths and characters } // check cache dir permission @@ -4753,11 +4779,11 @@ int main(int argc, char* argv[]) // our own certificate verification logic. // For now, this will be unsupported unless we get a request for it to // be supported. In that case, we have a couple of options: - // - implement a command line option that bypasses the verify host + // - implement a command line option that bypasses the verify host // but doesn't bypass verifying the certificate // - write our own host verification (this might be complex) // See issue #128strncasecmp - /* + /* if(1 == S3fsCurl::GetSslVerifyHostname()){ found = bucket.find_first_of("."); if(found != string::npos){ @@ -4842,7 +4868,7 @@ int main(int argc, char* argv[]) S3FS_PRN_EXIT("could not set signal handler for SIGUSR2."); exit(EXIT_FAILURE); } - + int result; if (EXIT_SUCCESS != (result = s3fs_check_service())) { S3FS_PRN_EXIT("bucket not exist, exiting...");