Merge pull request #173 from tencentyun/feature_huberyxxiao_ae9ba00c

新增单线程断点上传接口

Author: hubeytenctent

demo/object_op_demo/multi_put_object_demo.cpp

@@ -278,9 +278,9 @@ void AsyncPutObjectDemo(qcloud_cos::CosAPI& cos) {
 /*
  * 该 Demo 示范用户如何自行组合分块上传各接口进行对象上传
  * 分块操作基于初始化、上传分块、完成分块三个接口可以实现将对象切分为多个分块，然后再将这些分块上传到 cos，最后发起 Complete 完成分块上传
- * 与本节中的高级上传接口配置一样，可通过全局设置上传线程池大小、分块大小。该上传线程池上是每次上传独立的。
  * 本 Demo 中的上传分块接口 UploadPartData 仅支持传入流，最多支持10000分块，每个分块大小为1MB - 5GB，最后一个分块可以小于1MB
  */
+
 void PutPartDemo(qcloud_cos::CosAPI& cos) {
     std::string object_name = "big_file.txt";
 
@@ -295,19 +295,36 @@ void PutPartDemo(qcloud_cos::CosAPI& cos) {
 
     // 2. UploadPartData
     // UploadPartData 部分，可以根据实际选择分块数量和分块大小，这里以 2 个分块为例
+
+        // Complete 需要的两个列表：
     std::vector<std::string> etags;
     std::vector<uint64_t> part_numbers;
+
     std::string upload_id = init_resp.GetUploadId();
     {
         uint64_t part_number = 1;
-        // 模拟上传分块数据，这里以 1M 为例
-        std::vector<char> data(1024 * 1024, 'A');
+        // 模拟上传分块数据，这里以 100M 为例
+        uint64_t copy_size = 1024 * 1024 * 100;
+        std::vector<char> data(copy_size, 'A');
         std::string content(data.begin(), data.end());
         std::istringstream iss(content);
         qcloud_cos::UploadPartDataReq req(bucket_name, object_name, upload_id, iss);
         req.SetPartNumber(part_number);
+            // 限速上传对象，默认单位为 bit/s，限速值设置范围为 819200 - 838860800, 即800Kb/s-800Mb/s
+            uint64_t traffic_limit = 8192*1024*10; // 100MB 文件 5M
+            req.SetTrafficLimit(traffic_limit);
         qcloud_cos::UploadPartDataResp resp;
+        std::chrono::time_point<std::chrono::steady_clock> start_ts, end_ts;
+        start_ts = std::chrono::steady_clock::now();
         qcloud_cos::CosResult result = cos.UploadPartData(req, &resp);
+        end_ts = std::chrono::steady_clock::now();
+        auto time_consumed_ms =
+        std::chrono::duration_cast<std::chrono::milliseconds>(end_ts - start_ts)
+            .count();
+        float rate =
+          ((float)copy_size / 1024 / 1024) / ((float)time_consumed_ms / 1000);
+        SDK_LOG_ERR("send part_number: %d, send_size: %" PRIu64 " MB, time_consumed: %" PRIu64
+                    " ms, rate: %.2f MB/s , traffic_limit : %.2f MB", part_number, copy_size/ 1024 / 1024, time_consumed_ms, rate, traffic_limit/1024/1024/8.0);
         std::cout << "==================UploadPartDataResp1=====================" << std::endl;
         PrintResult(result, resp);
         std::cout << "==========================================================" << std::endl;
@@ -318,12 +335,38 @@ void PutPartDemo(qcloud_cos::CosAPI& cos) {
     }
     {
         uint64_t part_number = 2;
-        std::istringstream iss("The last part can be smaller than 1MB");
+        uint64_t copy_size = 1024 * 1024 * 100;
+        std::vector<char> data(copy_size, 'A');
+        std::string content(data.begin(), data.end());
+        std::istringstream iss(content);
         qcloud_cos::UploadPartDataReq req(bucket_name, object_name, upload_id, iss);
         req.SetPartNumber(part_number);
+            // 限速上传对象，默认单位为 bit/s，限速值设置范围为 819200 - 838860800, 即800Kb/s-800Mb/s
+        uint64_t traffic_limit = 8192 * 1024 * 5 ;
+        req.SetTrafficLimit(traffic_limit);
+        qcloud_cos::UploadPartDataResp resp;
+        qcloud_cos::CosResult result = cos.UploadPartData(req, &resp);
+        std::cout << "==================UploadPartDataResp2=====================" << std::endl;
+        PrintResult(result, resp);
+        std::cout << "==========================================================" << std::endl;
+        if (result.IsSucc()) {
+            part_numbers.push_back(part_number);
+            etags.push_back(resp.GetEtag());
+        }
+    }
+    {
+        uint64_t part_number = 3;
+        uint64_t copy_size = 1024 * 1024 * 10;
+        std::vector<char> data(copy_size, 'A');
+        std::string content(data.begin(), data.end());
+        std::istringstream iss(content);
+        qcloud_cos::UploadPartDataReq req(bucket_name, object_name, upload_id, iss);
+        req.SetPartNumber(part_number);
+            // 限速上传对象，默认单位为 bit/s，限速值设置范围为 819200 - 838860800, 即800Kb/s-800Mb/s
+            uint64_t traffic_limit = 8192 * 1024;
+            req.SetTrafficLimit(traffic_limit);
         qcloud_cos::UploadPartDataResp resp;
         qcloud_cos::CosResult result = cos.UploadPartData(req, &resp);
-
         std::cout << "==================UploadPartDataResp2=====================" << std::endl;
         PrintResult(result, resp);
         std::cout << "==========================================================" << std::endl;
@@ -348,11 +391,48 @@ void PutPartDemo(qcloud_cos::CosAPI& cos) {
 
     return;
 }
+void PutObjectResumableSingleThreadSyncDemo(qcloud_cos::CosAPI& cos) {
+    std::string local_file = "SingleThreadSync.txt";
+    std::string object_name = "SingleThreadSync.txt";
+
+    qcloud_cos::PutObjectResumableSingleSyncReq req(bucket_name, object_name, local_file);
+    req.AddHeader("x-cos-meta-ssss1","1xxxxxxx");
+    req.AddHeader("x-cos-meta-ssss2","2xxxxxxx");
+    req.AddHeader("x-cos-meta-ssss3","3xxxxxxx");
+    req.AddHeader("x-cos-meta-ssss4","4xxxxxxx");
+    uint64_t traffic_limit = 8192 * 1024;//1MB
+    req.SetTrafficLimit(traffic_limit);
+    //req.SetHttps();
+    //req.SetSSLCtxCallback(SslCtxCallback, nullptr);
+    qcloud_cos::PutObjectResumableSingleSyncResp resp;
+    qcloud_cos::CosResult result = cos.PutObjectResumableSingleThreadSync(req, &resp);
+    if (result.IsSucc()) {
+        std::cout << "MultiUpload Succ." << std::endl;
+        std::cout << resp.GetLocation() << std::endl;
+        std::cout << resp.GetKey() << std::endl;
+        std::cout << resp.GetBucket() << std::endl;
+        std::cout << resp.GetEtag() << std::endl;
+    } else {
+        std::cout << "MultiUpload Fail." << std::endl;
+        // 获取具体失败在哪一步
+        std::string resp_tag = resp.GetRespTag();
+        if ("Init" == resp_tag) {
+            // print result
+        } else if ("Upload" == resp_tag) {
+            // print result
+        } else if ("Complete" == resp_tag) {
+            // print result
+        }
+        PrintResult(result, resp);
+    }
+    std::cout << "===========================================================" << std::endl;
+}
 int main() {
     qcloud_cos::CosAPI cos = InitCosAPI();
     CosSysConfig::SetLogLevel((LOG_LEVEL)COS_LOG_ERR);
     MultiUploadObjectDemo(cos);
     AsyncMultiPutObjectDemo(cos);
     AsyncPutObjectDemo(cos);
     PutPartDemo(cos);
-}
+    PutObjectResumableSingleThreadSyncDemo(cos);
+}

include/cos_api.h

@@ -758,6 +758,9 @@ class CosAPI {
   CosResult MultiGetObject(const MultiGetObjectReq& req,
                            MultiGetObjectResp* resp);
 
+  CosResult PutObjectResumableSingleThreadSync(const PutObjectResumableSingleSyncReq& req,
+                           PutObjectResumableSingleSyncResp* resp);
+
   /* Resumable接口 */
 
   /// \brief 封装了初始化分块上传、分块上传、完成分块上传三步，支持断点续传

include/cos_defines.h

@@ -12,7 +12,7 @@
 
 namespace qcloud_cos {
 
-#define COS_CPP_SDK_VERSON "v5.5.15"
+#define COS_CPP_SDK_VERSON "v5.5.16"
 
 /// 路径分隔符
 const char kPathDelimiter[] = "/";

include/op/object_op.h

@@ -180,6 +180,12 @@ class ObjectOp : public BaseOp {
                               const SharedTransferHandler& handler = nullptr,
                               bool change_backup_domain = false);
 
+    /// \brief 单线程同步分块上传
+  ///
+  /// \return result
+  CosResult UploadObjectResumableSingleThreadSync(const PutObjectByFileReq& req,
+                              PutObjectResumableSingleSyncResp* resp);
+
   /// \brief 舍弃一个分块上传并删除已上传的块
   ///
   /// \param req  AbortMultiUpload请求
@@ -427,6 +433,11 @@ class ObjectOp : public BaseOp {
                     const SharedTransferHandler& handler = nullptr,
                     bool change_backup_domain = false);
 
+  CosResult SingleThreadUpload(const PutObjectByFileReq& req, const std::string& upload_id,
+                    const std::vector<std::string>& already_exist_parts, 
+                    bool resume_flag, std::vector<std::string>* etags_ptr, 
+                    std::vector<uint64_t>* part_numbers_ptr, PutObjectByFileResp* resp);
+
   /// \brief 读取文件内容, 并返回读取的长度
   // uint64_t GetContent(const std::string& src, std::string* file_content) const;
 

include/request/object_req.h

@@ -1722,6 +1722,17 @@ class MultiGetObjectReq : public GetObjectByFileReq {
   virtual ~MultiGetObjectReq() {}
 };
 
+class PutObjectResumableSingleSyncReq : public PutObjectByFileReq {
+ public:
+  PutObjectResumableSingleSyncReq(const std::string& bucket_name,
+                    const std::string& object_name,
+                    const std::string& local_file_path)
+      : PutObjectByFileReq(bucket_name, object_name, local_file_path) {
+        SetCheckCRC64(true);
+      }
+  virtual ~PutObjectResumableSingleSyncReq() {}
+};
+
 /* Async接口 */
 
 #if 0

include/response/object_resp.h

@@ -670,6 +670,41 @@ class MultiGetObjectResp  : public  GetObjectByFileResp {
   ~MultiGetObjectResp() {}
 };
 
+class PutObjectResumableSingleSyncResp  : public  GetObjectByFileResp {
+ public:
+  PutObjectResumableSingleSyncResp() {}
+
+  virtual ~PutObjectResumableSingleSyncResp() {}
+
+  std::string GetRespTag() { return m_resp_tag; }
+
+  std::string GetLocation() const { return m_location; }
+
+  std::string GetKey() const { return m_key; }
+
+  std::string GetBucket() const { return m_bucket; }
+
+  void CopyFrom(const InitMultiUploadResp& resp);
+
+  void CopyFrom(const PutObjectByFileResp& resp);
+
+  void CopyFrom(const CompleteMultiUploadResp& resp);
+
+  /// \brief Server端加密使用的算法
+  std::string GetXCosServerSideEncryption() const {
+    return GetHeader("x-cos-server-side-encryption");
+  }
+
+ private:
+  std::string m_location;  // Object的外网访问域名
+  std::string m_bucket;
+  std::string m_key;
+  std::string m_upload_id;
+
+  // FIXME(sevenyou) 先这么搞吧
+  std::string m_resp_tag;  // 用于区分是哪一种response
+};
+
 /* Async接口 */
 
 //typedef PutObjectByFileResp PutObjectAsyncResp;

src/cos_api.cpp

@@ -586,6 +586,11 @@ CosResult CosAPI::MultiPutObject(const MultiPutObjectReq& req,
   return result;
 }
 
+CosResult CosAPI::PutObjectResumableSingleThreadSync(const PutObjectResumableSingleSyncReq& req,
+                            PutObjectResumableSingleSyncResp* resp){
+  return m_object_op.UploadObjectResumableSingleThreadSync(static_cast<PutObjectByFileReq>(req), resp);
+}
+
 CosResult CosAPI::AbortMultiUpload(const AbortMultiUploadReq& req,
                                    AbortMultiUploadResp* resp) {
   CosResult result = m_object_op.AbortMultiUpload(req, resp);