fix upload

Author: dp-dingzhaohan

pkg/sumdb/sum.golang.org/latest

@@ -0,0 +1,5 @@
+go.sum database tree
+39073249
+CHoGLc3xiTLWJIkUvqwtoUC8uUi8tsi/AMKPaA1nDxo=
+
+— sum.golang.org Az3grieYv97xelotCMbg3+3zXebCIZkZfor40fTCTWo4rNjv+mQ8eEJFgiyyjPkL1h6GWziyoBzovPY6pFcfbssUNQU=

setup.py

@@ -11,7 +11,7 @@ def read_file(filename):
 
 setup(
     name="bohrium-sdk",
-    version="0.8.0",
+    version="0.10.0",
     author="dingzhaohan",
     author_email="[email protected]",
     url="https://github.com/dingzhaohan",

src/bohrium/_base_client.py

@@ -143,6 +143,8 @@ def _build_headers(self, custom_headers) -> httpx.Headers:
         headers_dict = _merge_mappings(
             self.default_headers, self._custom_headers, custom_headers
         )
+        # 过滤掉 value 为 None 的 header
+        headers_dict = {k: v for k, v in headers_dict.items() if v is not None}
         headers = httpx.Headers(headers_dict)
         return headers or dict()
 
@@ -177,7 +179,7 @@ def platform_headers(self) -> Dict[str, str]:
         exceptions=(httpx.RequestError,),
     )
     def _request(
-        self, method: str, path: str, json=None, headers=None, **kwargs
+        self, method: str, path: str, json=None, headers=None, data=None, **kwargs
     ) -> httpx.Response:
         url = urljoin(str(self._base_url), path)
         logger.info(f"Requesting {method} {url}")
@@ -188,6 +190,7 @@ def _request(
                 method.upper(),
                 url,
                 json=json,
+                data=data,
                 headers=merged_headers,
                 params=merged_params,
             )

src/bohrium/resources/job/job.py

@@ -58,7 +58,6 @@ def submit(
     ):
         # log.info(f"submit job {name},project_id:{project_id}")
         data = self.create_job(project_id, job_name, job_group_id)
-        print(data)
         if work_dir != "":
             if not os.path.exists(work_dir):
                 raise FileNotFoundError
@@ -97,36 +96,30 @@ def insert(self, **kwargs):
         if 'logFiles' in camel_data and not isinstance(camel_data['logFiles'], list):
             camel_data['logFiles'] = [camel_data['logFiles']]
         response = self._client.post("/openapi/v2/job/add", json=camel_data)
-        pprint(response.request)
-        print(response.json())
         return response.json().get("data")
 
     def delete(self, job_id):
         # log.info(f"delete job {job_id}")
         response = self._client.post(f"/openapi/v1/job/del/{job_id}")
-        pprint(response.request)
-        print(response.json())
+
 
     def terminate(self, job_id):
         # log.info(f"terminate job {job_id}")
         response = self._client.post(f"/openapi/v1/job/terminate/{job_id}")
-        pprint(response.request)
-        print(response.json())
+
 
     def kill(self, job_id):
         # log.info(f"kill job {job_id}")
         response = self._client.post(f"/openapi/v1/job/kill/{job_id}")
-        pprint(response.request)
-        print(response.json())
+    
 
     def log(self, job_id, log_file="STDOUTERR", page=-1, page_size=8192):
         # log.info(f"log job {job_id}")
         response = self._client.get(
             f"/openapi/v1/job/{job_id}/log",
             params={"logFile": log_file, "page": page, "pageSize": page_size},
         )
-        pprint(response.request)
-        print(response.json().get("data")["log"])
+
         return response.json().get("data")["log"]
 
     def create_job(
@@ -154,8 +147,7 @@ def create_job(
             "bohrGroupId": group_id,
         }
         response = self._client.post(f"/openapi/v1/job/create", json=data)
-        pprint(response.request)
-        print(response.json())
+
         return response.json().get("data")
 
     def create_job_group(self, project_id, job_group_name):
@@ -164,9 +156,8 @@ def create_job_group(self, project_id, job_group_name):
             "/openapi/v1/job_group/add",
             json={"name": job_group_name, "projectId": project_id},
         )
-        pprint(response.request)
-        print(response.json())
-
+        return response.json().get("data")
+    
     def upload(
         self,
         file_path: str,

src/bohrium/resources/tiefblue/tiefblue.py

@@ -71,7 +71,7 @@ def write(
         }
         headers[self.TIEFBLUE_HEADER_KEY] = self.encode_base64(param)
         req = self.client.post("/api/upload/binary", headers=headers, data=data)
-        pprint(req.request)
+
         return req
 
     def read(
@@ -103,40 +103,48 @@ def upload_from_file(
             parameter = Parameter()
         parameter.contentDisposition = f'attachment; filename="{disposition}"'
         with open(file_path, 'rb') as fp:
-            res = self.write(object_key=object_key, data=fp.read(), parameter=parameter)
+            res = self.write(object_key=object_key, data=fp.read(), parameter=parameter, token=token)
             return res
 
-    def init_upload_by_part(self, object_key: str, parameter=None):
+    def init_upload_by_part(self, object_key: str, parameter=None, token: str = ""):
         data = {
             'path': object_key
         }
         if parameter is not None:
             data['option'] = parameter.__dict__
+        headers = {
+            "Authorization": f"Bearer {token}",
+        }
         url = f"/api/upload/multipart/init"
-        return self.client.post(url, host=self.host, json=data)
+        return self.client.post(url, host=self.host, headers=headers, json=data)
 
-    def upload_by_part(self, object_key: str, initial_key: str, chunk_size: int, number: int, body):
+    def upload_by_part(self, object_key: str, initial_key: str, chunk_size: int, number: int, body, token: str = ""):
         param = {
             'initialKey': initial_key,
             'number': number,
             'partSize': chunk_size,
             'objectKey': object_key
         }
-        headers = {}
+        headers = {
+            "Authorization": f"Bearer {token}",
+        }
         headers[self.TIEFBLUE_HEADER_KEY] = self._dump_parameter(param)
         url = f"/api/upload/multipart/upload"
         resp = self.client.post(url, host=self.host, data=body, headers=headers)
         return resp
 
 
-    def complete_upload_by_part(self, object_key, initial_key, part_string):
+    def complete_upload_by_part(self, object_key, initial_key, part_string, token):
         data = {
             'path': object_key,
             'initialKey': initial_key,
             'partString': part_string
         }
+        headers = {
+            "Authorization": f"Bearer {token}",
+        }
         url = f"/api/upload/multipart/complete"
-        resp = self.client.post(url, host=self.host, json=data)
+        resp = self.client.post(url, host=self.host, headers=headers, json=data)
         return resp
 
     def upload_From_file_multi_part(
@@ -161,27 +169,34 @@ def upload_From_file_multi_part(
             parameter = Parameter()
         parameter.contentDisposition = f'attachment; filename="{disposition}"'
         bar_format = "{l_bar}{bar}| {n:.02f}/{total:.02f} %  [{elapsed}<{remaining}, {rate_fmt}{postfix}]"
-        with open(file_path, 'r') as f:
+        with open(file_path, 'rb') as f:
             pbar = tqdm(total=100, desc=f"Uploading {disposition}", smoothing=0.01, bar_format=bar_format,
                         disable=not progress_bar)
-            f.seek(0)
             if size < _DEFAULT_CHUNK_SIZE * 2:
-                self.write(object_key=object_key, data=f.buffer, parameter=parameter, token=token)
+
+                self.write(object_key=object_key, data=f.read(), parameter=parameter, token=token)
                 pbar.update(100)
                 pbar.close()
                 return
             chunks = split_size_by_part_size(size, chunk_size)
-            initial_key = self.init_upload_by_part(object_key, parameter).get('initialKey')
+            initial_key = self.init_upload_by_part(object_key, parameter, token).get('initialKey')
             part_string = []
+            uploaded = 0
             for c in chunks:
                 f.seek(c.Offset)
-                num_to_upload = min(chunk_size, size - c.Offset)
-                part_string.append(self.upload_by_part(object_key, initial_key, chunk_size=c.Size, number=c.Number,
-                                                       body=f.buffer.read(c.Size)).get('partString'))
-                percent = num_to_upload * 100 / (size + 1)
-                pbar.update(percent)
+                chunk_data = f.read(c.Size)
+                resp = self.upload_by_part(
+                    object_key, initial_key, chunk_size=c.Size, number=c.Number,
+                    body=chunk_data, token=token
+                )
+                part_string.append(resp.get('partString'))
+                uploaded += c.Size
+                percent = uploaded * 100 / size
+                pbar.n = percent
+                pbar.refresh()
+            pbar.update(100 - pbar.n)
             pbar.close()
-            return self.complete_upload_by_part(object_key, initial_key, part_string)
+            return self.complete_upload_by_part(object_key, initial_key, part_string, token)
 
     def download_from_file(self):