Change single file upload size to 5000 MiB (#503)

* As per the documentation on Azure: https://learn.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs#about-block-blobs, the maximum single blob size allowed is 5000 MiB and not 5 GiB. Due to this if a file of size above 4.9 GiB fails with the error: httpx.HTTPStatusError: Client error '413 Payload Too Large' for url '<upload_url_for_file>'

* Changes to make a release for the extractor-utils

---------

Co-authored-by: Nithin Bodanapu <nithin.bodanapu@cognite.com>

Author: nithinb

CHANGELOG.md

@@ -12,6 +12,11 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## 7.11.5
+
+### Fixed
+* Updated max single-chunk file size to 5000 MiB from 5 GiB. The earlier limit used to cause issues with Azure Blob Storage.
+
 
 ## 7.11.4
 

cognite/extractorutils/__init__.py

@@ -16,7 +16,7 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.11.4"
+__version__ = "7.11.5"
 from .base import Extractor
 
 __all__ = ["Extractor"]

cognite/extractorutils/uploader/files.py

@@ -59,10 +59,11 @@
 _QUEUES: int = 0
 _QUEUES_LOCK: threading.RLock = threading.RLock()
 
-# 5 GiB
-_MAX_SINGLE_CHUNK_FILE_SIZE = 5 * 1024 * 1024 * 1024
+# 5000 MiB
+_MAX_SINGLE_CHUNK_FILE_SIZE = 5 * 1000 * 1024 * 1024
+
 # 4000 MiB
-_MAX_FILE_CHUNK_SIZE = 4 * 1024 * 1024 * 1000
+_MAX_FILE_CHUNK_SIZE = 4 * 1000 * 1024 * 1024
 
 _CDF_ALPHA_VERSION_HEADER = {"cdf-version": "alpha"}
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite-extractor-utils"
-version = "7.11.4"
+version = "7.11.5"
 description = "Utilities for easier development of extractors for CDF"
 authors = [
     {name = "Mathias Lohne", email = "mathias.lohne@cognite.com"}