[GR-52784] Patch wheel to avoid 2GB bytes size limit

PullRequest: graalpython/3259

Author: msimacek

graalpython/lib-graalpython/patches/flake8/metadata.toml

@@ -1,3 +1,4 @@
 [[rules]]
 version = '== 6.0.0'
 patch = 'flake8-6.0.0.patch'
+subdir = 'src'

graalpython/lib-graalpython/patches/tensorflow/tensorflow-2.14.0.patch

@@ -317,6 +317,18 @@ index 68fd53d0..2871915e 100644
              auto *tb = reinterpret_cast<PyTracebackObject *>(m_trace.ptr());
 
              // Get the deepest trace possible.
+diff --git a/tools/generate_torch_version.py b/tools/generate_torch_version.py
+index 93689693..81e360dc 100644
+--- a/tools/generate_torch_version.py
++++ b/tools/generate_torch_version.py
+@@ -40,6 +40,7 @@ def get_tag(pytorch_root: Union[str, Path]) -> str:
+ 
+ 
+ def get_torch_version(sha: Optional[str] = None) -> str:
++    return '2.2.1'
+     pytorch_root = Path(__file__).parent.parent
+     version = open(pytorch_root / "version.txt").read().strip()
+ 
 diff --git a/torch/_dynamo/types.py b/torch/_dynamo/types.py
 index 0c1deddd..f60e9d72 100644
 --- a/torch/_dynamo/types.py

graalpython/lib-graalpython/patches/tensorflow/tensorflow-2.15.0.patch

@@ -1,17 +1,23 @@
 [[rules]]
 # Always pin a specific version. Our virtualenv seeder pins setuptools and pip to the bundled
 # ones, so it makes sense to always pin wheel too to avoid it getting out of sync with setuptools.
-version = '== 0.41.2'
-# Backport pypa/packaging PR for fixing graalpy architecture. Not strictly necessary for building wheels
-patch = 'wheel-0.41.2.patch'
+version = '== 0.43.0'
+# Add a workaround for building wheels with files larger than 2GB
+patch = 'wheel-0.43.patch'
 subdir = 'src'
 
+# Catch all rule for newer versions
 [[rules]]
-version = '>= 0.41.2'
-patch = 'wheel-0.41.2.patch'
+version = '>= 0.43.0'
+patch = 'wheel-0.43.patch'
 subdir = 'src'
 install-priority = 0
 
+[[rules]]
+version = '== 0.41.2'
+patch = 'wheel-0.41.2.patch'
+subdir = 'src'
+
 [[rules]]
 version = '>= 0.40, < 0.41.2'
 patch = 'wheel-0.40.patch'

graalpython/lib-graalpython/patches/torch/torch-2.2.1.patch

@@ -0,0 +1,82 @@
+diff --git a/wheel/wheelfile.py b/wheel/wheelfile.py
+index 6440e90..7ef7528 100644
+--- a/wheel/wheelfile.py
++++ b/wheel/wheelfile.py
+@@ -144,7 +144,9 @@ class WheelFile(ZipFile):
+     def write(self, filename, arcname=None, compress_type=None):
+         with open(filename, "rb") as f:
+             st = os.fstat(f.fileno())
+-            data = f.read()
++            data = []
++            while chunk := f.read(4194304):
++                data.append(chunk)
+ 
+         zinfo = ZipInfo(
+             arcname or filename, date_time=get_zipinfo_datetime(st.st_mtime)
+@@ -164,7 +166,10 @@ class WheelFile(ZipFile):
+         if isinstance(data, str):
+             data = data.encode("utf-8")
+ 
+-        ZipFile.writestr(self, zinfo_or_arcname, data, compress_type)
++        # GraalPy change
++        if not isinstance(data, list):
++            data = [data]
++        self.writestr_list(zinfo_or_arcname, data, compress_type)
+         fname = (
+             zinfo_or_arcname.filename
+             if isinstance(zinfo_or_arcname, ZipInfo)
+@@ -172,12 +177,52 @@ class WheelFile(ZipFile):
+         )
+         log.info(f"adding '{fname}'")
+         if fname != self.record_path:
+-            hash_ = self._default_algorithm(data)
++            hash_ = self._default_algorithm()
++            for chunk in data:
++                hash_.update(chunk)
+             self._file_hashes[fname] = (
+                 hash_.name,
+                 urlsafe_b64encode(hash_.digest()).decode("ascii"),
+             )
+-            self._file_sizes[fname] = len(data)
++            self._file_sizes[fname] = sum(map(len, data))
++
++    # GraalPy change: version that accepts data as a list of bytes chunks, to
++    # avoid running into the 2GB limit for bytes object size
++    def writestr_list(self, zinfo_or_arcname, data,
++                 compress_type=None, compresslevel=None):
++        if not isinstance(zinfo_or_arcname, ZipInfo):
++            zinfo = ZipInfo(filename=zinfo_or_arcname,
++                            date_time=time.localtime(time.time())[:6])
++            zinfo.compress_type = self.compression
++            zinfo._compresslevel = self.compresslevel
++            if zinfo.filename[-1] == '/':
++                zinfo.external_attr = 0o40775 << 16   # drwxrwxr-x
++                zinfo.external_attr |= 0x10           # MS-DOS directory flag
++            else:
++                zinfo.external_attr = 0o600 << 16     # ?rw-------
++        else:
++            zinfo = zinfo_or_arcname
++
++        if not self.fp:
++            raise ValueError(
++                "Attempt to write to ZIP archive that was already closed")
++        if self._writing:
++            raise ValueError(
++                "Can't write to ZIP archive while an open writing handle exists."
++            )
++
++        if compress_type is not None:
++            zinfo.compress_type = compress_type
++
++        if compresslevel is not None:
++            zinfo._compresslevel = compresslevel
++
++        zinfo.file_size = sum(map(len, data))            # Uncompressed size
++        with self._lock:
++            with self.open(zinfo, mode='w') as dest:
++                for chunk in data:
++                    dest.write(chunk)
++
+ 
+     def close(self):
+         # Write RECORD

graalpython/lib-graalpython/patches/wheel/metadata.toml

@@ -2106,6 +2106,7 @@ def _python_checkpatchfiles():
             # Empty license field. It's MIT
             'urllib3-2.patch',
             # Empty license field. It's MIT
+            'wheel-0.43.patch',
             'wheel-0.41.2.patch',
             'wheel-0.40.patch',
             'wheel-0.38.patch',