Add Deflate64 extraction support and bump version

Enable extraction of Deflate64-compressed ZIPs by adding zipfile-deflate64 and pyzipper dependencies and falling back to pyzipper when zipfile raises NotImplementedError. Detect Deflate64 entries and log a warning during extraction. Bump package version to 1.0.3 and add changelog entry. Also pass encoding to pyreadstat.read_sav, update test input to use .sav and adjust test flow (comment out vertical merge and disable download folder deletion).

Author: dirreno

CHANGELOG.md

@@ -7,6 +7,9 @@ The format is based on "Keep a Changelog" (https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased]
 - Prepare improvements and documentation updates.
 
+## [1.0.3] - 2026-02-23
+### Fixed
+- Extractor now decompress deflated64 
 
 ## [1.0.2] - 2026-02-23
 ### Added

requirements.txt

@@ -14,4 +14,6 @@ deep_translator
 transformers
 torch
 pytest
-geopandas
+geopandas
+pyzipper
+zipfile-deflate64

setup.py

@@ -39,7 +39,7 @@
     # For a discussion on single-sourcing the version across setup.py and the
     # project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version='1.0.2',  # Required
+    version='1.0.3',  # Required
 
     # This is a one-line description or tagline of what your project does. This
     # corresponds to the "Summary" metadata field:

src/socio4health/extractor.py

@@ -441,7 +441,7 @@ def _read_txt(self, filepath):
         return dd.read_csv(filepath, sep=self.sep or '\t', encoding=self.encoding, dtype=self.dtype or 'object')
 
     def _read_sav(self, filepath):
-        df, meta = pyreadstat.read_sav(filepath)
+        df, meta = pyreadstat.read_sav(filepath, encoding=self.encoding)
         return df
 
     def _read_file(self, filepath):

src/socio4health/utils/extractor_utils.py

@@ -1,3 +1,5 @@
+import zipfile_deflate64
+import pyzipper
 from scrapy.crawler import CrawlerProcess
 from .standard_spider import StandardSpider
 import zipfile
@@ -145,8 +147,21 @@ def compressed2files(input_archive, target_directory, down_ext, current_depth=0,
         try:
             # Extract the archive
             if zipfile.is_zipfile(input_archive):
-                with zipfile.ZipFile(input_archive, 'r') as zip_ref:
-                    zip_ref.extractall(temp_dir)
+                try:
+                    with zipfile.ZipFile(input_archive, 'r') as zip_ref:
+                        for zinfo in zip_ref.infolist():
+                            if getattr(zinfo, 'compress_type', None) == 9:
+                                logging.warning(f"Extracting Deflate64-compressed zip file: {input_archive}. This may take a while...")
+                                break
+                        zip_ref.extractall(temp_dir)
+                except NotImplementedError as e:
+                    logging.warning(f"zipfile failed for {input_archive}: {e}. Trying pyzipper fallback.")
+                    try:
+                        with pyzipper.ZipFile(input_archive, 'r') as zip_ref:
+                            zip_ref.extractall(temp_dir)
+                    except Exception as e2:
+                        logging.error(f"pyzipper extraction failed for {input_archive}: {e2}")
+                        return set()
             elif tarfile.is_tarfile(input_archive):
                 with tarfile.open(input_archive, 'r:*') as tar_ref:
                     tar_ref.extractall(temp_dir)

tests/mytest.py

@@ -48,8 +48,8 @@
 
 # Online extractors
 col_online_extractor = Extractor(
-    input_path="https://microdatos.dane.gov.co/index.php/catalog/771/get-microdata",
-    down_ext=['.csv', '.zip'],
+    input_path="https://microdatos.dane.gov.co/index.php/catalog/827/get-microdata",
+    down_ext=['.sav', '.zip'],
     sep=';',
     output_path="data",
     depth=0,
@@ -106,14 +106,15 @@ def test():
     print('Extracting data...')
     dfs = extractor.s4h_extract()
 
-    print('Vertical merge_____________________________________')
-    dfs = har.s4h_vertical_merge(dfs)
-
     for i, df in enumerate(dfs):
         print(f"DataFrame {i + 1} shape: {df.shape}")
         print(df.head())
         print("-" * 50)
 
+    """
+    print('Vertical merge_____________________________________')
+    dfs = har.s4h_vertical_merge(dfs)
+
     har.categories = ["Business"]
     har.key_col = 'DPTO'
     har.key_val = ['11']
@@ -123,7 +124,7 @@ def test():
 
     print(filtered_dask_dfs[0].head())
 
-    """
+    
     print('Horizontal merge___________________________________')
     joined_df = har.s4h_join_data(filtered_ddfs)
     available_cols = joined_df.columns.tolist()
@@ -136,7 +137,7 @@ def test():
     joined_df.to_csv('data/GEIH_2022_harmonized.csv', index=False)
     """
 
-    extractor.s4h_delete_download_folder()
+    #extractor.s4h_delete_download_folder()
 
 if __name__ == "__main__":
     test()