Unzip files after hash check.

Author: davidgamez

functions-python/batch_process_dataset/src/main.py

@@ -128,19 +128,7 @@ def download_content(self, temporary_file_path):
             logger=self.logger,
         )
         is_zip = zipfile.is_zipfile(temporary_file_path)
-        extracted_files_path = None
-        if is_zip:
-            extracted_files_path = os.path.join(
-                temporary_file_path.split(".")[0], "extracted"
-            )
-            # Create the directory for extracted files if it does not exist
-            os.makedirs(extracted_files_path, exist_ok=True)
-            with zipfile.ZipFile(temporary_file_path, "r") as zip_ref:
-                zip_ref.extractall(path=extracted_files_path)
-            # List all files in the extracted directory
-            extracted_files = os.listdir(extracted_files_path)
-            self.logger.info(f"Extracted files: {extracted_files}")
-        return file_hash, is_zip, extracted_files_path
+        return file_hash, is_zip
 
     def upload_file_to_storage(
         self, source_file_path, dataset_stable_id, extracted_files_path
@@ -209,12 +197,14 @@ def upload_dataset(self) -> DatasetFile or None:
             self.logger.info(
                 f"[{self.feed_stable_id}] File hash is {file_sha256_hash}."
             )
-
             if self.latest_hash != file_sha256_hash:
                 self.logger.info(
                     f"[{self.feed_stable_id}] Dataset has changed (hash {self.latest_hash}"
                     f"-> {file_sha256_hash}). Uploading new version."
                 )
+                extracted_files_path = self.unzip_files(
+                    extracted_files_path, temp_file_path
+                )
                 self.logger.info(
                     f"Creating file {self.feed_stable_id}/latest.zip in bucket {self.bucket_name}"
                 )
@@ -252,6 +242,18 @@ def upload_dataset(self) -> DatasetFile or None:
                 os.remove(temp_file_path)
         return None
 
+    def unzip_files(self, extracted_files_path, temp_file_path):
+        extracted_files_path = os.path.join(temp_file_path.split(".")[0], "extracted")
+        self.logger.info(f"Unzipping files to {extracted_files_path}")
+        # Create the directory for extracted files if it does not exist
+        os.makedirs(extracted_files_path, exist_ok=True)
+        with zipfile.ZipFile(temp_file_path, "r") as zip_ref:
+            zip_ref.extractall(path=extracted_files_path)
+        # List all files in the extracted directory
+        extracted_files = os.listdir(extracted_files_path)
+        self.logger.info(f"Extracted files: {extracted_files}")
+        return extracted_files_path
+
     def generate_temp_filename(self):
         """
         Generates a temporary filename

functions-python/batch_process_dataset/tests/test_batch_process_dataset_main.py

@@ -46,8 +46,9 @@ def create_cloud_event(mock_data):
 class TestDatasetProcessor(unittest.TestCase):
     @patch("main.DatasetProcessor.upload_file_to_storage")
     @patch("main.DatasetProcessor.download_content")
+    @patch("main.DatasetProcessor.unzip_files")
     def test_upload_dataset_diff_hash(
-        self, mock_download_url_content, upload_file_to_storage
+        self, mock_unzip_files, mock_download_url_content, upload_file_to_storage
     ):
         """
         Test upload_dataset method of DatasetProcessor class with different hash from the latest one
@@ -57,6 +58,7 @@ def test_upload_dataset_diff_hash(
         mock_blob.path = public_url
         upload_file_to_storage.return_value = mock_blob, []
         mock_download_url_content.return_value = file_hash, True, "path/file"
+        mock_unzip_files.return_value = [mock_blob, mock_blob]
 
         processor = DatasetProcessor(
             public_url,