add http status response code to the logs

Author: davidgamez

functions-python/batch_process_dataset/src/main.py

@@ -123,6 +123,7 @@ def download_content(self, temporary_file_path):
             authentication_type=self.authentication_type,
             api_key_parameter_name=self.api_key_parameter_name,
             credentials=self.feed_credentials,
+            logger=self.logger,
         )
         is_zip = zipfile.is_zipfile(temporary_file_path)
         return file_hash, is_zip

functions-python/helpers/utils.py

@@ -86,10 +86,12 @@ def download_and_get_hash(
     authentication_type=0,
     api_key_parameter_name=None,
     credentials=None,
+    logger=None,
 ):
     """
     Downloads the content of a URL and stores it in a file and returns the hash of the file
     """
+    logger = logger or logging.getLogger(__name__)
     try:
         hash_object = hashlib.new(hash_algorithm)
 
@@ -114,20 +116,28 @@ def download_and_get_hash(
 
         with urllib3.PoolManager(ssl_context=ctx) as http:
             with http.request(
-                "GET", url, preload_content=False, headers=headers
+                "GET", url, preload_content=False, headers=headers, redirect=True
             ) as r, open(file_path, "wb") as out_file:
-                while True:
-                    data = r.read(chunk_size)
-                    if not data:
-                        break
-                    hash_object.update(data)
-                    out_file.write(data)
-                r.release_conn()
+                if 200 <= r.status < 300:
+                    logger.info(f"HTTP response code: {r.status}")
+                    while True:
+                        data = r.read(chunk_size)
+                        if not data:
+                            break
+                        hash_object.update(data)
+                        out_file.write(data)
+                    r.release_conn()
+                else:
+                    raise ValueError(f"Invalid HTTP response code: {r.status}")
         return hash_object.hexdigest()
     except Exception as e:
-        print(e)
-        # Delete file if it exists
+        logger.error(e)
         if os.path.exists(file_path):
+            try:
+                with open(file_path, "r") as file:
+                    logger.error(f"File content: {file.read()}")
+            except Exception as read_error:
+                logger.error(f"Failed to read file content: {read_error}")
             os.remove(file_path)
         raise e