chore: add response http status code to the logs (#1208)

Author: davidgamez

functions-python/batch_process_dataset/src/main.py

@@ -123,6 +123,7 @@ def download_content(self, temporary_file_path):
             authentication_type=self.authentication_type,
             api_key_parameter_name=self.api_key_parameter_name,
             credentials=self.feed_credentials,
+            logger=self.logger,
         )
         is_zip = zipfile.is_zipfile(temporary_file_path)
         return file_hash, is_zip

functions-python/helpers/tests/test_helpers.py

@@ -53,6 +53,7 @@ def test_download_and_get_hash(self):
 
         mock_response = MagicMock()
         mock_response.read.side_effect = [mock_binary_data, b""]
+        mock_response.status = 200
         mock_response.__enter__.return_value = mock_response
 
         with patch("urllib3.PoolManager.request", return_value=mock_response):
@@ -80,6 +81,7 @@ def test_download_and_get_hash_auth_type_header(self):
         credentials = "Bearer token123"
 
         mock_response = MagicMock()
+        mock_response.status = 200
         mock_response.read.side_effect = [mock_binary_data, b""]
         mock_response.__enter__.return_value = mock_response
 
@@ -105,6 +107,7 @@ def test_download_and_get_hash_auth_type_header(self):
                     "User-Agent": expected_user_agent,
                     api_key_parameter_name: credentials,
                 },
+                redirect=True,
             )
 
             if os.path.exists(file_path):
@@ -113,28 +116,26 @@ def test_download_and_get_hash_auth_type_header(self):
     def test_download_and_get_hash_auth_type_api_key(self):
         """
         Test the download_and_get_hash function for authentication type 1 (API key).
-
-        This test verifies that the download_and_get_hash function correctly handles authentication type 1,
-        where the credentials are passed as a query parameter in the URL. It mocks the necessary components
-        and checks that the request is made with the appropriate URL containing the API key.
-
         """
         mock_binary_data = b"binary data for auth type 1"
         expected_hash = hashlib.sha256(mock_binary_data).hexdigest()
         file_path = "test_file.txt"
         base_url = "https://test.com"
         api_key_parameter_name = "api_key"
         credentials = "key123"
-
         modified_url = f"{base_url}?{api_key_parameter_name}={credentials}"
 
         mock_response = MagicMock()
         mock_response.read.side_effect = [mock_binary_data, b""]
+        mock_response.status = 200
+        mock_response.release_conn = MagicMock()
         mock_response.__enter__.return_value = mock_response
 
-        with patch(
-            "urllib3.PoolManager.request", return_value=mock_response
-        ) as mock_request:
+        mock_http = MagicMock()
+        mock_http.request.return_value = mock_response
+        mock_http.__enter__.return_value = mock_http
+
+        with patch("urllib3.PoolManager", return_value=mock_http):
             result_hash = download_and_get_hash(
                 base_url,
                 file_path,
@@ -148,15 +149,15 @@ def test_download_and_get_hash_auth_type_api_key(self):
             self.assertEqual(
                 result_hash,
                 expected_hash,
-                msg=f"Hash mismatch: got {result_hash},"
-                f" but expected {expected_hash}",
+                msg=f"Hash mismatch: got {result_hash}, but expected {expected_hash}",
             )
 
-            mock_request.assert_called_with(
+            mock_http.request.assert_called_with(
                 "GET",
                 modified_url,
                 preload_content=False,
                 headers={"User-Agent": expected_user_agent},
+                redirect=True,
             )
 
         if os.path.exists(file_path):

functions-python/helpers/utils.py

@@ -86,10 +86,12 @@ def download_and_get_hash(
     authentication_type=0,
     api_key_parameter_name=None,
     credentials=None,
+    logger=None,
 ):
     """
     Downloads the content of a URL and stores it in a file and returns the hash of the file
     """
+    logger = logger or logging.getLogger(__name__)
     try:
         hash_object = hashlib.new(hash_algorithm)
 
@@ -114,21 +116,27 @@ def download_and_get_hash(
 
         with urllib3.PoolManager(ssl_context=ctx) as http:
             with http.request(
-                "GET", url, preload_content=False, headers=headers
+                "GET", url, preload_content=False, headers=headers, redirect=True
             ) as r, open(file_path, "wb") as out_file:
-                while True:
-                    data = r.read(chunk_size)
-                    if not data:
-                        break
-                    hash_object.update(data)
-                    out_file.write(data)
-                r.release_conn()
+                if 200 <= r.status < 300:
+                    logger.info(f"HTTP response code: [{r.status}]")
+                    while True:
+                        data = r.read(chunk_size)
+                        if not data:
+                            break
+                        hash_object.update(data)
+                        out_file.write(data)
+                    r.release_conn()
+                else:
+                    raise ValueError(f"Invalid HTTP response code: [{r.status}]")
         return hash_object.hexdigest()
     except Exception as e:
-        print(e)
-        # Delete file if it exists
         if os.path.exists(file_path):
-            os.remove(file_path)
+            try:
+                os.remove(file_path)
+            except Exception:
+                logger.error(f"Delete file: [{file_path}]")
+
         raise e