add test for nan values

pnilan · pnilan · commit cd65e6e818ee · 2025-10-17T13:22:38.000-07:00
diff --git a/unit_tests/sources/declarative/extractors/decompressed_response.csv b/unit_tests/sources/declarative/extractors/decompressed_response.csv
@@ -1,3 +1,4 @@
+
 "EMAIL","FIRST_NAME","LAST_NAME","ADDRESS_LINE_1","ADDRESS_LINE_2","CITY","STATE_PROVINCE_REGION","POSTAL_CODE","COUNTRY","ALTERNATE_EMAILS","PHONE_NUMBER","WHATSAPP","LINE","FACEBOOK","UNIQUE_NAME","CREATED_AT","UPDATED_AT","CONTACT_ID"
 "fake_email_10@lmail.c","Fake contact","Lastname",,,,,"22341",,,,,,,,"2021-02-01T12:35:22Z","2021-02-01T12:35:51Z","eae8c5c8-f97e-40a8-8945-72acca457f5a"
 "fake_email_1@lmail.c","Fake contact","Lastname",,,,,"22341",,,,,,,,"2021-02-01T12:35:08Z","2021-02-01T12:35:38Z","198f959f-f441-4d15-a280-9e8f65a90ba5"
diff --git a/unit_tests/sources/declarative/extractors/nan_response.csv b/unit_tests/sources/declarative/extractors/nan_response.csv
@@ -0,0 +1,25 @@
+"EMAIL","FIRST_NAME","LAST_NAME","ADDRESS_LINE_1","ADDRESS_LINE_2","CITY","STATE_PROVINCE_REGION","POSTAL_CODE","COUNTRY","ALTERNATE_EMAILS","PHONE_NUMBER","WHATSAPP","LINE","FACEBOOK","UNIQUE_NAME","CREATED_AT","UPDATED_AT","CONTACT_ID"
+"fake_email_10@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:22Z","2021-02-01T12:35:51Z","eae8c5c8-f97e-40a8-8945-72acca457f5a"
+"fake_email_1@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:08Z","2021-02-01T12:35:38Z","198f959f-f441-4d15-a280-9e8f65a90ba5"
+"fake_email_12@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:24Z","2021-02-01T12:35:53Z","6975b74c-bb1e-4d54-a251-b934c4193ed4"
+"fake_email_8@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:19Z","2021-02-01T12:35:49Z","36ef1a2d-3cc4-4515-9c00-1615c5f860d0"
+"fake_email_18@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:30Z","2021-02-01T12:36:00Z","19163421-bb29-495d-950f-edede6218081"
+"fake_email_3@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:14Z","2021-02-01T12:35:43Z","d1211b88-e116-4a0b-a823-0361bf059a06"
+"fake_email_9@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:21Z","2021-02-01T12:35:50Z","ef4225b0-dff9-4756-af87-c4228d836d53"
+"fake_email_4@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:15Z","2021-02-01T12:35:44Z","9adef36c-fe51-421a-9653-6bd010962e98"
+"fake_email_2@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:13Z","2021-02-01T12:35:42Z","210d8004-d12a-4f01-815a-f90cfa9e4360"
+"fake_email_6@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:17Z","2021-02-01T12:35:46Z","76330f89-5645-4432-b3bb-9e33a9195273"
+"fake_email_14@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:26Z","2021-02-01T12:35:55Z","77200269-0b69-462c-bed1-9e6f912d4b83"
+"fake_email_13@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:25Z","2021-02-01T12:35:54Z","c91c993b-1dfa-4686-bcf0-31e4aeb2a1a9"
+"joepogbm@ggma.co",,,,,,,,,,,,,,,"2021-02-03T19:26:52Z","2021-02-03T19:27:21Z","a2a1f3f4-0170-4fbd-9152-ffe8cbcdb93d"
+"fake_email_17@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:29Z","2021-02-01T12:35:59Z","e45af829-de4e-44d6-9c89-bb0c7ce47925"
+"fake_email_15@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:27Z","2021-02-01T12:35:56Z","50b36a31-daf8-45c4-bc48-13e150f6746e"
+"fake_email_7@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:18Z","2021-02-01T12:35:47Z","353113b9-b41e-480a-bf98-72213350194c"
+"y.kurochkin@zazmic.com",,,,,,,,,,,,,,,"2021-02-03T19:34:41Z","2021-02-03T19:35:47Z","0b62947e-de93-419e-8c96-83572bf15ed1"
+"fake_email_19@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:31Z","2021-02-01T12:36:01Z","9932d677-1128-47e4-9d97-667c6155bfee"
+"joepogbum@ggma.co",,,,,,,,,,,,,,,"2021-02-03T19:22:41Z","2021-02-03T19:23:10Z","ba3c48d5-b63b-48e6-8687-c5034ed0a8dd"
+"fake_email_0@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:34:49Z","2021-02-01T12:35:19Z","44ec451f-d401-40d2-831d-3e3ce8a94f66"
+"avida.d3@gmail.com","dima","dima",,,,,,,,,,,,,"2021-09-08T09:02:22Z","2021-09-08T09:04:58Z","2f7b13f2-60d2-462a-bfb0-d30bb8eabed8"
+"fake_email_16@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:28Z","2021-02-01T12:35:57Z","c6cfd936-e327-48da-aa76-824076461d80"
+"fake_email_11@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:23Z","2021-02-01T12:35:52Z","4101feb2-2b07-4aef-8eb5-62878b612fcd"
+"fake_email_5@lmail.c","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:16Z","2021-02-01T12:35:45Z","32deb20d-9f8f-44b4-aed2-dc15d5bf45ba"
diff --git a/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py b/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py
@@ -1,5 +1,6 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 import csv
+import math
 import os
 from io import BytesIO
 from pathlib import Path
@@ -31,6 +32,22 @@ def test_text_response(self) -> None:
         extracted_records = list(self._extractor.extract_records(response))
         assert len(extracted_records) == 24
 
+    def test_nan_response(self) -> None:
+        # This test is used to ensure that the extractor is able to handle NaN values in the response.
+        response = self._mock_streamed_response_from_file(self._nan_response_path())
+        extracted_records = list(self._extractor.extract_records(response))
+        assert len(extracted_records) == 24
+
+        assert extracted_records[0]["CITY"] is None
+        assert extracted_records[0]["STATE_PROVINCE_REGION"] is None
+        assert extracted_records[0]["POSTAL_CODE"] is None
+        assert extracted_records[0]["COUNTRY"] is None
+        assert extracted_records[0]["ALTERNATE_EMAILS"] is None
+        assert extracted_records[0]["PHONE_NUMBER"] is None
+        assert extracted_records[0]["WHATSAPP"] is None
+        assert extracted_records[0]["FACEBOOK"] is None
+        assert extracted_records[0]["UNIQUE_NAME"] is None
+
     def test_text_response_with_null_bytes(self) -> None:
         csv_with_null_bytes = '"FIRST_\x00NAME","LAST_NAME"\n"a first n\x00ame","a last na\x00me"\n'
         response = self._mock_streamed_response(BytesIO(csv_with_null_bytes.encode("utf-8")))
@@ -48,6 +65,9 @@ def _compressed_response_path(self) -> Path:
     def _decompressed_response_path(self) -> Path:
         return self._test_folder_path() / "decompressed_response.csv"
 
+    def _nan_response_path(self) -> Path:
+        return self._test_folder_path() / "nan_response.csv"
+
     def _mock_streamed_response_from_file(self, path: Path) -> requests.Response:
         with path.open("rb") as f:
             return self._mock_streamed_response(f)  # type: ignore  # Could not find the right typing for file io

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+`
`1`	`2`	`"EMAIL","FIRST_NAME","LAST_NAME","ADDRESS_LINE_1","ADDRESS_LINE_2","CITY","STATE_PROVINCE_REGION","POSTAL_CODE","COUNTRY","ALTERNATE_EMAILS","PHONE_NUMBER","WHATSAPP","LINE","FACEBOOK","UNIQUE_NAME","CREATED_AT","UPDATED_AT","CONTACT_ID"`
`2`	`3`	`"[email protected]","Fake contact","Lastname",,,,,"22341",,,,,,,,"2021-02-01T12:35:22Z","2021-02-01T12:35:51Z","eae8c5c8-f97e-40a8-8945-72acca457f5a"`
`3`	`4`	`"[email protected]","Fake contact","Lastname",,,,,"22341",,,,,,,,"2021-02-01T12:35:08Z","2021-02-01T12:35:38Z","198f959f-f441-4d15-a280-9e8f65a90ba5"`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,25 @@ @@
 +"EMAIL","FIRST_NAME","LAST_NAME","ADDRESS_LINE_1","ADDRESS_LINE_2","CITY","STATE_PROVINCE_REGION","POSTAL_CODE","COUNTRY","ALTERNATE_EMAILS","PHONE_NUMBER","WHATSAPP","LINE","FACEBOOK","UNIQUE_NAME","CREATED_AT","UPDATED_AT","CONTACT_ID"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:22Z","2021-02-01T12:35:51Z","eae8c5c8-f97e-40a8-8945-72acca457f5a"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:08Z","2021-02-01T12:35:38Z","198f959f-f441-4d15-a280-9e8f65a90ba5"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:24Z","2021-02-01T12:35:53Z","6975b74c-bb1e-4d54-a251-b934c4193ed4"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:19Z","2021-02-01T12:35:49Z","36ef1a2d-3cc4-4515-9c00-1615c5f860d0"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:30Z","2021-02-01T12:36:00Z","19163421-bb29-495d-950f-edede6218081"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:14Z","2021-02-01T12:35:43Z","d1211b88-e116-4a0b-a823-0361bf059a06"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:21Z","2021-02-01T12:35:50Z","ef4225b0-dff9-4756-af87-c4228d836d53"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:15Z","2021-02-01T12:35:44Z","9adef36c-fe51-421a-9653-6bd010962e98"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:13Z","2021-02-01T12:35:42Z","210d8004-d12a-4f01-815a-f90cfa9e4360"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:17Z","2021-02-01T12:35:46Z","76330f89-5645-4432-b3bb-9e33a9195273"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:26Z","2021-02-01T12:35:55Z","77200269-0b69-462c-bed1-9e6f912d4b83"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:25Z","2021-02-01T12:35:54Z","c91c993b-1dfa-4686-bcf0-31e4aeb2a1a9"
 +"[email protected]",,,,,,,,,,,,,,,"2021-02-03T19:26:52Z","2021-02-03T19:27:21Z","a2a1f3f4-0170-4fbd-9152-ffe8cbcdb93d"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:29Z","2021-02-01T12:35:59Z","e45af829-de4e-44d6-9c89-bb0c7ce47925"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:27Z","2021-02-01T12:35:56Z","50b36a31-daf8-45c4-bc48-13e150f6746e"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:18Z","2021-02-01T12:35:47Z","353113b9-b41e-480a-bf98-72213350194c"
 +"[email protected]",,,,,,,,,,,,,,,"2021-02-03T19:34:41Z","2021-02-03T19:35:47Z","0b62947e-de93-419e-8c96-83572bf15ed1"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:31Z","2021-02-01T12:36:01Z","9932d677-1128-47e4-9d97-667c6155bfee"
 +"[email protected]",,,,,,,,,,,,,,,"2021-02-03T19:22:41Z","2021-02-03T19:23:10Z","ba3c48d5-b63b-48e6-8687-c5034ed0a8dd"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:34:49Z","2021-02-01T12:35:19Z","44ec451f-d401-40d2-831d-3e3ce8a94f66"
 +"[email protected]","dima","dima",,,,,,,,,,,,,"2021-09-08T09:02:22Z","2021-09-08T09:04:58Z","2f7b13f2-60d2-462a-bfb0-d30bb8eabed8"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:28Z","2021-02-01T12:35:57Z","c6cfd936-e327-48da-aa76-824076461d80"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:23Z","2021-02-01T12:35:52Z","4101feb2-2b07-4aef-8eb5-62878b612fcd"
 +"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:16Z","2021-02-01T12:35:45Z","32deb20d-9f8f-44b4-aed2-dc15d5bf45ba"