Skip to content

Commit cd65e6e

Browse files
committed
add test for nan values
1 parent abc0184 commit cd65e6e

File tree

3 files changed

+46
-0
lines changed

3 files changed

+46
-0
lines changed

unit_tests/sources/declarative/extractors/decompressed_response.csv

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
12
"EMAIL","FIRST_NAME","LAST_NAME","ADDRESS_LINE_1","ADDRESS_LINE_2","CITY","STATE_PROVINCE_REGION","POSTAL_CODE","COUNTRY","ALTERNATE_EMAILS","PHONE_NUMBER","WHATSAPP","LINE","FACEBOOK","UNIQUE_NAME","CREATED_AT","UPDATED_AT","CONTACT_ID"
23
"[email protected]","Fake contact","Lastname",,,,,"22341",,,,,,,,"2021-02-01T12:35:22Z","2021-02-01T12:35:51Z","eae8c5c8-f97e-40a8-8945-72acca457f5a"
34
"[email protected]","Fake contact","Lastname",,,,,"22341",,,,,,,,"2021-02-01T12:35:08Z","2021-02-01T12:35:38Z","198f959f-f441-4d15-a280-9e8f65a90ba5"
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
"EMAIL","FIRST_NAME","LAST_NAME","ADDRESS_LINE_1","ADDRESS_LINE_2","CITY","STATE_PROVINCE_REGION","POSTAL_CODE","COUNTRY","ALTERNATE_EMAILS","PHONE_NUMBER","WHATSAPP","LINE","FACEBOOK","UNIQUE_NAME","CREATED_AT","UPDATED_AT","CONTACT_ID"
2+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:22Z","2021-02-01T12:35:51Z","eae8c5c8-f97e-40a8-8945-72acca457f5a"
3+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:08Z","2021-02-01T12:35:38Z","198f959f-f441-4d15-a280-9e8f65a90ba5"
4+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:24Z","2021-02-01T12:35:53Z","6975b74c-bb1e-4d54-a251-b934c4193ed4"
5+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:19Z","2021-02-01T12:35:49Z","36ef1a2d-3cc4-4515-9c00-1615c5f860d0"
6+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:30Z","2021-02-01T12:36:00Z","19163421-bb29-495d-950f-edede6218081"
7+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:14Z","2021-02-01T12:35:43Z","d1211b88-e116-4a0b-a823-0361bf059a06"
8+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:21Z","2021-02-01T12:35:50Z","ef4225b0-dff9-4756-af87-c4228d836d53"
9+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:15Z","2021-02-01T12:35:44Z","9adef36c-fe51-421a-9653-6bd010962e98"
10+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:13Z","2021-02-01T12:35:42Z","210d8004-d12a-4f01-815a-f90cfa9e4360"
11+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:17Z","2021-02-01T12:35:46Z","76330f89-5645-4432-b3bb-9e33a9195273"
12+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:26Z","2021-02-01T12:35:55Z","77200269-0b69-462c-bed1-9e6f912d4b83"
13+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:25Z","2021-02-01T12:35:54Z","c91c993b-1dfa-4686-bcf0-31e4aeb2a1a9"
14+
"[email protected]",,,,,,,,,,,,,,,"2021-02-03T19:26:52Z","2021-02-03T19:27:21Z","a2a1f3f4-0170-4fbd-9152-ffe8cbcdb93d"
15+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:29Z","2021-02-01T12:35:59Z","e45af829-de4e-44d6-9c89-bb0c7ce47925"
16+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:27Z","2021-02-01T12:35:56Z","50b36a31-daf8-45c4-bc48-13e150f6746e"
17+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:18Z","2021-02-01T12:35:47Z","353113b9-b41e-480a-bf98-72213350194c"
18+
"[email protected]",,,,,,,,,,,,,,,"2021-02-03T19:34:41Z","2021-02-03T19:35:47Z","0b62947e-de93-419e-8c96-83572bf15ed1"
19+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:31Z","2021-02-01T12:36:01Z","9932d677-1128-47e4-9d97-667c6155bfee"
20+
"[email protected]",,,,,,,,,,,,,,,"2021-02-03T19:22:41Z","2021-02-03T19:23:10Z","ba3c48d5-b63b-48e6-8687-c5034ed0a8dd"
21+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:34:49Z","2021-02-01T12:35:19Z","44ec451f-d401-40d2-831d-3e3ce8a94f66"
22+
"[email protected]","dima","dima",,,,,,,,,,,,,"2021-09-08T09:02:22Z","2021-09-08T09:04:58Z","2f7b13f2-60d2-462a-bfb0-d30bb8eabed8"
23+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:28Z","2021-02-01T12:35:57Z","c6cfd936-e327-48da-aa76-824076461d80"
24+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:23Z","2021-02-01T12:35:52Z","4101feb2-2b07-4aef-8eb5-62878b612fcd"
25+
"[email protected]","Fake contact","Lastname",,,"","#NA","NaN","nan","N/A","#N/A","NA",,"NULL","null","2021-02-01T12:35:16Z","2021-02-01T12:35:45Z","32deb20d-9f8f-44b4-aed2-dc15d5bf45ba"

unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
22
import csv
3+
import math
34
import os
45
from io import BytesIO
56
from pathlib import Path
@@ -31,6 +32,22 @@ def test_text_response(self) -> None:
3132
extracted_records = list(self._extractor.extract_records(response))
3233
assert len(extracted_records) == 24
3334

35+
def test_nan_response(self) -> None:
36+
# This test is used to ensure that the extractor is able to handle NaN values in the response.
37+
response = self._mock_streamed_response_from_file(self._nan_response_path())
38+
extracted_records = list(self._extractor.extract_records(response))
39+
assert len(extracted_records) == 24
40+
41+
assert extracted_records[0]["CITY"] is None
42+
assert extracted_records[0]["STATE_PROVINCE_REGION"] is None
43+
assert extracted_records[0]["POSTAL_CODE"] is None
44+
assert extracted_records[0]["COUNTRY"] is None
45+
assert extracted_records[0]["ALTERNATE_EMAILS"] is None
46+
assert extracted_records[0]["PHONE_NUMBER"] is None
47+
assert extracted_records[0]["WHATSAPP"] is None
48+
assert extracted_records[0]["FACEBOOK"] is None
49+
assert extracted_records[0]["UNIQUE_NAME"] is None
50+
3451
def test_text_response_with_null_bytes(self) -> None:
3552
csv_with_null_bytes = '"FIRST_\x00NAME","LAST_NAME"\n"a first n\x00ame","a last na\x00me"\n'
3653
response = self._mock_streamed_response(BytesIO(csv_with_null_bytes.encode("utf-8")))
@@ -48,6 +65,9 @@ def _compressed_response_path(self) -> Path:
4865
def _decompressed_response_path(self) -> Path:
4966
return self._test_folder_path() / "decompressed_response.csv"
5067

68+
def _nan_response_path(self) -> Path:
69+
return self._test_folder_path() / "nan_response.csv"
70+
5171
def _mock_streamed_response_from_file(self, path: Path) -> requests.Response:
5272
with path.open("rb") as f:
5373
return self._mock_streamed_response(f) # type: ignore # Could not find the right typing for file io

0 commit comments

Comments
 (0)