Skip to content

Commit a4156fb

Browse files
committed
file-mode-api: uri make connectors pass the relative path of the file in the source rather the file to use the uri
1 parent 9dc319e commit a4156fb

File tree

2 files changed

+15
-23
lines changed

2 files changed

+15
-23
lines changed

airbyte_cdk/sources/file_based/file_based_stream_reader.py

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -178,21 +178,16 @@ def upload(
178178

179179
def _get_file_transfer_paths(
180180
self,
181-
file: RemoteFile,
182-
local_directory: str,
183-
parse_file_path_from_uri: Optional[Callable[[str], str]] = None,
181+
source_file_relative_path: str,
182+
local_directory: str
184183
) -> MutableMapping[str, Any]:
185184
preserve_directory_structure = self.preserve_directory_structure()
186-
if not parse_file_path_from_uri:
187-
file_path = file.uri
188-
else:
189-
file_path = parse_file_path_from_uri(file.uri)
190185

191-
file_name = path.basename(file_path)
192-
file_folder = path.dirname(file_path)
186+
file_name = path.basename(source_file_relative_path)
187+
file_folder = path.dirname(source_file_relative_path)
193188
if preserve_directory_structure:
194189
# Remove left slashes from source path format to make relative path for writing locally
195-
file_relative_path = file_path.lstrip("/")
190+
file_relative_path = source_file_relative_path.lstrip("/")
196191
else:
197192
file_relative_path = file_name
198193
local_file_path = path.join(local_directory, file_relative_path)
@@ -203,7 +198,6 @@ def _get_file_transfer_paths(
203198
self.FILE_RELATIVE_PATH: file_relative_path,
204199
self.LOCAL_FILE_PATH: local_file_path,
205200
self.FILE_NAME: file_name,
206-
self.FILE_FOLDER: file_folder,
207-
self.SOURCE_FILE_URI: file.uri,
201+
self.FILE_FOLDER: file_folder
208202
}
209203
return file_paths

unit_tests/sources/file_based/test_file_based_stream_reader.py

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,7 @@ def test_globs_and_prefixes_from_globs(
384384

385385

386386
@pytest.mark.parametrize(
387-
"config, source_file, expected_file_relative_path, expected_local_file_path",
387+
"config, source_file_path, expected_file_relative_path, expected_local_file_path",
388388
[
389389
pytest.param(
390390
{
@@ -430,23 +430,21 @@ def test_globs_and_prefixes_from_globs(
430430
)
431431
def test_preserve_sub_directories_scenarios(
432432
config: Mapping[str, Any],
433-
source_file: str,
433+
source_file_path: str,
434434
expected_file_relative_path: str,
435435
expected_local_file_path: str,
436436
) -> None:
437-
remote_file = RemoteFile(
438-
uri=source_file,
439-
last_modified=datetime(2025, 1, 9, 11, 27, 20),
440-
mime_type=None,
441-
)
437+
"""
438+
Test scenarios when preserve_directory_structure is True or False, the flag indicates whether we need to
439+
use a relative path to upload the file or simply place it in the root.
440+
"""
442441
reader = TestStreamReader()
443442
reader.config = TestSpec(**config)
444-
file_paths = reader._get_file_transfer_paths(remote_file, "/tmp/transfer-files/")
443+
file_paths = reader._get_file_transfer_paths(source_file_path, "/tmp/transfer-files/")
445444

446445
assert (
447446
file_paths[AbstractFileBasedStreamReader.FILE_RELATIVE_PATH] == expected_file_relative_path
448447
)
449448
assert file_paths[AbstractFileBasedStreamReader.LOCAL_FILE_PATH] == expected_local_file_path
450-
assert file_paths[AbstractFileBasedStreamReader.SOURCE_FILE_URI] == source_file
451-
assert file_paths[AbstractFileBasedStreamReader.FILE_NAME] == path.basename(source_file)
452-
assert file_paths[AbstractFileBasedStreamReader.FILE_FOLDER] == path.dirname(source_file)
449+
assert file_paths[AbstractFileBasedStreamReader.FILE_NAME] == path.basename(source_file_path)
450+
assert file_paths[AbstractFileBasedStreamReader.FILE_FOLDER] == path.dirname(source_file_path)

0 commit comments

Comments
 (0)