file-mode-api: add more tests and make internal check of include files to always return bool

Author: aldogonzalez8

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -179,14 +179,16 @@ def streams(
     def _get_include_files(
         stream_config: Dict[str, Any],
         catalog_with_streams_name: Mapping[str, ConfiguredAirbyteStream] | None,
-    ) -> Optional[bool]:
+    ) -> bool:
         """
         Returns the include_files for the stream if it exists in the catalog.
         """
         if catalog_with_streams_name:
             stream_name = stream_config.get("name")
-            configured_catalog_stream = catalog_with_streams_name.get(stream_name)
-            return configured_catalog_stream and configured_catalog_stream.include_files
+            configured_catalog_stream = (
+                catalog_with_streams_name.get(stream_name) if stream_name else None
+            )
+            return bool(configured_catalog_stream and configured_catalog_stream.include_files)
         return False
 
     @staticmethod

unit_tests/sources/declarative/file/test_file_stream.py

@@ -274,8 +274,8 @@ def test_get_article_attachments_with_filename_extractor(self) -> None:
             )
             assert file_reference.file_size_bytes == file_size
 
-    def test_get_article_attachments_without_include_files(self) -> None:
-        """Test that article attachments can be read without including files, it can be opt-out by configured catalog"""
+    def test_get_article_attachments_with_include_files_false(self) -> None:
+        """Test that article attachments can be read with including files False, it can be opt-out by configured catalog"""
         include_files = False
         with (
             HttpMocker() as http_mocker,
@@ -323,16 +323,52 @@ def test_get_article_attachments_without_include_files(self) -> None:
             # Ensure that NoopFileWriter is called to simulate file writing
             mock_noop_write.assert_called()
             file_reference = output.records[0].record.file_reference
-            assert file_reference
-            assert (
-                file_reference.staging_file_url
-                == "/tmp/airbyte-file-transfer/article_attachments/12138758717583/some_image_name.png"
+            assert file_reference.file_size_bytes == NoopFileWriter.NOOP_FILE_SIZE
+
+    def test_get_article_attachments_without_include_files(self) -> None:
+        """Test that article attachments can be read without including files, it can be opt-out by configured catalog"""
+        with (
+            HttpMocker() as http_mocker,
+            patch(
+                "airbyte_cdk.sources.declarative.retrievers.file_uploader.noop_file_writer.NoopFileWriter.write"
+            ) as mock_noop_write,
+            patch(
+                "airbyte_cdk.sources.declarative.retrievers.file_uploader.local_file_system_file_writer.LocalFileSystemFileWriter.write"
+            ) as mock_file_system_write,
+        ):
+            http_mocker.get(
+                HttpRequest(url=STREAM_URL),
+                HttpResponse(json.dumps(find_template("file_api/articles", __file__)), 200),
+            )
+            http_mocker.get(
+                HttpRequest(url=STREAM_ATTACHMENTS_URL),
+                HttpResponse(
+                    json.dumps(find_template("file_api/article_attachments", __file__)), 200
+                ),
+            )
+            http_mocker.get(
+                HttpRequest(url=STREAM_ATTACHMENT_CONTENT_URL),
+                HttpResponse(
+                    find_binary_response("file_api/article_attachment_content.png", __file__), 200
+                ),
             )
 
-            assert file_reference.source_file_relative_path
-            assert not re.match(
-                r"^article_attachments/[0-9a-fA-F-]{36}$", file_reference.source_file_relative_path
+            mock_noop_write.return_value = NoopFileWriter.NOOP_FILE_SIZE
+
+            output = read(
+                self._config(),
+                CatalogBuilder()
+                .with_stream(ConfiguredAirbyteStreamBuilder().with_name("article_attachments"))
+                .build(),
+                yaml_file="test_file_stream_with_filename_extractor.yaml",
             )
+
+            assert len(output.records) == 1
+            # Ensure that LocalFileSystemFileWriter is not called when include_files is False
+            mock_file_system_write.assert_not_called()
+            # Ensure that NoopFileWriter is called to simulate file writing
+            mock_noop_write.assert_called()
+            file_reference = output.records[0].record.file_reference
             assert file_reference.file_size_bytes == NoopFileWriter.NOOP_FILE_SIZE
 
     def test_get_article_attachments_messages_for_connector_builder(self) -> None: