updated compression types checks

Author: Oleksandr Bazarnov

airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -22,18 +22,6 @@
 logger = logging.getLogger("airbyte")
 
 
-COMPRESSION_TYPES = [
-    "gzip",
-    "x-gzip",
-    "gzip, deflate",
-    "x-gzip, deflate",
-    "application/zip",
-    "application/gzip",
-    "application/x-gzip",
-    "application/x-zip-compressed",
-]
-
-
 @dataclass
 class Parser(ABC):
     @abstractmethod
@@ -186,12 +174,6 @@ class CompositeRawDecoder(Decoder):
     def is_stream_response(self) -> bool:
         return self.stream_response
 
-    def is_compressed(self, response: requests.Response) -> bool:
-        """
-        Check if the response is compressed based on the Content-Encoding header.
-        """
-        return response.headers.get("Content-Encoding") in COMPRESSION_TYPES
-
     def decode(
         self,
         response: requests.Response,
@@ -202,7 +184,10 @@ def decode(
             # We have indeed observed some issues with CSV parsing.
             # Hence, we will manage the closing of the file ourselves until we find a better solution.
             response.raw.auto_close = False
-            yield from self.parser.parse(data=response.raw, compressed=self.is_compressed(response))  # type: ignore[arg-type]
+            yield from self.parser.parse(
+                data=response.raw,  # type: ignore[arg-type]
+                compressed=self.is_compressed_response(response),
+            )
             response.raw.close()
         else:
             yield from self.parser.parse(data=io.BytesIO(response.content))

airbyte_cdk/sources/declarative/decoders/decoder.py

@@ -8,6 +8,17 @@
 
 import requests
 
+COMPRESSION_RESPONSE_TYPES = [
+    "gzip",
+    "x-gzip",
+    "gzip, deflate",
+    "x-gzip, deflate",
+    "application/zip",
+    "application/gzip",
+    "application/x-gzip",
+    "application/x-zip-compressed",
+]
+
 
 @dataclass
 class Decoder:
@@ -30,3 +41,12 @@ def decode(
         :param response: the response to decode
         :return: Generator of Mapping describing the response
         """
+
+    def is_compressed_response(self, response: requests.Response) -> bool:
+        """
+        Check if the response is compressed based on the Content-Encoding header.
+        """
+        return (
+            response.headers.get("Content-Encoding") in COMPRESSION_RESPONSE_TYPES
+            or response.headers.get("Content-Type") in COMPRESSION_RESPONSE_TYPES
+        )

airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py

@@ -8,12 +8,11 @@
 from io import BytesIO
 from typing import Any, Generator, MutableMapping
 
-import orjson
 import requests
 
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.declarative.decoders import Decoder
-from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import COMPRESSION_TYPES, Parser
+from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import Parser
 from airbyte_cdk.utils import AirbyteTracedException
 
 logger = logging.getLogger("airbyte")
@@ -26,12 +25,6 @@ class ZipfileDecoder(Decoder):
     def is_stream_response(self) -> bool:
         return False
 
-    def is_compressed(self, response: requests.Response) -> bool:
-        """
-        Check if the response is compressed based on the Content-Encoding header.
-        """
-        return response.headers.get("Content-Encoding") in COMPRESSION_TYPES
-
     def decode(
         self, response: requests.Response
     ) -> Generator[MutableMapping[str, Any], None, None]:
@@ -43,7 +36,7 @@ def decode(
                     try:
                         yield from self.parser.parse(
                             buffered_content,
-                            compressed=self.is_compressed(response),
+                            compressed=self.is_compressed_response(response),
                         )
                     except Exception as e:
                         logger.error(