Merge pull request #46 from dataiku/bug/dss13-sc-216186-api-connect-add-application-atom-xml-to-xml

fix: [sc-216186] [API connect] Add application/atom+xml to XML content type

Author: alexbourret

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [Version 1.2.3](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.3) - Feature and bugfix release - 2024-11-25
+
+- Fix xml decoding for content type application/atom+xml
+- Dump returned content that can't be decoded
+- Fix: Empty cells in the recipe's input dataset now produce an empty string instead of a 'nan' string
+- Fix: UTF-8 encoding of raw mode body
+
 ## [Version 1.2.2](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.2) - Feature release - 2024-02-14
 
 - Handle XML and CSV endpoints

parameter-sets/secure-basic/parameter-set.json

@@ -15,15 +15,13 @@
             "label": "Azure Single Sign On",
             "credentialRequestSettings": {
                 "type": "BASIC"
-            },
-            "mandatory": true
+            }
         },
         {
             "name": "secure_domain",
             "label": "Domain",
             "description": "",
-            "type": "STRING",
-            "mandatory": true
+            "type": "STRING"
         },
         {
             "name": "login_type",

parameter-sets/secure-oauth/parameter-set.json

@@ -19,8 +19,7 @@
                 "authorizationEndpoint": " ",
                 "tokenEndpoint": " ",
                 "scope": " "
-            },
-            "mandatory": true
+            }
         },
         {
             "name": "authorizationEndpoint",
@@ -47,8 +46,7 @@
             "name": "secure_domain",
             "label": "Domain",
             "description": "",
-            "type": "STRING",
-            "mandatory": true
+            "type": "STRING"
         }
     ]
 }

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "api-connect",
-    "version": "1.2.2",
+    "version": "1.2.3",
     "meta": {
         "label": "API Connect",
         "description": "Retrieve data from any REST API",

python-connectors/api-connect_dataset/connector.py

@@ -4,7 +4,8 @@
 from rest_api_client import RestAPIClient
 from dku_utils import (
     get_dku_key_values, get_endpoint_parameters,
-    parse_keys_for_json, get_value_from_path, get_secure_credentials, decode_csv_data
+    parse_keys_for_json, get_value_from_path, get_secure_credentials,
+    decode_csv_data, decode_bytes
 )
 from dku_constants import DKUConstants
 import json
@@ -57,10 +58,16 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                 record_count += 1
                 yield self.format_output(data, metadata)
             else:
-                data = decode_csv_data(data)
-                record_count += len(data)
-                for row in data:
-                    yield self.format_output(row, metadata)
+                csv_data = decode_csv_data(data)
+                if csv_data:
+                    record_count += len(csv_data)
+                    for row in csv_data:
+                        yield self.format_output(row, metadata)
+                else:
+                    record_count += 1
+                    yield {
+                        DKUConstants.API_RESPONSE_KEY: "{}".format(decode_bytes(data))
+                    }
             if is_records_limit and record_count >= records_limit:
                 break
 

python-lib/dku_constants.py

@@ -2,6 +2,6 @@ class DKUConstants(object):
     API_RESPONSE_KEY = "api_response"
     FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"]
     FORM_DATA_BODY_FORMAT = "FORM_DATA"
-    PLUGIN_VERSION = "1.2.2"
+    PLUGIN_VERSION = "1.2.3"
     RAW_BODY_FORMAT = "RAW"
     REPONSE_ERROR_KEY = "dku_error"

python-lib/dku_utils.py

@@ -1,5 +1,6 @@
 import json
 import copy
+import math
 from jsonpath_ng.ext import parse
 from safe_logger import SafeLogger
 
@@ -125,7 +126,7 @@ def extract_key_using_json_path(json_dictionary, json_path):
 def is_reponse_xml(response):
     content_types = response.headers.get("Content-Type", "").split(";")
     for content_type in content_types:
-        if content_type in ["text/xml", "application/soap+xml", "application/xml"]:
+        if content_type in ["text/xml", "application/soap+xml", "application/xml", "application/atom+xml"]:
             return True
     return False
 
@@ -145,12 +146,24 @@ def decode_csv_data(data):
     import csv
     import io
     json_data = None
-    if isinstance(data, bytes):
-        data = data.decode("utf-8")
+    data = decode_bytes(data)
     try:
         reader = csv.DictReader(io.StringIO(data))
         json_data = list(reader)
     except Exception as error:
         logger.error("Could not extract csv data. Error={}".format(error))
         json_data = data
     return json_data
+
+
+def de_NaN(cell_content):
+    if isinstance(cell_content, float):
+        if math.isnan(cell_content):
+            return ''
+    return cell_content
+
+
+def decode_bytes(content):
+    if isinstance(content, bytes):
+        content = content.decode()
+    return content

python-lib/rest_api_client.py

@@ -128,6 +128,10 @@ def request(self, method, url, can_raise_exeption=True, **kwargs):
         error_message = None
         status_code = None
         response_headers = None
+        if "data" in kwargs:
+            data = kwargs.get("data")
+            if isinstance(data, str):
+                kwargs["data"] = data.encode("utf-8")
         try:
             response = self.request_with_redirect_retry(method, url, **kwargs)
             status_code = response.status_code

python-lib/rest_api_recipe_session.py

@@ -1,7 +1,7 @@
 from dataikuapi.utils import DataikuException
 from rest_api_client import RestAPIClient
 from safe_logger import SafeLogger
-from dku_utils import parse_keys_for_json, get_value_from_path, decode_csv_data
+from dku_utils import parse_keys_for_json, get_value_from_path, decode_csv_data, de_NaN
 from dku_constants import DKUConstants
 import copy
 import json
@@ -49,7 +49,11 @@ def process_dataframe(self, input_parameters_dataframe, is_raw_output):
             self.initial_parameter_columns = {}
             for column_name in self.column_to_parameter_dict:
                 parameter_name = self.column_to_parameter_dict[column_name]
-                self.initial_parameter_columns.update({parameter_name: input_parameters_row.get(column_name)})
+                self.initial_parameter_columns.update(
+                    {
+                        parameter_name: de_NaN(input_parameters_row.get(column_name))
+                    }
+                )
             updated_endpoint_parameters = copy.deepcopy(self.endpoint_parameters)
             updated_endpoint_parameters.update(self.initial_parameter_columns)
             logger.info("Processing row #{}, creating client with credential={}, updated_endpoint={}, custom_key_values={}".format(