Enable Attribute calculation for different data types (#6)

* check data type of calculated attributes

* put attribute value inbetween grave accents

* pr comment, float attributes can have datatype int or float

Author: FelixKirsch

Dockerfile

@@ -1,9 +1,11 @@
-FROM python:3.9-slim
+FROM python:3.10-slim
 
 RUN apt update && apt install -y curl
 
-COPY . .
+COPY requirements.txt .
 
 RUN pip3 install -r requirements.txt
 
+COPY . .
+
 ENTRYPOINT ["/run.sh"] 

run.sh

@@ -3,4 +3,4 @@
 /usr/bin/curl -s "$1" > docbin_full.json;
 /usr/bin/curl -s "$2" > attribute_calculators.py;
 
-/usr/local/bin/python run_ac.py "$3" "$4";
+/usr/local/bin/python -u run_ac.py "$3" "$4" "$5";

run_ac.py

@@ -5,6 +5,24 @@
 from spacy.tokens import DocBin
 
 
+def get_check_data_type_function(data_type):
+    if data_type == "INTEGER":
+        py_data_types = [int]
+    elif data_type == "FLOAT":
+        py_data_types = [int, float]
+    elif data_type == "BOOLEAN":
+        py_data_types = [bool]
+    elif data_type == "CATEGORY":
+        py_data_types = [str]
+    elif data_type == "TEXT":
+        py_data_types = [str]
+    else:
+        raise ValueError(f"Unknown data type: {data_type}")
+    return py_data_types, lambda f: any(
+        [isinstance(f, py_data_type) for py_data_type in py_data_types]
+    )
+
+
 def load_data_dict(record):
     if record["bytes"][:2] == "\\x":
         record["bytes"] = record["bytes"][2:]
@@ -33,7 +51,7 @@ def parse_data_to_record_dict(record_chunk):
 
 
 if __name__ == "__main__":
-    _, iso2_code, payload_url = sys.argv
+    _, iso2_code, payload_url, data_type = sys.argv
 
     print("Preparing data for attribute calculation.")
 
@@ -48,9 +66,18 @@ def parse_data_to_record_dict(record_chunk):
 
     record_dict_list = parse_data_to_record_dict(docbin_data)
 
+    py_data_types, check_data_type = get_check_data_type_function(data_type)
+
     print("Running attribute calculation.")
     calculated_attribute_by_record_id = {}
     for record_dict in record_dict_list:
+        attr_value = ac(record_dict["data"])
+        if not check_data_type(attr_value):
+            raise ValueError(
+                f"Attribute value `{attr_value}` is of type {type(attr_value)}, "
+                f"but data_type {data_type} requires "
+                f"{str(py_data_types) if len(py_data_types) > 1 else str(py_data_types[0])}."
+            )
         calculated_attribute_by_record_id[record_dict["id"]] = ac(record_dict["data"])
 
     print("Finished execution.")