Adds max size to integer columns & new category strategy (#178)

* Adds max size to integer columns & new category strategy

* Adds comment

Author: JWittmeyer

util/category.py

@@ -14,18 +14,24 @@ def infer_category(file_name: str) -> str:
 def infer_category_enum(df: pd.DataFrame, df_col: str) -> str:
     type_name = df[df_col].dtype.name
     if type_name == "int64":
+        if df[df_col].apply(lambda x: x > 2_147_483_647).sum() > 0:
+            # doesn't fit in database INTEGER type
+            # check all values instead of sample since it a simple integer column
+            return enums.DataTypes.TEXT.value
         return enums.DataTypes.INTEGER.value
     elif type_name == "float64":
         return enums.DataTypes.FLOAT.value
     elif type_name == "bool":
         return enums.DataTypes.BOOLEAN.value
     elif type_name == "object":
-        sample = df[df_col].sample(10) if len(df) > 10 else df[df_col]
-        if sample.apply(lambda x: len(str(x).split()) > 4).sum() > 0:
-            # if any of 10 randomly sampled texts contains more than 4 whitespaces, it is most likely text
-            return enums.DataTypes.TEXT.value
-        else:
+        # if the number of unique values is less than 20% of the number of rows
+        # & no value is longer than 50 characters then we assume category
+        if (
+            df[df_col].nunique() <= df[df_col].count() * 0.2
+            and df[df_col].str.len().max() < 50
+        ):
             return enums.DataTypes.CATEGORY.value
+        return enums.DataTypes.TEXT.value
     else:
         return enums.DataTypes.UNKNOWN.value