File tree Expand file tree Collapse file tree 1 file changed +13
-0
lines changed
Expand file tree Collapse file tree 1 file changed +13
-0
lines changed Original file line number Diff line number Diff line change @@ -16,6 +16,8 @@ def get_check_data_type_function(data_type):
1616 return [str ], __check_data_type_category
1717 elif data_type == "TEXT" :
1818 return [str ], __check_data_type_text
19+ elif data_type == "EMBEDDING_LIST" :
20+ return [list ], __check_data_type_embedding_list
1921 else :
2022 raise ValueError (f"Unknown data type: { data_type } " )
2123
@@ -55,9 +57,20 @@ def __check_data_type_text(attr_value):
5557 return False
5658 return True
5759
60+
61+ def __check_data_type_embedding_list (attr_value ):
62+ if not isinstance (attr_value , list ):
63+ return False
64+ for e in attr_value :
65+ if not isinstance (e , str ) or len (e ) == 0 :
66+ raise ValueError ("List entries need to be strings with a length > 0." )
67+ return True
68+
69+
5870def __print_progress (progress : float ) -> None :
5971 print (f"progress: { progress } " , flush = True )
6072
73+
6174def load_data_dict (record ):
6275 if record ["bytes" ][:2 ] == "\\ x" :
6376 record ["bytes" ] = record ["bytes" ][2 :]
You can’t perform that action at this time.
0 commit comments