fix various issues with unitary enums and return type conversions in opaque structs

Author: arik-so

src/binding_types.py

@@ -1,12 +1,13 @@
 class TypeInfo:
 	def __init__(self, is_native_primitive, rust_obj, swift_type, c_ty, is_const, passed_as_ptr, is_ptr, var_name,
-				 arr_len, arr_access, subty=None):
+				 arr_len, arr_access, subty=None, swift_raw_type=None):
 		self.is_native_primitive = is_native_primitive
 		self.rust_obj = rust_obj
 		# self.java_ty = java_ty
 		# self.java_hu_ty = java_hu_ty
 		# self.java_fn_ty_arg = java_fn_ty_arg
 		self.swift_type = swift_type
+		self.swift_raw_type = swift_raw_type if swift_raw_type is not None else swift_type
 		self.c_ty = c_ty
 		self.is_const = is_const
 		self.passed_as_ptr = passed_as_ptr

src/generators/opaque_struct_generator.py

@@ -13,7 +13,7 @@ def __init__(self) -> None:
 			template = template_handle.read()
 			self.template = template
 
-	def generate_opaque_struct(self, struct_name, struct_details):
+	def generate_opaque_struct(self, struct_name, struct_details, all_type_details = {}):
 		# method_names = ['openChannel', 'closeChannel']
 		# native_method_names = ['ChannelHandler_openChannel', 'ChannelHandler_closeChannel']
 
@@ -28,15 +28,26 @@ def generate_opaque_struct(self, struct_name, struct_details):
 			constructor_native_name = constructor_details['name']['native']
 			swift_arguments = []
 			native_arguments = []
+			constructor_argument_prep = ''
 			for current_argument_details in constructor_details['argument_types']:
 				argument_name = current_argument_details.var_name
 				passed_argument_name = argument_name
+				constructor_argument_conversion_method = None
+
+				if current_argument_details.rust_obj is not None and current_argument_details.rust_obj.startswith('LDK') and current_argument_details.swift_type.startswith('['):
+					constructor_argument_conversion_method = f'let converted_{argument_name} = Bindings.new_{current_argument_details.rust_obj}(array: {argument_name})'
+					constructor_argument_prep += constructor_argument_conversion_method
+					passed_argument_name = f'converted_{argument_name}'
+				elif current_argument_details.rust_obj == 'LDK'+current_argument_details.swift_type:
+					passed_argument_name += '.cOpaqueStruct!'
 
 				swift_arguments.append(f'{argument_name}: {current_argument_details.swift_type}')
 				native_arguments.append(f'{passed_argument_name}')
 
 			mutating_output_file_contents = mutating_output_file_contents.replace('swift_constructor_arguments',
 																				  ', '.join(swift_arguments))
+			mutating_output_file_contents = mutating_output_file_contents.replace('/* NATIVE_CONSTRUCTOR_PREP */',
+																				  constructor_argument_prep)
 			mutating_output_file_contents = mutating_output_file_contents.replace('native_constructor_arguments',
 																				  ', '.join(native_arguments))
 			mutating_output_file_contents = mutating_output_file_contents.replace(
@@ -59,9 +70,23 @@ def generate_opaque_struct(self, struct_name, struct_details):
 			current_native_method_name = current_method_details['name']['native']
 			current_method_name = current_method_details['name']['swift']
 			current_return_type = current_method_details['return_type'].swift_type
-			# current_method_name = current_native_method_name[len(method_prefix):]
+			# current_rust_return_type = current_method_details['return_type'].rust_obj
+
+			# if current_rust_return_type in all_type_details and all_type_details[current_rust_return_type].type.name == 'UNITARY_ENUM':
+			# 	current_return_type = current_rust_return_type
+			current_method_name = current_native_method_name[len(method_prefix):]
 
 			current_replacement = method_template
+
+			if current_method_details['return_type'].rust_obj is not None and current_method_details['return_type'].rust_obj.startswith('LDK') and current_method_details['return_type'].swift_type.startswith('['):
+				return_type_wrapper_prefix = f'Bindings.{current_method_details["return_type"].rust_obj}_to_array(byteType: '
+				return_type_wrapper_suffix = ')'
+				current_replacement = current_replacement.replace('return OpaqueStructType_methodName(native_arguments)', f'return {return_type_wrapper_prefix}OpaqueStructType_methodName(native_arguments){return_type_wrapper_suffix}')
+			elif current_method_details['return_type'].rust_obj == 'LDK' + current_method_details['return_type'].swift_type:
+				return_type_wrapper_prefix = f'{current_method_details["return_type"].swift_type}(pointer: '
+				return_type_wrapper_suffix = ')'
+				current_replacement = current_replacement.replace('return OpaqueStructType_methodName(native_arguments)', f'return {return_type_wrapper_prefix}OpaqueStructType_methodName(native_arguments){return_type_wrapper_suffix}')
+
 			current_replacement = current_replacement.replace('func methodName(', f'func {current_method_name}(')
 
 			is_clone_method = current_method_details['is_clone']
@@ -106,7 +131,12 @@ def generate_opaque_struct(self, struct_name, struct_details):
 
 				if not pass_instance:
 					swift_arguments.append(f'{argument_name}: {current_argument_details.swift_type}')
-				native_arguments.append(f'{passed_argument_name}')
+
+				# native_arguments.append(f'{passed_argument_name}')
+				if current_argument_details.rust_obj == 'LDK' + current_argument_details.swift_type and not current_argument_details.is_ptr:
+					native_arguments.append(f'{passed_argument_name}.cOpaqueStruct!')
+				else:
+					native_arguments.append(f'{passed_argument_name}')
 
 			current_replacement = current_replacement.replace('swift_arguments', ', '.join(swift_arguments))
 			if is_clone_method:

src/generators/trait_generator.py

@@ -19,70 +19,77 @@ def generate_trait(self, struct_name, struct_details):
 
 		swift_struct_name = struct_name[3:]
 
-		method_template_regex = re.compile(
-			"(\/\* STRUCT_METHODS_START \*\/\n)(.*)(\n[\t ]*\/\* STRUCT_METHODS_END \*\/)",
+		native_callback_template_regex = re.compile(
+			"(\/\* NATIVE_CALLBACKS_START \*\/\n)(.*)(\n[\t ]*\/\* NATIVE_CALLBACKS_END \*\/)",
 			flags=re.MULTILINE | re.DOTALL)
-		method_template = method_template_regex.search(self.template).group(2)
+		native_callback_template = native_callback_template_regex.search(self.template).group(2)
+
+		swift_callback_template_regex = re.compile(
+			"(\/\* SWIFT_CALLBACKS_START \*\/\n)(.*)(\n[\t ]*\/\* SWIFT_CALLBACKS_END \*\/)",
+			flags=re.MULTILINE | re.DOTALL)
+		swift_callback_template = swift_callback_template_regex.search(self.template).group(2)
 
 		method_prefix = swift_struct_name + '_'
-		struct_methods = ''
+		native_callbacks = ''
+		swift_callbacks = ''
+
+		instantiation_arguments = []
 
 		# fill templates
-		for current_method_details in struct_details.methods:
-			current_native_method_name = current_method_details['name']['native']
-			current_method_name = current_method_details['name']['swift']
-			# current_method_name = current_native_method_name[len(method_prefix):]
-
-			current_replacement = method_template
-			current_replacement = current_replacement.replace('func methodName(', f'func {current_method_name}(')
-			current_replacement = current_replacement.replace('OpaqueStructType_methodName(',
-															  f'{current_native_method_name}(')
-
-			# replace arguments
-			swift_arguments = []
-			native_arguments = ['self.cOpaqueStruct']
-			native_call_prep = ''
-			for current_argument_details in current_method_details['argument_types']:
-				argument_name = current_argument_details.var_name
-				passed_argument_name = argument_name
-				if argument_name == 'this_ptr':
-					# we already pass this much more elegantly
-					continue
-
-				if current_argument_details.passed_as_ptr:
-					passed_argument_name = argument_name+'Pointer'
-					# let managerPointer = withUnsafePointer(to: self.cChannelManager!) { (pointer: UnsafePointer<LDKChannelManager>) in
-					# 	pointer
-					# }
-					# the \n\t will add a bunch of extra lines, but this file will be easier to read
-					current_prep = f'''
-						\n\t	let {passed_argument_name} = withUnsafePointer(to: {argument_name}.cOpaqueStruct!) {{ (pointer: UnsafePointer<{current_argument_details.rust_obj}>) in
-							\n\t\t	pointer
-						\n\t	}}
-					'''
-					native_call_prep += current_prep
-
-				swift_arguments.append(f'{current_argument_details.java_hu_ty} {argument_name}')
-				native_arguments.append(f'{passed_argument_name}')
-
-			current_replacement = current_replacement.replace('swift_arguments', ', '.join(swift_arguments))
-			current_replacement = current_replacement.replace('native_arguments', ', '.join(native_arguments))
-			current_replacement = current_replacement.replace('/* NATIVE_CALL_PREP */', native_call_prep)
-
-			struct_methods += '\n' + current_replacement + '\n'
-
-		opaque_struct_file = self.template.replace('class OpaqueStructName {', f'class {swift_struct_name} {{')
-		opaque_struct_file = opaque_struct_file.replace('var cOpaqueStruct: OpaqueStructType?',
-														f'var cOpaqueStruct: {struct_name}?')
-		opaque_struct_file = opaque_struct_file.replace('self.cOpaqueStruct = OpaqueStructType()',
-														f'self.cOpaqueStruct = {struct_name}_new()')
-		opaque_struct_file = method_template_regex.sub(f'\g<1>{struct_methods}\g<3>', opaque_struct_file)
+		for current_lambda in struct_details.lambdas:
+			current_lambda_name = current_lambda['name']
+
+			current_native_callback_replacement = native_callback_template
+			current_native_callback_replacement = current_native_callback_replacement.replace('func methodNameCallback(', f'func {current_lambda_name}Callback(')
+			current_native_callback_replacement = current_native_callback_replacement.replace('instance: TraitName', f'instance: {swift_struct_name}')
+			current_native_callback_replacement = current_native_callback_replacement.replace('instance.callbackName(', f'instance.{current_lambda_name}(')
+
+			current_swift_callback_replacement = swift_callback_template
+			current_swift_callback_replacement = current_swift_callback_replacement.replace('func methodName(', f'func {current_lambda_name}(')
+
+			instantiation_arguments.append(f'{current_lambda_name}: {current_lambda_name}Callback')
+
+			# let's specify the correct return type
+			swift_raw_return_type = current_lambda['return_type'].swift_raw_type
+			current_native_callback_replacement = current_native_callback_replacement.replace(') -> Void {', f') -> {swift_raw_return_type} {{')
+
+			# let's get the current native arguments, i. e. the arguments we get from C into the native callback
+			native_arguments = []
+			swift_callback_arguments = []
+			swift_argument_string = ''
+			for current_argument in current_lambda['argument_types']:
+				native_arguments.append(f'{current_argument.var_name}: {current_argument.swift_raw_type}?')
+				swift_callback_arguments.append(f'{current_argument.var_name}: {current_argument.var_name}')
+			if len(native_arguments) > 0:
+				# add leading comma
+				swift_argument_string = ', '.join(native_arguments)
+				native_arguments.insert(0, '')
+
+			native_argument_string = ', '.join(native_arguments)
+			current_native_callback_replacement = current_native_callback_replacement.replace(', native_arguments', native_argument_string)
+			current_native_callback_replacement = current_native_callback_replacement.replace('swift_callback_arguments', ', '.join(swift_callback_arguments))
+			current_swift_callback_replacement = current_swift_callback_replacement.replace('swift_arguments', swift_argument_string)
+
+			if not current_lambda['is_constant']:
+				current_native_callback_replacement = current_native_callback_replacement.replace('(pointer: UnsafeRawPointer?', '(pointer: UnsafeMutableRawPointer?')
+
+			native_callbacks += '\n' + current_native_callback_replacement + '\n'
+			swift_callbacks += '\n' + current_swift_callback_replacement + '\n'
+
+		trait_file = self.template.replace('class TraitName {', f'class {swift_struct_name} {{')
+		trait_file = trait_file.replace('var cTrait: TraitType?',
+														f'var cTrait: {struct_name}?')
+		trait_file = trait_file.replace('self.cTrait = TraitType(',
+														f'self.cTrait = {struct_name}(')
+		trait_file = trait_file.replace('native_callback_instantiation_arguments', ', '.join(instantiation_arguments))
+		trait_file = native_callback_template_regex.sub(f'\g<1>{native_callbacks}\g<3>', trait_file)
+		trait_file = swift_callback_template_regex.sub(f'\g<1>{swift_callbacks}\g<3>', trait_file)
 
 		# store the output
 		output_path = f'{Config.OUTPUT_DIRECTORY_PATH}/traits/{swift_struct_name}.swift'
 		output_directory = os.path.dirname(output_path)
 		if not os.path.exists(output_directory):
 			os.makedirs(output_directory)
 		with open(output_path, "w") as f:
-			f.write(opaque_struct_file)
+			f.write(trait_file)
 		pass

src/generators/tuple_generator.py

@@ -15,7 +15,7 @@ def __init__(self) -> None:
 	def generate_tuple(self, tuple_name, tuple_details):
 		# method_names = ['openChannel', 'closeChannel']
 		# native_method_names = ['ChannelHandler_openChannel', 'ChannelHandler_closeChannel']
-
+		print(tuple_name)
 		swift_tuple_name = tuple_name[3:]
 
 		mutating_output_file_contents = self.template

src/generators/util_generators/vector_generator.py

@@ -12,11 +12,41 @@ def __init__(self) -> None:
 		self.loadTemplate()
 
 	def generate_vector(self, vector_name, vector_type_details):
-		if not vector_type_details.is_primitive:
-			# TODO: add non-primitive tuple support
-			return
+		dimensions = 1
+		conversion_call = None
+		is_primitive = vector_type_details.is_primitive
+
+		if is_primitive:
+			swift_primitive = vector_type_details.primitive_swift_counterpart
+		else:
+			deepest_iteratee = vector_type_details
+			while deepest_iteratee.iteratee is not None:
+				deepest_iteratee = deepest_iteratee.iteratee
+				dimensions += 1
+			shallowmost_iteratee = vector_type_details.iteratee
+			if deepest_iteratee.is_primitive:
+				swift_primitive = deepest_iteratee.primitive_swift_counterpart
+			else:
+				dimensions -= 1
+				swift_primitive = deepest_iteratee.name
+			if dimensions > 1:
+				conversion_call = f'let convertedEntry = {shallowmost_iteratee.name}_to_array(vector: currentEntry)'
+
 		mutating_current_vector_methods = self.template
+		for dim_delta in range(1, dimensions):
+			mutating_current_vector_methods = mutating_current_vector_methods.replace('[SwiftPrimitive]', '[[SwiftPrimitive]]')
 		mutating_current_vector_methods = mutating_current_vector_methods.replace('LDKCVec_rust_primitive', vector_name)
+
+		if conversion_call is not None:
+			mutating_current_vector_methods = mutating_current_vector_methods.replace('/* CONVERSION_PREP */', conversion_call)
+		else:
+			mutating_current_vector_methods = mutating_current_vector_methods.replace('(convertedEntry)', '(currentEntry)')
+
+		if not is_primitive:
+			mutating_current_vector_methods = mutating_current_vector_methods.replace('/* SWIFT_TO_RUST_START */', '/* SWIFT_TO_RUST_START ')
+			mutating_current_vector_methods = mutating_current_vector_methods.replace('/* SWIFT_TO_RUST_END */', 'SWIFT_TO_RUST_END */')
+
 		mutating_current_vector_methods = mutating_current_vector_methods.replace('SwiftPrimitive',
-																				  vector_type_details.primitive_swift_counterpart)
+																				  swift_primitive)
+
 		self.filled_template += "\n"+mutating_current_vector_methods+"\n"

src/lightning_header_parser.py

@@ -24,10 +24,12 @@ def __init__(self) -> None:
 		self.type = CTypes.OPAQUE_STRUCT
 		self.fields = []
 		self.methods = []
+		self.lambdas = []
 		self.constructor_method = None
 		self.free_method = None
 		self.is_primitive = False
 		self.primitive_swift_counterpart = None
+		self.iteratee = None
 
 
 class LightningHeaderParser():
@@ -256,18 +258,22 @@ def populate_type_details(self):
 					elif vec_ty is not None:
 						# TODO: vector type (each one needs to be mapped)
 						self.vec_types.add(struct_name)
-						vector_type_details = None
+						# vector_type_details = None
+						vector_type_details = TypeDetails() # iterator type
+						vector_type_details.type = CTypes.VECTOR
+						vector_type_details.name = struct_name
+
 						if vec_ty in self.type_details:
-							vector_type_details = self.type_details[vec_ty]
+							vectored_type_details = self.type_details[vec_ty]
+							# vector_type_details.name = struct_name
+							vector_type_details.is_primitive = False
+							vector_type_details.iteratee = vectored_type_details
 						else:
 							# it's a primitive
-							vector_type_details = TypeDetails()
-							vector_type_details.name = vec_ty
-							vector_type_details.type = CTypes.VECTOR
 							vector_type_details.is_primitive = True
 							vector_type_details.primitive_swift_counterpart = self.language_constants.c_type_map[vec_ty]
 						self.type_details[struct_name] = vector_type_details
-						pass
+						# pass
 					elif is_union_enum:
 						assert (struct_name.endswith("_Tag"))
 						struct_name = struct_name[:-4]
@@ -280,11 +286,13 @@ def populate_type_details(self):
 						pass
 					elif is_unitary_enum:
 						self.type_details[struct_name].type = CTypes.UNITARY_ENUM
-						# TODO: unitary enum
+						self.unitary_enums.add(struct_name)
+						# todo: unitary enums are to be used as is
 						pass
 					elif len(trait_fn_lines) > 0:
 						self.trait_structs.add(struct_name)
-					# TODO: trait
+						lambdas = self.parse_lambda_details(trait_fn_lines)
+						current_type_detail.lambdas = lambdas
 					elif struct_name == "LDKTxOut":
 						# TODO: why is this even a special case? It's Swift, we dgaf
 						pass
@@ -296,7 +304,7 @@ def populate_type_details(self):
 							print('irregular byte array struct type:', struct_name)
 							continue
 
-						print('byte array struct:', struct_name)
+						# print('byte array struct:', struct_name)
 						self.byte_arrays.add(struct_name)
 						self.type_details[struct_name].type = CTypes.BYTE_ARRAY
 
@@ -351,6 +359,30 @@ def populate_type_details(self):
 						# self.global_methods.add(method_details)
 						pass
 
+	def parse_lambda_details(self, trait_fn_lines):
+		lambdas = []
+		for fn_line in trait_fn_lines:
+			ret_ty_info = swift_type_mapper.map_types_to_swift(fn_line.group(2).strip() + " ret", None, False,
+															   self.tuple_types, self.unitary_enums,
+															   self.language_constants)
+			is_const = fn_line.group(4) is not None
+
+			arg_tys = []
+			for idx, arg in enumerate(fn_line.group(5).split(',')):
+				if arg == "":
+					continue
+				arg_conv_info = swift_type_mapper.map_types_to_swift(arg, None, False, self.tuple_types,
+																	 self.unitary_enums,
+																	 self.language_constants)
+				arg_tys.append(arg_conv_info)
+			lambdas.append({
+				'name': fn_line.group(3),
+				'is_constant': is_const,
+				'return_type': ret_ty_info,
+				'argument_types': arg_tys
+			})
+		return lambdas
+
 	def parse_function_details(self, line, re_match, ret_arr_len, c_call_string):
 		method_return_type = re_match.group(1)
 		method_name = re_match.group(2)