CreateNLSocket and CloseNLSocket should return gpointer (mono#15408)

In managed, these functions are used as `static external IntPtr`. This means that previously on arm64 the top bits of the return value were garbage. A comparison with 64-bit -1 like in LinuxNetworkChange.EnsureSocket would never be true, which was causing us to hit other assertions in the runtime.
Co-authored-by: Aleksey Kliger <[email protected]>

Author: 2 people

support/nl.c

@@ -163,7 +163,7 @@ value_to_name (value2name_t *tbl, int value)
 }
 #endif /* NL_DEBUG */
 
-int
+gpointer
 CreateNLSocket (void)
 {
 	int sock;
@@ -177,22 +177,22 @@ CreateNLSocket (void)
 		ret |= O_NONBLOCK;
 		ret = fcntl (sock, F_SETFL, ret);
 		if (ret < 0)
-			return -1;
+			return GINT_TO_POINTER (-1);
 	}
 
 	memset (&sa, 0, sizeof (sa));
 	if (sock < 0)
-		return -1;
+		return GINT_TO_POINTER (-1);
 	sa.nl_family = AF_NETLINK;
 	sa.nl_pid = getpid ();
 	sa.nl_groups = RTMGRP_IPV4_ROUTE | RTMGRP_IPV6_ROUTE | RTMGRP_NOTIFY;
 	/* RTNLGRP_IPV4_IFADDR | RTNLGRP_IPV6_IFADDR
 	 * RTMGRP_LINK */
 
 	if (bind (sock, (struct sockaddr *) &sa, sizeof (sa)) < 0)
-		return -1;
+		return GINT_TO_POINTER (-1);
 
-	return sock;
+	return GINT_TO_POINTER (sock);
 }
 
 int
@@ -359,10 +359,10 @@ ReadEvents (gpointer sock, gpointer buffer, gint32 count, gint32 size)
 	return result;
 }
 
-int
+gpointer
 CloseNLSocket (gpointer sock)
 {
-	return close (GPOINTER_TO_INT (sock));
+	return GINT_TO_POINTER (close (GPOINTER_TO_INT (sock)));
 }
 #else
 int
@@ -377,16 +377,16 @@ ReadEvents (gpointer sock, gpointer buffer, gint32 count, gint32 size)
 	return 0;
 }
 
-int
+gpointer
 CreateNLSocket (void)
 {
-	return -1;
+	return GINT_TO_POINTER (-1);
 }
 
-int
+gpointer
 CloseNLSocket (gpointer sock)
 {
-	return -1;
+	return GINT_TO_POINTER (-1);
 }
 #endif /* linux/netlink.h + linux/rtnetlink.h */
 

support/nl.h

@@ -3,9 +3,9 @@
 #include <glib.h>
 
 G_BEGIN_DECLS
-int CreateNLSocket (void);
+gpointer CreateNLSocket (void);
 int ReadEvents (gpointer sock, gpointer buffer, gint32 count, gint32 size);
-int CloseNLSocket (gpointer sock);
+gpointer CloseNLSocket (gpointer sock);
 G_END_DECLS
 
 #endif