[object] mono_string_new should not assert on UTF conversion failures (mono#6333)

Revert the embedding API behavior change introduced by dcdfb3c

mono_string_new will:
 * return NULL if the given byte sequence is not a valid UTF-8 sequence
 * assert if there is not enough memory to allocate a new MonoString.

This only changes the behavior of the API function.  The runtime will continue
to use mono_string_new_checked which sets MonoError* for both sorts of
failures.

Author: lambdageek

mono/metadata/object.c

@@ -6298,7 +6298,15 @@ mono_string_new (MonoDomain *domain, const char *text)
 	MonoError error;
 	MonoString *res = NULL;
 	res = mono_string_new_checked (domain, text, &error);
-	mono_error_assert_ok (&error);
+	if (!is_ok (&error)) {
+		/* Mono API compatability: assert on Out of Memory errors,
+		 * return NULL otherwise (most likely an invalid UTF-8 byte
+		 * sequence). */
+		if (mono_error_get_error_code (&error) == MONO_ERROR_OUT_OF_MEMORY)
+			mono_error_assert_ok (&error);
+		else
+			mono_error_cleanup (&error);
+	}
 	return res;
 }
 

mono/utils/mono-error-internals.h

@@ -63,7 +63,7 @@ struct _MonoErrorBoxed {
 void
 mono_error_assert_ok_pos (MonoError *error, const char* filename, int lineno) MONO_LLVM_INTERNAL;
 
-#define mono_error_assert_ok(e) mono_error_assert_ok_pos (e, __FILE__, __LINE__);
+#define mono_error_assert_ok(e) mono_error_assert_ok_pos (e, __FILE__, __LINE__)
 
 void
 mono_error_dup_strings (MonoError *error, gboolean dup_strings);